io.github.clearwsd.app.VerbNetClassifierUtils Maven / Gradle / Ivy
/*
* Copyright (C) 2017 James Gung
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see clusters = Lists.newArrayList("cluster-100", "cluster-320", "cluster-1000", "cluster-3200",
"cluster-10000");
static List CLUSTERS
= Arrays.asList("cluster-100", "cluster-320", "cluster-1000", "cluster-3200", "cluster-10000");
static String BROWN = "brown";
static List, DepNode>> windowUnigrams() {
return Arrays.asList(
window(0),
window(-1, 0),
window(0, 1),
window(-1, 1),
window(-1, 0, 1),
window(-2, -1, 0, 1),
window(-1, 0, 1, 2),
window(-2, -1, 1, 2),
window(-2, -1, 0, 1, 2)
);
}
static List, DepNode>> collocations() {
return Arrays.asList(
new CompositeContextFactory<>(
window(true, -2, -1),
window(true, 1, 2),
window(true, -1, 0, 1),
window(true, -3, -2, -1),
window(true, -2, -1, 0, 1),
window(true, -1, 0, 1, 2),
window(true, 1, 2, 3)
),
new CompositeContextFactory<>(
window(true, -2, -1),
window(true, 1, 2),
window(true, -1, 0, 1),
window(true, -3, -2, -1),
window(true, 1, 2, 3)
),
new CompositeContextFactory<>(
window(true, -2, -1),
window(true, 1, 2),
window(true, -1, 1)
),
new CompositeContextFactory<>(
window(true, -2, -1),
window(true, 1, 2)
),
new CompositeContextFactory<>(
window(true, -1, 0),
window(true, 0, 1)
)
);
}
static List, DepNode>> filteredContexts(int level) {
return Stream.of(
new DepChildrenContextFactory("dobj", "iobj", "nmod", "xcomp", "advmod"),
new DepChildrenContextFactory("dobj", "iobj", "nmod", "xcomp", "advmod", "nsubj", "nsubjpass"),
new DepChildrenContextFactory("dobj", "iobj", "nmod", "xcomp", "advmod", "nsubj", "nsubjpass", "advcl"),
new DepChildrenContextFactory("dobj", "nmod", "xcomp", "iobj"),
new DepChildrenContextFactory("dobj", "nmod"),
new DepChildrenContextFactory("dobj")
).map(f -> f.level(level)).collect(Collectors.toList());
}
private static FeatureFunctionFactory> getFactory() {
List, DepNode>> windowUnigrams = windowUnigrams();
List, DepNode>> windowBigrams = collocations();
List, DepNode>> allDeps =
singletonList(excludingDeps(Sets.newHashSet("punct")));
List, DepNode>> depContexts = filteredContexts(0);
NlpContextFactory, DepNode> rootPath = new RootPathContextFactory();
NlpContextFactory, DepNode> head = head();
StringExtractor text = lowerForm();
StringExtractor lemma = lowerLemma();
StringExtractor dep = lookup(Dep);
StringExtractor pos = lookup(Pos);
NlpFeaturePipelineFactory, DepNode> featureFunctionFactory = new NlpFeaturePipelineFactory<>(0);
featureFunctionFactory.addBias();
featureFunctionFactory.addFeatureFunctionFactory(windowUnigrams, text, true)
.addFeatureFunctionFactory(windowUnigrams, pos, true)
.addFeatureFunctionFactory(windowUnigrams, lemma, true)
.addFeatureFunctionFactory(windowUnigrams, dep, true);
featureFunctionFactory.addFeatureFunctionFactory(windowBigrams, text, true)
.addFeatureFunctionFactory(windowBigrams, pos, true)
.addFeatureFunctionFactory(windowBigrams, lemma, true)
.addFeatureFunctionFactory(windowBigrams, dep, true);
StringExtractor textDep = concat(text, dep);
StringExtractor posDep = concat(pos, dep);
StringExtractor lemmaDep = concat(lemma, dep);
featureFunctionFactory.addFeatureFunctionFactory(allDeps, textDep, true)
.addFeatureFunctionFactory(allDeps, posDep, true)
.addFeatureFunctionFactory(allDeps, lemmaDep, true)
.addFeatureFunctionFactory(rootPath, pos, true)
.addFeatureFunctionFactory(rootPath, dep, true)
.addFeatureFunctionFactory(rootPath, lemma, true)
.addFeatureFunctionFactory(head, pos, true)
.addFeatureFunctionFactory(head, dep, true)
.addFeatureFunctionFactory(head, lemma, true);
StringListExtractor wn = listConcat(listLookup(WN_KEY), dep);
StringListExtractor ddn = listConcat(listLookup(DDN_KEY), dep);
StringListExtractor brown = listConcat(listLookup(BROWN), dep);
StringListExtractor cluster100 = listConcat(listLookup(CLUSTERS.get(0)), dep);
StringListExtractor cluster320 = listConcat(listLookup(CLUSTERS.get(1)), dep);
StringListExtractor cluster1000 = listConcat(listLookup(CLUSTERS.get(2)), dep);
StringListExtractor cluster3200 = listConcat(listLookup(CLUSTERS.get(3)), dep);
StringListExtractor cluster10000 = listConcat(listLookup(CLUSTERS.get(4)), dep);
featureFunctionFactory
.addMultiFeatureFunctionFactory(depContexts, wn, true)
.addMultiFeatureFunctionFactory(depContexts, ddn, true)
.addMultiFeatureFunctionFactory(depContexts, brown, true)
.addMultiFeatureFunctionFactory(depContexts, cluster100, true)
.addMultiFeatureFunctionFactory(depContexts, cluster320, true)
.addMultiFeatureFunctionFactory(depContexts, cluster1000, true)
.addMultiFeatureFunctionFactory(depContexts, cluster3200, true)
.addMultiFeatureFunctionFactory(depContexts, cluster10000, true);
return featureFunctionFactory;
}
public static List>> annotators() {
List>> annotators = new ArrayList<>();
for (String cluster : clusters) {
annotators.add(new ListAnnotator<>(cluster, lowerForm()));
}
annotators.add(new ListAnnotator<>(BWC_KEY, form()));
annotators.add(new DepNodeListAnnotator<>(DDN_KEY, new DepChildrenContextFactory("dobj")));
annotators.add(new DepNodeListAnnotator<>(WN_KEY));
return annotators;
}
static FeatureResourceManager resourceManager() {
FeatureResourceManager resources = new DefaultFeatureResourceManager();
resources.registerInitializer(WN_KEY, new WordNetInitializer<>());
resources.registerInitializer(DDN_KEY, new DefaultTsvResourceInitializer(DDN_KEY, getURL(DDN_PATH))
.mappingFunction(lemma()));
resources.registerInitializer(BWC_KEY, new BrownClusterResourceInitializer<>(BWC_KEY, getURL(BWC_PATH)));
for (String cluster : clusters) {
resources.registerInitializer(cluster, new DefaultTsvResourceInitializer<>(cluster, getURL(CLUSTER_PATH + cluster))
.keyFunction(new LowercaseFunction()));
}
return resources;
}
private static URL getURL(String path) {
return VerbNetClassifierUtils.class.getClassLoader().getResource(path);
}
public static void main(String[] args) throws Throwable {
List> instances = new VerbNetReader().readInstances(new FileInputStream(args[0]));
List> devInstances = new VerbNetReader().readInstances(new FileInputStream(args[1]));
List> testInstances = new VerbNetReader().readInstances(new FileInputStream(args[2]));
AggregateAnnotator> annotator = new AggregateAnnotator<>(VerbNetClassifierUtils.annotators());
annotator.initialize(resourceManager());
instances.forEach(annotator::annotate);
devInstances.forEach(annotator::annotate);
testInstances.forEach(annotator::annotate);
FeatureFunctionFactory> factory = getFactory();
MultiClassifier, String> multi = new MultiClassifier<>(
(Serializable & Function, String>)
(i) -> i.focus().feature(FeatureType.Predicate.name()),
(Serializable & Supplier, String>>)
() -> new MetaModelTrainer<>(factory, LibLinearClassifier::new));
WordSenseClassifier classifier = new WordSenseClassifier(multi, new CountingSenseInventory(), new LemmaDictionary());
CrossValidation> cv = new CrossValidation<>(
(NlpFocus i) -> i.feature(FeatureType.Gold));
List>> folds = cv.createFolds(instances, 5, 0.8);
Evaluation overall = new Evaluation(cv.crossValidate(classifier, folds));
log.info("\n{}", overall.toString());
classifier.train(instances, devInstances);
Evaluation evaluation = new Evaluation();
for (NlpFocus instance : devInstances) {
evaluation.add(classifier.classify(instance), instance.feature(Gold));
}
log.info("\n{}", evaluation.toString());
evaluation = new Evaluation();
for (NlpFocus instance : testInstances) {
evaluation.add(classifier.classify(instance), instance.feature(Gold));
}
log.info("\n{}", evaluation.toString());
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy