gate.plugin.learningframework.engines.EngineMBMalletSeq Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of learningframework Show documentation
Show all versions of learningframework Show documentation
A GATE plugin that provides many different machine learning
algorithms for a wide range of NLP-related machine learning tasks like
text classification, tagging, or chunking.
/*
* Copyright (c) 2015-2016 The University Of Sheffield.
*
* This file is part of gateplugin-LearningFramework
* (see https://github.com/GateNLP/gateplugin-LearningFramework).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 2.1 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see applyModel(
AnnotationSet instanceAS, AnnotationSet inputAS, AnnotationSet sequenceAS,
String parms) {
// stop growth
CorpusRepresentationMalletSeq data = (CorpusRepresentationMalletSeq)corpusRepresentation;
data.stopGrowth();
List gcs = new ArrayList<>();
Transducer crf = (Transducer)model;
for(Annotation sequenceAnn : sequenceAS) {
int sequenceSpanId = sequenceAnn.getId();
Instance inst = data.getInstanceForSequence(
instanceAS, sequenceAnn, inputAS, null, null, TargetType.NONE, null, null);
//Always put the instance through the same pipe used for training.
inst = crf.getInputPipe().instanceFrom(inst);
SumLatticeDefault sl = new SumLatticeDefault(crf,
(FeatureVectorSequence) inst.getData());
List instanceAnnotations = gate.Utils.getContainedAnnotations(
instanceAS, sequenceAnn).inDocumentOrder();
//Sanity check that we're mapping the probs back onto the right anns.
//This being wrong might follow from errors reading in the data to mallet inst.
if (instanceAnnotations.size() != ((FeatureVectorSequence) inst.getData()).size()) {
LOGGER.warn("LearningFramework: CRF output length: "
+ ((FeatureVectorSequence) inst.getData()).size()
+ ", GATE instances: " + instanceAnnotations.size()
+ ". Can't assign.");
} else {
int i = 0;
for (Annotation instanceAnn : instanceAnnotations) {
i++;
String bestLabel = null;
double bestProb = 0.0;
//For each label option ..
// NOTE: for CRF we had this code:
//for (int j = 0; j < crf.getOutputAlphabet().size(); j++) {
// String label = crf.getOutputAlphabet().lookupObject(j).toString();
// but for Transducer we do not have the getOutputAlphabet method so we use
// model.getInputPipe().getTargetAlphabet() instead (this seems to be what
// is used inside CRF anyway.)
for (int j = 0; j < crf.getInputPipe().getTargetAlphabet().size(); j++) {
String label = crf.getInputPipe().getTargetAlphabet().lookupObject(j).toString();
//Get the probability of being in state j at position i+1
//Note that the plus one is because the labels are on the
//transitions. Positions are between transitions.
double marg = sl.getGammaProbability(i, crf.getState(j));
if (marg > bestProb) {
bestLabel = label;
bestProb = marg;
}
}
ModelApplication gc = new ModelApplication(
instanceAnn, bestLabel, bestProb, sequenceSpanId);
gcs.add(gc);
}
}
}
data.startGrowth();
return gcs;
}
@Override
protected void loadModel(URL directory, String parms) {
URL modelFile = newURL(directory, FILENAME_MODEL);
Transducer classifier;
try (InputStream is = modelFile.openStream();
ObjectInputStream ois = new ObjectInputStream(is)) {
// TODO: check cast!
classifier = (CRF) ois.readObject();
model=classifier;
} catch (Exception ex) {
throw new GateRuntimeException("Could not load Mallet model", ex);
}
}
@Override
// NOTE: this evaluates only the classification problem generated from the original chunking problem,
// so as for classification, we get accuracy estimates, not precision/recall/F-measure.
// We do not have anything in the LearningFramework for doing F-measure evaluation, this has to
// be done outside of the LF in some kind of wrapper or script that invokes the proper LF methods.
public EvaluationResult evaluate(String algorithmParameters, EvaluationMethod evaluationMethod, int numberOfFolds, double trainingFraction, int numberOfRepeats) {
EvaluationResult ret = null;
Parms parms = new Parms(algorithmParameters,"s:seed:i");
int seed = (Integer)parms.getValueOrElse("seed", 1);
if(evaluationMethod == EvaluationMethod.CROSSVALIDATION) {
InstanceList.CrossValidationIterator cvi = corpusRepresentation.getRepresentationMallet().crossValidationIterator(numberOfFolds, seed);
if(algorithm instanceof AlgorithmClassification) {
double sumOfAccs = 0.0;
while(cvi.hasNext()) {
InstanceList[] il = cvi.nextSplit();
InstanceList trainSet = il[0];
InstanceList testSet = il[1];
Transducer crf = trainModel(trainSet, algorithmParameters);
sumOfAccs += crf.averageTokenAccuracy(testSet);
}
EvaluationResultClXval e = new EvaluationResultClXval();
// e.internalEvaluationResult = null;
e.accuracyEstimate = sumOfAccs/numberOfFolds;
e.nrFolds = numberOfFolds;
ret = e;
} else {
throw new GateRuntimeException("Mallet evaluation: not available for regression!");
}
} else {
if(algorithm instanceof AlgorithmClassification) {
Random rnd = new Random(seed);
double sumOfAccs = 0.0;
for(int i = 0; i