gate.plugin.learningframework.engines.EngineMBWekaWrapper Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of learningframework Show documentation
Show all versions of learningframework Show documentation
A GATE plugin that provides many different machine learning
algorithms for a wide range of NLP-related machine learning tasks like
text classification, tagging, or chunking.
/*
* Copyright (c) 2015-2016 The University Of Sheffield.
*
* This file is part of gateplugin-LearningFramework
* (see https://github.com/GateNLP/gateplugin-LearningFramework).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 2.1 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see finalCommand = new ArrayList<>();
// TODO: for now, we only allow URLs which are file: URLs here.
// This is because the script wrapping Weka is currently not able to access
// the model from any other location. Also, we need to export the
// data and currently this is done into the directoryURL.
// At some later point, we may be able to e.g. copy the model into
// a temporary directory and use the demporary directory also to store
// the data!
File directoryFile = null;
if("file".equals(directoryURL.getProtocol())) {
directoryFile = Files.fileFromURL(directoryURL);
} else {
throw new GateRuntimeException("The dataDirectory for WekaWrapper must be a file: URL");
}
// Instead of loading a model, this establishes a connection with the
// external weka process. For this, we expect an additional file in the
// directory, weka.yaml, which describes how to run the weka wrapper
File commandFile = findWrapperCommand(directoryFile, true);
// If the directoryURL
String modelFileName = new File(directoryFile,FILENAME_MODEL).getAbsolutePath();
if(!new File(modelFileName).exists()) {
throw new GateRuntimeException("File not found: "+modelFileName);
}
String header = new File(directoryFile,"header.arff").getAbsolutePath();
if(!new File(header).exists()) {
throw new GateRuntimeException("File not found: "+header);
}
if(shellcmd != null) {
finalCommand.add(shellcmd);
if(shellparms != null) {
String[] sps = shellparms.trim().split("\\s+");
for(String sp : sps) { finalCommand.add(sp); }
}
}
finalCommand.add(commandFile.getAbsolutePath());
finalCommand.add(modelFileName);
finalCommand.add(header);
//System.err.println("Running: "+finalCommand);
// Create a fake Model jsut to make LF_Apply... happy which checks if this is null
model = "ExternalWekaWrapperModel";
Map env = new HashMap<>();
env.put(ENV_WRAPPER_HOME,wrapperhome);
// NOTE: if the directoryFile is null, the current Java process' directory is used
process = Process4ObjectStream.create(directoryFile,env,finalCommand);
}
@Override
protected void saveModel(File directory) {
// NOTE: we do not need to save the model here because the external
// WekaWrapper command does this.
// However we still need to make sure a usable info file is saved!
info.engineClass = EngineMBWekaWrapper.class.getName();
info.save(directory);
}
@Override
public void trainModel(File dataDirectory, String instanceType, String parms) {
ArrayList finalCommand = new ArrayList<>();
// TODO: invoke the weka wrapper
// NOTE: for this the first word in parms must be the full weka class name, the rest are parms
if(parms == null || parms.trim().isEmpty()) {
throw new GateRuntimeException("Cannot train using WekaWrapper, algorithmParameter must contain Weka algorithm class as first word");
}
String wekaClass;
String wekaParms = "";
parms = parms.trim();
int spaceIdx = parms.indexOf(" ");
if(spaceIdx<0) {
wekaClass = parms;
} else {
wekaClass = parms.substring(0,spaceIdx);
wekaParms = parms.substring(spaceIdx).trim();
}
File commandFile = findWrapperCommand(dataDirectory, false);
// Export the data
// Note: any scaling was already done in the PR before calling this method!
// find out if we train classification or regression
// NOTE: not sure if classification/regression matters here as long as
// the actual exporter class does the right thing based on the corpus representation!
// Was previously:
//Exporter.export(corpusRepresentation,
// Exporter.ARFF_CL_MR, dataDirectory, instanceType, parms);
corpusExporter.export();
String dataFileName = new File(dataDirectory,Globals.dataBasename+".arff").getAbsolutePath();
String modelFileName = new File(dataDirectory, FILENAME_MODEL).getAbsolutePath();
if(shellcmd != null) {
finalCommand.add(shellcmd);
if(shellparms != null) {
String[] sps = shellparms.trim().split("\\s+");
for(String sp : sps) { finalCommand.add(sp); }
}
}
finalCommand.add(commandFile.getAbsolutePath());
finalCommand.add(dataFileName);
finalCommand.add(modelFileName);
finalCommand.add(wekaClass);
if(!wekaParms.isEmpty()) {
String[] tmp = wekaParms.split("\\s+",-1);
finalCommand.addAll(Arrays.asList(tmp));
}
// Create a fake Model jsut to make LF_Apply... happy which checks if this is null
model = "ExternalWekaWrapperModel";
model = "ExternalWekaWrapperModel";
Map env = new HashMap<>();
process = ProcessSimple.create(dataDirectory,env,finalCommand);
process.waitFor();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
info.modelWhenTrained = sdf.format(new Date());
info.save(dataDirectory);
featureInfo.save(dataDirectory);
}
@Override
public EvaluationResult evaluate(String algorithmParameters, EvaluationMethod evaluationMethod, int numberOfFolds, double trainingFraction, int numberOfRepeats) {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
@Override
public List applyModel(AnnotationSet instanceAS, AnnotationSet inputAS,
AnnotationSet sequenceAS, String parms) {
CorpusRepresentationMalletTarget data = (CorpusRepresentationMalletTarget)corpusRepresentation;
data.stopGrowth();
//System.err.println("Running EngineWeka.applyModel on document "+instanceAS.getDocument().getName());
List gcs = new ArrayList<>();
LFPipe pipe = (LFPipe)data.getRepresentationMallet().getPipe();
for(Annotation instAnn : instanceAS.inDocumentOrder()) {
Instance inst = data.extractIndependentFeatures(instAnn, inputAS);
//FeatureVector fv = (FeatureVector)inst.getData();
//System.out.println("Mallet instance, fv: "+fv.toString(true)+", len="+fv.numLocations());
inst = pipe.instanceFrom(inst);
FeatureVector fv = (FeatureVector)inst.getData();
//System.out.println("Mallet instance, fv: "+fv.toString(true)+", len="+fv.numLocations());
double weight = Double.NaN;
Object weightObj = inst.getProperty("instanceWeight");
if(weightObj != null) {
weight = (double)weightObj;
}
// Convert to the sparse vector we use to send to the weka process
int locs = fv.numLocations();
SparseDoubleVector sdv = new SparseDoubleVector(locs);
sdv.setInstanceWeight(weight);
int[] locations = sdv.getLocations();
double[] values = sdv.getValues();
for(int i=0;i= 2
if(ret.length < 2) {
throw new RuntimeException("We think we have classification but Weka process sent a ret of length "+ret.length);
}
double bestprob = 0.0;
int bestlabel = 0;
/*
System.err.print("DEBUG: got classes from pipe: ");
Object[] cls = pipe.getTargetAlphabet().toArray();
boolean first = true;
for(Object cl : cls) {
if(first) { first = false; } else { System.err.print(", "); }
System.err.print(">"+cl+"<");
}
System.err.println();
*/
List classList = new ArrayList<>();
List confidenceList = new ArrayList<>();
for (int i = 0; i < ret.length; i++) {
int thislabel = i;
double thisprob = ret[i];
String labelstr = pipe.getTargetAlphabet().lookupObject(thislabel).toString();
classList.add(labelstr);
confidenceList.add(thisprob);
if (thisprob > bestprob) {
bestlabel = thislabel;
bestprob = thisprob;
}
} // end for i < predictionDistribution.length
String cl
= pipe.getTargetAlphabet().lookupObject(bestlabel).toString();
gc = new ModelApplication(
instAnn, cl, bestprob, classList, confidenceList);
}
gcs.add(gc);
}
data.startGrowth();
return gcs;
}
@Override
public void initializeAlgorithm(Algorithm algorithm, String parms) {
// do not do anything
}
}