gate.plugin.learningframework.engines.EngineMBSklearnBase Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of learningframework Show documentation
Show all versions of learningframework Show documentation
A GATE plugin that provides many different machine learning
algorithms for a wide range of NLP-related machine learning tasks like
text classification, tagging, or chunking.
/*
* Copyright (c) 2015-2016 The University Of Sheffield.
*
* This file is part of gateplugin-LearningFramework
* (see https://github.com/GateNLP/gateplugin-LearningFramework).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 2.1 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see finalCommand = new ArrayList<>();
// Instead of loading a model, this establishes a connection with the
// external sklearn process.
if(!"file".equals(directoryURL.getProtocol())) {
throw new GateRuntimeException("The dataDirectory URL must be a file: URL for sklearn");
}
File directory = Files.fileFromURL(directoryURL);
File commandFile = findWrapperCommand(directory, true);
String modelFileName = new File(directory,MODEL_BASENAME).getAbsolutePath();
finalCommand.add(commandFile.getAbsolutePath());
finalCommand.add(modelFileName);
// if we have a shell command prepend that, and if we have shell parms too, include them
if(shellcmd != null) {
finalCommand.add(0,shellcmd);
if(shellparms != null) {
String[] sps = shellparms.trim().split("\\s+");
int i=0; for(String sp : sps) { finalCommand.add(++i,sp); }
}
}
//System.err.println("Running: "+finalCommand);
// Create a fake Model jsut to make LF_Apply... happy which checks if this is null
model = MODEL_INSTANCE;
Map env = new HashMap<>();
env.put(ENV_WRAPPER_HOME, wrapperhome);
process = Process4JsonStream.create(directory,env,finalCommand);
}
@Override
protected void saveModel(File directory) {
// NOTE: we do not need to save the model here because the external
// sklearnWrapper command does this.
// However we still need to make sure a usable info file is saved!
info.engineClass = this.getClass().getName();
info.save(directory);
}
@Override
public void trainModel(File dataDirectory, String instanceType, String parms) {
ArrayList finalCommand = new ArrayList();
// invoke the sklearn wrapper for training
// NOTE: for this the first word in parms must be the full sklearn class name, the rest are parms
if(parms == null || parms.trim().isEmpty()) {
throw new GateRuntimeException(WRAPPER_NAME+": Cannot train, algorithmParameter must contain full algorithm class name as first word");
}
String sklearnClass;
String sklearnParms = "";
parms = parms.trim();
int spaceIdx = parms.indexOf(" ");
if(spaceIdx<0) {
sklearnClass = parms;
} else {
sklearnClass = parms.substring(0,spaceIdx);
sklearnParms = parms.substring(spaceIdx).trim();
}
File commandFile = findWrapperCommand(dataDirectory, false);
// Export the data
// Note: any scaling was already done in the PR before calling this method!
// find out if we train classification or regression
// TODO: NOTE: not sure if classification/regression matters here as long as
// the actual exporter class does the right thing based on the corpus representation!
// was previously:
//Exporter.export(corpusRepresentation,
// Exporter.MatrixMarket2_CL_MR, dataDirectory, instanceType, parms);
corpusExporter.export();
String dataFileName = dataDirectory.getAbsolutePath()+File.separator;
String modelFileName = new File(dataDirectory, MODEL_BASENAME).getAbsolutePath();
finalCommand.add(commandFile.getAbsolutePath());
finalCommand.add(dataFileName);
finalCommand.add(modelFileName);
finalCommand.add(sklearnClass);
if(!sklearnParms.isEmpty()) {
String[] tmp = sklearnParms.trim().split("\\s+",-1);
finalCommand.addAll(Arrays.asList(tmp));
}
// if we have a shell command prepend that, and if we have shell parms too, include them
if(shellcmd != null) {
finalCommand.add(0,shellcmd);
if(shellparms != null) {
String[] sps = shellparms.trim().split("\\s+");
int i=0; for(String sp : sps) { finalCommand.add(++i,sp); }
}
}
//System.err.println("Running: ");
//for(int i=0; i"+finalCommand.get(i)+"<");
//}
// Create a fake Model jsut to make LF_Apply... happy which checks if this is null
model = MODEL_INSTANCE;
Map env = new HashMap<>();
env.put(ENV_WRAPPER_HOME,wrapperhome);
process = ProcessSimple.create(dataDirectory,env,finalCommand);
process.waitFor();
updateInfo();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
info.modelWhenTrained = sdf.format(new Date());
info.algorithmParameters = parms;
info.save(dataDirectory);
featureInfo.save(dataDirectory);
}
@Override
public EvaluationResult evaluate(String algorithmParameters, EvaluationMethod evaluationMethod, int numberOfFolds, double trainingFraction, int numberOfRepeats) {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
@Override
@SuppressWarnings("unchecked")
public List applyModel(AnnotationSet instanceAS, AnnotationSet inputAS,
AnnotationSet sequenceAS, String parms) {
CorpusRepresentationMalletTarget data = (CorpusRepresentationMalletTarget)corpusRepresentation;
data.stopGrowth();
int nrCols = data.getPipe().getDataAlphabet().size();
//System.err.println("Running EngineSklearn.applyModel on document "+instanceAS.getDocument().getName());
List gcs = new ArrayList<>();
LFPipe pipe = (LFPipe)data.getRepresentationMallet().getPipe();
ArrayList classList = null;
// If we have a classification problem, pre-calculate the class label list
if(pipe.getTargetAlphabet() != null) {
classList = new ArrayList<>();
for(int i = 0; i map = new HashMap<>();
map.put("cmd", "CSR1");
ArrayList values = new ArrayList<>();
ArrayList rowinds = new ArrayList<>();
ArrayList colinds = new ArrayList<>();
int rowIndex = 0;
List instances = instanceAS.inDocumentOrder();
for(Annotation instAnn : instances) {
Instance inst = data.extractIndependentFeatures(instAnn, inputAS);
//FeatureVector fv = (FeatureVector)inst.getData();
//System.out.println("Mallet instance, fv: "+fv.toString(true)+", len="+fv.numLocations());
inst = pipe.instanceFrom(inst);
FeatureVector fv = (FeatureVector)inst.getData();
//System.out.println("Mallet instance, fv: "+fv.toString(true)+", len="+fv.numLocations());
// Convert to the sparse vector we use to send to the weka process
int locs = fv.numLocations();
for(int i=0;i response = null;
if(ret instanceof Map) {
@SuppressWarnings("unchecked")
Map tmpresponse = (Map)ret;
response = tmpresponse;
}
if(response == null) {
throw new RuntimeException("Got a response from Sklearn process which cannot be used: "+response);
}
// the response has the following format:
// - status: should be "OK" or an error message
// - targets: a vector of target indices/values
// - probas: if probabilities are supported, a vector of vectors of class probabilities, otherwise null
String status = (String)response.get("status");
if(status == null || !status.equals("OK")) {
throw new RuntimeException("Status of response is not OK but "+status);
}
@SuppressWarnings("unchecked")
ArrayList targets = (ArrayList)response.get("targets");
@SuppressWarnings("unchecked")
ArrayList> probas = (ArrayList>)response.get("probas");
ModelApplication gc;
// now check if the mallet representation and the weka process agree
// on if we have regression or classification
if(pipe.getTargetAlphabet() == null) {
// we expect a regression result, i.e probas should be null
if(probas != null) {
throw new RuntimeException("We think we have regression but the Sklearn process sent probabilities");
}
}
// now go through all the instances again and do the target assignment from the vector(s) we got
int instNr = 0;
for(Annotation instAnn : instances) {
if(pipe.getTargetAlphabet() == null) { // we have regression
gc = new ModelApplication(instAnn, targets.get(instNr));
} else {
int bestlabel = targets.get(instNr).intValue();
String cl
= pipe.getTargetAlphabet().lookupObject(bestlabel).toString();
double bestprob = Double.NaN;
if(probas != null) {
bestprob = Collections.max(probas.get(instNr));
}
gc = new ModelApplication(
instAnn, cl, bestprob, classList, probas.get(instNr));
}
gcs.add(gc);
instNr++;
}
data.startGrowth();
return gcs;
}
@Override
public void initializeAlgorithm(Algorithm algorithm, String parms) {
// do not do anything
}
}