weka.knowledgeflow.steps.Appender Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of weka-dev Show documentation
Show all versions of weka-dev Show documentation
The Waikato Environment for Knowledge Analysis (WEKA), a machine
learning workbench. This version represents the developer version, the
"bleeding edge" of development, you could say. New functionality gets added
to this version.
/*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see m_completed;
/** Handles on temp files used to store batches of instances in batch mode */
protected Map m_tempBatchFiles;
/** Used to hold the final header in the case of incremental operation */
protected Instances m_completeHeader;
/** Gets decremented for each incoming instance stream that has finished */
protected AtomicInteger m_streamingCountDown;
/**
* Holds savers used for incrementally saving incoming instance streams. After
* we've seen the structure from each incoming connection we can create the
* final output structure, pull any saved instances from the temp files and
* discard these savers as they will no longer be needed.
*/
protected transient Map m_incrementalSavers;
/** Holds a files in play for incremental incoming connections */
protected transient Map m_incrementalFiles;
/** Re-usable data object for streaming mode */
protected Data m_streamingData;
/** True if this step has been reset */
protected boolean m_isReset;
/**
* Initialize the step
*
* @throws WekaException if a problem occurs
*/
@Override
public void stepInit() throws WekaException {
m_isReset = true;
m_completed = new HashMap();
m_tempBatchFiles = new HashMap();
m_completeHeader = null;
m_incrementalSavers = new HashMap();
m_incrementalFiles = new HashMap();
m_streamingCountDown = new AtomicInteger(
getStepManager().numIncomingConnectionsOfType(StepManager.CON_INSTANCE));
m_streamingData = new Data(StepManager.CON_INSTANCE);
}
/**
* Get the incoming connection types accepted by this step at this time
*
* @return a list of incoming connection types
*/
@Override
public List getIncomingConnectionTypes() {
List result = new ArrayList();
if (getStepManager().numIncomingConnections() == 0 || getStepManager()
.numIncomingConnectionsOfType(StepManager.CON_INSTANCE) == 0) {
result.addAll(Arrays.asList(StepManager.CON_DATASET,
StepManager.CON_TRAININGSET, StepManager.CON_TESTSET));
}
if (getStepManager().numIncomingConnections() == 0 || getStepManager()
.numIncomingConnectionsOfType(StepManager.CON_INSTANCE) > 0) {
result.add(StepManager.CON_INSTANCE);
}
return result;
}
/**
* Get a list of outgoing connection types that this step can produce at this
* time
*
* @return a list of outgoing connection types
*/
@Override
public List getOutgoingConnectionTypes() {
List result = new ArrayList();
if (getStepManager()
.numIncomingConnectionsOfType(StepManager.CON_INSTANCE) > 0) {
result.add(StepManager.CON_INSTANCE);
} else {
result.addAll(Arrays.asList(StepManager.CON_DATASET,
StepManager.CON_TRAININGSET, StepManager.CON_TESTSET));
}
return result;
}
/**
* Process an incoming data payload (if the step accepts incoming connections)
*
* @param data the data to process
* @throws WekaException if a problem occurs
*/
@Override
public void processIncoming(Data data) throws WekaException {
if (m_isReset
&& !data.getConnectionName().equals(StepManager.CON_INSTANCE)) {
getStepManager().processing();
m_isReset = false;
}
if (data.getConnectionName().equals(StepManager.CON_INSTANCE)) {
processStreaming(data);
if (m_streamingCountDown.get() == 0) {
// all done
m_streamingData.clearPayload();
getStepManager().throughputFinished(m_streamingData);
}
} else {
processBatch(data);
if (m_completed.size() == getStepManager().numIncomingConnections()) {
// done
getStepManager().finished();
// save memory
m_completed.clear();
m_tempBatchFiles.clear();
}
}
if (isStopRequested()) {
getStepManager().interrupted();
// save memory
m_completed.clear();
m_tempBatchFiles.clear();
m_incrementalSavers.clear();
m_incrementalFiles.clear();
}
}
/**
* Process batch data
*
* @param data the data to process
* @throws WekaException if a problem occurs
*/
protected synchronized void processBatch(Data data) throws WekaException {
Integer setNum =
data.getPayloadElement(StepManager.CON_AUX_DATA_SET_NUM, 1);
Integer maxSetNum =
data.getPayloadElement(StepManager.CON_AUX_DATA_MAX_SET_NUM, 1);
Instances insts = data.getPrimaryPayload();
if (setNum > 1 || maxSetNum > 1) {
// can't accept more than one dataset/batch from a particular source
throw new WekaException("Source " + data.getSourceStep().getName() + " "
+ "is generating more than one " + data.getConnectionName() + " "
+ "in a batch");
}
Instances header = new Instances(insts, 0);
m_completed.put(data.getSourceStep(), header);
// write these instances (serialized) to a temp file
try {
File tmpF =
File.createTempFile("weka", SerializedInstancesLoader.FILE_EXTENSION);
// tmpF.deleteOnExit();
ObjectOutputStream oos = new ObjectOutputStream(
new BufferedOutputStream(new FileOutputStream(tmpF)));
oos.writeObject(insts);
oos.flush();
oos.close();
m_tempBatchFiles.put(data.getSourceStep(), tmpF);
} catch (IOException e1) {
throw new WekaException(e1);
}
if (isStopRequested()) {
return;
}
// have we seen a dataset from every incoming connection?
if (m_completed.size() == getStepManager().numIncomingConnections()) {
// process all headers and create mongo header for new output.
// missing values will fill columns that don't exist in particular data
// sets
Instances output = makeOutputHeader();
getStepManager().logDetailed("Making output header structure");
try {
for (File f : m_tempBatchFiles.values()) {
ObjectInputStream ois = new ObjectInputStream(
new BufferedInputStream(new FileInputStream(f)));
Instances temp = (Instances) ois.readObject();
ois.close();
// copy each instance over
for (int i = 0; i < temp.numInstances(); i++) {
Instance converted = makeOutputInstance(output, temp.instance(i));
output.add(converted);
}
}
Data outputD = new Data(data.getConnectionName(), output);
outputD.setPayloadElement(StepManager.CON_AUX_DATA_SET_NUM, 1);
outputD.setPayloadElement(StepManager.CON_AUX_DATA_MAX_SET_NUM, 1);
getStepManager().outputData(outputD);
} catch (Exception ex) {
throw new WekaException(ex);
}
}
}
/**
* Process streaming data
*
* @param data the data to process
* @throws WekaException if a problem occurs
*/
protected synchronized void processStreaming(Data data) throws WekaException {
if (isStopRequested()) {
return;
}
Step source = data.getSourceStep();
Instance inst = data.getPrimaryPayload();
if (!m_completed.containsKey(source)) {
m_completed.put(source, inst.dataset());
}
if (m_completed.size() == getStepManager().numIncomingConnections()
&& m_completeHeader == null) {
// create mondo header...
getStepManager().logDetailed("Creating output header structure");
m_completeHeader = makeOutputHeader();
// now check for any buffered instances
if (m_incrementalSavers.size() > 0) {
// read in and convert these instances now
for (Map.Entry e : m_incrementalSavers
.entrySet()) {
// for (ObjectOutputStream s : m_incrementalSavers.values()) {
ObjectOutputStream s = e.getValue();
// finish off the saving process first
try {
// s.writeIncremental(null);
s.flush();
s.close();
// File tmpFile = s.retrieveFile();
File tmpFile = m_incrementalFiles.get(e.getKey());
ObjectInputStream ois = new ObjectInputStream(
new BufferedInputStream(new FileInputStream(tmpFile)));
Instance tmpLoaded = null;
do {
try {
tmpLoaded = (Instance) ois.readObject();
Instance converted =
makeOutputInstance(m_completeHeader, tmpLoaded);
m_streamingData.setPayloadElement(StepManager.CON_INSTANCE,
converted);
getStepManager().outputData(m_streamingData);
} catch (Exception ex) {
// EOF
ois.close();
break;
}
} while (tmpLoaded != null);
/*
* ArffLoader loader = new ArffLoader(); loader.setFile(tmpFile);
* Instances tempStructure = loader.getStructure(); Instance
* tempLoaded = loader.getNextInstance(tempStructure); while
* (tempLoaded != null) { Instance converted =
* makeOutputInstance(m_completeHeader, tempLoaded);
* m_streamingData.setPayloadElement(StepManager.CON_INSTANCE,
* converted); getStepManager().outputData(data);
*
* tempLoaded = loader.getNextInstance(tempStructure); }
*/
} catch (Exception ex) {
throw new WekaException(ex);
}
}
m_incrementalSavers.clear();
m_incrementalFiles.clear();
}
}
if (isStopRequested()) {
return;
}
if (getStepManager().isStreamFinished(data)) {
m_streamingCountDown.decrementAndGet();
return;
}
if (m_completeHeader == null) {
ObjectOutputStream saver = m_incrementalSavers.get(data.getSourceStep());
if (saver == null) {
try {
File tmpFile = File.createTempFile("weka", ".arff");
saver = new ObjectOutputStream(
new BufferedOutputStream(new FileOutputStream(tmpFile)));
m_incrementalSavers.put(data.getSourceStep(), saver);
m_incrementalFiles.put(data.getSourceStep(), tmpFile);
} catch (IOException ex) {
throw new WekaException(ex);
}
}
// ArffSaver saver = m_incrementalSavers.get(data.getSourceStep());
// if (saver == null) {
/*
* saver = new ArffSaver(); try { File tmpFile =
* File.createTempFile("weka", ".arff"); saver.setFile(tmpFile);
* saver.setRetrieval(weka.core.converters.Saver.INCREMENTAL);
* saver.setInstances(new Instances(inst.dataset(), 0));
* m_incrementalSavers.put(data.getSourceStep(), saver); } catch
* (IOException e1) { throw new WekaException(e1); }
*/
try {
// saver.writeIncremental(inst);
saver.writeObject(inst);
} catch (IOException e1) {
throw new WekaException(e1);
}
// }
} else {
Instance newI = makeOutputInstance(m_completeHeader, inst);
m_streamingData.setPayloadElement(StepManager.CON_INSTANCE, newI);
getStepManager().outputData(m_streamingData);
}
}
/**
* Makes an output instance
*
* @param output the structure of the output
* @param source the source instance
* @return an output instance
*/
private Instance makeOutputInstance(Instances output, Instance source) {
double[] newVals = new double[output.numAttributes()];
for (int i = 0; i < newVals.length; i++) {
newVals[i] = Utils.missingValue();
}
for (int i = 0; i < source.numAttributes(); i++) {
if (!source.isMissing(i)) {
Attribute s = source.attribute(i);
int outputIndex = output.attribute(s.name()).index();
if (s.isNumeric()) {
newVals[outputIndex] = source.value(s);
} else if (s.isString()) {
String sVal = source.stringValue(s);
newVals[outputIndex] =
output.attribute(outputIndex).addStringValue(sVal);
} else if (s.isRelationValued()) {
Instances rVal = source.relationalValue(s);
newVals[outputIndex] =
output.attribute(outputIndex).addRelation(rVal);
} else if (s.isNominal()) {
String nomVal = source.stringValue(s);
newVals[outputIndex] =
output.attribute(outputIndex).indexOfValue(nomVal);
}
}
}
Instance newInst = new DenseInstance(source.weight(), newVals);
newInst.setDataset(output);
return newInst;
}
/**
* Create the structure of the output
*
* @return the structure of the output as a header-only set of instances
* @throws WekaException if a problem occurs
*/
protected Instances makeOutputHeader() throws WekaException {
return makeOutputHeader(m_completed.values());
}
/**
* Create the structure of the output given a collection of input structures
*
* @param headers a collection of incoming instance structures
* @return the structure of the output as a header-only set of instances
* @throws WekaException if a problem occurs
*/
protected Instances makeOutputHeader(Collection headers)
throws WekaException {
// process each header in turn...
Map attLookup = new HashMap();
List attList = new ArrayList();
Map> nominalLookups =
new HashMap>();
for (Instances h : headers) {
for (int i = 0; i < h.numAttributes(); i++) {
Attribute a = h.attribute(i);
if (!attLookup.containsKey(a.name())) {
attLookup.put(a.name(), a);
attList.add(a);
if (a.isNominal()) {
TreeSet nVals = new TreeSet();
for (int j = 0; j < a.numValues(); j++) {
nVals.add(a.value(j));
}
nominalLookups.put(a.name(), nVals);
}
} else {
Attribute storedVersion = attLookup.get(a.name());
// mismatched types between headers - can't continue
if (storedVersion.type() != a.type()) {
throw new WekaException("Conflicting types for attribute "
+ "name '" + a.name() + "' between incoming " + "instance sets");
}
if (storedVersion.isNominal()) {
Set storedVals = nominalLookups.get(a.name());
for (int j = 0; j < a.numValues(); j++) {
storedVals.add(a.value(j));
}
}
}
}
}
ArrayList finalAttList = new ArrayList();
for (Attribute a : attList) {
Attribute newAtt = null;
if (a.isDate()) {
newAtt = new Attribute(a.name(), a.getDateFormat());
} else if (a.isNumeric()) {
newAtt = new Attribute(a.name());
} else if (a.isRelationValued()) {
newAtt = new Attribute(a.name(), a.relation());
} else if (a.isNominal()) {
Set vals = nominalLookups.get(a.name());
List newVals = new ArrayList();
for (String v : vals) {
newVals.add(v);
}
newAtt = new Attribute(a.name(), newVals);
} else if (a.isString()) {
newAtt = new Attribute(a.name(), (List) null);
}
finalAttList.add(newAtt);
}
return new Instances(
"Appended_" + getStepManager().numIncomingConnections() + "_sets",
finalAttList, 0);
}
/**
* If possible, get the output structure for the named connection type as a
* header-only set of instances. Can return null if the specified connection
* type is not representable as Instances or cannot be determined at present.
*
* @param connectionName the name of the connection to get the output structure for
* @return the output structure or null if it can't be produced
* @throws WekaException if a problem occurs
*/
@Override
public Instances outputStructureForConnectionType(String connectionName)
throws WekaException {
if (getStepManager().numIncomingConnections() > 0) {
List incomingHeaders = new ArrayList();
for (Map.Entry> e : getStepManager()
.getIncomingConnections().entrySet()) {
if (e.getValue().size() > 0) {
String incomingConType = e.getKey();
for (StepManager sm : e.getValue()) {
Instances incomingStruc = getStepManager()
.getIncomingStructureFromStep(sm, incomingConType);
if (incomingStruc == null) {
// can't determine final output structure if any incoming
// structures are null at present
return null;
}
incomingHeaders.add(incomingStruc);
}
}
}
if (incomingHeaders.size() > 0) {
return makeOutputHeader(incomingHeaders);
}
}
return null;
}
}