weka.knowledgeflow.steps.StorePropertiesInEnvironment Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of weka-dev Show documentation
Show all versions of weka-dev Show documentation
The Waikato Environment for Knowledge Analysis (WEKA), a machine
learning workbench. This version represents the developer version, the
"bleeding edge" of development, you could say. New functionality gets added
to this version.
/*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see > m_propsToSetFromIncomingInstances =
new LinkedHashMap<>();
/** True if the structure has been checked */
protected boolean m_structureCheckComplete;
/**
* OK if there is at least one specified attribute in the incoming instance
* structure
*/
protected boolean m_structureOK;
/** Internal string-based representation of property configs */
protected String m_internalRep = "";
protected boolean m_raiseErrorWhenValueMissing;
@ProgrammaticProperty
public void setPropsInternalRep(String rep) {
m_internalRep = rep;
}
public String getPropsInternalRep() {
return m_internalRep;
}
@Override
public void stepInit() throws WekaException {
m_structureCheckComplete = false;
m_structureOK = false;
m_propsToSetFromIncomingInstances = internalDynamicToMap(m_internalRep);
Environment currentEnv =
getStepManager().getExecutionEnvironment().getEnvironmentVariables();
if (currentEnv == null) {
throw new WekaException(
"The execution environment doesn't seem to have any support for variables");
}
if (!(currentEnv instanceof JobEnvironment)) {
currentEnv = new JobEnvironment(currentEnv);
getStepManager().getExecutionEnvironment().setEnvironmentVariables(
currentEnv);
}
if (getStepManager().numIncomingConnections() > 0
&& m_propsToSetFromIncomingInstances.size() == 0) {
getStepManager().logWarning(
"Incoming data detected, but no properties to "
+ "set from incoming instances have been defined.");
}
}
@Override
public void processIncoming(Data data) throws WekaException {
if (!m_structureCheckComplete) {
m_structureCheckComplete = true;
Instances structure = null;
if (data.getConnectionName().equals(StepManager.CON_INSTANCE)) {
structure = ((Instance) data.getPrimaryPayload()).dataset();
} else if (data.getConnectionName().equals(StepManager.CON_ENVIRONMENT)) {
structure =
((Instance) data.getPayloadElement(StepManager.CON_AUX_DATA_INSTANCE))
.dataset();
} else {
structure = data.getPrimaryPayload();
}
checkStructure(structure);
}
getStepManager().processing();
if (data.getConnectionName().equals(StepManager.CON_INSTANCE)
|| data.getConnectionName().equals(StepManager.CON_ENVIRONMENT)) {
if (isStopRequested()) {
getStepManager().interrupted();
return;
}
if (getStepManager().isStreamFinished(data)) {
Data finished = new Data(StepManager.CON_ENVIRONMENT);
if (data.getConnectionName().equals(StepManager.CON_ENVIRONMENT)) {
finished
.setPayloadElement(
StepManager.CON_AUX_DATA_ENVIRONMENT_VARIABLES,
data
.getPayloadElement(StepManager.CON_AUX_DATA_ENVIRONMENT_VARIABLES));
finished
.setPayloadElement(
StepManager.CON_AUX_DATA_ENVIRONMENT_PROPERTIES,
data
.getPayloadElement(StepManager.CON_AUX_DATA_ENVIRONMENT_PROPERTIES));
}
getStepManager().throughputFinished(finished);
return;
}
Instance toProcess =
(Instance) (data.getConnectionName().equals(StepManager.CON_INSTANCE) ? data
.getPrimaryPayload() : data
.getPayloadElement(StepManager.CON_AUX_DATA_INSTANCE));
getStepManager().throughputUpdateStart();
processInstance(toProcess,
data.getConnectionName().equals(StepManager.CON_ENVIRONMENT) ? data
: null);
getStepManager().throughputUpdateEnd();
} else {
Instances insts = data.getPrimaryPayload();
for (int i = 0; i < insts.numInstances(); i++) {
if (isStopRequested()) {
break;
}
processInstance(insts.instance(i), null);
Data finished = new Data(StepManager.CON_ENVIRONMENT);
getStepManager().throughputFinished(finished);
}
if (isStopRequested()) {
getStepManager().interrupted();
}
}
}
protected void processInstance(Instance inst, Data existingEnv)
throws WekaException {
Map> props = new HashMap<>();
for (Map.Entry> e : m_propsToSetFromIncomingInstances
.entrySet()) {
String attName = environmentSubstitute(e.getKey());
Attribute current = inst.dataset().attribute(attName);
int index = -1;
if (current != null) {
index = current.index();
} else {
// try as a 1-based index
try {
index = Integer.parseInt(attName);
index--; // make zero-based
} catch (NumberFormatException ex) {
// ignore
}
}
if (index != -1) {
String stepName = environmentSubstitute(e.getValue().get(0));
String propToSet = environmentSubstitute(e.getValue().get(1));
String val = environmentSubstitute(e.getValue().get(2));
if (inst.isMissing(index)) {
if (val.length() == 0 && m_raiseErrorWhenValueMissing) {
throw new WekaException("Value of attribute '"
+ inst.attribute(index).name()
+ "' was missing in current instance and no default value has "
+ "been specified");
}
} else {
val = inst.stringValue(index);
}
Map propsForStep = props.get(stepName);
if (propsForStep == null) {
propsForStep = new LinkedHashMap<>();
props.put(stepName, propsForStep);
}
propsForStep.put(propToSet, val);
getStepManager().logDebug(
"Storing property '" + propToSet + "' for step " + "'" + stepName
+ "' with value '" + val + "'");
}
}
JobEnvironment env =
(JobEnvironment) getStepManager().getExecutionEnvironment()
.getEnvironmentVariables();
env.addToStepProperties(props);
if (existingEnv != null) {
Map> existingProps =
existingEnv
.getPayloadElement(StepManager.CON_AUX_DATA_ENVIRONMENT_PROPERTIES);
if (existingProps != null) {
props.putAll(existingProps);
}
}
Data output = new Data(StepManager.CON_ENVIRONMENT);
output.setPayloadElement(StepManager.CON_AUX_DATA_ENVIRONMENT_PROPERTIES,
props);
if (existingEnv != null) {
output.setPayloadElement(StepManager.CON_AUX_DATA_ENVIRONMENT_VARIABLES,
existingEnv
.getPayloadElement(StepManager.CON_AUX_DATA_ENVIRONMENT_VARIABLES));
}
output.setPayloadElement(StepManager.CON_AUX_DATA_INSTANCE, inst);
output.setPayloadElement(StepManager.CON_AUX_DATA_IS_INCREMENTAL, true);
getStepManager().outputData(output);
}
protected void checkStructure(Instances structure) {
List notFoundInIncoming = new ArrayList<>();
for (String attName : m_propsToSetFromIncomingInstances.keySet()) {
if (structure.attribute(attName) == null) {
notFoundInIncoming.add(attName);
} else {
m_structureOK = true;
}
}
if (notFoundInIncoming.size() == m_propsToSetFromIncomingInstances.size()) {
getStepManager().logWarning(
"None of the specified attributes appear to be "
+ "in the incoming instance structure");
return;
}
for (String s : notFoundInIncoming) {
getStepManager().logWarning(
"Attribute '" + s + "' was not found in the "
+ "incoming instance structure");
}
}
@Override
public List getIncomingConnectionTypes() {
if (getStepManager().numIncomingConnections() == 0) {
return Arrays.asList(StepManager.CON_DATASET,
StepManager.CON_TRAININGSET, StepManager.CON_TESTSET,
StepManager.CON_INSTANCE, StepManager.CON_ENVIRONMENT);
}
return new ArrayList<>();
}
@Override
public List getOutgoingConnectionTypes() {
if (getStepManager().numIncomingConnections() != 0) {
return Arrays.asList(StepManager.CON_ENVIRONMENT);
}
return new ArrayList<>();
}
/**
* Return the fully qualified name of a custom editor component (JComponent)
* to use for editing the properties of the step. This method can return null,
* in which case the system will dynamically generate an editor using the
* GenericObjectEditor
*
* @return the fully qualified name of a step editor component
*/
@Override
public String getCustomEditorForStep() {
return "weka.gui.knowledgeflow.steps.StorePropertiesInEnvironmentStepEditorDialog";
}
public static Map> internalDynamicToMap(
String internalRep) {
Map> propsToSet = new LinkedHashMap<>();
if (internalRep != null && internalRep.length() > 0) {
String[] parts = internalRep.split(SEP1);
for (String p : parts) {
String[] attVal = p.split(SEP2);
if (attVal.length == 4) {
String attName = attVal[0].trim();
String stepName = attVal[1].trim();
String propName = attVal[2].trim();
String defVal = attVal[3].trim();
if (attName.length() > 0 && stepName.length() > 0) {
List stepAndDefL = new ArrayList<>();
stepAndDefL.add(stepName);
stepAndDefL.add(propName);
stepAndDefL.add(defVal);
propsToSet.put(attName, stepAndDefL);
}
}
}
}
return propsToSet;
}
}