org.apache.sysml.api.mlcontext.MLContextUtil Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of systemml Show documentation
Show all versions of systemml Show documentation
Declarative Machine Learning
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.sysml.api.mlcontext;
import java.io.FileNotFoundException;
import java.net.URL;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Scanner;
import java.util.Set;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.text.WordUtils;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.linalg.VectorUDT;
import org.apache.spark.rdd.RDD;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.parser.ParseException;
import org.apache.sysml.parser.Statement;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.caching.FrameObject;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.instructions.cp.BooleanObject;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.DoubleObject;
import org.apache.sysml.runtime.instructions.cp.IntObject;
import org.apache.sysml.runtime.instructions.cp.StringObject;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
/**
* Utility class containing methods for working with the MLContext API.
*
*/
public final class MLContextUtil {
/**
* Basic data types supported by the MLContext API
*/
@SuppressWarnings("rawtypes")
public static final Class[] BASIC_DATA_TYPES = { Integer.class, Boolean.class, Double.class, String.class };
/**
* Complex data types supported by the MLContext API
*/
@SuppressWarnings("rawtypes")
public static final Class[] COMPLEX_DATA_TYPES = { JavaRDD.class, RDD.class, DataFrame.class,
BinaryBlockMatrix.class, BinaryBlockFrame.class, Matrix.class, Frame.class, (new double[][] {}).getClass(),
MatrixBlock.class, URL.class };
/**
* All data types supported by the MLContext API
*/
@SuppressWarnings("rawtypes")
public static final Class[] ALL_SUPPORTED_DATA_TYPES = (Class[]) ArrayUtils.addAll(BASIC_DATA_TYPES,
COMPLEX_DATA_TYPES);
/**
* Compare two version strings (ie, "1.4.0" and "1.4.1").
*
* @param versionStr1
* First version string.
* @param versionStr2
* Second version string.
* @return If versionStr1 is less than versionStr2, return {@code -1}. If
* versionStr1 equals versionStr2, return {@code 0}. If versionStr1
* is greater than versionStr2, return {@code 1}.
* @throws MLContextException
* if versionStr1 or versionStr2 is {@code null}
*/
private static int compareVersion(String versionStr1, String versionStr2) {
if (versionStr1 == null) {
throw new MLContextException("First version argument to compareVersion() is null");
}
if (versionStr2 == null) {
throw new MLContextException("Second version argument to compareVersion() is null");
}
Scanner scanner1 = null;
Scanner scanner2 = null;
try {
scanner1 = new Scanner(versionStr1);
scanner2 = new Scanner(versionStr2);
scanner1.useDelimiter("\\.");
scanner2.useDelimiter("\\.");
while (scanner1.hasNextInt() && scanner2.hasNextInt()) {
int version1 = scanner1.nextInt();
int version2 = scanner2.nextInt();
if (version1 < version2) {
return -1;
} else if (version1 > version2) {
return 1;
}
}
return scanner1.hasNextInt() ? 1 : 0;
} finally {
scanner1.close();
scanner2.close();
}
}
/**
* Determine whether the Spark version is supported.
*
* @param sparkVersion
* Spark version string (ie, "1.5.0").
* @return {@code true} if Spark version supported; otherwise {@code false}.
*/
public static boolean isSparkVersionSupported(String sparkVersion) {
return compareVersion(sparkVersion, MLContext.SYSTEMML_MINIMUM_SPARK_VERSION) >= 0;
}
/**
* Check that the Spark version is supported. If it isn't supported, throw
* an MLContextException.
*
* @param sc
* SparkContext
* @throws MLContextException
* thrown if Spark version isn't supported
*/
public static void verifySparkVersionSupported(SparkContext sc) {
if (!MLContextUtil.isSparkVersionSupported(sc.version())) {
throw new MLContextException(
"SystemML requires Spark " + MLContext.SYSTEMML_MINIMUM_SPARK_VERSION + " or greater");
}
}
/**
* Set default SystemML configuration properties.
*/
public static void setDefaultConfig() {
ConfigurationManager.setGlobalConfig(new DMLConfig());
}
/**
* Set SystemML configuration properties based on a configuration file.
*
* @param configFilePath
* Path to configuration file.
* @throws MLContextException
* if configuration file was not found or a parse exception
* occurred
*/
public static void setConfig(String configFilePath) {
try {
DMLConfig config = new DMLConfig(configFilePath);
ConfigurationManager.setGlobalConfig(config);
} catch (ParseException e) {
throw new MLContextException("Parse Exception when setting config", e);
} catch (FileNotFoundException e) {
throw new MLContextException("File not found (" + configFilePath + ") when setting config", e);
}
}
/**
* Set SystemML compiler configuration properties for MLContext
*/
public static void setCompilerConfig() {
CompilerConfig compilerConfig = new CompilerConfig();
compilerConfig.set(ConfigType.IGNORE_UNSPECIFIED_ARGS, true);
compilerConfig.set(ConfigType.REJECT_READ_WRITE_UNKNOWNS, false);
compilerConfig.set(ConfigType.ALLOW_CSE_PERSISTENT_READS, false);
compilerConfig.set(ConfigType.MLCONTEXT, true);
ConfigurationManager.setGlobalConfig(compilerConfig);
}
/**
* Verify that the types of input values are supported.
*
* @param inputs
* Map of String/Object pairs
* @throws MLContextException
* if an input value type is not supported
*/
public static void checkInputValueTypes(Map inputs) {
for (Entry entry : inputs.entrySet()) {
checkInputValueType(entry.getKey(), entry.getValue());
}
}
/**
* Verify that the type of input value is supported.
*
* @param name
* The name of the input
* @param value
* The value of the input
* @throws MLContextException
* if the input value type is not supported
*/
public static void checkInputValueType(String name, Object value) {
if (name == null) {
throw new MLContextException("No input name supplied");
} else if (value == null) {
throw new MLContextException("No input value supplied");
}
Object o = value;
boolean supported = false;
for (Class clazz : ALL_SUPPORTED_DATA_TYPES) {
if (o.getClass().equals(clazz)) {
supported = true;
break;
} else if (clazz.isAssignableFrom(o.getClass())) {
supported = true;
break;
}
}
if (!supported) {
throw new MLContextException("Input name (\"" + value + "\") value type not supported: " + o.getClass());
}
}
/**
* Verify that the type of input parameter value is supported.
*
* @param parameterName
* The name of the input parameter
* @param parameterValue
* The value of the input parameter
* @throws MLContextException
* if the input parameter value type is not supported
*/
public static void checkInputParameterType(String parameterName, Object parameterValue) {
if (parameterName == null) {
throw new MLContextException("No parameter name supplied");
} else if (parameterValue == null) {
throw new MLContextException("No parameter value supplied");
} else if (!parameterName.startsWith("$")) {
throw new MLContextException("Input parameter name must start with a $");
}
Object o = parameterValue;
boolean supported = false;
for (Class clazz : BASIC_DATA_TYPES) {
if (o.getClass().equals(clazz)) {
supported = true;
break;
} else if (clazz.isAssignableFrom(o.getClass())) {
supported = true;
break;
}
}
if (!supported) {
throw new MLContextException(
"Input parameter (\"" + parameterName + "\") value type not supported: " + o.getClass());
}
}
/**
* Is the object one of the supported basic data types? (Integer, Boolean,
* Double, String)
*
* @param object
* the object type to be examined
* @return {@code true} if type is a basic data type; otherwise
* {@code false}.
*/
public static boolean isBasicType(Object object) {
for (Class clazz : BASIC_DATA_TYPES) {
if (object.getClass().equals(clazz)) {
return true;
} else if (clazz.isAssignableFrom(object.getClass())) {
return true;
}
}
return false;
}
/**
* Obtain the SystemML scalar value type string equivalent of an accepted
* basic type (Integer, Boolean, Double, String)
*
* @param object
* the object type to be examined
* @return a String representing the type as a SystemML scalar value type
*/
public static String getBasicTypeString(Object object) {
if (!isBasicType(object)) {
throw new MLContextException("Type (" + object.getClass() + ") not a recognized basic type");
}
Class clazz = object.getClass();
if (clazz.equals(Integer.class)) {
return Statement.INT_VALUE_TYPE;
} else if (clazz.equals(Boolean.class)) {
return Statement.BOOLEAN_VALUE_TYPE;
} else if (clazz.equals(Double.class)) {
return Statement.DOUBLE_VALUE_TYPE;
} else if (clazz.equals(String.class)) {
return Statement.STRING_VALUE_TYPE;
} else {
return null;
}
}
/**
* Is the object one of the supported complex data types? (JavaRDD, RDD,
* DataFrame, BinaryBlockMatrix, Matrix, double[][], MatrixBlock, URL)
*
* @param object
* the object type to be examined
* @return {@code true} if type is a complex data type; otherwise
* {@code false}.
*/
public static boolean isComplexType(Object object) {
for (Class clazz : COMPLEX_DATA_TYPES) {
if (object.getClass().equals(clazz)) {
return true;
} else if (clazz.isAssignableFrom(object.getClass())) {
return true;
}
}
return false;
}
/**
* Converts non-string basic input parameter values to strings to pass to
* the parser.
*
* @param basicInputParameterMap
* map of input parameters
* @param scriptType
* {@code ScriptType.DML} or {@code ScriptType.PYDML}
* @return map of String/String name/value pairs
*/
public static Map convertInputParametersForParser(Map basicInputParameterMap,
ScriptType scriptType) {
if (basicInputParameterMap == null) {
return null;
}
if (scriptType == null) {
throw new MLContextException("ScriptType needs to be specified");
}
Map convertedMap = new HashMap();
for (Entry entry : basicInputParameterMap.entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
if (value == null) {
throw new MLContextException("Input parameter value is null for: " + entry.getKey());
} else if (value instanceof Integer) {
convertedMap.put(key, Integer.toString((Integer) value));
} else if (value instanceof Boolean) {
if (scriptType == ScriptType.DML) {
convertedMap.put(key, String.valueOf((Boolean) value).toUpperCase());
} else {
convertedMap.put(key, WordUtils.capitalize(String.valueOf((Boolean) value)));
}
} else if (value instanceof Double) {
convertedMap.put(key, Double.toString((Double) value));
} else if (value instanceof String) {
convertedMap.put(key, (String) value);
} else {
throw new MLContextException("Incorrect type for input parameters");
}
}
return convertedMap;
}
/**
* Convert input types to internal SystemML representations
*
* @param parameterName
* The name of the input parameter
* @param parameterValue
* The value of the input parameter
* @return input in SystemML data representation
*/
public static Data convertInputType(String parameterName, Object parameterValue) {
return convertInputType(parameterName, parameterValue, null);
}
/**
* Convert input types to internal SystemML representations
*
* @param parameterName
* The name of the input parameter
* @param parameterValue
* The value of the input parameter
* @param metadata
* matrix/frame metadata
* @return input in SystemML data representation
*/
public static Data convertInputType(String parameterName, Object parameterValue, Metadata metadata) {
String name = parameterName;
Object value = parameterValue;
boolean hasMetadata = (metadata != null) ? true : false;
boolean hasMatrixMetadata = hasMetadata && (metadata instanceof MatrixMetadata) ? true : false;
boolean hasFrameMetadata = hasMetadata && (metadata instanceof FrameMetadata) ? true : false;
if (name == null) {
throw new MLContextException("Input parameter name is null");
} else if (value == null) {
throw new MLContextException("Input parameter value is null for: " + parameterName);
} else if (value instanceof JavaRDD) {
@SuppressWarnings("unchecked")
JavaRDD javaRDD = (JavaRDD) value;
if (hasMatrixMetadata) {
MatrixMetadata matrixMetadata = (MatrixMetadata) metadata;
if (matrixMetadata.getMatrixFormat() == MatrixFormat.IJV) {
return MLContextConversionUtil.javaRDDStringIJVToMatrixObject(name, javaRDD, matrixMetadata);
} else {
return MLContextConversionUtil.javaRDDStringCSVToMatrixObject(name, javaRDD, matrixMetadata);
}
} else if (hasFrameMetadata) {
FrameMetadata frameMetadata = (FrameMetadata) metadata;
if (frameMetadata.getFrameFormat() == FrameFormat.IJV) {
return MLContextConversionUtil.javaRDDStringIJVToFrameObject(name, javaRDD, frameMetadata);
} else {
return MLContextConversionUtil.javaRDDStringCSVToFrameObject(name, javaRDD, frameMetadata);
}
} else if (!hasMetadata) {
String firstLine = javaRDD.first();
boolean isAllNumbers = isCSVLineAllNumbers(firstLine);
if (isAllNumbers) {
return MLContextConversionUtil.javaRDDStringCSVToMatrixObject(name, javaRDD);
} else {
return MLContextConversionUtil.javaRDDStringCSVToFrameObject(name, javaRDD);
}
}
} else if (value instanceof RDD) {
@SuppressWarnings("unchecked")
RDD rdd = (RDD) value;
if (hasMatrixMetadata) {
MatrixMetadata matrixMetadata = (MatrixMetadata) metadata;
if (matrixMetadata.getMatrixFormat() == MatrixFormat.IJV) {
return MLContextConversionUtil.rddStringIJVToMatrixObject(name, rdd, matrixMetadata);
} else {
return MLContextConversionUtil.rddStringCSVToMatrixObject(name, rdd, matrixMetadata);
}
} else if (hasFrameMetadata) {
FrameMetadata frameMetadata = (FrameMetadata) metadata;
if (frameMetadata.getFrameFormat() == FrameFormat.IJV) {
return MLContextConversionUtil.rddStringIJVToFrameObject(name, rdd, frameMetadata);
} else {
return MLContextConversionUtil.rddStringCSVToFrameObject(name, rdd, frameMetadata);
}
} else if (!hasMetadata) {
String firstLine = rdd.first();
boolean isAllNumbers = isCSVLineAllNumbers(firstLine);
if (isAllNumbers) {
return MLContextConversionUtil.rddStringCSVToMatrixObject(name, rdd);
} else {
return MLContextConversionUtil.rddStringCSVToFrameObject(name, rdd);
}
}
} else if (value instanceof MatrixBlock) {
MatrixBlock matrixBlock = (MatrixBlock) value;
return MLContextConversionUtil.matrixBlockToMatrixObject(name, matrixBlock, (MatrixMetadata) metadata);
} else if (value instanceof FrameBlock) {
FrameBlock frameBlock = (FrameBlock) value;
return MLContextConversionUtil.frameBlockToFrameObject(name, frameBlock, (FrameMetadata) metadata);
} else if (value instanceof DataFrame) {
DataFrame dataFrame = (DataFrame) value;
if (hasMatrixMetadata) {
return MLContextConversionUtil.dataFrameToMatrixObject(name, dataFrame, (MatrixMetadata) metadata);
} else if (hasFrameMetadata) {
return MLContextConversionUtil.dataFrameToFrameObject(name, dataFrame, (FrameMetadata) metadata);
} else if (!hasMetadata) {
boolean looksLikeMatrix = doesDataFrameLookLikeMatrix(dataFrame);
if (looksLikeMatrix) {
return MLContextConversionUtil.dataFrameToMatrixObject(name, dataFrame);
} else {
return MLContextConversionUtil.dataFrameToFrameObject(name, dataFrame);
}
}
} else if (value instanceof BinaryBlockMatrix) {
BinaryBlockMatrix binaryBlockMatrix = (BinaryBlockMatrix) value;
if (metadata == null) {
metadata = binaryBlockMatrix.getMatrixMetadata();
}
JavaPairRDD binaryBlocks = binaryBlockMatrix.getBinaryBlocks();
return MLContextConversionUtil.binaryBlocksToMatrixObject(name, binaryBlocks, (MatrixMetadata) metadata);
} else if (value instanceof BinaryBlockFrame) {
BinaryBlockFrame binaryBlockFrame = (BinaryBlockFrame) value;
if (metadata == null) {
metadata = binaryBlockFrame.getFrameMetadata();
}
JavaPairRDD binaryBlocks = binaryBlockFrame.getBinaryBlocks();
return MLContextConversionUtil.binaryBlocksToFrameObject(name, binaryBlocks, (FrameMetadata) metadata);
} else if (value instanceof Matrix) {
Matrix matrix = (Matrix) value;
return matrix.toMatrixObject();
} else if (value instanceof Frame) {
Frame frame = (Frame) value;
return frame.toFrameObject();
} else if (value instanceof double[][]) {
double[][] doubleMatrix = (double[][]) value;
return MLContextConversionUtil.doubleMatrixToMatrixObject(name, doubleMatrix, (MatrixMetadata) metadata);
} else if (value instanceof URL) {
URL url = (URL) value;
return MLContextConversionUtil.urlToMatrixObject(name, url, (MatrixMetadata) metadata);
} else if (value instanceof Integer) {
return new IntObject((Integer) value);
} else if (value instanceof Double) {
return new DoubleObject((Double) value);
} else if (value instanceof String) {
return new StringObject((String) value);
} else if (value instanceof Boolean) {
return new BooleanObject((Boolean) value);
}
return null;
}
/**
* If no metadata is supplied for an RDD or JavaRDD, this method can be used
* to determine whether the data appears to be matrix (or a frame)
*
* @param line
* a line of the RDD
* @return {@code true} if all the csv-separated values are numbers,
* {@code false} otherwise
*/
public static boolean isCSVLineAllNumbers(String line) {
if (StringUtils.isBlank(line)) {
return false;
}
String[] parts = line.split(",");
for (int i = 0; i < parts.length; i++) {
String part = parts[i].trim();
try {
Double.parseDouble(part);
} catch (NumberFormatException e) {
return false;
}
}
return true;
}
/**
* Examine the DataFrame schema to determine whether the data appears to be
* a matrix.
*
* @param df
* the DataFrame
* @return {@code true} if the DataFrame appears to be a matrix,
* {@code false} otherwise
*/
public static boolean doesDataFrameLookLikeMatrix(DataFrame df) {
StructType schema = df.schema();
StructField[] fields = schema.fields();
if (fields == null) {
return true;
}
for (StructField field : fields) {
DataType dataType = field.dataType();
if ((dataType != DataTypes.DoubleType) && (dataType != DataTypes.IntegerType)
&& (dataType != DataTypes.LongType) && (!(dataType instanceof VectorUDT))) {
// uncomment if we support arrays of doubles for matrices
// if (dataType instanceof ArrayType) {
// ArrayType arrayType = (ArrayType) dataType;
// if (arrayType.elementType() == DataTypes.DoubleType) {
// continue;
// }
// }
return false;
}
}
return true;
}
/**
* Return a double-quoted string with inner single and double quotes
* escaped.
*
* @param str
* the original string
* @return double-quoted string with inner single and double quotes escaped
*/
public static String quotedString(String str) {
if (str == null) {
return null;
}
StringBuilder sb = new StringBuilder();
sb.append("\"");
for (int i = 0; i < str.length(); i++) {
char ch = str.charAt(i);
if ((ch == '\'') || (ch == '"')) {
if ((i > 0) && (str.charAt(i - 1) != '\\')) {
sb.append('\\');
} else if (i == 0) {
sb.append('\\');
}
}
sb.append(ch);
}
sb.append("\"");
return sb.toString();
}
/**
* Display the keys and values in a Map
*
* @param mapName
* the name of the map
* @param map
* Map of String keys and Object values
* @return the keys and values in the Map as a String
*/
public static String displayMap(String mapName, Map map) {
StringBuilder sb = new StringBuilder();
sb.append(mapName);
sb.append(":\n");
Set keys = map.keySet();
if (keys.isEmpty()) {
sb.append("None\n");
} else {
int count = 0;
for (String key : keys) {
sb.append(" [");
sb.append(++count);
sb.append("] ");
sb.append(key);
sb.append(": ");
sb.append(map.get(key));
sb.append("\n");
}
}
return sb.toString();
}
/**
* Display the values in a Set
*
* @param setName
* the name of the Set
* @param set
* Set of String values
* @return the values in the Set as a String
*/
public static String displaySet(String setName, Set set) {
StringBuilder sb = new StringBuilder();
sb.append(setName);
sb.append(":\n");
if (set.isEmpty()) {
sb.append("None\n");
} else {
int count = 0;
for (String value : set) {
sb.append(" [");
sb.append(++count);
sb.append("] ");
sb.append(value);
sb.append("\n");
}
}
return sb.toString();
}
/**
* Display the keys and values in the symbol table
*
* @param name
* the name of the symbol table
* @param symbolTable
* the LocalVariableMap
* @return the keys and values in the symbol table as a String
*/
public static String displaySymbolTable(String name, LocalVariableMap symbolTable) {
StringBuilder sb = new StringBuilder();
sb.append(name);
sb.append(":\n");
sb.append(displaySymbolTable(symbolTable));
return sb.toString();
}
/**
* Display the keys and values in the symbol table
*
* @param symbolTable
* the LocalVariableMap
* @return the keys and values in the symbol table as a String
*/
public static String displaySymbolTable(LocalVariableMap symbolTable) {
StringBuilder sb = new StringBuilder();
Set keys = symbolTable.keySet();
if (keys.isEmpty()) {
sb.append("None\n");
} else {
int count = 0;
for (String key : keys) {
sb.append(" [");
sb.append(++count);
sb.append("]");
sb.append(" (");
sb.append(determineOutputTypeAsString(symbolTable, key));
sb.append(") ");
sb.append(key);
sb.append(": ");
sb.append(symbolTable.get(key));
sb.append("\n");
}
}
return sb.toString();
}
/**
* Obtain a symbol table output type as a String
*
* @param symbolTable
* the symbol table
* @param outputName
* the name of the output variable
* @return the symbol table output type for a variable as a String
*/
public static String determineOutputTypeAsString(LocalVariableMap symbolTable, String outputName) {
Data data = symbolTable.get(outputName);
if (data instanceof BooleanObject) {
return "Boolean";
} else if (data instanceof DoubleObject) {
return "Double";
} else if (data instanceof IntObject) {
return "Long";
} else if (data instanceof StringObject) {
return "String";
} else if (data instanceof MatrixObject) {
return "Matrix";
} else if (data instanceof FrameObject) {
return "Frame";
}
return "Unknown";
}
/**
* Obtain a display of script inputs.
*
* @param name
* the title to display for the inputs
* @param map
* the map of inputs
* @param symbolTable the symbol table
* @return the script inputs represented as a String
*/
public static String displayInputs(String name, Map map, LocalVariableMap symbolTable) {
StringBuilder sb = new StringBuilder();
sb.append(name);
sb.append(":\n");
Set keys = map.keySet();
if (keys.isEmpty()) {
sb.append("None\n");
} else {
int count = 0;
for (String key : keys) {
Object object = map.get(key);
@SuppressWarnings("rawtypes")
Class clazz = object.getClass();
String type = clazz.getSimpleName();
if (object instanceof JavaRDD) {
type = "JavaRDD";
} else if (object instanceof RDD) {
type = "RDD";
}
sb.append(" [");
sb.append(++count);
sb.append("]");
sb.append(" (");
sb.append(type);
if (doesSymbolTableContainMatrixObject(symbolTable, key)) {
sb.append(" as Matrix");
} else if (doesSymbolTableContainFrameObject(symbolTable, key)) {
sb.append(" as Frame");
}
sb.append(") ");
sb.append(key);
sb.append(": ");
String str = object.toString();
str = StringUtils.abbreviate(str, 100);
sb.append(str);
sb.append("\n");
}
}
return sb.toString();
}
/**
* Obtain a display of the script outputs.
*
* @param name
* the title to display for the outputs
* @param outputNames
* the names of the output variables
* @param symbolTable
* the symbol table
* @return the script outputs represented as a String
*
*/
public static String displayOutputs(String name, Set outputNames, LocalVariableMap symbolTable) {
StringBuilder sb = new StringBuilder();
sb.append(name);
sb.append(":\n");
sb.append(displayOutputs(outputNames, symbolTable));
return sb.toString();
}
/**
* Obtain a display of the script outputs.
*
* @param outputNames
* the names of the output variables
* @param symbolTable
* the symbol table
* @return the script outputs represented as a String
*
*/
public static String displayOutputs(Set outputNames, LocalVariableMap symbolTable) {
StringBuilder sb = new StringBuilder();
if (outputNames.isEmpty()) {
sb.append("None\n");
} else {
int count = 0;
for (String outputName : outputNames) {
sb.append(" [");
sb.append(++count);
sb.append("] ");
if (symbolTable.get(outputName) != null) {
sb.append("(");
sb.append(determineOutputTypeAsString(symbolTable, outputName));
sb.append(") ");
}
sb.append(outputName);
if (symbolTable.get(outputName) != null) {
sb.append(": ");
sb.append(symbolTable.get(outputName));
}
sb.append("\n");
}
}
return sb.toString();
}
/**
* The SystemML welcome message
*
* @return the SystemML welcome message
*/
public static String welcomeMessage() {
StringBuilder sb = new StringBuilder();
sb.append("\nWelcome to Apache SystemML!\n");
return sb.toString();
}
/**
* Generate a String history entry for a script.
*
* @param script
* the script
* @param when
* when the script was executed
* @return a script history entry as a String
*/
public static String createHistoryForScript(Script script, long when) {
DateFormat dateFormat = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss.SSS");
StringBuilder sb = new StringBuilder();
sb.append("Script Name: " + script.getName() + "\n");
sb.append("When: " + dateFormat.format(new Date(when)) + "\n");
sb.append(script.displayInputs());
sb.append(script.displayOutputs());
sb.append(script.displaySymbolTable());
return sb.toString();
}
/**
* Generate a String listing of the script execution history.
*
* @param scriptHistory
* the list of script history entries
* @return the listing of the script execution history as a String
*/
public static String displayScriptHistory(List scriptHistory) {
StringBuilder sb = new StringBuilder();
sb.append("MLContext Script History:\n");
if (scriptHistory.isEmpty()) {
sb.append("None");
}
int i = 1;
for (String history : scriptHistory) {
sb.append("--------------------------------------------\n");
sb.append("#" + (i++) + ":\n");
sb.append(history);
}
return sb.toString();
}
/**
* Obtain the Spark Context
*
* @param mlContext
* the SystemML MLContext
* @return the Spark Context
*/
public static SparkContext getSparkContext(MLContext mlContext) {
return mlContext.getSparkContext();
}
/**
* Obtain the Java Spark Context
*
* @param mlContext
* the SystemML MLContext
* @return the Java Spark Context
*/
public static JavaSparkContext getJavaSparkContext(MLContext mlContext) {
return new JavaSparkContext(mlContext.getSparkContext());
}
/**
* Determine if the symbol table contains a FrameObject with the given
* variable name.
*
* @param symbolTable
* the LocalVariableMap
* @param variableName
* the variable name
* @return {@code true} if the variable in the symbol table is a
* FrameObject, {@code false} otherwise.
*/
public static boolean doesSymbolTableContainFrameObject(LocalVariableMap symbolTable, String variableName) {
return (symbolTable != null && symbolTable.keySet().contains(variableName)
&& symbolTable.get(variableName) instanceof FrameObject);
}
/**
* Determine if the symbol table contains a MatrixObject with the given
* variable name.
*
* @param symbolTable
* the LocalVariableMap
* @param variableName
* the variable name
* @return {@code true} if the variable in the symbol table is a
* MatrixObject, {@code false} otherwise.
*/
public static boolean doesSymbolTableContainMatrixObject(LocalVariableMap symbolTable, String variableName) {
return (symbolTable != null && symbolTable.keySet().contains(variableName)
&& symbolTable.get(variableName) instanceof MatrixObject);
}
}