org.apache.sysml.hops.Hop Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of systemml Show documentation
Show all versions of systemml Show documentation
Declarative Machine Learning
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.sysml.hops;
import java.util.ArrayList;
import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.lops.CSVReBlock;
import org.apache.sysml.lops.Checkpoint;
import org.apache.sysml.lops.Compression;
import org.apache.sysml.lops.Data;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.lops.LopsException;
import org.apache.sysml.lops.MultipleCP;
import org.apache.sysml.lops.ReBlock;
import org.apache.sysml.lops.UnaryCP;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.util.UtilFunctions;
public abstract class Hop
{
protected static final Log LOG = LogFactory.getLog(Hop.class.getName());
public static final long CPThreshold = 2000;
public enum VisitStatus {
DONE,
NOTVISITED
}
/**
* Optional hop interface, to be implemented by multi-threaded hops.
*/
public interface MultiThreadedHop {
public abstract void setMaxNumThreads( int k );
public abstract int getMaxNumThreads();
}
// static variable to assign an unique ID to every hop that is created
private static IDSequence _seqHopID = new IDSequence();
protected long _ID;
protected String _name;
protected DataType _dataType;
protected ValueType _valueType;
protected VisitStatus _visited = VisitStatus.NOTVISITED;
protected long _dim1 = -1;
protected long _dim2 = -1;
protected long _rows_in_block = -1;
protected long _cols_in_block = -1;
protected long _nnz = -1;
protected UpdateType _updateType = UpdateType.COPY;
protected ArrayList _parent = new ArrayList();
protected ArrayList _input = new ArrayList();
protected ExecType _etype = null; //currently used exec type
protected ExecType _etypeForced = null; //exec type forced via platform or external optimizer
// Estimated size for the output produced from this Hop
protected double _outputMemEstimate = OptimizerUtils.INVALID_SIZE;
// Estimated size for the entire operation represented by this Hop
// It includes the memory required for all inputs as well as the output
protected double _memEstimate = OptimizerUtils.INVALID_SIZE;
protected double _processingMemEstimate = 0;
protected double _spBroadcastMemEstimate = 0;
// indicates if there are unknowns during compilation
// (in that case re-complication ensures robustness and efficiency)
protected boolean _requiresRecompile = false;
// indicates if the output of this hop needs to be reblocked
// (usually this happens on persistent reads dataops)
protected boolean _requiresReblock = false;
// indicates if the output of this hop needs to be compressed
// (this happens on persistent reads after reblock but before checkpoint)
protected boolean _requiresCompression = false;
// indicates if the output of this hop needs to be checkpointed (cached)
// (the default storage level for caching is not yet exposed here)
protected boolean _requiresCheckpoint = false;
// indicates if the output of this hops needs to contain materialized empty blocks
// if those exists; otherwise only blocks w/ non-zero values are materialized
protected boolean _outputEmptyBlocks = true;
private Lop _lops = null;
protected Hop(){
//default constructor for clone
}
public Hop(String l, DataType dt, ValueType vt) {
_ID = getNextHopID();
setName(l);
setDataType(dt);
setValueType(vt);
}
private static long getNextHopID() {
return _seqHopID.getNextID();
}
public long getHopID() {
return _ID;
}
public ExecType getExecType()
{
return _etype;
}
public void resetExecType()
{
_etype = null;
}
public ExecType getForcedExecType()
{
return _etypeForced;
}
public void setForcedExecType(ExecType etype)
{
_etypeForced = etype;
}
public abstract boolean allowsAllExecTypes();
/**
* Defines if this operation is transpose-safe, which means that
* the result of op(input) is equivalent to op(t(input)).
* Usually, this applies to aggregate operations with fixed output
* dimension. Finally, this information is very useful in order to
* safely optimize the plan for sparse vectors, which otherwise
* would be (currently) always represented dense.
*
*
* @return always returns false
*/
public boolean isTransposeSafe()
{
//by default: its conservatively define as unsafe
return false;
}
public void checkAndSetForcedPlatform()
{
if ( DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE )
_etypeForced = ExecType.CP;
else if ( DMLScript.rtplatform == RUNTIME_PLATFORM.HADOOP )
_etypeForced = ExecType.MR;
else if ( DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK )
_etypeForced = ExecType.SPARK;
}
public void checkAndSetInvalidCPDimsAndSize()
{
if( _etype == ExecType.CP )
{
boolean invalid = false;
//Step 1: check dimensions of output and all inputs (INTEGER)
invalid |= !OptimizerUtils.isValidCPDimensions(_dim1, _dim2);
for( Hop in : getInput() )
invalid |= !OptimizerUtils.isValidCPDimensions(in._dim1, in._dim2);
//Step 2: check valid output and input sizes for cp (<16GB for DENSE)
//(if the memory estimate is smaller than max_numcells we are guaranteed to have it in sparse representation)
invalid |= !( OptimizerUtils.isValidCPMatrixSize(_dim1, _dim2, OptimizerUtils.getSparsity(_dim1, _dim2, _nnz))
|| getOutputMemEstimate() < OptimizerUtils.MAX_NUMCELLS_CP_DENSE );
for( Hop in : getInput() )
invalid |= !( OptimizerUtils.isValidCPMatrixSize(in._dim1, in._dim2, OptimizerUtils.getSparsity(in._dim1, in._dim2, in._nnz))
|| in.getOutputMemEstimate() < OptimizerUtils.MAX_NUMCELLS_CP_DENSE);
//force exec type mr if necessary
if( invalid ) {
if( DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID )
_etype = ExecType.MR;
else if( DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
_etype = ExecType.SPARK;
}
}
}
public void setRequiresReblock(boolean flag) {
_requiresReblock = flag;
}
public void setRequiresCompression(boolean flag) {
_requiresCompression = flag;
}
public boolean hasMatrixInputWithDifferentBlocksizes()
{
for( Hop c : getInput() ) {
if( c.getDataType()==DataType.MATRIX
&&(getRowsInBlock() != c.getRowsInBlock()
|| getColsInBlock() != c.getColsInBlock()) )
{
return true;
}
}
return false;
}
public void setOutputBlocksizes( long brlen, long bclen )
{
setRowsInBlock( brlen );
setColsInBlock( bclen );
}
public boolean requiresReblock()
{
return _requiresReblock;
}
public void setRequiresCheckpoint(boolean flag)
{
_requiresCheckpoint = flag;
}
public boolean requiresCheckpoint()
{
return _requiresCheckpoint;
}
public void constructAndSetLopsDataFlowProperties()
throws HopsException
{
//Step 1: construct reblock lop if required (output of hop)
constructAndSetReblockLopIfRequired();
//Step 2: construct compression lop if required
constructAndSetCompressionLopIfRequired();
//Step 3: construct checkpoint lop if required (output of hop or reblock)
constructAndSetCheckpointLopIfRequired();
}
private void constructAndSetReblockLopIfRequired()
throws HopsException
{
//determine execution type
ExecType et = ExecType.CP;
if( DMLScript.rtplatform != RUNTIME_PLATFORM.SINGLE_NODE
&& !(getDataType()==DataType.SCALAR) )
{
et = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
}
//add reblock lop to output if required
if( _requiresReblock && et != ExecType.CP )
{
Lop input = getLops();
Lop reblock = null;
try
{
if( (this instanceof DataOp // CSV
&& ((DataOp)this).getDataOpType() == DataOpTypes.PERSISTENTREAD
&& ((DataOp)this).getInputFormatType() == FileFormatTypes.CSV )
|| (this instanceof ParameterizedBuiltinOp
&& ((ParameterizedBuiltinOp)this).getOp() == ParamBuiltinOp.TRANSFORM) )
{
reblock = new CSVReBlock( input, getRowsInBlock(), getColsInBlock(),
getDataType(), getValueType(), et);
}
else //TEXT / MM / BINARYBLOCK / BINARYCELL
{
reblock = new ReBlock( input, getRowsInBlock(), getColsInBlock(),
getDataType(), getValueType(), _outputEmptyBlocks, et);
}
}
catch( LopsException ex ) {
throw new HopsException(ex);
}
setOutputDimensions( reblock );
setLineNumbers( reblock );
setLops( reblock );
}
}
@SuppressWarnings("unused") //see CHECKPOINT_SPARSE_CSR
private void constructAndSetCheckpointLopIfRequired()
throws HopsException
{
//determine execution type
ExecType et = ExecType.CP;
if( OptimizerUtils.isSparkExecutionMode()
&& getDataType()!=DataType.SCALAR )
{
//conditional checkpoint based on memory estimate in order to
//(1) avoid unnecessary persist and unpersist calls, and
//(2) avoid unnecessary creation of spark context (incl executors)
if( OptimizerUtils.isHybridExecutionMode()
&& !OptimizerUtils.exceedsCachingThreshold(getDim2(), _outputMemEstimate)
|| _etypeForced == ExecType.CP )
{
et = ExecType.CP;
}
else //default case
{
et = ExecType.SPARK;
}
}
//add checkpoint lop to output if required
if( _requiresCheckpoint && et != ExecType.CP )
{
try
{
//investigate need for serialized storage of large sparse matrices
//(compile- instead of runtime-level for better debugging)
boolean serializedStorage = false;
if( getDataType()==DataType.MATRIX && dimsKnown(true) && !Checkpoint.CHECKPOINT_SPARSE_CSR ) {
double matrixPSize = OptimizerUtils.estimatePartitionedSizeExactSparsity(_dim1, _dim2, _rows_in_block, _cols_in_block, _nnz);
double dataCache = SparkExecutionContext.getDataMemoryBudget(true, true);
serializedStorage = (MatrixBlock.evalSparseFormatInMemory(_dim1, _dim2, _nnz)
&& matrixPSize > dataCache ); //sparse in-memory does not fit in agg mem
}
else if( !dimsKnown(true) ) {
setRequiresRecompile();
}
//construct checkpoint w/ right storage level
Lop input = getLops();
Lop chkpoint = new Checkpoint(input, getDataType(), getValueType(),
serializedStorage ? Checkpoint.getSerializeStorageLevelString() :
Checkpoint.getDefaultStorageLevelString() );
setOutputDimensions( chkpoint );
setLineNumbers( chkpoint );
setLops( chkpoint );
}
catch( LopsException ex ) {
throw new HopsException(ex);
}
}
}
private void constructAndSetCompressionLopIfRequired()
throws HopsException
{
//determine execution type
ExecType et = ExecType.CP;
if( OptimizerUtils.isSparkExecutionMode()
&& getDataType()!=DataType.SCALAR )
{
//conditional checkpoint based on memory estimate in order to avoid unnecessary
//persist and unpersist calls (4x the memory budget is conservative)
if( OptimizerUtils.isHybridExecutionMode()
&& 2*_outputMemEstimate < OptimizerUtils.getLocalMemBudget()
|| _etypeForced == ExecType.CP )
{
et = ExecType.CP;
}
else //default case
{
et = ExecType.SPARK;
}
}
//add reblock lop to output if required
if( _requiresCompression )
{
try
{
Lop compress = new Compression(getLops(), getDataType(), getValueType(), et);
setOutputDimensions( compress );
setLineNumbers( compress );
setLops( compress );
}
catch( LopsException ex ) {
throw new HopsException(ex);
}
}
}
public static Lop createOffsetLop( Hop hop, boolean repCols )
throws HopsException, LopsException
{
Lop offset = null;
if( ConfigurationManager.isDynamicRecompilation() && hop.dimsKnown() )
{
// If dynamic recompilation is enabled and dims are known, we can replace the ncol with
// a literal in order to increase the piggybacking potential. This is safe because append
// is always marked for recompilation and hence, we have propagated the exact dimensions.
offset = Data.createLiteralLop(ValueType.INT, String.valueOf(repCols ? hop.getDim2() : hop.getDim1()));
}
else
{
offset = new UnaryCP(hop.constructLops(),
repCols ? UnaryCP.OperationTypes.NCOL : UnaryCP.OperationTypes.NROW,
DataType.SCALAR, ValueType.INT);
}
offset.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
offset.setAllPositions(hop.getBeginLine(), hop.getBeginColumn(), hop.getEndLine(), hop.getEndColumn());
return offset;
}
public void setOutputEmptyBlocks(boolean flag)
{
_outputEmptyBlocks = flag;
}
public boolean isOutputEmptyBlocks()
{
return _outputEmptyBlocks;
}
/**
* Returns the memory estimate for the output produced from this Hop.
* It must be invoked only within Hops. From outside Hops, one must
* only use getMemEstimate(), which gives memory required to store
* all inputs and the output.
*
* @return output size memory estimate
*/
protected double getOutputSize()
{
return _outputMemEstimate;
}
protected double getInputSize()
{
double sum = 0;
int len = _input.size();
for( int i=0; i 1024*1024 ) {//for relevant sizes
//check if already included in estimate (if an input is used
//multiple times it is still only required once in memory)
//(not that this check benefits from common subexpression elimination)
boolean flag = false;
for( int j=0; jpos )
ret = _input.get(pos)._outputMemEstimate;
return ret;
}
protected double getIntermediateSize() {
return _processingMemEstimate;
}
/**
* NOTES:
* * Purpose: Whenever the output dimensions / sparsity of a hop are unknown, this hop
* should store its worst-case output statistics (if known) in that table. Subsequent
* hops can then
* * Invocation: Intended to be called for ALL root nodes of one Hops DAG with the same
* (initially empty) memo table.
*
* @return memory estimate
*/
public double getMemEstimate()
{
if ( OptimizerUtils.isMemoryBasedOptLevel() ) {
if ( ! isMemEstimated() ) {
//LOG.warn("Nonexisting memory estimate - reestimating w/o memo table.");
computeMemEstimate( new MemoTable() );
}
return _memEstimate;
}
else {
return OptimizerUtils.INVALID_SIZE;
}
}
/**
* Sets memory estimate in bytes
*
* @param mem memory estimate
*/
public void setMemEstimate( double mem )
{
_memEstimate = mem;
}
public void clearMemEstimate()
{
_memEstimate = OptimizerUtils.INVALID_SIZE;
}
public boolean isMemEstimated()
{
return (_memEstimate != OptimizerUtils.INVALID_SIZE);
}
//wrappers for meaningful public names to memory estimates.
public double getInputMemEstimate()
{
return getInputSize();
}
public double getOutputMemEstimate()
{
return getOutputSize();
}
public double getIntermediateMemEstimate()
{
return getIntermediateSize();
}
public double getSpBroadcastSize()
{
return _spBroadcastMemEstimate;
}
/**
* Computes the estimate of memory required to store the input/output of this hop in memory.
* This is the default implementation (orchestration of hop-specific implementation)
* that should suffice for most hops. If a hop requires more control, this method should
* be overwritten with awareness of (1) output estimates, and (2) propagation of worst-case
* matrix characteristics (dimensions, sparsity).
*
* TODO remove memo table and, on constructor refresh, inference in refresh, single compute mem,
* maybe general computeMemEstimate, flags to indicate if estimate or not.
*
* @param memo memory table
*/
public void computeMemEstimate( MemoTable memo )
{
long[] wstats = null;
////////
//Step 1) Compute hop output memory estimate (incl size inference)
switch( getDataType() )
{
case SCALAR:
{
//memory estimate always known
if( getValueType()== ValueType.DOUBLE) //default case
_outputMemEstimate = OptimizerUtils.DOUBLE_SIZE;
else //literalops, dataops
_outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, _nnz );
break;
}
case FRAME:
case MATRIX:
{
//1a) mem estimate based on exactly known dimensions and sparsity
if( dimsKnown(true) ) {
//nnz always exactly known (see dimsKnown(true))
_outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, _nnz );
}
//1b) infer output statistics and mem estimate based on these statistics
else if( memo.hasInputStatistics(this) )
{
//infer the output stats
wstats = inferOutputCharacteristics(memo);
if( wstats != null ) {
//use worst case characteristics to estimate mem
long lnnz = ((wstats[2]>=0)?wstats[2]:wstats[0]*wstats[1]);
_outputMemEstimate = computeOutputMemEstimate( wstats[0], wstats[1], lnnz );
//propagate worst-case estimate
memo.memoizeStatistics(getHopID(), wstats[0], wstats[1], wstats[2]);
}
else if( dimsKnown() ) {
//nnz unknown, estimate mem as dense
long lnnz = _dim1*_dim2;
_outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, lnnz );
}
else {
//unknown output size
_outputMemEstimate = OptimizerUtils.DEFAULT_SIZE;
}
}
//1c) mem estimate based on exactly known dimensions and unknown sparsity
//(required e.g., for datagenops w/o any input statistics)
else if( dimsKnown() ) {
//nnz unknown, estimate mem as dense
long lnnz = _dim1*_dim2;
_outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, lnnz );
}
//1d) fallback: unknown output size
else {
_outputMemEstimate = OptimizerUtils.DEFAULT_SIZE;
}
break;
}
case OBJECT:
case UNKNOWN:
{
//memory estimate always unknown
_outputMemEstimate = OptimizerUtils.DEFAULT_SIZE;
break;
}
}
////////
//Step 2) Compute hop intermediate memory estimate
//note: ensure consistency w/ step 1 (for simplified debugging)
if( dimsKnown(true) ) { //incl scalar output
//nnz always exactly known (see dimsKnown(true))
_processingMemEstimate = computeIntermediateMemEstimate( _dim1, _dim2, _nnz );
}
else if( wstats!=null ) {
//use worst case characteristics to estimate mem
long lnnz = ((wstats[2]>=0)?wstats[2]:wstats[0]*wstats[1]);
_processingMemEstimate = computeIntermediateMemEstimate( wstats[0], wstats[1], lnnz );
}
else if( dimsKnown() ){
//nnz unknown, estimate mem as dense
long lnnz = _dim1 * _dim2;
_processingMemEstimate = computeIntermediateMemEstimate(_dim1, _dim2, lnnz);
}
////////
//Step 3) Compute final hop memory estimate
//final estimate (sum of inputs/intermediates/output)
_memEstimate = getInputOutputSize();
}
/**
* Computes the hop-specific output memory estimate in bytes. Should be 0 if not
* applicable.
*
* @param dim1 dimension 1
* @param dim2 dimension 2
* @param nnz number of non-zeros
* @return memory estimate
*/
protected abstract double computeOutputMemEstimate( long dim1, long dim2, long nnz );
/**
* Computes the hop-specific intermediate memory estimate in bytes. Should be 0 if not
* applicable.
*
* @param dim1 dimension 1
* @param dim2 dimension 2
* @param nnz number of non-zeros
* @return memory estimate
*/
protected abstract double computeIntermediateMemEstimate( long dim1, long dim2, long nnz );
/**
* Computes the output matrix characteristics (rows, cols, nnz) based on worst-case output
* and/or input estimates. Should return null if dimensions are unknown.
*
* @param memo memory table
* @return output characteristics as a long array
*/
protected abstract long[] inferOutputCharacteristics( MemoTable memo );
/**
* Recursively computes memory estimates for all the Hops in the DAG rooted at the
* current hop pointed by this
.
*
* @param memo memory table
*/
public void refreshMemEstimates( MemoTable memo ) {
if (getVisited() == VisitStatus.DONE)
return;
for (Hop h : this.getInput())
h.refreshMemEstimates( memo );
this.computeMemEstimate( memo );
this.setVisited(VisitStatus.DONE);
}
/**
* This method determines the execution type (CP, MR) based ONLY on the
* estimated memory footprint required for this operation, which includes
* memory for all inputs and the output represented by this Hop.
*
* It is used when OptimizationType = MEMORY_BASED
.
* This optimization schedules an operation to CP whenever inputs+output
* fit in memory -- note that this decision MAY NOT be optimal in terms of
* execution time.
*
* @return execution type
*/
protected ExecType findExecTypeByMemEstimate() {
ExecType et = null;
char c = ' ';
if ( getMemEstimate() < OptimizerUtils.getLocalMemBudget() ) {
et = ExecType.CP;
}
else {
if( DMLScript.rtplatform == DMLScript.RUNTIME_PLATFORM.HYBRID )
et = ExecType.MR;
else if( DMLScript.rtplatform == DMLScript.RUNTIME_PLATFORM.HYBRID_SPARK )
et = ExecType.SPARK;
c = '*';
}
if (LOG.isDebugEnabled()){
String s = String.format(" %c %-5s %-8s (%s,%s) %s", c, getHopID(), getOpString(), OptimizerUtils.toMB(_outputMemEstimate), OptimizerUtils.toMB(_memEstimate), et);
//System.out.println(s);
LOG.debug(s);
}
return et;
}
protected ExecType findGPUExecTypeByMemEstimate(ExecType et) {
if(DMLScript.USE_ACCELERATOR && (DMLScript.FORCE_ACCELERATOR || getMemEstimate() < OptimizerUtils.GPU_MEMORY_BUDGET)) {
return ExecType.GPU;
}
return et;
}
public ArrayList getParent() {
return _parent;
}
public ArrayList getInput() {
return _input;
}
public long getRowsInBlock() {
return _rows_in_block;
}
public void setRowsInBlock(long rowsInBlock) {
_rows_in_block = rowsInBlock;
}
public long getColsInBlock() {
return _cols_in_block;
}
public void setColsInBlock(long colsInBlock) {
_cols_in_block = colsInBlock;
}
public void setNnz(long nnz){
_nnz = nnz;
}
public long getNnz(){
return _nnz;
}
public void setUpdateType(UpdateType update){
_updateType = update;
}
public UpdateType getUpdateType(){
return _updateType;
}
public abstract Lop constructLops()
throws HopsException, LopsException;
protected abstract ExecType optFindExecType()
throws HopsException;
public abstract String getOpString();
protected boolean isVector() {
return (dimsKnown() && (_dim1 == 1 || _dim2 == 1) );
}
protected boolean areDimsBelowThreshold() {
return (dimsKnown() && _dim1 <= Hop.CPThreshold && _dim2 <= Hop.CPThreshold );
}
public boolean dimsKnown() {
return ( _dataType == DataType.SCALAR
|| ((_dataType==DataType.MATRIX || _dataType==DataType.FRAME)
&& _dim1 > 0 && _dim2 > 0) );
}
public boolean dimsKnown(boolean includeNnz) {
return ( _dataType == DataType.SCALAR
|| ((_dataType==DataType.MATRIX || _dataType==DataType.FRAME)
&& _dim1 > 0 && _dim2 > 0 && ((includeNnz)? _nnz>=0 : true)));
}
public boolean dimsKnownAny() {
return ( _dataType == DataType.SCALAR
|| ((_dataType==DataType.MATRIX || _dataType==DataType.FRAME)
&& (_dim1 > 0 || _dim2 > 0)) );
}
public static void resetVisitStatus( ArrayList hops )
{
if( hops != null )
for( Hop hopRoot : hops )
hopRoot.resetVisitStatus();
}
public void resetVisitStatus()
{
if ( getVisited() == Hop.VisitStatus.NOTVISITED )
return;
for (Hop h : this.getInput())
h.resetVisitStatus();
setVisited(Hop.VisitStatus.NOTVISITED);
}
public static void resetRecompilationFlag( ArrayList hops, ExecType et )
{
resetVisitStatus( hops );
for( Hop hopRoot : hops )
hopRoot.resetRecompilationFlag( et );
}
public static void resetRecompilationFlag( Hop hops, ExecType et )
{
hops.resetVisitStatus();
hops.resetRecompilationFlag( et );
}
private void resetRecompilationFlag( ExecType et )
{
if( getVisited() == VisitStatus.DONE )
return;
//process child hops
for (Hop h : getInput())
h.resetRecompilationFlag( et );
//reset recompile flag
if( et == null || getExecType() == et || getExecType()==null )
_requiresRecompile = false;
this.setVisited(VisitStatus.DONE);
}
public void printMe() throws HopsException {
if (LOG.isDebugEnabled()) {
StringBuilder s = new StringBuilder("");
s.append(getClass().getSimpleName() + " " + getHopID() + "\n");
s.append(" Label: " + getName() + "; DataType: " + _dataType + "; ValueType: " + _valueType + "\n");
s.append(" Parent: ");
for (Hop h : getParent()) {
s.append(h.hashCode() + "; ");
}
;
s.append("\n Input: ");
for (Hop h : getInput()) {
s.append(h.getHopID() + "; ");
}
s.append("\n dims [" + _dim1 + "," + _dim2 + "] blk [" + _rows_in_block + "," + _cols_in_block + "] nnz: " + _nnz + " UpdateInPlace: " + _updateType);
s.append(" MemEstimate = Out " + (_outputMemEstimate/1024/1024) + " MB, In&Out " + (_memEstimate/1024/1024) + " MB" );
LOG.debug(s.toString());
}
}
public long getDim1() {
return _dim1;
}
public void setDim1(long dim1) {
_dim1 = dim1;
}
public long getDim2() {
return _dim2;
}
public void setDim2(long dim2) {
_dim2 = dim2;
}
protected void setOutputDimensions(Lop lop)
throws HopsException
{
lop.getOutputParameters().setDimensions(
getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz(), getUpdateType());
}
public Lop getLops() {
return _lops;
}
public void setLops(Lop lops) {
_lops = lops;
}
public VisitStatus getVisited() {
return _visited;
}
public DataType getDataType() {
return _dataType;
}
public void setDataType( DataType dt ) {
_dataType = dt;
}
public void setVisited(VisitStatus visited) {
_visited = visited;
}
public void setName(String _name) {
this._name = _name;
}
public String getName() {
return _name;
}
public ValueType getValueType() {
return _valueType;
}
public void setValueType(ValueType vt) {
_valueType = vt;
}
public enum OpOp1 {
NOT, ABS, SIN, COS, TAN, ASIN, ACOS, ATAN, SIGN, SQRT, LOG, EXP,
CAST_AS_SCALAR, CAST_AS_MATRIX, CAST_AS_FRAME, CAST_AS_DOUBLE, CAST_AS_INT, CAST_AS_BOOLEAN,
PRINT, EIGEN, NROW, NCOL, LENGTH, ROUND, IQM, STOP, CEIL, FLOOR, MEDIAN, INVERSE, CHOLESKY,
//cumulative sums, products, extreme values
CUMSUM, CUMPROD, CUMMIN, CUMMAX,
//fused ML-specific operators for performance
SPROP, //sample proportion: P * (1 - P)
SIGMOID, //sigmoid function: 1 / (1 + exp(-X))
SELP, //select positive: X * (X>0)
LOG_NZ, //sparse-safe log; ppred(X,0,"!=")*log(X)
}
// Operations that require two operands
public enum OpOp2 {
PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, LESS, LESSEQUAL, GREATER, GREATEREQUAL, EQUAL, NOTEQUAL,
MIN, MAX, AND, OR, LOG, POW, PRINT, CONCAT, QUANTILE, INTERQUANTILE, IQM,
CENTRALMOMENT, COVARIANCE, CBIND, RBIND, SOLVE, MEDIAN, INVALID,
//fused ML-specific operators for performance
MINUS_NZ, //sparse-safe minus: X-(mean*ppred(X,0,!=))
LOG_NZ, //sparse-safe log; ppred(X,0,"!=")*log(X,0.5)
MINUS1_MULT, //1-X*Y
};
// Operations that require 3 operands
public enum OpOp3 {
QUANTILE, INTERQUANTILE, CTABLE, CENTRALMOMENT, COVARIANCE, PLUS_MULT, MINUS_MULT
};
// Operations that require 4 operands
public enum OpOp4 {
WSLOSS, //weighted sloss mm
WSIGMOID, //weighted sigmoid mm
WDIVMM, //weighted divide mm
WCEMM, //weighted cross entropy mm
WUMM //weighted unary mm
};
// Operations that require a variable number of operands
public enum MultipleOperandOperation {
PRINTF
}
public enum AggOp {
SUM, SUM_SQ, MIN, MAX, TRACE, PROD, MEAN, VAR, MAXINDEX, MININDEX
};
public enum ReOrgOp {
TRANSPOSE, DIAG, RESHAPE, SORT, REV
//Note: Diag types are invalid because for unknown sizes this would
//create incorrect plans (now we try to infer it for memory estimates
//and rewrites but the final choice is made during runtime)
//DIAG_V2M, DIAG_M2V,
};
public enum ConvOp {
MAX_POOLING, MAX_POOLING_BACKWARD,
DIRECT_CONV2D, DIRECT_CONV2D_BACKWARD_FILTER, DIRECT_CONV2D_BACKWARD_DATA,
BIAS_ADD
};
public enum DataGenMethod {
RAND, SEQ, SINIT, SAMPLE, INVALID
};
public enum ParamBuiltinOp {
INVALID, CDF, INVCDF, GROUPEDAGG, RMEMPTY, REPLACE, REXPAND,
TRANSFORM, TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMMETA,
TOSTRING
};
public enum FileFormatTypes {
TEXT, BINARY, MM, CSV
};
public enum DataOpTypes {
PERSISTENTREAD, PERSISTENTWRITE, TRANSIENTREAD, TRANSIENTWRITE, FUNCTIONOUTPUT
};
public enum Direction {
RowCol, Row, Col
};
protected static final HashMap HopsData2Lops;
static {
HopsData2Lops = new HashMap();
HopsData2Lops.put(DataOpTypes.PERSISTENTREAD, org.apache.sysml.lops.Data.OperationTypes.READ);
HopsData2Lops.put(DataOpTypes.PERSISTENTWRITE, org.apache.sysml.lops.Data.OperationTypes.WRITE);
HopsData2Lops.put(DataOpTypes.TRANSIENTWRITE, org.apache.sysml.lops.Data.OperationTypes.WRITE);
HopsData2Lops.put(DataOpTypes.TRANSIENTREAD, org.apache.sysml.lops.Data.OperationTypes.READ);
}
protected static final HashMap HopsAgg2Lops;
static {
HopsAgg2Lops = new HashMap();
HopsAgg2Lops.put(AggOp.SUM, org.apache.sysml.lops.Aggregate.OperationTypes.KahanSum);
HopsAgg2Lops.put(AggOp.SUM_SQ, org.apache.sysml.lops.Aggregate.OperationTypes.KahanSumSq);
HopsAgg2Lops.put(AggOp.TRACE, org.apache.sysml.lops.Aggregate.OperationTypes.KahanTrace);
HopsAgg2Lops.put(AggOp.MIN, org.apache.sysml.lops.Aggregate.OperationTypes.Min);
HopsAgg2Lops.put(AggOp.MAX, org.apache.sysml.lops.Aggregate.OperationTypes.Max);
HopsAgg2Lops.put(AggOp.MAXINDEX, org.apache.sysml.lops.Aggregate.OperationTypes.MaxIndex);
HopsAgg2Lops.put(AggOp.MININDEX, org.apache.sysml.lops.Aggregate.OperationTypes.MinIndex);
HopsAgg2Lops.put(AggOp.PROD, org.apache.sysml.lops.Aggregate.OperationTypes.Product);
HopsAgg2Lops.put(AggOp.MEAN, org.apache.sysml.lops.Aggregate.OperationTypes.Mean);
HopsAgg2Lops.put(AggOp.VAR, org.apache.sysml.lops.Aggregate.OperationTypes.Var);
}
protected static final HashMap HopsTransf2Lops;
static {
HopsTransf2Lops = new HashMap();
HopsTransf2Lops.put(ReOrgOp.TRANSPOSE, org.apache.sysml.lops.Transform.OperationTypes.Transpose);
HopsTransf2Lops.put(ReOrgOp.REV, org.apache.sysml.lops.Transform.OperationTypes.Rev);
HopsTransf2Lops.put(ReOrgOp.DIAG, org.apache.sysml.lops.Transform.OperationTypes.Diag);
HopsTransf2Lops.put(ReOrgOp.RESHAPE, org.apache.sysml.lops.Transform.OperationTypes.Reshape);
HopsTransf2Lops.put(ReOrgOp.SORT, org.apache.sysml.lops.Transform.OperationTypes.Sort);
}
protected static final HashMap HopsConv2Lops;
static {
HopsConv2Lops = new HashMap();
HopsConv2Lops.put(ConvOp.MAX_POOLING, org.apache.sysml.lops.ConvolutionTransform.OperationTypes.MAX_POOLING);
HopsConv2Lops.put(ConvOp.MAX_POOLING_BACKWARD, org.apache.sysml.lops.ConvolutionTransform.OperationTypes.MAX_POOLING_BACKWARD);
HopsConv2Lops.put(ConvOp.DIRECT_CONV2D, org.apache.sysml.lops.ConvolutionTransform.OperationTypes.DIRECT_CONV2D);
HopsConv2Lops.put(ConvOp.BIAS_ADD, org.apache.sysml.lops.ConvolutionTransform.OperationTypes.BIAS_ADD);
HopsConv2Lops.put(ConvOp.DIRECT_CONV2D_BACKWARD_FILTER, org.apache.sysml.lops.ConvolutionTransform.OperationTypes.DIRECT_CONV2D_BACKWARD_FILTER);
HopsConv2Lops.put(ConvOp.DIRECT_CONV2D_BACKWARD_DATA, org.apache.sysml.lops.ConvolutionTransform.OperationTypes.DIRECT_CONV2D_BACKWARD_DATA);
}
protected static final HashMap HopsDirection2Lops;
static {
HopsDirection2Lops = new HashMap();
HopsDirection2Lops.put(Direction.RowCol, org.apache.sysml.lops.PartialAggregate.DirectionTypes.RowCol);
HopsDirection2Lops.put(Direction.Col, org.apache.sysml.lops.PartialAggregate.DirectionTypes.Col);
HopsDirection2Lops.put(Direction.Row, org.apache.sysml.lops.PartialAggregate.DirectionTypes.Row);
}
protected static final HashMap HopsOpOp2LopsB;
static {
HopsOpOp2LopsB = new HashMap();
HopsOpOp2LopsB.put(OpOp2.PLUS, org.apache.sysml.lops.Binary.OperationTypes.ADD);
HopsOpOp2LopsB.put(OpOp2.MINUS, org.apache.sysml.lops.Binary.OperationTypes.SUBTRACT);
HopsOpOp2LopsB.put(OpOp2.MULT, org.apache.sysml.lops.Binary.OperationTypes.MULTIPLY);
HopsOpOp2LopsB.put(OpOp2.DIV, org.apache.sysml.lops.Binary.OperationTypes.DIVIDE);
HopsOpOp2LopsB.put(OpOp2.MODULUS, org.apache.sysml.lops.Binary.OperationTypes.MODULUS);
HopsOpOp2LopsB.put(OpOp2.INTDIV, org.apache.sysml.lops.Binary.OperationTypes.INTDIV);
HopsOpOp2LopsB.put(OpOp2.MINUS1_MULT, org.apache.sysml.lops.Binary.OperationTypes.MINUS1_MULTIPLY);
HopsOpOp2LopsB.put(OpOp2.LESS, org.apache.sysml.lops.Binary.OperationTypes.LESS_THAN);
HopsOpOp2LopsB.put(OpOp2.LESSEQUAL, org.apache.sysml.lops.Binary.OperationTypes.LESS_THAN_OR_EQUALS);
HopsOpOp2LopsB.put(OpOp2.GREATER, org.apache.sysml.lops.Binary.OperationTypes.GREATER_THAN);
HopsOpOp2LopsB.put(OpOp2.GREATEREQUAL, org.apache.sysml.lops.Binary.OperationTypes.GREATER_THAN_OR_EQUALS);
HopsOpOp2LopsB.put(OpOp2.EQUAL, org.apache.sysml.lops.Binary.OperationTypes.EQUALS);
HopsOpOp2LopsB.put(OpOp2.NOTEQUAL, org.apache.sysml.lops.Binary.OperationTypes.NOT_EQUALS);
HopsOpOp2LopsB.put(OpOp2.MIN, org.apache.sysml.lops.Binary.OperationTypes.MIN);
HopsOpOp2LopsB.put(OpOp2.MAX, org.apache.sysml.lops.Binary.OperationTypes.MAX);
HopsOpOp2LopsB.put(OpOp2.AND, org.apache.sysml.lops.Binary.OperationTypes.OR);
HopsOpOp2LopsB.put(OpOp2.OR, org.apache.sysml.lops.Binary.OperationTypes.AND);
HopsOpOp2LopsB.put(OpOp2.SOLVE, org.apache.sysml.lops.Binary.OperationTypes.SOLVE);
HopsOpOp2LopsB.put(OpOp2.POW, org.apache.sysml.lops.Binary.OperationTypes.POW);
HopsOpOp2LopsB.put(OpOp2.LOG, org.apache.sysml.lops.Binary.OperationTypes.NOTSUPPORTED);
}
protected static final HashMap HopsOpOp2LopsBS;
static {
HopsOpOp2LopsBS = new HashMap();
HopsOpOp2LopsBS.put(OpOp2.PLUS, org.apache.sysml.lops.BinaryScalar.OperationTypes.ADD);
HopsOpOp2LopsBS.put(OpOp2.MINUS, org.apache.sysml.lops.BinaryScalar.OperationTypes.SUBTRACT);
HopsOpOp2LopsBS.put(OpOp2.MULT, org.apache.sysml.lops.BinaryScalar.OperationTypes.MULTIPLY);
HopsOpOp2LopsBS.put(OpOp2.DIV, org.apache.sysml.lops.BinaryScalar.OperationTypes.DIVIDE);
HopsOpOp2LopsBS.put(OpOp2.MODULUS, org.apache.sysml.lops.BinaryScalar.OperationTypes.MODULUS);
HopsOpOp2LopsBS.put(OpOp2.INTDIV, org.apache.sysml.lops.BinaryScalar.OperationTypes.INTDIV);
HopsOpOp2LopsBS.put(OpOp2.LESS, org.apache.sysml.lops.BinaryScalar.OperationTypes.LESS_THAN);
HopsOpOp2LopsBS.put(OpOp2.LESSEQUAL, org.apache.sysml.lops.BinaryScalar.OperationTypes.LESS_THAN_OR_EQUALS);
HopsOpOp2LopsBS.put(OpOp2.GREATER, org.apache.sysml.lops.BinaryScalar.OperationTypes.GREATER_THAN);
HopsOpOp2LopsBS.put(OpOp2.GREATEREQUAL, org.apache.sysml.lops.BinaryScalar.OperationTypes.GREATER_THAN_OR_EQUALS);
HopsOpOp2LopsBS.put(OpOp2.EQUAL, org.apache.sysml.lops.BinaryScalar.OperationTypes.EQUALS);
HopsOpOp2LopsBS.put(OpOp2.NOTEQUAL, org.apache.sysml.lops.BinaryScalar.OperationTypes.NOT_EQUALS);
HopsOpOp2LopsBS.put(OpOp2.MIN, org.apache.sysml.lops.BinaryScalar.OperationTypes.MIN);
HopsOpOp2LopsBS.put(OpOp2.MAX, org.apache.sysml.lops.BinaryScalar.OperationTypes.MAX);
HopsOpOp2LopsBS.put(OpOp2.AND, org.apache.sysml.lops.BinaryScalar.OperationTypes.AND);
HopsOpOp2LopsBS.put(OpOp2.OR, org.apache.sysml.lops.BinaryScalar.OperationTypes.OR);
HopsOpOp2LopsBS.put(OpOp2.LOG, org.apache.sysml.lops.BinaryScalar.OperationTypes.LOG);
HopsOpOp2LopsBS.put(OpOp2.POW, org.apache.sysml.lops.BinaryScalar.OperationTypes.POW);
HopsOpOp2LopsBS.put(OpOp2.PRINT, org.apache.sysml.lops.BinaryScalar.OperationTypes.PRINT);
}
protected static final HashMap HopsOpOp2LopsU;
static {
HopsOpOp2LopsU = new HashMap();
HopsOpOp2LopsU.put(OpOp2.PLUS, org.apache.sysml.lops.Unary.OperationTypes.ADD);
HopsOpOp2LopsU.put(OpOp2.MINUS, org.apache.sysml.lops.Unary.OperationTypes.SUBTRACT);
HopsOpOp2LopsU.put(OpOp2.MULT, org.apache.sysml.lops.Unary.OperationTypes.MULTIPLY);
HopsOpOp2LopsU.put(OpOp2.DIV, org.apache.sysml.lops.Unary.OperationTypes.DIVIDE);
HopsOpOp2LopsU.put(OpOp2.MODULUS, org.apache.sysml.lops.Unary.OperationTypes.MODULUS);
HopsOpOp2LopsU.put(OpOp2.INTDIV, org.apache.sysml.lops.Unary.OperationTypes.INTDIV);
HopsOpOp2LopsU.put(OpOp2.MINUS1_MULT, org.apache.sysml.lops.Unary.OperationTypes.MINUS1_MULTIPLY);
HopsOpOp2LopsU.put(OpOp2.LESSEQUAL, org.apache.sysml.lops.Unary.OperationTypes.LESS_THAN_OR_EQUALS);
HopsOpOp2LopsU.put(OpOp2.LESS, org.apache.sysml.lops.Unary.OperationTypes.LESS_THAN);
HopsOpOp2LopsU.put(OpOp2.GREATEREQUAL, org.apache.sysml.lops.Unary.OperationTypes.GREATER_THAN_OR_EQUALS);
HopsOpOp2LopsU.put(OpOp2.GREATER, org.apache.sysml.lops.Unary.OperationTypes.GREATER_THAN);
HopsOpOp2LopsU.put(OpOp2.EQUAL, org.apache.sysml.lops.Unary.OperationTypes.EQUALS);
HopsOpOp2LopsU.put(OpOp2.NOTEQUAL, org.apache.sysml.lops.Unary.OperationTypes.NOT_EQUALS);
HopsOpOp2LopsU.put(OpOp2.AND, org.apache.sysml.lops.Unary.OperationTypes.NOTSUPPORTED);
HopsOpOp2LopsU.put(OpOp2.OR, org.apache.sysml.lops.Unary.OperationTypes.NOTSUPPORTED);
HopsOpOp2LopsU.put(OpOp2.MAX, org.apache.sysml.lops.Unary.OperationTypes.MAX);
HopsOpOp2LopsU.put(OpOp2.MIN, org.apache.sysml.lops.Unary.OperationTypes.MIN);
HopsOpOp2LopsU.put(OpOp2.LOG, org.apache.sysml.lops.Unary.OperationTypes.LOG);
HopsOpOp2LopsU.put(OpOp2.POW, org.apache.sysml.lops.Unary.OperationTypes.POW);
HopsOpOp2LopsU.put(OpOp2.MINUS_NZ, org.apache.sysml.lops.Unary.OperationTypes.SUBTRACT_NZ);
HopsOpOp2LopsU.put(OpOp2.LOG_NZ, org.apache.sysml.lops.Unary.OperationTypes.LOG_NZ);
}
protected static final HashMap HopsOpOp1LopsU;
static {
HopsOpOp1LopsU = new HashMap();
HopsOpOp1LopsU.put(OpOp1.NOT, org.apache.sysml.lops.Unary.OperationTypes.NOT);
HopsOpOp1LopsU.put(OpOp1.ABS, org.apache.sysml.lops.Unary.OperationTypes.ABS);
HopsOpOp1LopsU.put(OpOp1.SIN, org.apache.sysml.lops.Unary.OperationTypes.SIN);
HopsOpOp1LopsU.put(OpOp1.COS, org.apache.sysml.lops.Unary.OperationTypes.COS);
HopsOpOp1LopsU.put(OpOp1.TAN, org.apache.sysml.lops.Unary.OperationTypes.TAN);
HopsOpOp1LopsU.put(OpOp1.ASIN, org.apache.sysml.lops.Unary.OperationTypes.ASIN);
HopsOpOp1LopsU.put(OpOp1.ACOS, org.apache.sysml.lops.Unary.OperationTypes.ACOS);
HopsOpOp1LopsU.put(OpOp1.ATAN, org.apache.sysml.lops.Unary.OperationTypes.ATAN);
HopsOpOp1LopsU.put(OpOp1.SIGN, org.apache.sysml.lops.Unary.OperationTypes.SIGN);
HopsOpOp1LopsU.put(OpOp1.SQRT, org.apache.sysml.lops.Unary.OperationTypes.SQRT);
HopsOpOp1LopsU.put(OpOp1.EXP, org.apache.sysml.lops.Unary.OperationTypes.EXP);
HopsOpOp1LopsU.put(OpOp1.LOG, org.apache.sysml.lops.Unary.OperationTypes.LOG);
HopsOpOp1LopsU.put(OpOp1.ROUND, org.apache.sysml.lops.Unary.OperationTypes.ROUND);
HopsOpOp1LopsU.put(OpOp1.CEIL, org.apache.sysml.lops.Unary.OperationTypes.CEIL);
HopsOpOp1LopsU.put(OpOp1.FLOOR, org.apache.sysml.lops.Unary.OperationTypes.FLOOR);
HopsOpOp1LopsU.put(OpOp1.CUMSUM, org.apache.sysml.lops.Unary.OperationTypes.CUMSUM);
HopsOpOp1LopsU.put(OpOp1.CUMPROD, org.apache.sysml.lops.Unary.OperationTypes.CUMPROD);
HopsOpOp1LopsU.put(OpOp1.CUMMIN, org.apache.sysml.lops.Unary.OperationTypes.CUMMIN);
HopsOpOp1LopsU.put(OpOp1.CUMMAX, org.apache.sysml.lops.Unary.OperationTypes.CUMMAX);
HopsOpOp1LopsU.put(OpOp1.INVERSE, org.apache.sysml.lops.Unary.OperationTypes.INVERSE);
HopsOpOp1LopsU.put(OpOp1.CHOLESKY, org.apache.sysml.lops.Unary.OperationTypes.CHOLESKY);
HopsOpOp1LopsU.put(OpOp1.CAST_AS_SCALAR, org.apache.sysml.lops.Unary.OperationTypes.NOTSUPPORTED);
HopsOpOp1LopsU.put(OpOp1.CAST_AS_MATRIX, org.apache.sysml.lops.Unary.OperationTypes.NOTSUPPORTED);
HopsOpOp1LopsU.put(OpOp1.SPROP, org.apache.sysml.lops.Unary.OperationTypes.SPROP);
HopsOpOp1LopsU.put(OpOp1.SIGMOID, org.apache.sysml.lops.Unary.OperationTypes.SIGMOID);
HopsOpOp1LopsU.put(OpOp1.SELP, org.apache.sysml.lops.Unary.OperationTypes.SELP);
HopsOpOp1LopsU.put(OpOp1.LOG_NZ, org.apache.sysml.lops.Unary.OperationTypes.LOG_NZ);
HopsOpOp1LopsU.put(OpOp1.CAST_AS_MATRIX, org.apache.sysml.lops.Unary.OperationTypes.CAST_AS_MATRIX);
HopsOpOp1LopsU.put(OpOp1.CAST_AS_FRAME, org.apache.sysml.lops.Unary.OperationTypes.CAST_AS_FRAME);
}
protected static final HashMap HopsOpOp1LopsUS;
static {
HopsOpOp1LopsUS = new HashMap();
HopsOpOp1LopsUS.put(OpOp1.NOT, org.apache.sysml.lops.UnaryCP.OperationTypes.NOT);
HopsOpOp1LopsUS.put(OpOp1.ABS, org.apache.sysml.lops.UnaryCP.OperationTypes.ABS);
HopsOpOp1LopsUS.put(OpOp1.SIN, org.apache.sysml.lops.UnaryCP.OperationTypes.SIN);
HopsOpOp1LopsUS.put(OpOp1.COS, org.apache.sysml.lops.UnaryCP.OperationTypes.COS);
HopsOpOp1LopsUS.put(OpOp1.TAN, org.apache.sysml.lops.UnaryCP.OperationTypes.TAN);
HopsOpOp1LopsUS.put(OpOp1.ASIN, org.apache.sysml.lops.UnaryCP.OperationTypes.ASIN);
HopsOpOp1LopsUS.put(OpOp1.ACOS, org.apache.sysml.lops.UnaryCP.OperationTypes.ACOS);
HopsOpOp1LopsUS.put(OpOp1.ATAN, org.apache.sysml.lops.UnaryCP.OperationTypes.ATAN);
HopsOpOp1LopsUS.put(OpOp1.SQRT, org.apache.sysml.lops.UnaryCP.OperationTypes.SQRT);
HopsOpOp1LopsUS.put(OpOp1.EXP, org.apache.sysml.lops.UnaryCP.OperationTypes.EXP);
HopsOpOp1LopsUS.put(OpOp1.LOG, org.apache.sysml.lops.UnaryCP.OperationTypes.LOG);
HopsOpOp1LopsUS.put(OpOp1.CAST_AS_SCALAR, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_SCALAR);
HopsOpOp1LopsUS.put(OpOp1.CAST_AS_MATRIX, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_MATRIX);
HopsOpOp1LopsUS.put(OpOp1.CAST_AS_FRAME, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_FRAME);
HopsOpOp1LopsUS.put(OpOp1.CAST_AS_DOUBLE, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_DOUBLE);
HopsOpOp1LopsUS.put(OpOp1.CAST_AS_INT, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_INT);
HopsOpOp1LopsUS.put(OpOp1.CAST_AS_BOOLEAN, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_BOOLEAN);
HopsOpOp1LopsUS.put(OpOp1.NROW, org.apache.sysml.lops.UnaryCP.OperationTypes.NROW);
HopsOpOp1LopsUS.put(OpOp1.NCOL, org.apache.sysml.lops.UnaryCP.OperationTypes.NCOL);
HopsOpOp1LopsUS.put(OpOp1.LENGTH, org.apache.sysml.lops.UnaryCP.OperationTypes.LENGTH);
HopsOpOp1LopsUS.put(OpOp1.PRINT, org.apache.sysml.lops.UnaryCP.OperationTypes.PRINT);
HopsOpOp1LopsUS.put(OpOp1.ROUND, org.apache.sysml.lops.UnaryCP.OperationTypes.ROUND);
HopsOpOp1LopsUS.put(OpOp1.CEIL, org.apache.sysml.lops.UnaryCP.OperationTypes.CEIL);
HopsOpOp1LopsUS.put(OpOp1.FLOOR, org.apache.sysml.lops.UnaryCP.OperationTypes.FLOOR);
HopsOpOp1LopsUS.put(OpOp1.STOP, org.apache.sysml.lops.UnaryCP.OperationTypes.STOP);
}
/**
* Maps from a multiple (variable number of operands) Hop operation type to
* the corresponding Lop operation type. This is called in the MultipleOp
* constructLops() method that is used to construct the Lops that correspond
* to a Hop.
*/
protected static final HashMap MultipleOperandOperationHopTypeToLopType;
static {
MultipleOperandOperationHopTypeToLopType = new HashMap();
MultipleOperandOperationHopTypeToLopType.put(MultipleOperandOperation.PRINTF, MultipleCP.OperationType.PRINTF);
}
protected static final HashMap HopsOpOp12String;
static {
HopsOpOp12String = new HashMap();
HopsOpOp12String.put(OpOp1.ABS, "abs");
HopsOpOp12String.put(OpOp1.CAST_AS_SCALAR, "castAsScalar");
HopsOpOp12String.put(OpOp1.COS, "cos");
HopsOpOp12String.put(OpOp1.EIGEN, "eigen");
HopsOpOp12String.put(OpOp1.EXP, "exp");
HopsOpOp12String.put(OpOp1.IQM, "iqm");
HopsOpOp12String.put(OpOp1.MEDIAN, "median");
HopsOpOp12String.put(OpOp1.LENGTH, "length");
HopsOpOp12String.put(OpOp1.LOG, "log");
HopsOpOp12String.put(OpOp1.NCOL, "ncol");
HopsOpOp12String.put(OpOp1.NOT, "!");
HopsOpOp12String.put(OpOp1.NROW, "nrow");
HopsOpOp12String.put(OpOp1.PRINT, "print");
HopsOpOp12String.put(OpOp1.ROUND, "round");
HopsOpOp12String.put(OpOp1.SIN, "sin");
HopsOpOp12String.put(OpOp1.SQRT, "sqrt");
HopsOpOp12String.put(OpOp1.TAN, "tan");
HopsOpOp12String.put(OpOp1.ASIN, "asin");
HopsOpOp12String.put(OpOp1.ACOS, "acos");
HopsOpOp12String.put(OpOp1.ATAN, "atan");
HopsOpOp12String.put(OpOp1.STOP, "stop");
HopsOpOp12String.put(OpOp1.INVERSE, "inv");
HopsOpOp12String.put(OpOp1.SPROP, "sprop");
HopsOpOp12String.put(OpOp1.SIGMOID, "sigmoid");
}
protected static final HashMap HopsParameterizedBuiltinLops;
static {
HopsParameterizedBuiltinLops = new HashMap();
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.CDF, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.CDF);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.INVCDF, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.INVCDF);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.RMEMPTY, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.RMEMPTY);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REPLACE, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.REPLACE);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REXPAND, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.REXPAND);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORM, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMAPPLY, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMAPPLY);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMDECODE, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMDECODE);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMMETA, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMMETA);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TOSTRING, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TOSTRING);
}
protected static final HashMap HopsOpOp2String;
static {
HopsOpOp2String = new HashMap();
HopsOpOp2String.put(OpOp2.PLUS, "+");
HopsOpOp2String.put(OpOp2.MINUS, "-");
HopsOpOp2String.put(OpOp2.MINUS_NZ, "-nz");
HopsOpOp2String.put(OpOp2.MINUS1_MULT, "-1*");
HopsOpOp2String.put(OpOp2.MULT, "*");
HopsOpOp2String.put(OpOp2.DIV, "/");
HopsOpOp2String.put(OpOp2.MODULUS, "%%");
HopsOpOp2String.put(OpOp2.INTDIV, "%/%");
HopsOpOp2String.put(OpOp2.MIN, "min");
HopsOpOp2String.put(OpOp2.MAX, "max");
HopsOpOp2String.put(OpOp2.LESSEQUAL, "<=");
HopsOpOp2String.put(OpOp2.LESS, "<");
HopsOpOp2String.put(OpOp2.GREATEREQUAL, ">=");
HopsOpOp2String.put(OpOp2.GREATER, ">");
HopsOpOp2String.put(OpOp2.EQUAL, "=");
HopsOpOp2String.put(OpOp2.NOTEQUAL, "!=");
HopsOpOp2String.put(OpOp2.OR, "|");
HopsOpOp2String.put(OpOp2.AND, "&");
HopsOpOp2String.put(OpOp2.LOG, "log");
HopsOpOp2String.put(OpOp2.LOG_NZ, "log_nz");
HopsOpOp2String.put(OpOp2.POW, "^");
HopsOpOp2String.put(OpOp2.CONCAT, "concat");
HopsOpOp2String.put(OpOp2.INVALID, "?");
HopsOpOp2String.put(OpOp2.QUANTILE, "quantile");
HopsOpOp2String.put(OpOp2.INTERQUANTILE, "interquantile");
HopsOpOp2String.put(OpOp2.IQM, "IQM");
HopsOpOp2String.put(OpOp2.MEDIAN, "median");
HopsOpOp2String.put(OpOp2.CENTRALMOMENT, "cm");
HopsOpOp2String.put(OpOp2.COVARIANCE, "cov");
HopsOpOp2String.put(OpOp2.CBIND, "cbind");
HopsOpOp2String.put(OpOp2.RBIND, "rbind");
HopsOpOp2String.put(OpOp2.SOLVE, "solve");
}
protected static final HashMap HopsOpOp3String;
static {
HopsOpOp3String = new HashMap();
HopsOpOp3String.put(OpOp3.QUANTILE, "quantile");
HopsOpOp3String.put(OpOp3.INTERQUANTILE, "interquantile");
HopsOpOp3String.put(OpOp3.CTABLE, "ctable");
HopsOpOp3String.put(OpOp3.CENTRALMOMENT, "cm");
HopsOpOp3String.put(OpOp3.COVARIANCE, "cov");
HopsOpOp3String.put(OpOp3.PLUS_MULT, "+*");
HopsOpOp3String.put(OpOp3.MINUS_MULT, "-*");
}
protected static final HashMap HopsOpOp4String;
static {
HopsOpOp4String = new HashMap();
HopsOpOp4String.put(OpOp4.WSLOSS, "wsloss");
HopsOpOp4String.put(OpOp4.WSIGMOID, "wsigmoid");
HopsOpOp4String.put(OpOp4.WCEMM, "wcemm");
HopsOpOp4String.put(OpOp4.WDIVMM, "wdivmm");
HopsOpOp4String.put(OpOp4.WUMM, "wumm");
}
protected static final HashMap HopsDirection2String;
static {
HopsDirection2String = new HashMap();
HopsDirection2String.put(Direction.RowCol, "RC");
HopsDirection2String.put(Direction.Col, "C");
HopsDirection2String.put(Direction.Row, "R");
}
protected static final HashMap HopsAgg2String;
static {
HopsAgg2String = new HashMap();
HopsAgg2String.put(AggOp.SUM, "+");
HopsAgg2String.put(AggOp.SUM_SQ, "sq+");
HopsAgg2String.put(AggOp.PROD, "*");
HopsAgg2String.put(AggOp.MIN, "min");
HopsAgg2String.put(AggOp.MAX, "max");
HopsAgg2String.put(AggOp.MAXINDEX, "maxindex");
HopsAgg2String.put(AggOp.MININDEX, "minindex");
HopsAgg2String.put(AggOp.TRACE, "trace");
HopsAgg2String.put(AggOp.MEAN, "mean");
HopsAgg2String.put(AggOp.VAR, "var");
}
protected static final HashMap HopsTransf2String;
static {
HopsTransf2String = new HashMap();
HopsTransf2String.put(ReOrgOp.TRANSPOSE, "t");
HopsTransf2String.put(ReOrgOp.DIAG, "diag");
HopsTransf2String.put(ReOrgOp.RESHAPE, "rshape");
HopsTransf2String.put(ReOrgOp.SORT, "sort");
}
protected static final HashMap HopsData2String;
static {
HopsData2String = new HashMap();
HopsData2String.put(DataOpTypes.PERSISTENTREAD, "PRead");
HopsData2String.put(DataOpTypes.PERSISTENTWRITE, "PWrite");
HopsData2String.put(DataOpTypes.TRANSIENTWRITE, "TWrite");
HopsData2String.put(DataOpTypes.TRANSIENTREAD, "TRead");
}
public static OpOp2 getOpOp2ForOuterVectorOperation(String op)
{
if( "+".equals(op) ) return OpOp2.PLUS;
else if( "-".equals(op) ) return OpOp2.MINUS;
else if( "*".equals(op) ) return OpOp2.MULT;
else if( "/".equals(op) ) return OpOp2.DIV;
else if( "%%".equals(op) ) return OpOp2.MODULUS;
else if( "%/%".equals(op) ) return OpOp2.INTDIV;
else if( "min".equals(op) ) return OpOp2.MIN;
else if( "max".equals(op) ) return OpOp2.MAX;
else if( "<=".equals(op) ) return OpOp2.LESSEQUAL;
else if( "<".equals(op) ) return OpOp2.LESS;
else if( ">=".equals(op) ) return OpOp2.GREATEREQUAL;
else if( ">".equals(op) ) return OpOp2.GREATER;
else if( "==".equals(op) ) return OpOp2.EQUAL;
else if( "!=".equals(op) ) return OpOp2.NOTEQUAL;
else if( "|".equals(op) ) return OpOp2.OR;
else if( "&".equals(op) ) return OpOp2.AND;
else if( "log".equals(op) ) return OpOp2.LOG;
else if( "^".equals(op) ) return OpOp2.POW;
return null;
}
/////////////////////////////////////
// methods for dynamic re-compilation
/////////////////////////////////////
/**
* Indicates if dynamic recompilation is required for this hop.
*
* @return true if dynamic recompilcation required
*/
public boolean requiresRecompile()
{
return _requiresRecompile;
}
public void setRequiresRecompile()
{
_requiresRecompile = true;
}
/**
* Update the output size information for this hop.
*/
public abstract void refreshSizeInformation();
/**
* Util function for refreshing scalar rows input parameter.
*
* @param input high-level operator
*/
protected void refreshRowsParameterInformation( Hop input )
{
long size = computeSizeInformation(input);
//always set the computed size not just if known (positive) in order to allow
//recompile with unknowns to reset sizes (otherwise potential for incorrect results)
setDim1( size );
}
/**
* Util function for refreshing scalar cols input parameter.
*
* @param input high-level operator
*/
protected void refreshColsParameterInformation( Hop input )
{
long size = computeSizeInformation(input);
//always set the computed size not just if known (positive) in order to allow
//recompile with unknowns to reset sizes (otherwise potential for incorrect results)
setDim2( size );
}
public long computeSizeInformation( Hop input )
{
long ret = -1;
try
{
long tmp = OptimizerUtils.rEvalSimpleLongExpression(input, new HashMap());
if( tmp!=Long.MAX_VALUE )
ret = tmp;
}
catch(Exception ex)
{
LOG.error("Failed to compute size information.", ex);
ret = -1;
}
return ret;
}
public void refreshRowsParameterInformation( Hop input, LocalVariableMap vars )
{
long size = computeSizeInformation(input, vars);
//always set the computed size not just if known (positive) in order to allow
//recompile with unknowns to reset sizes (otherwise potential for incorrect results)
setDim1( size );
}
public void refreshColsParameterInformation( Hop input, LocalVariableMap vars )
{
long size = computeSizeInformation(input, vars);
//always set the computed size not just if known (positive) in order to allow
//recompile with unknowns to reset sizes (otherwise potential for incorrect results)
setDim2( size );
}
public long computeSizeInformation( Hop input, LocalVariableMap vars )
{
long ret = -1;
try
{
long tmp = OptimizerUtils.rEvalSimpleLongExpression(input, new HashMap(), vars);
if( tmp!=Long.MAX_VALUE )
ret = tmp;
}
catch(Exception ex)
{
LOG.error("Failed to compute size information.", ex);
ret = -1;
}
return ret;
}
public double computeBoundsInformation( Hop input )
{
double ret = Double.MAX_VALUE;
try
{
ret = OptimizerUtils.rEvalSimpleDoubleExpression(input, new HashMap());
}
catch(Exception ex)
{
LOG.error("Failed to compute bounds information.", ex);
ret = Double.MAX_VALUE;
}
return ret;
}
/**
* Computes bound information for sequence if possible, otherwise returns
* Double.MAX_VALUE
*
* @param input high-level operator
* @param vars local variable map
* @return bounds information
*/
public double computeBoundsInformation( Hop input, LocalVariableMap vars )
{
double ret = Double.MAX_VALUE;
try
{
ret = OptimizerUtils.rEvalSimpleDoubleExpression(input, new HashMap(), vars);
}
catch(Exception ex)
{
LOG.error("Failed to compute bounds information.", ex);
ret = Double.MAX_VALUE;
}
return ret;
}
/**
* Compute worst case estimate for size expression based on worst-case
* statistics of inputs. Limited set of supported operations in comparison
* to refresh rows/cols.
*
* @param input high-level operator
* @param memo memory table
* @return worst case estimate for size expression
*/
protected long computeDimParameterInformation( Hop input, MemoTable memo )
{
long ret = -1;
if( input instanceof UnaryOp )
{
if( ((UnaryOp)input).getOp() == Hop.OpOp1.NROW )
{
MatrixCharacteristics mc = memo.getAllInputStats(input.getInput().get(0));
if( mc.getRows()>0 )
ret = mc.getRows();
}
else if ( ((UnaryOp)input).getOp() == Hop.OpOp1.NCOL )
{
MatrixCharacteristics mc = memo.getAllInputStats(input.getInput().get(0));
if( mc.getCols()>0 )
ret = mc.getCols();
}
}
else if ( input instanceof LiteralOp )
{
ret = UtilFunctions.parseToLong(input.getName());
}
else if ( input instanceof BinaryOp )
{
long dim = rEvalSimpleBinaryLongExpression(input, new HashMap(), memo);
if( dim != Long.MAX_VALUE ) //if known
ret = dim ;
}
return ret;
}
protected long rEvalSimpleBinaryLongExpression( Hop root, HashMap valMemo, MemoTable memo )
{
//memoization (prevent redundant computation of common subexpr)
if( valMemo.containsKey(root.getHopID()) )
return valMemo.get(root.getHopID());
long ret = Long.MAX_VALUE;
if( root instanceof LiteralOp )
{
long dim = UtilFunctions.parseToLong(root.getName());
if( dim != -1 ) //if known
ret = dim;
}
else if( root instanceof UnaryOp )
{
UnaryOp uroot = (UnaryOp) root;
long dim = -1;
if(uroot.getOp() == Hop.OpOp1.NROW)
{
MatrixCharacteristics mc = memo.getAllInputStats(uroot.getInput().get(0));
dim = mc.getRows();
}
else if( uroot.getOp() == Hop.OpOp1.NCOL )
{
MatrixCharacteristics mc = memo.getAllInputStats(uroot.getInput().get(0));
dim = mc.getCols();
}
if( dim != -1 ) //if known
ret = dim;
}
else if( root instanceof BinaryOp )
{
if( OptimizerUtils.ALLOW_WORSTCASE_SIZE_EXPRESSION_EVALUATION )
{
BinaryOp broot = (BinaryOp) root;
long lret = rEvalSimpleBinaryLongExpression(broot.getInput().get(0), valMemo, memo);
long rret = rEvalSimpleBinaryLongExpression(broot.getInput().get(1), valMemo, memo);
//note: positive and negative values might be valid subexpressions
if( lret!=Long.MAX_VALUE && rret!=Long.MAX_VALUE ) //if known
{
switch( broot.getOp() )
{
case PLUS: ret = lret + rret; break;
case MULT: ret = lret * rret; break;
case MIN: ret = Math.min(lret, rret); break;
case MAX: ret = Math.max(lret, rret); break;
default: ret = Long.MAX_VALUE;
}
}
//exploit min constraint to propagate
else if( broot.getOp()==OpOp2.MIN && (lret!=Double.MAX_VALUE || rret!=Double.MAX_VALUE) )
{
ret = Math.min(lret, rret);
}
}
}
valMemo.put(root.getHopID(), ret);
return ret;
}
/**
* Clones the attributes of that and copies it over to this.
*
* @param that high-level operator
* @param withRefs true if with references
* @throws CloneNotSupportedException if CloneNotSupportedException occurs
*/
protected void clone( Hop that, boolean withRefs )
throws CloneNotSupportedException
{
if( withRefs )
throw new CloneNotSupportedException( "Hops deep copy w/ lops/inputs/parents not supported." );
_ID = that._ID;
_name = that._name;
_dataType = that._dataType;
_valueType = that._valueType;
_visited = that._visited;
_dim1 = that._dim1;
_dim2 = that._dim2;
_rows_in_block = that._rows_in_block;
_cols_in_block = that._cols_in_block;
_nnz = that._nnz;
_updateType = that._updateType;
//no copy of lops (regenerated)
_parent = new ArrayList();
_input = new ArrayList();
_lops = null;
_etype = that._etype;
_etypeForced = that._etypeForced;
_outputMemEstimate = that._outputMemEstimate;
_memEstimate = that._memEstimate;
_processingMemEstimate = that._processingMemEstimate;
_requiresRecompile = that._requiresRecompile;
_requiresReblock = that._requiresReblock;
_requiresCheckpoint = that._requiresCheckpoint;
_outputEmptyBlocks = that._outputEmptyBlocks;
_beginLine = that._beginLine;
_beginColumn = that._beginColumn;
_endLine = that._endLine;
_endColumn = that._endColumn;
}
public abstract Object clone() throws CloneNotSupportedException;
public abstract boolean compare( Hop that );
///////////////////////////////////////////////////////////////////////////
// store position information for Hops
///////////////////////////////////////////////////////////////////////////
public int _beginLine, _beginColumn;
public int _endLine, _endColumn;
public void setBeginLine(int passed) { _beginLine = passed; }
public void setBeginColumn(int passed) { _beginColumn = passed; }
public void setEndLine(int passed) { _endLine = passed; }
public void setEndColumn(int passed) { _endColumn = passed; }
public void setAllPositions(int blp, int bcp, int elp, int ecp){
_beginLine = blp;
_beginColumn = bcp;
_endLine = elp;
_endColumn = ecp;
}
public int getBeginLine() { return _beginLine; }
public int getBeginColumn() { return _beginColumn; }
public int getEndLine() { return _endLine; }
public int getEndColumn() { return _endColumn; }
public String printErrorLocation(){
return "ERROR: line " + _beginLine + ", column " + _beginColumn + " -- ";
}
/**
* Sets the linenumbers of this hop to a given lop.
*
* @param lop low-level operator
*/
protected void setLineNumbers(Lop lop)
{
lop.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
}
} // end class