org.apache.sysml.runtime.matrix.mapred.MMRJMRMapper Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of systemml Show documentation
Declarative Machine Learning
There is a newer version: 1.2.0
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */


package org.apache.sysml.runtime.matrix.mapred;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;

import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;

import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
import org.apache.sysml.runtime.instructions.mr.AggregateBinaryInstruction;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.TaggedMatrixValue;
import org.apache.sysml.runtime.matrix.data.TripleIndexes;


public class MMRJMRMapper extends MapperBase 
implements Mapper
{
		
	//the aggregate binary instruction for this mmcj job
	private TripleIndexes triplebuffer=new TripleIndexes();
	private TaggedMatrixValue taggedValue=null;
	private HashMap numRepeats=new HashMap();
	private HashSet aggBinInput1s=new HashSet();
	private HashSet aggBinInput2s=new HashSet();
	
	@Override
	protected void specialOperationsForActualMap(int index,
			OutputCollector out, Reporter reporter)
			throws IOException {
		//apply all instructions
		processMapperInstructionsForMatrix(index);
		
		for(byte output: outputIndexes.get(index))
		{
			ArrayList blkList = cachedValues.get(output);
			if( blkList != null )
				for(IndexedMatrixValue result : blkList )
				{
					if(result==null)
						continue;
					
					//output the left matrix
					if(aggBinInput1s.contains(output))
					{
						for(long j=0; j out, Reporter reporter)
			throws IOException {
		commonMap(rawKey, rawValue, out, reporter);
		
	}
	
	public void configure(JobConf job)
	{
		super.configure(job);
		taggedValue=TaggedMatrixValue.createObject(valueClass);
		AggregateBinaryInstruction[] aggBinInstructions;
		try {
			aggBinInstructions = MRJobConfiguration.getAggregateBinaryInstructions(job);
		} catch (DMLUnsupportedOperationException e) {
			throw new RuntimeException(e);
		} catch (DMLRuntimeException e) {
			throw new RuntimeException(e);
		}
		
		for(AggregateBinaryInstruction aggBinInstruction: aggBinInstructions)
		{
			MatrixCharacteristics mc=MRJobConfiguration.getMatrixCharactristicsForBinAgg(job, aggBinInstruction.input2);
			long matrixNumColumn=mc.getCols();
			int blockNumColumn=mc.getColsPerBlock();
			numRepeats.put(aggBinInstruction.input1, (long)Math.ceil((double)matrixNumColumn/(double)blockNumColumn));
			
			mc=MRJobConfiguration.getMatrixCharactristicsForBinAgg(job, aggBinInstruction.input1);
			long matrixNumRow=mc.getRows();
			int blockNumRow=mc.getRowsPerBlock();
			numRepeats.put(aggBinInstruction.input2, (long)Math.ceil((double)matrixNumRow/(double)blockNumRow));
			aggBinInput1s.add(aggBinInstruction.input1);
			aggBinInput2s.add(aggBinInstruction.input2);
		}
	}
}