org.apache.hadoop.mapred.lib.ChainReducer Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapred.lib;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.*;
import java.io.IOException;
import java.util.Iterator;
/**
* The ChainReducer class allows to chain multiple Mapper classes after a
* Reducer within the Reducer task.
*
* For each record output by the Reducer, the Mapper classes are invoked in a
* chained (or piped) fashion, the output of the first becomes the input of the
* second, and so on until the last Mapper, the output of the last Mapper will
* be written to the task's output.
*
* The key functionality of this feature is that the Mappers in the chain do not
* need to be aware that they are executed after the Reducer or in a chain.
* This enables having reusable specialized Mappers that can be combined to
* perform composite operations within a single task.
*
* Special care has to be taken when creating chains that the key/values output
* by a Mapper are valid for the following Mapper in the chain. It is assumed
* all Mappers and the Reduce in the chain use maching output and input key and
* value classes as no conversion is done by the chaining code.
*
* Using the ChainMapper and the ChainReducer classes is possible to compose
* Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And
* immediate benefit of this pattern is a dramatic reduction in disk IO.
*
* IMPORTANT: There is no need to specify the output key/value classes for the
* ChainReducer, this is done by the setReducer or the addMapper for the last
* element in the chain.
*
* ChainReducer usage pattern:
*
*
* ...
* conf.setJobName("chain");
* conf.setInputFormat(TextInputFormat.class);
* conf.setOutputFormat(TextOutputFormat.class);
*
* JobConf mapAConf = new JobConf(false);
* ...
* ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
* Text.class, Text.class, true, mapAConf);
*
* JobConf mapBConf = new JobConf(false);
* ...
* ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
* LongWritable.class, Text.class, false, mapBConf);
*
* JobConf reduceConf = new JobConf(false);
* ...
* ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
* Text.class, Text.class, true, reduceConf);
*
* ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
* LongWritable.class, Text.class, false, null);
*
* ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
* LongWritable.class, LongWritable.class, true, null);
*
* FileInputFormat.setInputPaths(conf, inDir);
* FileOutputFormat.setOutputPath(conf, outDir);
* ...
*
* JobClient jc = new JobClient(conf);
* RunningJob job = jc.submitJob(conf);
* ...
*
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
public class ChainReducer implements Reducer {
/**
* Sets the Reducer class to the chain job's JobConf.
*
* It has to be specified how key and values are passed from one element of
* the chain to the next, by value or by reference. If a Reducer leverages the
* assumed semantics that the key and values are not modified by the collector
* 'by value' must be used. If the Reducer does not expect this semantics, as
* an optimization to avoid serialization and deserialization 'by reference'
* can be used.
*
* For the added Reducer the configuration given for it,
* reducerConf, have precedence over the job's JobConf. This
* precedence is in effect when the task is running.
*
* IMPORTANT: There is no need to specify the output key/value classes for the
* ChainReducer, this is done by the setReducer or the addMapper for the last
* element in the chain.
*
* @param job job's JobConf to add the Reducer class.
* @param klass the Reducer class to add.
* @param inputKeyClass reducer input key class.
* @param inputValueClass reducer input value class.
* @param outputKeyClass reducer output key class.
* @param outputValueClass reducer output value class.
* @param byValue indicates if key/values should be passed by value
* to the next Mapper in the chain, if any.
* @param reducerConf a JobConf with the configuration for the Reducer
* class. It is recommended to use a JobConf without default values using the
* JobConf(boolean loadDefaults) constructor with FALSE.
*/
public static void setReducer(JobConf job,
Class> klass,
Class inputKeyClass,
Class inputValueClass,
Class outputKeyClass,
Class outputValueClass,
boolean byValue, JobConf reducerConf) {
job.setReducerClass(ChainReducer.class);
job.setOutputKeyClass(outputKeyClass);
job.setOutputValueClass(outputValueClass);
Chain.setReducer(job, klass, inputKeyClass, inputValueClass, outputKeyClass,
outputValueClass, byValue, reducerConf);
}
/**
* Adds a Mapper class to the chain job's JobConf.
*
* It has to be specified how key and values are passed from one element of
* the chain to the next, by value or by reference. If a Mapper leverages the
* assumed semantics that the key and values are not modified by the collector
* 'by value' must be used. If the Mapper does not expect this semantics, as
* an optimization to avoid serialization and deserialization 'by reference'
* can be used.
*
* For the added Mapper the configuration given for it,
* mapperConf, have precedence over the job's JobConf. This
* precedence is in effect when the task is running.
*
* IMPORTANT: There is no need to specify the output key/value classes for the
* ChainMapper, this is done by the addMapper for the last mapper in the chain
* .
*
* @param job chain job's JobConf to add the Mapper class.
* @param klass the Mapper class to add.
* @param inputKeyClass mapper input key class.
* @param inputValueClass mapper input value class.
* @param outputKeyClass mapper output key class.
* @param outputValueClass mapper output value class.
* @param byValue indicates if key/values should be passed by value
* to the next Mapper in the chain, if any.
* @param mapperConf a JobConf with the configuration for the Mapper
* class. It is recommended to use a JobConf without default values using the
* JobConf(boolean loadDefaults) constructor with FALSE.
*/
public static void addMapper(JobConf job,
Class> klass,
Class inputKeyClass,
Class inputValueClass,
Class outputKeyClass,
Class outputValueClass,
boolean byValue, JobConf mapperConf) {
job.setOutputKeyClass(outputKeyClass);
job.setOutputValueClass(outputValueClass);
Chain.addMapper(false, job, klass, inputKeyClass, inputValueClass,
outputKeyClass, outputValueClass, byValue, mapperConf);
}
private Chain chain;
/**
* Constructor.
*/
public ChainReducer() {
chain = new Chain(false);
}
/**
* Configures the ChainReducer, the Reducer and all the Mappers in the chain.
*
* If this method is overriden super.configure(...) should be
* invoked at the beginning of the overwriter method.
*/
public void configure(JobConf job) {
chain.configure(job);
}
/**
* Chains the reduce(...) method of the Reducer with the
* map(...) methods of the Mappers in the chain.
*/
@SuppressWarnings({"unchecked"})
public void reduce(Object key, Iterator values, OutputCollector output,
Reporter reporter) throws IOException {
Reducer reducer = chain.getReducer();
if (reducer != null) {
reducer.reduce(key, values, chain.getReducerCollector(output, reporter),
reporter);
}
}
/**
* Closes the ChainReducer, the Reducer and all the Mappers in the chain.
*
* If this method is overriden super.close() should be
* invoked at the end of the overwriter method.
*/
public void close() throws IOException {
chain.close();
}
}