org.apache.hadoop.hive.ql.exec.tez.RecordProcessor Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.exec.tez;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.ObjectCache;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.tez.TezProcessor.TezKVOutputCollector;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.tez.mapreduce.processor.MRTaskReporter;
import org.apache.tez.runtime.api.LogicalInput;
import org.apache.tez.runtime.api.LogicalOutput;
import org.apache.tez.runtime.api.ProcessorContext;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
/**
* Process input from tez LogicalInput and write output
* It has different subclasses for map and reduce processing
*/
public abstract class RecordProcessor {
protected final JobConf jconf;
protected Map inputs;
protected Map outputs;
protected Map outMap;
protected final ProcessorContext processorContext;
public static final Log l4j = LogFactory.getLog(RecordProcessor.class);
// used to log memory usage periodically
protected boolean isLogInfoEnabled = false;
protected boolean isLogTraceEnabled = false;
protected MRTaskReporter reporter;
protected PerfLogger perfLogger = PerfLogger.getPerfLogger();
protected String CLASS_NAME = RecordProcessor.class.getName();
public RecordProcessor(JobConf jConf, ProcessorContext processorContext) {
this.jconf = jConf;
this.processorContext = processorContext;
}
/**
* Common initialization code for RecordProcessors
* @param mrReporter
* @param inputs map of Input names to {@link LogicalInput}s
* @param outputs map of Output names to {@link LogicalOutput}s
* @throws Exception
*/
void init(MRTaskReporter mrReporter,
Map inputs, Map outputs) throws Exception {
this.reporter = mrReporter;
this.inputs = inputs;
this.outputs = outputs;
isLogInfoEnabled = l4j.isInfoEnabled();
isLogTraceEnabled = l4j.isTraceEnabled();
//log classpaths
try {
if (l4j.isDebugEnabled()) {
l4j.debug("conf classpath = "
+ Arrays.asList(((URLClassLoader) jconf.getClassLoader()).getURLs()));
l4j.debug("thread classpath = "
+ Arrays.asList(((URLClassLoader) Thread.currentThread()
.getContextClassLoader()).getURLs()));
}
} catch (Exception e) {
l4j.info("cannot get classpath: " + e.getMessage());
}
}
/**
* start processing the inputs and writing output
* @throws Exception
*/
abstract void run() throws Exception;
abstract void close();
protected void createOutputMap() {
Preconditions.checkState(outMap == null, "Outputs should only be setup once");
outMap = Maps.newHashMap();
for (Entry entry : outputs.entrySet()) {
TezKVOutputCollector collector = new TezKVOutputCollector(entry.getValue());
outMap.put(entry.getKey(), collector);
}
}
public List getMergeWorkList(final JobConf jconf, String key, String queryId,
ObjectCache cache, List cacheKeys) throws HiveException {
String prefixes = jconf.get(DagUtils.TEZ_MERGE_WORK_FILE_PREFIXES);
if (prefixes != null) {
List mergeWorkList = new ArrayList();
for (final String prefix : prefixes.split(",")) {
if (prefix == null || prefix.isEmpty()) {
continue;
}
key = queryId + prefix;
cacheKeys.add(key);
mergeWorkList.add((BaseWork) cache.retrieve(key, new Callable