eu.stratosphere.hadoopcompatibility.HadoopOutputFormatWrapper Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hadoop-compatibility Show documentation
The newest version!
/***********************************************************************************************************************
 * Copyright (C) 2010-2014 by the Stratosphere project (http://stratosphere.eu)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 **********************************************************************************************************************/

package eu.stratosphere.hadoopcompatibility;

import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;

import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordWriter;
import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.util.ReflectionUtils;

import eu.stratosphere.api.common.io.OutputFormat;
import eu.stratosphere.configuration.Configuration;
import eu.stratosphere.hadoopcompatibility.datatypes.StratosphereTypeConverter;
import eu.stratosphere.types.Record;


public class HadoopOutputFormatWrapper implements OutputFormat {

	private static final long serialVersionUID = 1L;

	public JobConf jobConf;

	public org.apache.hadoop.mapred.OutputFormat hadoopOutputFormat;

	private String hadoopOutputFormatName;

	public RecordWriter recordWriter;

	public StratosphereTypeConverter converter;

	public FileOutputCommitterWrapper fileOutputCommitterWrapper;

	public HadoopOutputFormatWrapper(org.apache.hadoop.mapred.OutputFormat hadoopFormat, JobConf job, StratosphereTypeConverter conv) {
		super();
		this.hadoopOutputFormat = hadoopFormat;
		this.hadoopOutputFormatName = hadoopFormat.getClass().getName();
		this.converter = conv;
		this.fileOutputCommitterWrapper = new FileOutputCommitterWrapper();
		HadoopConfiguration.mergeHadoopConf(job);
		this.jobConf = job;
	}

	@Override
	public void configure(Configuration parameters) {
	}

	/**
	 * create the temporary output file for hadoop RecordWriter.
	 * @param taskNumber The number of the parallel instance.
	 * @param numTasks The number of parallel tasks.
	 * @throws IOException
	 */
	@Override
	public void open(int taskNumber, int numTasks) throws IOException {
		this.fileOutputCommitterWrapper.setupJob(this.jobConf);
		if (Integer.toString(taskNumber + 1).length() <= 6) {
			this.jobConf.set("mapred.task.id", "attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s"," ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0");
			//compatible for hadoop 2.2.0, the temporary output directory is different from hadoop 1.2.1
			this.jobConf.set("mapreduce.task.output.dir", this.fileOutputCommitterWrapper.getTempTaskOutputPath(this.jobConf,TaskAttemptID.forName(this.jobConf.get("mapred.task.id"))).toString());
		} else {
			throw new IOException("task id too large");
		}
		this.recordWriter = this.hadoopOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new DummyHadoopProgressable());
	}


	@Override
	public void writeRecord(Record record) throws IOException {
		K key = this.converter.convertKey(record);
		V value = this.converter.convertValue(record);
		this.recordWriter.write(key, value);
	}

	/**
	 * commit the task by moving the output file out from the temporary directory.
	 * @throws IOException
	 */
	@Override
	public void close() throws IOException {
		this.recordWriter.close(new DummyHadoopReporter());
		if (this.fileOutputCommitterWrapper.needsTaskCommit(this.jobConf, TaskAttemptID.forName(this.jobConf.get("mapred.task.id")))) {
			this.fileOutputCommitterWrapper.commitTask(this.jobConf, TaskAttemptID.forName(this.jobConf.get("mapred.task.id")));
		}
	//TODO: commitjob when all the tasks are finished
	}


	/**
	 * Custom serialization methods.
	 *  @see http://docs.oracle.com/javase/7/docs/api/java/io/Serializable.html
	 */
	private void writeObject(ObjectOutputStream out) throws IOException {
		out.writeUTF(hadoopOutputFormatName);
		jobConf.write(out);
		out.writeObject(converter);
		out.writeObject(fileOutputCommitterWrapper);
	}

	@SuppressWarnings("unchecked")
	private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
		hadoopOutputFormatName = in.readUTF();
		if(jobConf == null) {
			jobConf = new JobConf();
		}
		jobConf.readFields(in);
		try {
			this.hadoopOutputFormat = (org.apache.hadoop.mapred.OutputFormat) Class.forName(this.hadoopOutputFormatName).newInstance();
		} catch (Exception e) {
			throw new RuntimeException("Unable to instantiate the hadoop output format", e);
		}
		ReflectionUtils.setConf(hadoopOutputFormat, jobConf);
		converter = (StratosphereTypeConverter) in.readObject();
		fileOutputCommitterWrapper = (FileOutputCommitterWrapper) in.readObject();
	}


	public void setJobConf(JobConf job) {
		this.jobConf = job;
	}

	public JobConf getJobConf() {
		return jobConf;
	}

	public org.apache.hadoop.mapred.OutputFormat getHadoopOutputFormat() {
		return hadoopOutputFormat;
	}

	public void setHadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat hadoopOutputFormat) {
		this.hadoopOutputFormat = hadoopOutputFormat;
	}

}