eu.stratosphere.hadoopcompatibility.HadoopDataSink Maven / Gradle / Ivy

Go to download
/***********************************************************************************************************************
 * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 **********************************************************************************************************************/

package eu.stratosphere.hadoopcompatibility;

import java.util.List;

import eu.stratosphere.types.Record;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputFormat;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;

import eu.stratosphere.api.java.record.operators.GenericDataSink;
import eu.stratosphere.api.common.operators.Operator;
import eu.stratosphere.compiler.contextcheck.Validatable;
import eu.stratosphere.hadoopcompatibility.datatypes.DefaultStratosphereTypeConverter;
import eu.stratosphere.hadoopcompatibility.datatypes.StratosphereTypeConverter;

/**
 * The HadoopDataSink is a generic wrapper for all Hadoop OutputFormats.
 *
 * Example usage:
 *  * 		HadoopDataSink out = new HadoopDataSink(new org.apache.hadoop.mapred.TextOutputFormat(), new JobConf(), "Hadoop TextOutputFormat",reducer, Text.class,IntWritable.class);
 *		org.apache.hadoop.mapred.TextOutputFormat.setOutputPath(out.getJobConf(), new Path(output));
 * 
 *
 * Note that it is possible to provide custom data type converter.
 *
 * The HadoopDataSink provides a default converter: {@link eu.stratosphere.hadoopcompatibility.datatypes.DefaultStratosphereTypeConverter}
 **/
public class HadoopDataSink extends GenericDataSink implements Validatable {

	private static String DEFAULT_NAME = "";

	private JobConf jobConf;

	public HadoopDataSink(OutputFormat hadoopFormat, JobConf jobConf, String name, Operator input, StratosphereTypeConverter conv, Class keyClass, Class valueClass) {
		this(hadoopFormat, jobConf, name, ImmutableList.>of(input), conv, keyClass, valueClass);
	}

	public HadoopDataSink(OutputFormat hadoopFormat, JobConf jobConf, String name, Operator input, Class keyClass, Class valueClass) {
		this(hadoopFormat, jobConf, name, input, new DefaultStratosphereTypeConverter(keyClass, valueClass), keyClass, valueClass);
	}

	public HadoopDataSink(OutputFormat hadoopFormat, JobConf jobConf, Operator input, Class keyClass, Class valueClass) {
		this(hadoopFormat, jobConf, DEFAULT_NAME, input, new DefaultStratosphereTypeConverter(keyClass, valueClass), keyClass, valueClass);
	}

	public HadoopDataSink(OutputFormat hadoopFormat, Operator input, Class keyClass, Class valueClass) {
		this(hadoopFormat, new JobConf(), DEFAULT_NAME, input, new DefaultStratosphereTypeConverter(keyClass, valueClass), keyClass, valueClass);
	}



	@SuppressWarnings("deprecation")
	public HadoopDataSink(OutputFormat hadoopFormat, JobConf jobConf, String name, List> input, StratosphereTypeConverter conv, Class keyClass, Class valueClass) {
		super(new HadoopOutputFormatWrapper(hadoopFormat, jobConf, conv),input, name);
		Preconditions.checkNotNull(hadoopFormat);
		Preconditions.checkNotNull(jobConf);
		this.name = name;
		this.jobConf = jobConf;
		jobConf.setOutputKeyClass(keyClass);
		jobConf.setOutputValueClass(valueClass);
	}

	public HadoopDataSink(OutputFormat hadoopFormat, JobConf jobConf, String name, List> input, Class keyClass, Class valueClass) {
		this(hadoopFormat, jobConf, name, input, new DefaultStratosphereTypeConverter(keyClass, valueClass), keyClass, valueClass);
	}

	public HadoopDataSink(OutputFormat hadoopFormat, JobConf jobConf, List> input, Class keyClass, Class valueClass) {
		this(hadoopFormat, jobConf, DEFAULT_NAME, input, new DefaultStratosphereTypeConverter(keyClass, valueClass), keyClass, valueClass);
	}

	public HadoopDataSink(OutputFormat hadoopFormat, List> input, Class keyClass, Class valueClass) {
		this(hadoopFormat, new JobConf(), DEFAULT_NAME, input, new DefaultStratosphereTypeConverter(keyClass, valueClass), keyClass, valueClass);
	}

	public JobConf getJobConf() {
		return this.jobConf;
	}

	@Override
	public void check() {
		// see for more details https://github.com/stratosphere/stratosphere/pull/531
		Preconditions.checkNotNull(FileOutputFormat.getOutputPath(jobConf), "The HadoopDataSink currently expects a correct outputPath.");
	}

}