org.apache.hadoop.mapred.OutputCommitter Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of hadoop-apache Show documentation
Show all versions of hadoop-apache Show documentation
Shaded version of Apache Hadoop for Presto
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapred;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* OutputCommitter describes the commit of task output for a
* Map-Reduce job.
*
* The Map-Reduce framework relies on the OutputCommitter of
* the job to:
*
* -
* Setup the job during initialization. For example, create the temporary
* output directory for the job during the initialization of the job.
*
* -
* Cleanup the job after the job completion. For example, remove the
* temporary output directory after the job completion.
*
* -
* Setup the task temporary output.
*
* -
* Check whether a task needs a commit. This is to avoid the commit
* procedure if a task does not need commit.
*
* -
* Commit of the task output.
*
* -
* Discard the task commit.
*
*
* The methods in this class can be called from several different processes and
* from several different contexts. It is important to know which process and
* which context each is called from. Each method should be marked accordingly
* in its documentation. It is also important to note that not all methods are
* guaranteed to be called once and only once. If a method is not guaranteed to
* have this property the output committer needs to handle this appropriately.
* Also note it will only be in rare situations where they may be called
* multiple times for the same task.
*
* @see FileOutputCommitter
* @see JobContext
* @see TaskAttemptContext
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
public abstract class OutputCommitter
extends org.apache.hadoop.mapreduce.OutputCommitter {
/**
* For the framework to setup the job output during initialization. This is
* called from the application master process for the entire job. This will be
* called multiple times, once per job attempt.
*
* @param jobContext Context of the job whose output is being written.
* @throws IOException if temporary output could not be created
*/
public abstract void setupJob(JobContext jobContext) throws IOException;
/**
* For cleaning up the job's output after job completion. This is called
* from the application master process for the entire job. This may be called
* multiple times.
*
* @param jobContext Context of the job whose output is being written.
* @throws IOException
* @deprecated Use {@link #commitJob(JobContext)} or
* {@link #abortJob(JobContext, int)} instead.
*/
@Deprecated
public void cleanupJob(JobContext jobContext) throws IOException { }
/**
* For committing job's output after successful job completion. Note that this
* is invoked for jobs with final runstate as SUCCESSFUL. This is called
* from the application master process for the entire job. This is guaranteed
* to only be called once. If it throws an exception the entire job will
* fail.
*
* @param jobContext Context of the job whose output is being written.
* @throws IOException
*/
public void commitJob(JobContext jobContext) throws IOException {
cleanupJob(jobContext);
}
/**
* For aborting an unsuccessful job's output. Note that this is invoked for
* jobs with final runstate as {@link JobStatus#FAILED} or
* {@link JobStatus#KILLED}. This is called from the application
* master process for the entire job. This may be called multiple times.
*
* @param jobContext Context of the job whose output is being written.
* @param status final runstate of the job
* @throws IOException
*/
public void abortJob(JobContext jobContext, int status)
throws IOException {
cleanupJob(jobContext);
}
/**
* Sets up output for the task. This is called from each individual task's
* process that will output to HDFS, and it is called just for that task. This
* may be called multiple times for the same task, but for different task
* attempts.
*
* @param taskContext Context of the task whose output is being written.
* @throws IOException
*/
public abstract void setupTask(TaskAttemptContext taskContext)
throws IOException;
/**
* Check whether task needs a commit. This is called from each individual
* task's process that will output to HDFS, and it is called just for that
* task.
*
* @param taskContext
* @return true/false
* @throws IOException
*/
public abstract boolean needsTaskCommit(TaskAttemptContext taskContext)
throws IOException;
/**
* To promote the task's temporary output to final output location.
* If {@link #needsTaskCommit(TaskAttemptContext)} returns true and this
* task is the task that the AM determines finished first, this method
* is called to commit an individual task's output. This is to mark
* that tasks output as complete, as {@link #commitJob(JobContext)} will
* also be called later on if the entire job finished successfully. This
* is called from a task's process. This may be called multiple times for the
* same task, but different task attempts. It should be very rare for this to
* be called multiple times and requires odd networking failures to make this
* happen. In the future the Hadoop framework may eliminate this race.
*
* @param taskContext Context of the task whose output is being written.
* @throws IOException if commit is not
*/
public abstract void commitTask(TaskAttemptContext taskContext)
throws IOException;
/**
* Discard the task output. This is called from a task's process to clean
* up a single task's output that can not yet been committed. This may be
* called multiple times for the same task, but for different task attempts.
*
* @param taskContext
* @throws IOException
*/
public abstract void abortTask(TaskAttemptContext taskContext)
throws IOException;
/**
* This method implements the new interface by calling the old method. Note
* that the input types are different between the new and old apis and this is
* a bridge between the two.
*
* @deprecated Use {@link #isRecoverySupported(JobContext)} instead.
*/
@Deprecated
@Override
public boolean isRecoverySupported() {
return false;
}
/**
* Is task output recovery supported for restarting jobs?
*
* If task output recovery is supported, job restart can be done more
* efficiently.
*
* @param jobContext
* Context of the job whose output is being written.
* @return true if task output recovery is supported,
* false otherwise
* @throws IOException
* @see #recoverTask(TaskAttemptContext)
*/
public boolean isRecoverySupported(JobContext jobContext) throws IOException {
return isRecoverySupported();
}
/**
* Returns true if an in-progress job commit can be retried. If the MR AM is
* re-run then it will check this value to determine if it can retry an
* in-progress commit that was started by a previous version.
* Note that in rare scenarios, the previous AM version might still be running
* at that time, due to system anomalies. Hence if this method returns true
* then the retry commit operation should be able to run concurrently with
* the previous operation.
*
* If repeatable job commit is supported, job restart can tolerate previous
* AM failures during job commit.
*
* By default, it is not supported. Extended classes (like:
* FileOutputCommitter) should explicitly override it if provide support.
*
* @param jobContext
* Context of the job whose output is being written.
* @return true repeatable job commit is supported,
* false otherwise
* @throws IOException
*/
public boolean isCommitJobRepeatable(JobContext jobContext) throws
IOException {
return false;
}
@Override
public boolean isCommitJobRepeatable(org.apache.hadoop.mapreduce.JobContext
jobContext) throws IOException {
return isCommitJobRepeatable((JobContext) jobContext);
}
/**
* Recover the task output.
*
* The retry-count for the job will be passed via the
* {@link MRConstants#APPLICATION_ATTEMPT_ID} key in
* {@link TaskAttemptContext#getConfiguration()} for the
* OutputCommitter. This is called from the application master
* process, but it is called individually for each task.
*
* If an exception is thrown the task will be attempted again.
*
* @param taskContext Context of the task whose output is being recovered
* @throws IOException
*/
public void recoverTask(TaskAttemptContext taskContext)
throws IOException {
}
/**
* This method implements the new interface by calling the old method. Note
* that the input types are different between the new and old apis and this
* is a bridge between the two.
*/
@Override
public final void setupJob(org.apache.hadoop.mapreduce.JobContext jobContext
) throws IOException {
setupJob((JobContext) jobContext);
}
/**
* This method implements the new interface by calling the old method. Note
* that the input types are different between the new and old apis and this
* is a bridge between the two.
* @deprecated Use {@link #commitJob(org.apache.hadoop.mapreduce.JobContext)}
* or {@link #abortJob(org.apache.hadoop.mapreduce.JobContext, org.apache.hadoop.mapreduce.JobStatus.State)}
* instead.
*/
@Override
@Deprecated
public final void cleanupJob(org.apache.hadoop.mapreduce.JobContext context
) throws IOException {
cleanupJob((JobContext) context);
}
/**
* This method implements the new interface by calling the old method. Note
* that the input types are different between the new and old apis and this
* is a bridge between the two.
*/
@Override
public final void commitJob(org.apache.hadoop.mapreduce.JobContext context
) throws IOException {
commitJob((JobContext) context);
}
/**
* This method implements the new interface by calling the old method. Note
* that the input types are different between the new and old apis and this
* is a bridge between the two.
*/
@Override
public final void abortJob(org.apache.hadoop.mapreduce.JobContext context,
org.apache.hadoop.mapreduce.JobStatus.State runState)
throws IOException {
int state = JobStatus.getOldNewJobRunState(runState);
if (state != JobStatus.FAILED && state != JobStatus.KILLED) {
throw new IOException ("Invalid job run state : " + runState.name());
}
abortJob((JobContext) context, state);
}
/**
* This method implements the new interface by calling the old method. Note
* that the input types are different between the new and old apis and this
* is a bridge between the two.
*/
@Override
public final
void setupTask(org.apache.hadoop.mapreduce.TaskAttemptContext taskContext
) throws IOException {
setupTask((TaskAttemptContext) taskContext);
}
/**
* This method implements the new interface by calling the old method. Note
* that the input types are different between the new and old apis and this
* is a bridge between the two.
*/
@Override
public final boolean
needsTaskCommit(org.apache.hadoop.mapreduce.TaskAttemptContext taskContext
) throws IOException {
return needsTaskCommit((TaskAttemptContext) taskContext);
}
/**
* This method implements the new interface by calling the old method. Note
* that the input types are different between the new and old apis and this
* is a bridge between the two.
*/
@Override
public final
void commitTask(org.apache.hadoop.mapreduce.TaskAttemptContext taskContext
) throws IOException {
commitTask((TaskAttemptContext) taskContext);
}
/**
* This method implements the new interface by calling the old method. Note
* that the input types are different between the new and old apis and this
* is a bridge between the two.
*/
@Override
public final
void abortTask(org.apache.hadoop.mapreduce.TaskAttemptContext taskContext
) throws IOException {
abortTask((TaskAttemptContext) taskContext);
}
/**
* This method implements the new interface by calling the old method. Note
* that the input types are different between the new and old apis and this
* is a bridge between the two.
*/
@Override
public final
void recoverTask(org.apache.hadoop.mapreduce.TaskAttemptContext taskContext
) throws IOException {
recoverTask((TaskAttemptContext) taskContext);
}
/**
* This method implements the new interface by calling the old method. Note
* that the input types are different between the new and old apis and this is
* a bridge between the two.
*/
@Override
public final boolean isRecoverySupported(
org.apache.hadoop.mapreduce.JobContext context) throws IOException {
return isRecoverySupported((JobContext) context);
}
}