All Downloads are FREE. Search and download functionalities are using the official Maven repository.

alluxio.examples.keyvalue.hadoop.CloneStoreMapReduce Maven / Gradle / Ivy

/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the “License”). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.examples.keyvalue.hadoop;

import alluxio.hadoop.mapreduce.KeyValueInputFormat;
import alluxio.hadoop.mapreduce.KeyValueOutputFormat;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**
 * This MapReduce job reads a key-value store and saves the same key-value data to another
 * key-value store with a different URI.
 */
public final class CloneStoreMapReduce {
  /**
   * The mapper emits all key-value pairs it receives to reducers.
   */
  public static class CloneStoreMapper
      extends Mapper {

    @Override
    public void map(BytesWritable key, BytesWritable value, Context context)
        throws IOException, InterruptedException {
      context.write(key, value);
    }
  }

  /**
   * The reducer writes all key-value pairs it receives to the new key-value store.
   */
  public static class CloneStoreReducer
      extends Reducer {

    @Override
    public void reduce(BytesWritable key, Iterable values, Context context)
        throws IOException, InterruptedException {
      for (BytesWritable value : values) {
        context.write(key, value);
      }
    }
  }

  /**
   * @param args two parameters, the first is the input key-value store path, the second is the
   *    output key-value store path
   * @throws Exception if any exception happens
   */
  public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();

    // NOTE(binfan): we are using the deprecated constructor of Job instance to compile with
    // hadoop-1.0. If this is not a concern, a better way is
    //     Job job = Job.getInstance(conf);
    Job job = new Job(conf);

    job.setJobName("CloneStoreMapReduce");
    job.setJarByClass(CloneStoreMapReduce.class);

    job.setOutputKeyClass(BytesWritable.class);
    job.setOutputValueClass(BytesWritable.class);

    job.setMapperClass(CloneStoreMapper.class);
    job.setReducerClass(CloneStoreReducer.class);

    job.setInputFormatClass(KeyValueInputFormat.class);
    job.setOutputFormatClass(KeyValueOutputFormat.class);

    FileInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    System.exit(job.waitForCompletion(true) ? 0 : 1);
  }
}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy