org.apache.hadoop.examples.pi.Util Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of hadoop-mapreduce-examples Show documentation
Show all versions of hadoop-mapreduce-examples Show documentation
Apache Hadoop MapReduce Examples
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.examples.pi;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import org.apache.hadoop.examples.pi.DistSum.Machine;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.ToolRunner;
import com.google.common.base.Charsets;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
/** Utility methods */
public class Util {
/** Output stream */
public static final PrintStream out = System.out;
/** Error stream */
public static final PrintStream err = System.out;
/** Timer */
public static class Timer {
private final boolean isAccumulative;
private final long start = System.currentTimeMillis();
private long previous = start;
/** Timer constructor
* @param isAccumulative Is accumulating the time duration?
*/
public Timer(boolean isAccumulative) {
this.isAccumulative = isAccumulative;
final StackTraceElement[] stack = Thread.currentThread().getStackTrace();
final StackTraceElement e = stack[stack.length - 1];
out.println(e + " started at " + new Date(start));
}
/** Same as tick(null). */
public long tick() {return tick(null);}
/**
* Tick
* @param s Output message. No output if it is null.
* @return delta
*/
public synchronized long tick(String s) {
final long t = System.currentTimeMillis();
final long delta = t - (isAccumulative? start: previous);
if (s != null) {
out.format("%15dms (=%-15s: %s%n", delta, millis2String(delta) + ")", s);
out.flush();
}
previous = t;
return delta;
}
}
/** Covert milliseconds to a String. */
public static String millis2String(long n) {
if (n < 0)
return "-" + millis2String(-n);
else if (n < 1000)
return n + "ms";
final StringBuilder b = new StringBuilder();
final int millis = (int)(n % 1000L);
if (millis != 0)
b.append(String.format(".%03d", millis));
if ((n /= 1000) < 60)
return b.insert(0, n).append("s").toString();
b.insert(0, String.format(":%02d", (int)(n % 60L)));
if ((n /= 60) < 60)
return b.insert(0, n).toString();
b.insert(0, String.format(":%02d", (int)(n % 60L)));
if ((n /= 60) < 24)
return b.insert(0, n).toString();
b.insert(0, n % 24L);
final int days = (int)((n /= 24) % 365L);
b.insert(0, days == 1? " day ": " days ").insert(0, days);
if ((n /= 365L) > 0)
b.insert(0, n == 1? " year ": " years ").insert(0, n);
return b.toString();
}
/** Covert a String to a long.
* This support comma separated number format.
*/
public static long string2long(String s) {
return Long.parseLong(s.trim().replace(",", ""));
}
/** Covert a long to a String in comma separated number format. */
public static String long2string(long n) {
if (n < 0)
return "-" + long2string(-n);
final StringBuilder b = new StringBuilder();
for(; n >= 1000; n = n/1000)
b.insert(0, String.format(",%03d", n % 1000));
return n + b.toString();
}
/** Parse a variable. */
public static long parseLongVariable(final String name, final String s) {
return string2long(parseStringVariable(name, s));
}
/** Parse a variable. */
public static String parseStringVariable(final String name, final String s) {
if (!s.startsWith(name + '='))
throw new IllegalArgumentException("!s.startsWith(name + '='), name="
+ name + ", s=" + s);
return s.substring(name.length() + 1);
}
/** Execute the callables by a number of threads */
public static > void execute(int nThreads, List callables
) throws InterruptedException, ExecutionException {
final ExecutorService executor = HadoopExecutors.newFixedThreadPool(
nThreads);
final List> futures = executor.invokeAll(callables);
for(Future f : futures)
f.get();
}
/** Print usage messages */
public static int printUsage(String[] args, String usage) {
err.println("args = " + Arrays.asList(args));
err.println();
err.println("Usage: java " + usage);
err.println();
ToolRunner.printGenericCommandUsage(err);
return -1;
}
/** Combine a list of items. */
public static > List combine(Collection items) {
final List sorted = new ArrayList(items);
if (sorted.size() <= 1)
return sorted;
Collections.sort(sorted);
final List combined = new ArrayList(items.size());
T prev = sorted.get(0);
for(int i = 1; i < sorted.size(); i++) {
final T curr = sorted.get(i);
final T c = curr.combine(prev);
if (c != null)
prev = c;
else {
combined.add(prev);
prev = curr;
}
}
combined.add(prev);
return combined;
}
/** Check local directory. */
public static void checkDirectory(File dir) {
if (!dir.exists())
if (!dir.mkdirs())
throw new IllegalArgumentException("!dir.mkdirs(), dir=" + dir);
if (!dir.isDirectory())
throw new IllegalArgumentException("dir (=" + dir + ") is not a directory.");
}
/** Create a writer of a local file. */
public static PrintWriter createWriter(File dir, String prefix) throws IOException {
checkDirectory(dir);
SimpleDateFormat dateFormat = new SimpleDateFormat("-yyyyMMdd-HHmmssSSS");
for(;;) {
final File f = new File(dir,
prefix + dateFormat.format(new Date(System.currentTimeMillis())) + ".txt");
if (!f.exists())
return new PrintWriter(new OutputStreamWriter(new FileOutputStream(f), Charsets.UTF_8));
try {Thread.sleep(10);} catch (InterruptedException e) {}
}
}
/** Print a "bits skipped" message. */
public static void printBitSkipped(final long b) {
out.println();
out.println("b = " + long2string(b)
+ " (" + (b < 2? "bit": "bits") + " skipped)");
}
/** Convert a pi value to a String. */
public static String pi2string(final double pi, final long terms) {
final long value = (long)(pi * (1L << DOUBLE_PRECISION));
final int acc_bit = accuracy(terms, false);
final int acc_hex = acc_bit/4;
final int shift = DOUBLE_PRECISION - acc_bit;
return String.format("%0" + acc_hex + "X %0" + (13-acc_hex) + "X (%d hex digits)",
value >> shift, value & ((1 << shift) - 1), acc_hex);
}
static final int DOUBLE_PRECISION = 52; //mantissa size
static final int MACHEPS_EXPONENT = DOUBLE_PRECISION + 1;
/** Estimate accuracy. */
public static int accuracy(final long terms, boolean print) {
final double error = terms <= 0? 2: (Math.log(terms) / Math.log(2)) / 2;
final int bits = MACHEPS_EXPONENT - (int)Math.ceil(error);
if (print)
out.println("accuracy: bits=" + bits + ", terms=" + long2string(terms) + ", error exponent=" + error);
return bits - bits%4;
}
private static final String JOB_SEPARATION_PROPERTY = "pi.job.separation.seconds";
private static final Semaphore JOB_SEMAPHORE = new Semaphore(1);
/** Run a job. */
static void runJob(String name, Job job, Machine machine, String startmessage, Util.Timer timer) {
JOB_SEMAPHORE.acquireUninterruptibly();
Long starttime = null;
try {
try {
starttime = timer.tick("starting " + name + " ...\n " + startmessage);
//initialize and submit a job
machine.init(job);
job.submit();
// Separate jobs
final long sleeptime = 1000L * job.getConfiguration().getInt(JOB_SEPARATION_PROPERTY, 10);
if (sleeptime > 0) {
Util.out.println(name + "> sleep(" + Util.millis2String(sleeptime) + ")");
Thread.sleep(sleeptime);
}
} finally {
JOB_SEMAPHORE.release();
}
if (!job.waitForCompletion(false))
throw new RuntimeException(name + " failed.");
} catch(Exception e) {
throw e instanceof RuntimeException? (RuntimeException)e: new RuntimeException(e);
} finally {
if (starttime != null)
timer.tick(name + "> timetaken=" + Util.millis2String(timer.tick() - starttime));
}
}
/** Read job outputs */
static List readJobOutputs(FileSystem fs, Path outdir) throws IOException {
final List results = new ArrayList();
for(FileStatus status : fs.listStatus(outdir)) {
if (status.getPath().getName().startsWith("part-")) {
final BufferedReader in = new BufferedReader(
new InputStreamReader(fs.open(status.getPath()), Charsets.UTF_8));
try {
for(String line; (line = in.readLine()) != null; )
results.add(TaskResult.valueOf(line));
}
finally {
in.close();
}
}
}
if (results.isEmpty())
throw new IOException("Output not found");
return results;
}
/** Write results */
static void writeResults(String name, List results, FileSystem fs, String dir) throws IOException {
final Path outfile = new Path(dir, name + ".txt");
Util.out.println(name + "> writing results to " + outfile);
final PrintWriter out = new PrintWriter(new OutputStreamWriter(fs.create(outfile), Charsets.UTF_8), true);
try {
for(TaskResult r : results)
out.println(r);
}
finally {
out.close();
}
}
/** Create a directory. */
static boolean createNonexistingDirectory(FileSystem fs, Path dir) throws IOException {
if (fs.exists(dir)) {
Util.err.println("dir (= " + dir + ") already exists.");
return false;
} else if (!fs.mkdirs(dir)) {
throw new IOException("Cannot create working directory " + dir);
}
fs.setPermission(dir, new FsPermission((short)0777));
return true;
}
}