org.apache.hadoop.examples.pi.Parser Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hadoop-mapreduce-examples Show documentation
Apache Hadoop MapReduce Examples
There is a newer version: 3.4.1
Show newest version
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.examples.pi;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import org.apache.hadoop.examples.pi.math.Bellard;
import org.apache.hadoop.examples.pi.math.Bellard.Parameter;

import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;

/** A class for parsing outputs */
public final class Parser {
  static final String VERBOSE_PROPERTY = "pi.parser.verbose";

  final boolean isVerbose;
  
  public Parser(boolean isVerbose) {
    this.isVerbose = isVerbose;
  }
  
  private void println(String s) {
    if (isVerbose)
      Util.out.println(s);
  }

  /** Parse a line */
  private static void parseLine(final String line, Map> m) {
//      LOG.info("line = " + line);
    final Map.Entry e = DistSum.string2TaskResult(line);
    if (e != null) {
      final List sums = m.get(Parameter.get(e.getKey()));
      if (sums == null)
        throw new IllegalArgumentException("sums == null, line=" + line + ", e=" + e);
      sums.add(e.getValue());
    }
  }

  /** Parse a file or a directory tree */
  private void parse(File f, Map> sums) throws IOException {
    if (f.isDirectory()) {
      println("Process directory " + f);
      File[] files = f.listFiles();
      if (files != null) {
        for(File child : files) {
          parse(child, sums);
        }
      }
    } else if (f.getName().endsWith(".txt")) {
      println("Parse file " + f);
      final Map> m = new TreeMap>();    
      for(Parameter p : Parameter.values())
        m.put(p, new ArrayList());

      final BufferedReader in = new BufferedReader(
          new InputStreamReader(new FileInputStream(f), Charsets.UTF_8)); 
      try {
        for(String line; (line = in.readLine()) != null; )
          try {
            parseLine(line, m);
          } catch(RuntimeException e) {
            Util.err.println("line = " + line);
            throw e;
          }
      } finally {
        in.close();
      }

      for(Parameter p : Parameter.values()) {
        final List combined = Util.combine(m.get(p));
        if (!combined.isEmpty()) {
          println(p + " (size=" + combined.size() + "):");
          for(TaskResult r : combined)
            println("  " + r);
        }
        sums.get(p).addAll(m.get(p));
      }
    }
  }

  /** Parse a path */
  private Map> parse(String f) throws IOException {
    final Map> m = new TreeMap>();
    for(Parameter p : Parameter.values())
      m.put(p, new ArrayList());
    parse(new File(f), m);

    //LOG.info("m=" + m.toString().replace(", ", ",\n  "));
    for(Parameter p : Parameter.values())
      m.put(p, m.get(p));
    return m;
  }

  /** Parse input and re-write results. */
  Map> parse(String inputpath, String outputdir
      ) throws IOException {
    //parse input
    Util.out.print("\nParsing " + inputpath + " ... ");
    Util.out.flush();
    final Map> parsed = parse(inputpath);
    Util.out.println("DONE");

    //re-write the results
    if (outputdir != null) {
      Util.out.print("\nWriting to " + outputdir + " ...");
      Util.out.flush();
      for(Parameter p : Parameter.values()) {
        final List results = parsed.get(p);
        Collections.sort(results);

        final PrintWriter out = new PrintWriter(
            new OutputStreamWriter(new FileOutputStream(
                new File(outputdir, p + ".txt")), Charsets.UTF_8), true);
        try {
          for(int i = 0; i < results.size(); i++)
            out.println(DistSum.taskResult2string(p + "." + i, results.get(i)));
        }
        finally {
          out.close();
        }
      }
      Util.out.println("DONE");
    }
    return parsed;
  }

  /** Combine results */
  static > Map combine(Map> m) {
    final Map combined = new TreeMap();
    for(Parameter p : Parameter.values()) {
      //note: results would never be null due to the design of Util.combine
      final List results = Util.combine(m.get(p));
      Util.out.format("%-6s => ", p); 
      if (results.size() != 1)
        Util.out.println(results.toString().replace(", ", ",\n           "));
      else {
        final T r = results.get(0);
        combined.put(p, r); 
        Util.out.println(r);
      }
    }
    return combined;
  }

  /** main */
  public static void main(String[] args) throws IOException {
    if (args.length < 2 || args.length > 3)
      Util.printUsage(args, Parser.class.getName()
          + "   []");

    int i = 0;
    final long b = Util.string2long(args[i++]);
    final String inputpath = args[i++];
    final String outputdir = args.length >= 3? args[i++]: null;

    //read input
    final Map> parsed = new Parser(true).parse(inputpath, outputdir);
    final Map combined = combine(parsed);
    long duration = 0;
    for(TaskResult r : combined.values())
      duration += r.getDuration();

    //print pi
    final double pi = Bellard.computePi(b, combined);
    Util.printBitSkipped(b);
    Util.out.println(Util.pi2string(pi, Bellard.bit2terms(b)));
    Util.out.println("cpu time = " + Util.millis2String(duration));
  }
}