• Home
• uk.ac.gate
• gate-core
• 9.0.1
• source code
• PRTimeReporter.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

gate.util.reporting.PRTimeReporter Maven / Gradle / Ivy

/*
 *  PRTimeReporter.java
 *
 *  Copyright (c) 2008-2009, Intelius, Inc. 
 *
 *  This file is part of GATE (see http://gate.ac.uk/), and is free
 *  software, licenced under the GNU Library General Public License,
 *  Version 2, June 1991 (in the distribution as file licence.html,
 *  and also available at http://gate.ac.uk/gate/licence.html).
 *
 *  Chirag Viradiya & Andrew Borthwick, 30/Sep/2009
 *
 *  $Id$
 */

package gate.util.reporting;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.StringTokenizer;
import java.util.Timer;
import java.util.TimerTask;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import gate.util.reporting.exceptions.BenchmarkReportExecutionException;
import gate.util.reporting.exceptions.BenchmarkReportInputFileFormatException;
import gnu.getopt.Getopt;

/**
 * A reporter class to generate a report on total time taken by each processing
 * element across corpus.
 */
public class PRTimeReporter implements BenchmarkReportable {
  /**
   * This string constant when set as print media indicates that the report is
   * printed in TEXT format.
   */
  public static final String MEDIA_TEXT = "text";
  /**
   * This string constant when set as print media indicates that the report is
   * printed in HTML format.
   */
  public static final String MEDIA_HTML = "html";
  /**
   * This string constant when set as sort order indicates that the processing
   * elements are sorted in the order of their execution.
   */
  public static final String SORT_EXEC_ORDER = "exec_order";
  /**
   * This string constant when set as sort order indicates that the processing
   * elements are sorted in the descending order of processing time taken by a
   * particular element.
   */
  public static final String SORT_TIME_TAKEN = "time_taken";
  /** A Hashtable storing the time taken by each pipeline. */
  private Hashtable globalTotal = new Hashtable();
  /** An ArrayList containing the lines to be printed in the final text report. */
  private ArrayList printLines = new ArrayList();
  /** An OS independent line separator. */
  private static final String NL = System.getProperty("line.separator");
  /**
   * A String containing the HTML code to generate collapsible tree for
   * processing elements.
   */
  private String htmlElementTree =
    "" + NL +
    "
" + NL +
    "
" + NL;
  /**
   * A String containing the HTML code to generate collapsible tree for time
   * taken by each processing elements.
   */
  private String htmlTimeTree =
    "" + NL +
    "
" + NL;
  /**
   * A String containing the HTML code to generate collapsible tree for time
   * taken by each processing elements (in %).
   */
  private String htmlTimeInPercentTree =
    "" + NL +
    "
" + NL;
  /** A integer to track tree depth level. */
  private int level = 1;
  /** Place holder for storing the total time taken by a pipeline. */
  private double globalValue = 0;
  /** Status flag for normal exit. */
  private static final int STATUS_NORMAL = 0;
  /** Status flag for error exit. */
  private static final int STATUS_ERROR = 1;
  /**
   * An integer containing the count of total valid log entries present in input
   * file provided.
   */
  public int validEntries = 0;
  /** Chunk size in which file will be read. */
  private static final int FILE_CHUNK_SIZE = 2000;
  /**
   * Names of the given pipeline for which the entries are present in given
   * benchmark file.
   */
  public HashSet pipelineNames = new HashSet();

  /** A handle to the input benchmark file (benchmark.txt). */
  private File benchmarkFile = new File("benchmark.txt");
  /** Indicate whether or not to show 0 millisecond entries. */
  private boolean suppressZeroTimeEntries = true;
  /** Report media. */
  private String printMedia = MEDIA_HTML;
  /**
   * A String specifying the sorting order to be used while displaying the
   * report.
   */
  private String sortOrder = SORT_EXEC_ORDER;
  /** Path where to save the report file. */
  private File reportFile;
  /** A marker indicating the start of current logical run. */
  private String logicalStart = null;

  /**
   * No Argument constructor.
   */
  public PRTimeReporter() {
  }

  /**
   * A constructor to be used while executing from the command line.
   *
   * @param args array containing command line arguments.
   */
  PRTimeReporter(String[] args) {
    parseArguments(args);
  }

  /**
   * Stores GATE processing elements and the time taken by them in an in-memory
   * data structure for report generation.
   *
   * @param inputFile
   *          A File handle of the input log file.
   *
   * @return An Object of type LinkedHashMap<String, Object> containing the
   *         processing elements (with time in milliseconds) in hierarchical
   *         structure. Null if there was an error.
   */
  @Override
  public Object store(File inputFile)
      throws BenchmarkReportInputFileFormatException {
    LinkedHashMap globalStore =
      new LinkedHashMap();
    long fromPos = 0;
    RandomAccessFile in = null;
    try {
      if (getLogicalStart() != null) {
        fromPos = tail(inputFile, FILE_CHUNK_SIZE);
      }
      in = new RandomAccessFile(inputFile, "r");
      if (getLogicalStart() != null) {
        in.seek(fromPos);
      }
      ArrayList startTokens = new ArrayList();
      String logEntry;
      String docName = null;
      Pattern pattern = Pattern.compile("(\\d+) (\\d+) (.*) (.*) \\{(.*)\\}");
      while ((logEntry = in.readLine()) != null) {
        Matcher matcher = pattern.matcher(logEntry);
        // Skip the statistics for the event documentLoaded
        if (logEntry.matches(".*documentLoaded.*"))
          continue;
        if (logEntry.matches(".*START.*")) {
          String[] splittedStartEntry = logEntry.split("\\s");
          String startToken = (splittedStartEntry.length > 2) ? splittedStartEntry[2]
              : null;
          if (startToken == null) {
            throw new BenchmarkReportInputFileFormatException(
                getBenchmarkFile().getAbsolutePath() + " is invalid.");
          }
          startTokens.add(startToken);
          if (startToken.endsWith("Start"))
            continue;
          organizeEntries(globalStore, startToken.split("\\."), "0");
        }

        if (matcher != null) {
          if (matcher.matches()) {
            if (validateLogEntry(matcher.group(3), startTokens)) {
              String[] splittedBIDs = matcher.group(3).split("\\.");
              if (splittedBIDs.length > 1) {
                docName = splittedBIDs[1];
                pipelineNames.add(splittedBIDs[0]);
              }
              organizeEntries(globalStore, (matcher.group(3).replaceFirst(
                  Pattern.quote(docName) + ".", "")).split("\\."), matcher.group(2));
            }
          }
        }
      }

    } catch (IOException e) {
      e.printStackTrace();
      globalStore = null;

    } finally {
      try {
        if (in != null) { in.close(); }
      } catch (IOException e) {
        e.printStackTrace();
        globalStore = null;
      }
    }

    if (validEntries == 0) {
      if (logicalStart != null) {
        throw new BenchmarkReportInputFileFormatException(
          "No valid log entries present in " +
          getBenchmarkFile().getAbsolutePath() +
          " does not contain a marker named " + logicalStart + ".");
      } else {
        throw new BenchmarkReportInputFileFormatException(
          "No valid log entries present in " +
          getBenchmarkFile().getAbsolutePath());
      }
    }
    return globalStore;
  }

  /**
   * Generates a tree like structure made up of LinkedHashMap containing the
   * processing elements and time taken by each element totaled at leaf level
   * over corpus.
   *
   * @param store
   *          An Object of type LinkedHashMap containing the
   *          processing elements (with time in milliseconds) in hierarchical
   *          structure.
   * @param tokens
   *          An array consisting of remaining benchmarkID tokens except the one
   *          being processed.
   * @param bTime
   *          time(in milliseconds) of the benchmarkID token being processed.
   */
  @SuppressWarnings("unchecked")
  private void organizeEntries(LinkedHashMap store,
                               String[] tokens, String bTime) {
      if (tokens.length > 0 && store.containsKey(tokens[0])) {
        if (tokens.length > 1) {
          String[] tempArr = new String[tokens.length - 1];
          System.arraycopy(tokens, 1, tempArr, 0, tokens.length - 1);
          if (store.get(tokens[0]) instanceof LinkedHashMap) {
            organizeEntries((LinkedHashMap) (store
                .get(tokens[0])), tempArr, bTime);
          } else {
            if (store.get(tokens[0]) != null) {
              store.put(tokens[0], new LinkedHashMap());
            } else {
              store.put(tokens[0], bTime);
            }
          }
        } else {
          if (store.get(tokens[0]) != null) {
            if (!(store.get(tokens[0]) instanceof LinkedHashMap)) {
              int total = Integer.parseInt((String) (store.get(tokens[0])))
                  + Integer.parseInt(bTime);
              store.put(tokens[0], Integer.toString(total));
            } else {
              int total = Integer.parseInt(bTime);
              if (((java.util.LinkedHashMap) (store
                  .get(tokens[0]))).get("systotal") != null) {
                total = total
                    + Integer
                        .parseInt((String) ((java.util.LinkedHashMap) (store
                            .get(tokens[0]))).get("systotal"));
              }
              ((java.util.LinkedHashMap) (store.get(tokens[0])))
                  .put("systotal", Integer.toString(total));
            }
          }
        }
      } else {
        if (tokens.length - 1 == 0) {
          store.put(tokens[0], bTime);
        } else {
          store.put(tokens[0], new LinkedHashMap());
          String[] tempArr = new String[tokens.length - 1];
          System.arraycopy(tokens, 1, tempArr, 0, tokens.length - 1);
          organizeEntries(
              (LinkedHashMap) (store.get(tokens[0])), tempArr,
              bTime);
        }
      }
  }

  /**
   * Sorts the processing element entries inside tree like structure made up of
   * LinkedHashMap. Entries will be sorted in descending order of time taken.
   *
   * @param gStore
   *          An Object of type LinkedHashMap containing the
   *          processing elements (with time in milliseconds) in hierarchical
   *          structure.
   *
   * @return An Object of type LinkedHashMap containing the
   *         processing elements sorted in descending order of processing time
   *         taken.
   */
  @SuppressWarnings("unchecked")
  private LinkedHashMap sortReport(
    LinkedHashMap gStore) {
    Iterator i = gStore.keySet().iterator();
    LinkedHashMap sortedReport = new LinkedHashMap();
    LinkedHashMap mapperReport = new LinkedHashMap();
    LinkedHashMap unsortedReport = new LinkedHashMap();
    while (i.hasNext()) {
      Object key = i.next();
      if (gStore.get(key) instanceof LinkedHashMap) {
        int systotal = 0;
        if (((LinkedHashMap) (gStore.get(key)))
            .get("systotal") != null) {
          systotal = Integer
              .parseInt((String) ((LinkedHashMap) (gStore
                  .get(key))).get("systotal"));
        }
        if (systotal >= 0) {
          unsortedReport.put((String) key, Integer.toString(systotal));
        }
        mapperReport.put((String) key,
            sortReport((LinkedHashMap) (gStore.get(key))));

      } else {
        if (!(key.equals("total") || key.equals("systotal"))) {
          if (Integer.parseInt((String) (gStore.get(key))) >= 0) {
            unsortedReport.put((String) key, new Integer((String) gStore
                .get(key)).toString());
          }
        }
      }
    }
    LinkedHashMap tempOutLHM =
      sortHashMapByValues(unsortedReport);

    Iterator itr = tempOutLHM.keySet().iterator();
    while (itr.hasNext()) {
      Object tempKey = itr.next();
      sortedReport.put((String) tempKey, tempOutLHM.get(tempKey));
      if (mapperReport.containsKey(tempKey)) {
        sortedReport
            .put((String) tempKey, mapperReport.get(tempKey));
      }
    }
    sortedReport.put("total", gStore.get("total"));
    if (gStore.get("systotal") != null) {
      sortedReport.put("systotal", gStore.get("systotal"));
    }
    return sortedReport;
  }

  /**
   * Sorts LinkedHashMap by its values(natural descending order). keeps the
   * duplicates as it is.
   *
   * @param passedMap
   *          An Object of type LinkedHashMap to be sorted by its values.
   *
   * @return An Object containing the sorted LinkedHashMap.
   */
  private LinkedHashMap sortHashMapByValues(LinkedHashMap passedMap) {
      List mapKeys = new ArrayList(passedMap.keySet());
      List mapValues = new ArrayList(passedMap.values());

      Collections.sort(mapValues, new ValueComparator());
      Collections.sort(mapKeys);
      Collections.reverse(mapValues);
      LinkedHashMap sortedMap = new LinkedHashMap();

      Iterator valueIt = mapValues.iterator();
      while (valueIt.hasNext()) {
        String val = valueIt.next();
        Iterator keyIt = mapKeys.iterator();
        while (keyIt.hasNext()) {
          String key = keyIt.next();
          String comp1 = passedMap.get(key);
          String comp2 = val;

          if (comp1.equals(comp2)) {
            passedMap.remove(key);
            mapKeys.remove(key);
            sortedMap.put(key, val);
            break;
          }
        }
      }
      return sortedMap;
  }

  /**
   * Calculates the sub totals at each level.
   *
   * @param reportContainer
   *          An Object of type LinkedHashMap<String, Object> containing the
   *          processing elements (with time in milliseconds) in hierarchical
   *          structure.
   *
   * @return An Object containing modified hierarchical structure of processing
   *         elements with totals and All others embedded in it.
   */
  @SuppressWarnings("unchecked")
  @Override
  public Object calculate(Object reportContainer) {
    LinkedHashMap globalStore =
      (LinkedHashMap) reportContainer;
    Iterator iter = globalStore.keySet().iterator();
    int total = 0;
    while (iter.hasNext()) {
      String key = iter.next();
      total = getTotal((LinkedHashMap) (globalStore.get(key)));
      globalTotal.put(key, Integer.toString(total));
    }
    return globalStore;
  }

  /**
   * Calculates the total of the time taken by processing element at each leaf
   * level. Also calculates the difference between the actual time taken by the
   * resources and system noted time.
   *
   * @param reportContainer
   *          An Object of type LinkedHashMap containing the
   *          processing elements (with time in milliseconds) in hierarchical
   *          structure.
   *
   * @return An integer containing the sub total.
   */

  @SuppressWarnings("unchecked")
  private int getTotal(LinkedHashMap reportContainer) {
    int total = 0;
    int diff = 0;
    int systotal = 0;
    int subLevelTotal = 0;
    Iterator i = reportContainer.keySet().iterator();
    while (i.hasNext()) {
      Object key = i.next();

      if (reportContainer.get(key) instanceof LinkedHashMap) {
        subLevelTotal = getTotal((LinkedHashMap) (reportContainer
            .get(key)));
        total = total + subLevelTotal;
      } else {
        if (!key.equals("systotal")) {
          total = total
              + Integer.parseInt((String) (reportContainer.get(key)));
        }
      }
    }
    if (reportContainer.get("systotal") != null) {
      systotal = Integer.parseInt((String) (reportContainer.get("systotal")));
    }
    diff = systotal - total;
    reportContainer.put("total", Integer.toString(total));
    reportContainer.put("All others", Integer.toString(diff));
    total += diff;
    return total;
  }

  /**
   * Prints a report as per the value provided for print media option.
   *
   * @param reportSource
   *          An Object of type LinkedHashMap<String, Object> containing the
   *          processing elements (with time in milliseconds) in hierarchical
   *          structure.
   * @param outputFile
   *          Path where to save the report.
   */
  @SuppressWarnings("unchecked")
  @Override
  public void printReport(Object reportSource, File outputFile) {
    if (printMedia.equalsIgnoreCase(MEDIA_TEXT)) {
      printToText(reportSource, outputFile, suppressZeroTimeEntries);
    } else if (printMedia.equalsIgnoreCase(MEDIA_HTML)) {
      printToHTML((LinkedHashMap) reportSource,
        outputFile, suppressZeroTimeEntries);
    }
  }

  /**
   * Prints benchmark report in text format.
   *
   * @param reportContainer
   *          An Object of type LinkedHashMap containing the
   *          processing elements (with time in milliseconds) in hierarchical
   *          structure.
   * @param outputFile
   *          An object of type File representing the output report file.
   * @param suppressZeroTimeEntries
   *          Indicate whether or not to show 0 millisecond entries.
   */
  @SuppressWarnings("unchecked")
  private void printToText(Object reportContainer, File outputFile,
                           boolean suppressZeroTimeEntries) {
    LinkedHashMap globalStore =
      (LinkedHashMap) reportContainer;
    prettyPrint(globalStore, "\t", suppressZeroTimeEntries);
    BufferedWriter out = null;
    try {
      out = new BufferedWriter(new FileWriter(outputFile));
      for (String line : printLines) {
        out.write(line);
        out.newLine();
      }
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      try {
        if (out != null) { out.close(); }
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
  }

  /**
   * Prints a processing elements structure in a tree like format with time
   * taken by each element in milliseconds and in %.
   *
   * @param gStore
   *          An Object of type LinkedHashMap containing the
   *          processing elements (with time in milliseconds) in hierarchical
   *          structure.
   * @param separator
   *          A String separator to indent the processing elements in tree like
   *          structure.
   * @param suppressZeroTimeEntries
   *          Indicate whether or not to show 0 millisecond entries.
   */
  @SuppressWarnings("unchecked")
  private void prettyPrint(LinkedHashMap gStore,
                           String separator, boolean suppressZeroTimeEntries) {

    Iterator i = gStore.keySet().iterator();
    while (i.hasNext()) {
      Object key = i.next();
      if (globalTotal.containsKey(key))
        globalValue = Integer.parseInt(globalTotal.get(key));
      if (gStore.get(key) instanceof LinkedHashMap) {
        int systotal = 0;
        if (((LinkedHashMap) gStore.get(key))
            .containsKey("systotal")) {
          systotal = Integer
              .parseInt((String) ((LinkedHashMap) (gStore
                  .get(key))).get("systotal"));
        }
        if (suppressZeroTimeEntries) {
          if (systotal > 0)
            printLines.add(separator + key + " (" + systotal / 1000.0 + ") ["
                + Math.round(((systotal / globalValue) * 100) * 10) / 10.0
                + "%]");
        } else {
          printLines
              .add(separator + key + " (" + systotal / 1000.0 + ") ["
                  + Math.round(((systotal / globalValue) * 100) * 10) / 10.0
                  + "%]");
        }

        prettyPrint((LinkedHashMap) (gStore.get(key)),
            separator + "\t", suppressZeroTimeEntries);
      } else {
        if (!(key.equals("total") || key.equals("systotal"))) {
          if (suppressZeroTimeEntries) {
            if (Integer.parseInt((String) (gStore.get(key))) != 0) {
              printLines
                  .add(separator
                      + key
                      + " ("
                      + Integer.parseInt((String) (gStore.get(key)))
                      / 1000.0
                      + ") ["
                      + Math
                          .round(((Integer.parseInt((String) (gStore.get(key))) / globalValue) * 100) * 10)
                      / 10.0 + "%]");
            }
          } else {
            printLines
                .add(separator
                    + key
                    + " ("
                    + Integer.parseInt((String) (gStore.get(key)))
                    / 1000.0
                    + ") ["
                    + Math
                        .round(((Integer.parseInt((String) (gStore.get(key))) / globalValue) * 100) * 10)
                    / 10.0 + "%]");
          }
        }
      }
    }
  }

  /**
   * Prints a report in HTML format. The output report will be represented as
   * collapsible ul/li structure.
   *
   * @param gStore
   *          An Object of type LinkedHashMap containing the
   *          processing elements (with time in milliseconds) in hierarchical
   *          structure.
   * @param outputFile
   *          An object of type File representing the output report file to
   *          which the HTML report is to be written.
   * @param suppressZeroTimeEntries
   *          Indicate whether or not to show 0 millisecond entries.
   */
  private void printToHTML(LinkedHashMap gStore,
                           File outputFile, boolean suppressZeroTimeEntries) {
    String htmlPipelineNames = "
";
    for (String pipeline : pipelineNames) {
      htmlPipelineNames += "
" + pipeline + "
" + NL;
    }
    htmlPipelineNames += "
" + NL;
    String htmlReport =
      "" + NL +
      "" + NL +
      "" + NL +
      "" + NL +
      "" + NL +
      "" + NL +
      "" + NL +
      "" + NL +
      "" + NL +
      "" + NL +
      "";
    generateCollapsibleHTMLTree(gStore, suppressZeroTimeEntries);
    htmlElementTree += "" + NL;
    htmlTimeTree += "" + NL;
    htmlTimeInPercentTree += "" + NL;
    htmlReport += htmlElementTree + htmlTimeTree + htmlTimeInPercentTree +
      "
Processing elements of following pipelines" + NL +
      htmlPipelineNames + NL + "
Time in seconds
% time taken
" + NL +
      "";
    // write the html string in the specified output html file
    BufferedWriter out = null;
    try {
      out = new BufferedWriter(new FileWriter(outputFile));
      out.write(htmlReport);
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      try {
        if (out != null) { out.close(); }
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
  }

  /**
   * Creates three tree like ul/li structures 1. A tree to represent processing
   * elements 2. A tree to represent time taken by processing elements 3. A tree
   * to represent time taken by processing elements in %.
   *
   * @param gStore
   *          An Object of type LinkedHashMap containing the
   *          processing elements (with time in milliseconds) in hierarchical
   *          structure.
   * @param suppressZeroTimeEntries
   *          Indicate whether or not to show 0 millisecond entries.
   */
  @SuppressWarnings("unchecked")
  private void generateCollapsibleHTMLTree(LinkedHashMap gStore,
                                           boolean suppressZeroTimeEntries) {
    Iterator i = gStore.keySet().iterator();
    while (i.hasNext()) {
      Object key = i.next();
      if (globalTotal.containsKey(key))
        globalValue = Integer.parseInt(globalTotal.get(key));

      if (gStore.get(key) instanceof LinkedHashMap) {
        int systotal = 0;
        if (((LinkedHashMap) gStore.get(key))
            .containsKey("systotal")) {
          systotal = Integer
              .parseInt((String) ((LinkedHashMap) (gStore
                  .get(key))).get("systotal"));
        }

        if (suppressZeroTimeEntries) {
          if (systotal > 0) {
            htmlElementTree += "
" +
              "[+]" +
              " " + key + "
" + NL;
            htmlTimeTree += "
" + NL +
              systotal / 1000.0 + NL + "
" + NL;
            htmlTimeInPercentTree += "
" + NL
                + Math.round(((systotal / globalValue) * 100) * 10) / 10.0
                + "
" + NL;
            level++;
            generateCollapsibleHTMLTree((LinkedHashMap) (gStore
                .get(key)), suppressZeroTimeEntries);
            htmlElementTree += "
" + NL;
            htmlTimeTree += "
" + NL;
            htmlTimeInPercentTree += "
" + NL;
          }
        } else {
          htmlElementTree += "
" +
            "[+]" +
            " " + key + "
" + NL;
          htmlTimeTree += "
" + NL +
            systotal / 1000.0 + "
" + NL;
          htmlTimeInPercentTree += "
" + NL
              + Math.round(((systotal / globalValue) * 100) * 10) / 10.0
              + "
" + NL;
          level++;
          generateCollapsibleHTMLTree((LinkedHashMap) (gStore
              .get(key)), suppressZeroTimeEntries);
          htmlElementTree += "
" + NL;
          htmlTimeTree += "