org.apache.hadoop.hive.ql.log.PerfLogger Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.log;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.session.SessionState;
import java.util.HashMap;
import java.util.Map;
/**
* PerfLogger.
*
* Can be used to measure and log the time spent by a piece of code.
*/
public class PerfLogger {
public static final String ACQUIRE_READ_WRITE_LOCKS = "acquireReadWriteLocks";
public static final String COMPILE = "compile";
public static final String PARSE = "parse";
public static final String ANALYZE = "semanticAnalyze";
public static final String DO_AUTHORIZATION = "doAuthorization";
public static final String DRIVER_EXECUTE = "Driver.execute";
public static final String INPUT_SUMMARY = "getInputSummary";
public static final String GET_SPLITS = "getSplits";
public static final String RUN_TASKS = "runTasks";
public static final String SERIALIZE_PLAN = "serializePlan";
public static final String DESERIALIZE_PLAN = "deserializePlan";
public static final String CLONE_PLAN = "clonePlan";
public static final String TASK = "task.";
public static final String RELEASE_LOCKS = "releaseLocks";
public static final String PRUNE_LISTING = "prune-listing";
public static final String PARTITION_RETRIEVING = "partition-retrieving";
public static final String PRE_HOOK = "PreHook.";
public static final String POST_HOOK = "PostHook.";
public static final String FAILURE_HOOK = "FailureHook.";
public static final String DRIVER_RUN = "Driver.run";
public static final String TIME_TO_SUBMIT = "TimeToSubmit";
public static final String TEZ_SUBMIT_TO_RUNNING = "TezSubmitToRunningDag";
public static final String TEZ_BUILD_DAG = "TezBuildDag";
public static final String TEZ_SUBMIT_DAG = "TezSubmitDag";
public static final String TEZ_RUN_DAG = "TezRunDag";
public static final String TEZ_CREATE_VERTEX = "TezCreateVertex.";
public static final String TEZ_RUN_VERTEX = "TezRunVertex.";
public static final String TEZ_INITIALIZE_PROCESSOR = "TezInitializeProcessor";
public static final String TEZ_RUN_PROCESSOR = "TezRunProcessor";
public static final String TEZ_INIT_OPERATORS = "TezInitializeOperators";
public static final String LOAD_HASHTABLE = "LoadHashtable";
public static final String ORC_GET_SPLITS = "OrcGetSplits";
public static final String SPARK_SUBMIT_TO_RUNNING = "SparkSubmitToRunning";
public static final String SPARK_BUILD_PLAN = "SparkBuildPlan";
public static final String SPARK_BUILD_RDD_GRAPH = "SparkBuildRDDGraph";
public static final String SPARK_SUBMIT_JOB = "SparkSubmitJob";
public static final String SPARK_RUN_JOB = "SparkRunJob";
public static final String SPARK_CREATE_TRAN = "SparkCreateTran.";
public static final String SPARK_RUN_STAGE = "SparkRunStage.";
public static final String SPARK_INIT_OPERATORS = "SparkInitializeOperators";
public static final String SPARK_GENERATE_TASK_TREE = "SparkGenerateTaskTree";
public static final String SPARK_OPTIMIZE_OPERATOR_TREE = "SparkOptimizeOperatorTree";
public static final String SPARK_OPTIMIZE_TASK_TREE = "SparkOptimizeTaskTree";
public static final String SPARK_FLUSH_HASHTABLE = "SparkFlushHashTable.";
protected static final ThreadLocal perfLogger = new ThreadLocal();
protected final Map startTimes = new HashMap();
protected final Map endTimes = new HashMap();
static final private Log LOG = LogFactory.getLog(PerfLogger.class.getName());
public PerfLogger() {
// Use getPerfLogger to get an instance of PerfLogger
}
public static PerfLogger getPerfLogger() {
return getPerfLogger(false);
}
/**
* Call this function to get an instance of PerfLogger.
*
* Use resetPerfLogger to require a new instance. Useful at the beginning of execution.
*
* @return Session perflogger if there's a sessionstate, otherwise return the thread local instance
*/
public static PerfLogger getPerfLogger(boolean resetPerfLogger) {
if (SessionState.get() == null) {
if (perfLogger.get() == null || resetPerfLogger) {
perfLogger.set(new PerfLogger());
}
return perfLogger.get();
} else {
return SessionState.get().getPerfLogger(resetPerfLogger);
}
}
/**
* Call this function when you start to measure time spent by a piece of code.
* @param _log the logging object to be used.
* @param method method or ID that identifies this perf log element.
*/
public void PerfLogBegin(String callerName, String method) {
long startTime = System.currentTimeMillis();
LOG.info("");
startTimes.put(method, new Long(startTime));
}
/**
* Call this function in correspondence of PerfLogBegin to mark the end of the measurement.
* @param _log
* @param method
* @return long duration the difference between now and startTime, or -1 if startTime is null
*/
public long PerfLogEnd(String callerName, String method) {
Long startTime = startTimes.get(method);
long endTime = System.currentTimeMillis();
long duration = -1;
endTimes.put(method, new Long(endTime));
StringBuilder sb = new StringBuilder(" ");
LOG.info(sb);
return duration;
}
/**
* Call this function at the end of processing a query (any time after the last call to PerfLogEnd
* for a given query) to run any cleanup/final steps that need to be run
* @param _log
*/
public void close(Log _log, QueryPlan queryPlan) {
}
public Long getStartTime(String method) {
long startTime = 0L;
if (startTimes.containsKey(method)) {
startTime = startTimes.get(method);
}
return startTime;
}
public Long getEndTime(String method) {
long endTime = 0L;
if (endTimes.containsKey(method)) {
endTime = endTimes.get(method);
}
return endTime;
}
public boolean startTimeHasMethod(String method) {
return startTimes.containsKey(method);
}
public boolean endTimeHasMethod(String method) {
return endTimes.containsKey(method);
}
public Long getDuration(String method) {
long duration = 0;
if (startTimes.containsKey(method) && endTimes.containsKey(method)) {
duration = endTimes.get(method) - startTimes.get(method);
}
return duration;
}
}