org.apache.hadoop.yarn.client.cli.LogsCLI Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in org.apache.hadoop.shaded.com.liance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org.apache.hadoop.shaded.org.licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.shaded.org.apache.hadoop.yarn.client.cli;
import org.apache.hadoop.shaded.org.apache.hadoop.thirdparty.org.apache.hadoop.shaded.com.google.org.apache.hadoop.shaded.com.on.annotations.VisibleForTesting;
import org.apache.hadoop.shaded.com.sun.jersey.api.client.Client;
import org.apache.hadoop.shaded.com.sun.jersey.api.client.ClientHandlerException;
import org.apache.hadoop.shaded.com.sun.jersey.api.client.ClientRequest;
import org.apache.hadoop.shaded.com.sun.jersey.api.client.ClientResponse;
import org.apache.hadoop.shaded.com.sun.jersey.api.client.UniformInterfaceException;
import org.apache.hadoop.shaded.com.sun.jersey.api.client.WebResource;
import org.apache.hadoop.shaded.com.sun.jersey.api.client.WebResource.Builder;
import org.apache.hadoop.shaded.com.sun.jersey.api.client.filter.ClientFilter;
import org.apache.hadoop.shaded.com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
import org.apache.hadoop.shaded.com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import java.org.apache.hadoop.shaded.io.File;
import java.org.apache.hadoop.shaded.io.IOException;
import java.org.apache.hadoop.shaded.io.InputStream;
import java.org.apache.hadoop.shaded.io.PrintStream;
import java.org.apache.hadoop.shaded.net.ConnectException;
import java.org.apache.hadoop.shaded.net.HttpURLConnection;
import java.org.apache.hadoop.shaded.net.SocketException;
import java.org.apache.hadoop.shaded.net.SocketTimeoutException;
import java.org.apache.hadoop.shaded.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Pattern;
import org.apache.hadoop.shaded.javax.ws.rs.core.MediaType;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.cli.CommandLine;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.cli.CommandLineParser;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.cli.GnuParser;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.cli.HelpFormatter;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.cli.Option;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.cli.Options;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.cli.ParseException;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.org.apache.hadoop.shaded.io.IOUtils;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.lang3.StringUtils;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.math3.util.Pair;
import org.apache.hadoop.shaded.org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.shaded.org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.shaded.org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.shaded.org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.shaded.org.apache.hadoop.conf.Configured;
import org.apache.hadoop.shaded.org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.shaded.org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.shaded.org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.shaded.org.apache.hadoop.util.Tool;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.logaggregation.LogToolUtils;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import org.apache.hadoop.shaded.org.codehaus.jettison.json.JSONArray;
import org.apache.hadoop.shaded.org.codehaus.jettison.json.JSONException;
import org.apache.hadoop.shaded.org.codehaus.jettison.json.JSONObject;
@Public
@Evolving
public class LogsCLI extends Configured implements Tool {
private static final String CONTAINER_ID_OPTION = "containerId";
private static final String APPLICATION_ID_OPTION = "applicationId";
private static final String CLUSTER_ID_OPTION = "clusterId";
private static final String NODE_ADDRESS_OPTION = "nodeAddress";
private static final String APP_OWNER_OPTION = "appOwner";
private static final String AM_CONTAINER_OPTION = "am";
private static final String PER_CONTAINER_LOG_FILES_OPTION = "log_files";
private static final String PER_CONTAINER_LOG_FILES_OLD_OPTION = "logFiles";
private static final String PER_CONTAINER_LOG_FILES_REGEX_OPTION
= "log_files_pattern";
private static final String LIST_NODES_OPTION = "list_nodes";
private static final String SHOW_APPLICATION_LOG_INFO
= "show_application_log_info";
private static final String SHOW_CONTAINER_LOG_INFO
= "show_container_log_info";
private static final String OUT_OPTION = "out";
private static final String SIZE_OPTION = "size";
private static final String CLIENT_MAX_RETRY_OPTION = "client_max_retries";
private static final String CLIENT_RETRY_INTERVAL_OPTION
= "client_retry_interval_ms";
public static final String HELP_CMD = "help";
private static final String SIZE_LIMIT_OPTION = "size_limit_mb";
private PrintStream outStream = System.out;
private YarnClient yarnClient = null;
private Client webServiceClient = null;
private static final int DEFAULT_MAX_RETRIES = 30;
private static final long DEFAULT_RETRY_INTERVAL = 1000;
private static final long LOG_SIZE_LIMIT_DEFAULT = 10240L;
private long logSizeLeft = LOG_SIZE_LIMIT_DEFAULT * 1024 * 1024;
private long specifedLogLimits = LOG_SIZE_LIMIT_DEFAULT;
@Private
@VisibleForTesting
ClientConnectionRetry connectionRetry;
@Override
public int run(String[] args) throws Exception {
try {
webServiceClient = new Client(new URLConnectionClientHandler(
new HttpURLConnectionFactory() {
@Override
public HttpURLConnection getHttpURLConnection(URL url)
throws IOException {
AuthenticatedURL.Token token = new AuthenticatedURL.Token();
HttpURLConnection conn = null;
try {
conn = new AuthenticatedURL().openConnection(url, token);
} catch (AuthenticationException e) {
throw new IOException(e);
}
return conn;
}
}));
return runCommand(args);
} finally {
if (yarnClient != null) {
yarnClient.close();
}
if (webServiceClient != null) {
webServiceClient.destroy();
}
}
}
private int runCommand(String[] args) throws Exception {
Options opts = createCommandOpts();
Options printOpts = createPrintOpts(opts);
if (args.length < 1) {
printHelpMessage(printOpts);
return -1;
}
if (args[0].equals("-help")) {
printHelpMessage(printOpts);
return 0;
}
CommandLineParser parser = new GnuParser();
String appIdStr = null;
String clusterIdStr = null;
String containerIdStr = null;
String nodeAddress = null;
String appOwner = null;
boolean getAMContainerLogs = false;
boolean nodesList = false;
boolean showApplicationLogInfo = false;
boolean showContainerLogInfo = false;
boolean useRegex = false;
String[] logFiles = null;
String[] logFilesRegex = null;
List amContainersList = new ArrayList();
String localDir = null;
long bytes = Long.MAX_VALUE;
boolean ignoreSizeLimit = false;
int maxRetries = DEFAULT_MAX_RETRIES;
long retryInterval = DEFAULT_RETRY_INTERVAL;
try {
CommandLine org.apache.hadoop.shaded.com.andLine = parser.parse(opts, args, false);
appIdStr = org.apache.hadoop.shaded.com.andLine.getOptionValue(APPLICATION_ID_OPTION);
containerIdStr = org.apache.hadoop.shaded.com.andLine.getOptionValue(CONTAINER_ID_OPTION);
nodeAddress = org.apache.hadoop.shaded.com.andLine.getOptionValue(NODE_ADDRESS_OPTION);
appOwner = org.apache.hadoop.shaded.com.andLine.getOptionValue(APP_OWNER_OPTION);
getAMContainerLogs = org.apache.hadoop.shaded.com.andLine.hasOption(AM_CONTAINER_OPTION);
nodesList = org.apache.hadoop.shaded.com.andLine.hasOption(LIST_NODES_OPTION);
localDir = org.apache.hadoop.shaded.com.andLine.getOptionValue(OUT_OPTION);
showApplicationLogInfo = org.apache.hadoop.shaded.com.andLine.hasOption(
SHOW_APPLICATION_LOG_INFO);
showContainerLogInfo = org.apache.hadoop.shaded.com.andLine.hasOption(SHOW_CONTAINER_LOG_INFO);
if (getAMContainerLogs) {
try {
amContainersList = parseAMContainer(org.apache.hadoop.shaded.com.andLine, printOpts);
} catch (NumberFormatException ex) {
System.err.println(ex.getMessage());
return -1;
}
}
if (org.apache.hadoop.shaded.com.andLine.hasOption(CLUSTER_ID_OPTION)) {
clusterIdStr = org.apache.hadoop.shaded.com.andLine.getOptionValue(CLUSTER_ID_OPTION);
getConf().set(YarnConfiguration.RM_CLUSTER_ID, clusterIdStr);
}
if (org.apache.hadoop.shaded.com.andLine.hasOption(PER_CONTAINER_LOG_FILES_OPTION)) {
logFiles = org.apache.hadoop.shaded.com.andLine.getOptionValues(PER_CONTAINER_LOG_FILES_OPTION);
} else {
// For backward org.apache.hadoop.shaded.com.atibility, we need to check for the old form of this
// org.apache.hadoop.shaded.com.and line option as well. New form takes precedent.
if (org.apache.hadoop.shaded.com.andLine.hasOption(PER_CONTAINER_LOG_FILES_OLD_OPTION)) {
logFiles = org.apache.hadoop.shaded.com.andLine.getOptionValues(PER_CONTAINER_LOG_FILES_OLD_OPTION);
}
}
if (org.apache.hadoop.shaded.com.andLine.hasOption(PER_CONTAINER_LOG_FILES_REGEX_OPTION)) {
logFilesRegex = org.apache.hadoop.shaded.com.andLine.getOptionValues(
PER_CONTAINER_LOG_FILES_REGEX_OPTION);
useRegex = true;
}
if (org.apache.hadoop.shaded.com.andLine.hasOption(SIZE_OPTION)) {
bytes = Long.parseLong(org.apache.hadoop.shaded.com.andLine.getOptionValue(SIZE_OPTION));
}
if (org.apache.hadoop.shaded.com.andLine.hasOption(CLIENT_MAX_RETRY_OPTION)) {
maxRetries = Integer.parseInt(org.apache.hadoop.shaded.com.andLine.getOptionValue(
CLIENT_MAX_RETRY_OPTION));
}
if (org.apache.hadoop.shaded.com.andLine.hasOption(CLIENT_RETRY_INTERVAL_OPTION)) {
retryInterval = Long.parseLong(org.apache.hadoop.shaded.com.andLine.getOptionValue(
CLIENT_RETRY_INTERVAL_OPTION));
}
if (org.apache.hadoop.shaded.com.andLine.hasOption(SIZE_LIMIT_OPTION)) {
specifedLogLimits = Long.parseLong(org.apache.hadoop.shaded.com.andLine.getOptionValue(
SIZE_LIMIT_OPTION));
logSizeLeft = specifedLogLimits * 1024 * 1024;
}
if (logSizeLeft < 0L) {
ignoreSizeLimit = true;
}
} catch (ParseException e) {
System.err.println("options parsing failed: " + e.getMessage());
printHelpMessage(printOpts);
return -1;
}
if (appIdStr == null && containerIdStr == null) {
System.err.println("Both applicationId and containerId are missing, "
+ " one of them must be specified.");
printHelpMessage(printOpts);
return -1;
}
ApplicationId appId = null;
if (appIdStr != null) {
try {
appId = ApplicationId.fromString(appIdStr);
} catch (Exception e) {
System.err.println("Invalid ApplicationId specified");
return -1;
}
}
if (containerIdStr != null) {
try {
ContainerId containerId = ContainerId.fromString(containerIdStr);
if (appId == null) {
appId = containerId.getApplicationAttemptId().getApplicationId();
} else if (!containerId.getApplicationAttemptId().getApplicationId()
.equals(appId)) {
System.err.println("The Application:" + appId
+ " does not have the container:" + containerId);
return -1;
}
} catch (Exception e) {
System.err.println("Invalid ContainerId specified");
return -1;
}
}
if (showApplicationLogInfo && showContainerLogInfo) {
System.err.println("Invalid options. Can only accept one of "
+ "show_application_log_info/show_container_log_info.");
return -1;
}
if (logFiles != null && logFiles.length > 0 && logFilesRegex != null
&& logFilesRegex.length > 0) {
System.err.println("Invalid options. Can only accept one of "
+ "log_files/log_files_pattern.");
return -1;
}
if (localDir != null) {
File file = new File(localDir);
if (file.exists() && file.isFile()) {
System.err.println("Invalid value for -out option. "
+ "Please provide a directory.");
return -1;
}
}
// Set up Retry WebService Client
connectionRetry = new ClientConnectionRetry(maxRetries, retryInterval);
ClientJerseyRetryFilter retryFilter = new ClientJerseyRetryFilter();
webServiceClient.addFilter(retryFilter);
LogCLIHelpers logCliHelper = new LogCLIHelpers();
logCliHelper.setConf(getConf());
yarnClient = createYarnClient();
YarnApplicationState appState = YarnApplicationState.NEW;
ApplicationReport appReport = null;
try {
appReport = getApplicationReport(appId);
appState = appReport.getYarnApplicationState();
if (appState == YarnApplicationState.NEW
|| appState == YarnApplicationState.NEW_SAVING
|| appState == YarnApplicationState.SUBMITTED) {
System.err.println("Logs are not available right now.");
return -1;
}
} catch (IOException | YarnException e) {
// If we can not get appReport from either RM or ATS
// We will assume that this app has already finished.
appState = YarnApplicationState.FINISHED;
System.err.println("Unable to get ApplicationState."
+ " Attempting to fetch logs directly from the filesystem.");
}
if (appOwner == null || appOwner.isEmpty()) {
appOwner = guessAppOwner(appReport, appId);
if (appOwner == null) {
System.err.println("Can not find the appOwner. "
+ "Please specify the correct appOwner");
System.err.println("Could not locate application logs for " + appId);
return -1;
}
}
Set logs = new HashSet();
if (fetchAllLogFiles(logFiles, logFilesRegex)) {
logs.add("ALL");
} else if (logFiles != null && logFiles.length > 0) {
logs.addAll(Arrays.asList(logFiles));
} else if (logFilesRegex != null && logFilesRegex.length > 0) {
logs.addAll(Arrays.asList(logFilesRegex));
}
ContainerLogsRequest request = new ContainerLogsRequest(appId, null,
Apps.isApplicationFinalState(appState), appOwner, nodeAddress,
null, containerIdStr, localDir, logs, bytes, null);
if (showContainerLogInfo) {
return showContainerLogInfo(request, logCliHelper);
}
if (nodesList) {
return showNodeLists(request, logCliHelper);
}
if (showApplicationLogInfo) {
return showApplicationLogInfo(request, logCliHelper);
}
// To get am logs
if (getAMContainerLogs) {
return fetchAMContainerLogs(request, amContainersList,
logCliHelper, useRegex, ignoreSizeLimit);
}
int resultCode = 0;
if (containerIdStr != null) {
return fetchContainerLogs(request, logCliHelper, useRegex,
ignoreSizeLimit);
} else {
if (nodeAddress == null) {
resultCode = fetchApplicationLogs(request, logCliHelper, useRegex,
ignoreSizeLimit);
} else {
System.err.println("Should at least provide ContainerId!");
printHelpMessage(printOpts);
resultCode = -1;
}
}
return resultCode;
}
private ApplicationReport getApplicationReport(ApplicationId appId)
throws IOException, YarnException {
return yarnClient.getApplicationReport(appId);
}
@VisibleForTesting
protected YarnClient createYarnClient() {
YarnClient client = YarnClient.createYarnClient();
client.init(getConf());
client.start();
return client;
}
public static void main(String[] args) throws Exception {
Configuration conf = new YarnConfiguration();
LogsCLI logDumper = new LogsCLI();
logDumper.setConf(conf);
int exitCode = logDumper.run(args);
System.exit(exitCode);
}
private void printHelpMessage(Options options) {
outStream.println("Retrieve logs for YARN applications.");
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("yarn logs -applicationId [OPTIONS]",
new Options());
formatter.setSyntaxPrefix("");
formatter.printHelp("general options are:", options);
}
protected List getAMContainerInfoForRMWebService(
Configuration conf, String appId) throws Exception {
return WebAppUtils.execOnActiveRM(conf, this::getAMContainerInfoFromRM,
appId);
}
private List getAMContainerInfoFromRM(
String webAppAddress, String appId) throws ClientHandlerException,
UniformInterfaceException, JSONException {
List amContainersList = new ArrayList();
ClientResponse response = null;
try {
Builder builder = webServiceClient.resource(webAppAddress)
.path("ws").path("v1").path("cluster")
.path("apps").path(appId).path("appattempts")
.accept(MediaType.APPLICATION_JSON);
response = builder.get(ClientResponse.class);
JSONObject json = response.getEntity(JSONObject.class)
.getJSONObject("appAttempts");
JSONArray requests = json.getJSONArray("appAttempt");
for (int j = 0; j < requests.length(); j++) {
amContainersList.add(requests.getJSONObject(j));
}
return amContainersList;
} finally {
if (response != null) {
response.close();
}
}
}
private List getAMContainerInfoForAHSWebService(
Configuration conf, String appId) throws ClientHandlerException,
UniformInterfaceException, JSONException {
String webAppAddress =
WebAppUtils.getHttpSchemePrefix(conf)
+ WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
WebResource webResource = webServiceClient.resource(webAppAddress);
ClientResponse response =
webResource.path("ws").path("v1").path("applicationhistory")
.path("apps").path(appId).path("appattempts")
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
JSONObject json = response.getEntity(JSONObject.class);
JSONArray requests = json.getJSONArray("appAttempt");
List amContainersList = new ArrayList();
for (int i = 0; i < requests.length(); i++) {
amContainersList.add(requests.getJSONObject(i));
}
Collections.reverse(amContainersList);
return amContainersList;
}
private boolean fetchAllLogFiles(String[] logFiles, String[] logFilesRegex) {
// If no value is specified for the PER_CONTAINER_LOG_FILES_OPTION option
// and PER_CONTAINER_LOG_FILES_REGEX_OPTION
// we will assume all logs.
if ((logFiles == null || logFiles.length == 0) && (
logFilesRegex == null || logFilesRegex.length == 0)) {
return true;
}
if (logFiles != null && logFiles.length > 0) {
List logs = Arrays.asList(logFiles);
if (logs.contains("ALL") || logs.contains("*")) {
return true;
}
}
if (logFilesRegex != null && logFilesRegex.length > 0) {
List logsRegex = Arrays.asList(logFilesRegex);
if (logsRegex.contains(".*")) {
return true;
}
}
return false;
}
private List> getContainerLogFiles(
Configuration conf, String containerIdStr, String nodeHttpAddress)
throws IOException {
List> logFileInfos
= new ArrayList<>();
try {
WebResource webResource = webServiceClient
.resource(WebAppUtils.getHttpSchemePrefix(conf) + nodeHttpAddress);
ClientResponse response =
webResource.path("ws").path("v1").path("node").path("containers")
.path(containerIdStr).path("logs")
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
if (response.getStatusInfo().getStatusCode() ==
ClientResponse.Status.OK.getStatusCode()) {
try {
JSONArray array = new JSONArray();
JSONObject json = response.getEntity(JSONObject.class);
if (!json.has("containerLogsInfo")) {
return logFileInfos;
}
Object logsInfoObj = json.get("containerLogsInfo");
if (logsInfoObj instanceof JSONObject) {
array.put((JSONObject)logsInfoObj);
} else if (logsInfoObj instanceof JSONArray) {
JSONArray logsArray = (JSONArray)logsInfoObj;
for (int i=0; i < logsArray.length(); i++) {
array.put(logsArray.getJSONObject(i));
}
}
for (int i = 0; i < array.length(); i++) {
JSONObject log = array.getJSONObject(i);
String aggregateType = log.has("logAggregationType") ?
log.getString("logAggregationType") : "N/A";
if (!log.has("containerLogInfo")) {
continue;
}
Object ob = log.get("containerLogInfo");
if (ob instanceof JSONArray) {
JSONArray obArray = (JSONArray)ob;
for (int j = 0; j < obArray.length(); j++) {
logFileInfos.add(new Pair(
generatePerContainerLogFileInfoFromJSON(
obArray.getJSONObject(j)), aggregateType));
}
} else if (ob instanceof JSONObject) {
logFileInfos.add(new Pair(
generatePerContainerLogFileInfoFromJSON(
(JSONObject)ob), aggregateType));
}
}
} catch (Exception e) {
System.err.println("Unable to parse json from webservice. Error:");
System.err.println(e.getMessage());
throw new IOException(e);
}
}
} catch (ClientHandlerException | UniformInterfaceException ex) {
System.err.println("Unable to fetch log files list");
throw new IOException(ex);
}
return logFileInfos;
}
private ContainerLogFileInfo generatePerContainerLogFileInfoFromJSON(
JSONObject meta) throws JSONException {
String fileName = meta.has("fileName") ?
meta.getString("fileName") : "N/A";
String fileSize = meta.has("fileSize") ?
meta.getString("fileSize") : "N/A";
String lastModificationTime = meta.has("lastModifiedTime") ?
meta.getString("lastModifiedTime") : "N/A";
return new ContainerLogFileInfo(fileName, fileSize,
lastModificationTime);
}
@Private
@VisibleForTesting
public int printContainerLogsFromRunningApplication(Configuration conf,
ContainerLogsRequest request, LogCLIHelpers logCliHelper,
boolean useRegex, boolean ignoreSizeLimit) throws IOException {
String containerIdStr = request.getContainerId().toString();
String localDir = request.getOutputLocalDir();
String nodeId = request.getNodeId();
PrintStream out = LogToolUtils.createPrintStream(localDir, nodeId,
containerIdStr);
try {
boolean foundAnyLogs = false;
byte[] buffer = new byte[65536];
for (String logFile : request.getLogTypes()) {
InputStream is = null;
try {
ClientResponse response = getResponseFromNMWebService(conf,
webServiceClient, request, logFile);
if (response != null && response.getStatusInfo().getStatusCode() ==
ClientResponse.Status.OK.getStatusCode()) {
is = response.getEntityInputStream();
int len = 0;
while((len = is.read(buffer)) != -1) {
out.write(buffer, 0, len);
}
out.println();
} else {
out.println("Can not get any logs for the log file: " + logFile);
String msg = "Response from the NodeManager:" + nodeId +
" WebService is " + ((response == null) ? "null":
"not successful," + " HTTP error code: " +
response.getStatus() + ", Server response:\n" +
response.getEntity(String.class));
out.println(msg);
}
out.flush();
foundAnyLogs = true;
} catch (ClientHandlerException | UniformInterfaceException ex) {
System.err.println("Can not find the log file:" + logFile
+ " for the container:" + containerIdStr + " in NodeManager:"
+ nodeId);
} finally {
IOUtils.closeQuietly(is);
}
}
if (foundAnyLogs) {
return 0;
} else {
return -1;
}
} finally {
logCliHelper.closePrintStream(out);
}
}
@Private
@VisibleForTesting
public ContainerReport getContainerReport(String containerIdStr)
throws YarnException, IOException {
return yarnClient.getContainerReport(
ContainerId.fromString(containerIdStr));
}
private int printAMContainerLogs(Configuration conf,
ContainerLogsRequest request, List amContainers,
LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit)
throws Exception {
List amContainersList = null;
List requests =
new ArrayList();
boolean getAMContainerLists = false;
String appId = request.getAppId().toString();
StringBuilder errorMessage = new StringBuilder();
// We will call RM webservice to get all AppAttempts information.
// If we get nothing, we will try to call AHS webservice to get AppAttempts
// which includes nodeAddress for the AM Containers.
try {
amContainersList = getAMContainerInfoForRMWebService(conf, appId);
if (amContainersList != null && !amContainersList.isEmpty()) {
getAMContainerLists = true;
for (JSONObject amContainer : amContainersList) {
ContainerLogsRequest amRequest = new ContainerLogsRequest(request);
amRequest.setContainerId(amContainer.getString("containerId"));
String httpAddress = amContainer.getString("nodeHttpAddress");
if (httpAddress != null && !httpAddress.isEmpty()) {
amRequest.setNodeHttpAddress(httpAddress);
}
amRequest.setNodeId(amContainer.getString("nodeId"));
requests.add(amRequest);
}
}
} catch (Exception ex) {
errorMessage.append(ex.getMessage() + "\n");
if (request.isAppFinished()) {
if (!conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
errorMessage.append("Please enable the timeline service "
+ "and make sure the timeline server is running.");
} else {
try {
if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
try {
amContainersList =
getAMContainerInfoFromTimelineReader(conf, appId);
getAMContainerLists =
createContainerLogsRequestForMasterContainer(requests,
request, amContainersList,
AppAttemptMetricsConstants.MASTER_CONTAINER_INFO);
} catch (Exception e) {
System.err.println(
"Unable to get AM container informations from "
+ "TimelineReader for the application:" + appId);
if (YarnConfiguration.timelineServiceV1Enabled(conf)
|| YarnConfiguration.timelineServiceV15Enabled(conf)) {
getAMContainerLists =
getAMContainerInfoForAHSWebService(conf, appId, requests,
request);
} else {
throw e;
}
}
} else {
getAMContainerLists =
getAMContainerInfoForAHSWebService(conf, appId, requests,
request);
}
} catch (Exception e) {
errorMessage.append(e.getMessage());
}
}
}
}
if (!getAMContainerLists) {
System.err.println("Unable to get AM container informations "
+ "for the application:" + appId);
System.err.println(errorMessage);
System.err.println("Can not get AMContainers logs for "
+ "the application:" + appId + " with the appOwner:"
+ request.getAppOwner());
return -1;
}
List candidates = new ArrayList<>();
if (amContainers.contains("ALL")) {
candidates.addAll(requests);
outStream.println();
outStream.println("Specified ALL for -am option. "
+ "Printed logs for all am containers.");
} else {
for (String amContainer : amContainers) {
int amContainerId = Integer.parseInt(amContainer.trim());
if (amContainerId == -1) {
candidates.add(requests.get(requests.size() - 1));
} else {
if (amContainerId <= requests.size()) {
candidates.add(requests.get(amContainerId - 1));
} else {
System.err.println(String.format("ERROR: Specified AM containerId"
+ " (%s) exceeds the number of AM containers (%s).",
amContainerId, requests.size()));
return -1;
}
}
}
}
Map newOptions = new HashMap<>();
if (request.isAppFinished()) {
newOptions = getMatchedLogTypesForFinishedApp(candidates,
logCliHelper, useRegex, ignoreSizeLimit);
} else {
newOptions = getMatchedLogTypesForRunningApp(candidates, useRegex,
ignoreSizeLimit);
}
for (Entry amRequest
: newOptions.entrySet()) {
outputAMContainerLogs(amRequest.getValue(), conf, logCliHelper,
useRegex, ignoreSizeLimit);
}
return 0;
}
private boolean getAMContainerInfoForAHSWebService(Configuration conf,
String appId, List requests,
ContainerLogsRequest request) throws JSONException {
List amContainersList =
getAMContainerInfoForAHSWebService(conf, appId);
return createContainerLogsRequestForMasterContainer(requests, request,
amContainersList, "amContainerId");
}
private boolean createContainerLogsRequestForMasterContainer(
List requests, ContainerLogsRequest request,
List amContainersList, String masterContainerInfo)
throws JSONException {
boolean getAMContainerLists = false;
if (amContainersList != null && !amContainersList.isEmpty()) {
getAMContainerLists = true;
for (JSONObject amContainer : amContainersList) {
ContainerLogsRequest amRequest = new ContainerLogsRequest(request);
amRequest.setContainerId(amContainer.getString(masterContainerInfo));
requests.add(amRequest);
}
}
return getAMContainerLists;
}
private List getAMContainerInfoFromTimelineReader(
Configuration conf, String appId)
throws IOException, ClientHandlerException, UniformInterfaceException,
JSONException {
ClientResponse response = getClientResponseFromTimelineReader(conf, appId);
JSONArray appAttemptEntities = response.getEntity(JSONArray.class);
List amContainersList = new ArrayList();
for (int i = 0; i < appAttemptEntities.length(); i++) {
JSONObject appAttemptEntity = appAttemptEntities.getJSONObject(i);
JSONObject infoField = appAttemptEntity.getJSONObject("info");
amContainersList.add(infoField);
}
Collections.reverse(amContainersList);
return amContainersList;
}
protected ClientResponse getClientResponseFromTimelineReader(
Configuration conf, String appId) throws IOException {
String webAppAddress = WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils
.getTimelineReaderWebAppURLWithoutScheme(conf);
WebResource webResource = webServiceClient.resource(webAppAddress);
ClientResponse response =
webResource.path("ws").path("v2").path("timeline").path("clusters")
.path(conf.get(YarnConfiguration.RM_CLUSTER_ID)).path("apps")
.path(appId).path("entities")
.path(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString())
.queryParam("fields", "INFO").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
if (response == null
|| response.getStatusInfo().getStatusCode() != ClientResponse.Status.OK
.getStatusCode()) {
String msg =
"Response from the timeline reader server is " + ((response == null) ?
"null" :
"not successful," + " HTTP error code: " + response.getStatus()
+ ", Server response:\n" + response.getEntity(String.class));
System.out.println(msg);
throw new IOException(msg);
}
return response;
}
private void outputAMContainerLogs(ContainerLogsRequest request,
Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex,
boolean ignoreSizeLimit) throws Exception {
String nodeHttpAddress = request.getNodeHttpAddress();
String containerId = request.getContainerId();
String nodeId = request.getNodeId();
if (request.isAppFinished()) {
if (containerId != null && !containerId.isEmpty()) {
if (nodeId != null && !nodeId.isEmpty()) {
logCliHelper.dumpAContainerLogsForLogType(request);
} else {
logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId(
request);
}
}
} else {
if (nodeHttpAddress != null && containerId != null
&& !nodeHttpAddress.isEmpty() && !containerId.isEmpty()) {
ContainerState containerState = getContainerReport(containerId)
.getContainerState();
request.setContainerState(containerState);
printContainerLogsFromRunningApplication(conf,
request, logCliHelper, useRegex, ignoreSizeLimit);
}
}
}
private int showContainerLogInfo(ContainerLogsRequest request,
LogCLIHelpers logCliHelper) throws IOException, YarnException,
ClientHandlerException, UniformInterfaceException, JSONException {
if (!request.isAppFinished()) {
return printContainerInfoFromRunningApplication(request, logCliHelper);
} else {
return logCliHelper.printAContainerLogMetadata(
request, System.out, System.err);
}
}
private int showNodeLists(ContainerLogsRequest request,
LogCLIHelpers logCliHelper) throws IOException {
if (!request.isAppFinished()) {
System.err.println("The -list_nodes org.apache.hadoop.shaded.com.and can be only used with "
+ "finished applications");
return -1;
} else {
logCliHelper.printNodesList(request, System.out, System.err);
return 0;
}
}
private int showApplicationLogInfo(ContainerLogsRequest request,
LogCLIHelpers logCliHelper) throws IOException, YarnException {
String appState = "Application State: "
+ (request.isAppFinished() ? "Completed." : "Running.");
if (!request.isAppFinished()) {
List reports =
getContainerReportsFromRunningApplication(request);
List filterReports = filterContainersInfo(
request, reports);
if (filterReports.isEmpty()) {
System.err.println("Can not find any containers for the application:"
+ request.getAppId() + ".");
return -1;
}
outStream.println(appState);
for (ContainerReport report : filterReports) {
outStream.println(String.format(LogCLIHelpers.CONTAINER_ON_NODE_PATTERN,
report.getContainerId(), report.getAssignedNode()));
}
return 0;
} else {
outStream.println(appState);
logCliHelper.printContainersList(request, System.out, System.err);
return 0;
}
}
/**
* Create Command Options.
* @return the org.apache.hadoop.shaded.com.and options
*/
private Options createCommandOpts() {
Options opts = new Options();
opts.addOption(HELP_CMD, false, "Displays help for all org.apache.hadoop.shaded.com.ands.");
Option appIdOpt =
new Option(APPLICATION_ID_OPTION, true, "ApplicationId (required)");
opts.addOption(appIdOpt);
opts.addOption(CONTAINER_ID_OPTION, true, "ContainerId. "
+ "By default, it will print all available logs."
+ " Work with -log_files to get only specific logs. If specified, the"
+ " applicationId can be omitted");
opts.addOption(CLUSTER_ID_OPTION, true, "ClusterId. "
+ "By default, it will take default cluster id from the RM");
opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress in the format "
+ "nodename:port");
opts.addOption(APP_OWNER_OPTION, true,
"AppOwner (assumed to be current user if not specified)");
Option amOption = new Option(AM_CONTAINER_OPTION, true,
"Prints the AM Container logs for this application. "
+ "Specify org.apache.hadoop.shaded.com.a-separated value to get logs for related AM "
+ "Container. For example, If we specify -am 1,2, we will get "
+ "the logs for the first AM Container as well as the second "
+ "AM Container. To get logs for all AM Containers, use -am ALL. "
+ "To get logs for the latest AM Container, use -am -1. "
+ "By default, it will print all available logs. Work with -log_files "
+ "to get only specific logs.");
amOption.setValueSeparator(',');
amOption.setArgs(Option.UNLIMITED_VALUES);
amOption.setArgName("AM Containers");
opts.addOption(amOption);
Option logFileOpt = new Option(PER_CONTAINER_LOG_FILES_OPTION, true,
"Specify org.apache.hadoop.shaded.com.a-separated value "
+ "to get exact matched log files. Use \"ALL\" or \"*\" to "
+ "fetch all the log files for the container.");
logFileOpt.setValueSeparator(',');
logFileOpt.setArgs(Option.UNLIMITED_VALUES);
logFileOpt.setArgName("Log File Name");
opts.addOption(logFileOpt);
Option oldLogFileOpt = new Option(PER_CONTAINER_LOG_FILES_OLD_OPTION, true,
"Deprecated name for log_files, please use log_files option instead");
oldLogFileOpt.setValueSeparator(',');
oldLogFileOpt.setArgs(Option.UNLIMITED_VALUES);
oldLogFileOpt.setArgName("Log File Name");
opts.addOption(oldLogFileOpt);
Option logFileRegexOpt = new Option(PER_CONTAINER_LOG_FILES_REGEX_OPTION,
true, "Specify org.apache.hadoop.shaded.com.a-separated value "
+ "to get matched log files by using java regex. Use \".*\" to "
+ "fetch all the log files for the container.");
logFileRegexOpt.setValueSeparator(',');
logFileRegexOpt.setArgs(Option.UNLIMITED_VALUES);
logFileRegexOpt.setArgName("Log File Pattern");
opts.addOption(logFileRegexOpt);
opts.addOption(SHOW_CONTAINER_LOG_INFO, false,
"Show the container log metadata, "
+ "including log-file names, the size of the log files. "
+ "You can org.apache.hadoop.shaded.com.ine this with --containerId to get log metadata for "
+ "the specific container, or with --nodeAddress to get log metadata "
+ "for all the containers on the specific NodeManager.");
opts.addOption(SHOW_APPLICATION_LOG_INFO, false, "Show the "
+ "containerIds which belong to the specific Application. "
+ "You can org.apache.hadoop.shaded.com.ine this with --nodeAddress to get containerIds "
+ "for all the containers on the specific NodeManager.");
opts.addOption(LIST_NODES_OPTION, false,
"Show the list of nodes that successfully aggregated logs. "
+ "This option can only be used with finished applications.");
opts.addOption(OUT_OPTION, true, "Local directory for storing individual "
+ "container logs. The container logs will be stored based on the "
+ "node the container ran on.");
opts.addOption(SIZE_OPTION, true, "Prints the log file's first 'n' bytes "
+ "or the last 'n' bytes. Use negative values as bytes to read from "
+ "the end and positive values as bytes to read from the beginning.");
opts.addOption(CLIENT_MAX_RETRY_OPTION, true, "Set max retry number for a"
+ " retry client to get the container logs for the running "
+ "applications. Use a negative value to make retry forever. "
+ "The default value is 30.");
opts.addOption(CLIENT_RETRY_INTERVAL_OPTION, true,
"Work with --client_max_retries to create a retry client. "
+ "The default value is 1000.");
opts.addOption(SIZE_LIMIT_OPTION, true, "Use this option to limit "
+ "the size of the total logs which could be fetched. "
+ "By default, we only allow to fetch at most "
+ LOG_SIZE_LIMIT_DEFAULT + " MB logs. If the total log size is "
+ "larger than the specified number, the CLI would fail. "
+ "The user could specify -1 to ignore the size limit "
+ "and fetch all logs.");
opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID");
opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID");
opts.getOption(CLUSTER_ID_OPTION).setArgName("Cluster ID");
opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address");
opts.getOption(APP_OWNER_OPTION).setArgName("Application Owner");
opts.getOption(AM_CONTAINER_OPTION).setArgName("AM Containers");
opts.getOption(OUT_OPTION).setArgName("Local Directory");
opts.getOption(SIZE_OPTION).setArgName("size");
opts.getOption(CLIENT_MAX_RETRY_OPTION).setArgName("Max Retries");
opts.getOption(CLIENT_RETRY_INTERVAL_OPTION)
.setArgName("Retry Interval");
opts.getOption(SIZE_LIMIT_OPTION).setArgName("Size Limit");
return opts;
}
/**
* Create Print options for helper message.
* @param org.apache.hadoop.shaded.com.andOpts the options
* @return the print options
*/
private Options createPrintOpts(Options org.apache.hadoop.shaded.com.andOpts) {
Options printOpts = new Options();
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(HELP_CMD));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(CONTAINER_ID_OPTION));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(CLUSTER_ID_OPTION));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(NODE_ADDRESS_OPTION));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(APP_OWNER_OPTION));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(AM_CONTAINER_OPTION));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(PER_CONTAINER_LOG_FILES_OPTION));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(LIST_NODES_OPTION));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(SHOW_APPLICATION_LOG_INFO));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(SHOW_CONTAINER_LOG_INFO));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(OUT_OPTION));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(SIZE_OPTION));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(
PER_CONTAINER_LOG_FILES_REGEX_OPTION));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(CLIENT_MAX_RETRY_OPTION));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(CLIENT_RETRY_INTERVAL_OPTION));
printOpts.addOption(org.apache.hadoop.shaded.com.andOpts.getOption(SIZE_LIMIT_OPTION));
return printOpts;
}
private List parseAMContainer(CommandLine org.apache.hadoop.shaded.com.andLine,
Options printOpts) throws NumberFormatException {
List amContainersList = new ArrayList();
String[] amContainers = org.apache.hadoop.shaded.com.andLine.getOptionValues(AM_CONTAINER_OPTION);
for (String am : amContainers) {
boolean errorInput = false;
if (!am.trim().equalsIgnoreCase("ALL")) {
try {
int id = Integer.parseInt(am.trim());
if (id != -1 && id <= 0) {
errorInput = true;
}
} catch (NumberFormatException ex) {
errorInput = true;
}
if (errorInput) {
String errMessage =
"Invalid input for option -am. Valid inputs are 'ALL', -1 "
+ "and any other integer which is larger than 0.";
printHelpMessage(printOpts);
throw new NumberFormatException(errMessage);
}
amContainersList.add(am.trim());
} else {
amContainersList.add("ALL");
break;
}
}
return amContainersList;
}
private int fetchAMContainerLogs(ContainerLogsRequest request,
List amContainersList, LogCLIHelpers logCliHelper,
boolean useRegex, boolean ignoreSizeLimit) throws Exception {
return printAMContainerLogs(getConf(), request, amContainersList,
logCliHelper, useRegex, ignoreSizeLimit);
}
private int fetchContainerLogs(ContainerLogsRequest request,
LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit)
throws IOException, ClientHandlerException, UniformInterfaceException,
JSONException {
String appIdStr = request.getAppId().toString();
String containerIdStr = request.getContainerId();
String nodeAddress = request.getNodeId();
String appOwner = request.getAppOwner();
boolean isAppFinished = request.isAppFinished();
// if the application is in the final state,
// we could directly get logs from HDFS.
if (isAppFinished) {
// if user specified "ALL" as the logFiles param, pass empty list
// to logCliHelper so that it fetches all the logs
ContainerLogsRequest newOptions = getMatchedLogOptions(
request, logCliHelper, useRegex, ignoreSizeLimit);
if (newOptions == null) {
System.err.println("Can not find any log file matching the pattern: "
+ request.getLogTypes() + " for the container: "
+ request.getContainerId() + " within the application: "
+ request.getAppId());
return -1;
}
if (nodeAddress != null && !nodeAddress.isEmpty()) {
return logCliHelper.dumpAContainerLogsForLogType(newOptions);
} else {
return logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId(
newOptions);
}
}
String nodeHttpAddress = null;
String nodeId = null;
try {
// If the nodeAddress is not provided, we will try to get
// the ContainerReport. In the containerReport, we could get
// nodeAddress and nodeHttpAddress
ContainerReport report = getContainerReport(containerIdStr);
nodeHttpAddress = report.getNodeHttpAddress();
if (nodeHttpAddress != null && !nodeHttpAddress.isEmpty()) {
nodeHttpAddress = nodeHttpAddress.replaceFirst(
WebAppUtils.getHttpSchemePrefix(getConf()), "");
request.setNodeHttpAddress(nodeHttpAddress);
}
nodeId = report.getAssignedNode().toString();
request.setNodeId(nodeId);
request.setContainerState(report.getContainerState());
} catch (IOException | YarnException ex) {
nodeHttpAddress = getNodeHttpAddressFromRMWebString(request);
if (nodeHttpAddress != null && !nodeHttpAddress.isEmpty()) {
request.setNodeHttpAddress(nodeHttpAddress);
} else {
// for the case, we have already uploaded partial logs in HDFS
int result = -1;
ContainerLogsRequest newOptions = getMatchedLogOptions(
request, logCliHelper, useRegex, ignoreSizeLimit);
if (newOptions == null) {
System.err.println("Can not find any log file matching the pattern: "
+ request.getLogTypes() + " for the container: "
+ request.getContainerId() + " within the application: "
+ request.getAppId());
} else {
if (nodeAddress != null && !nodeAddress.isEmpty()) {
result = logCliHelper.dumpAContainerLogsForLogType(newOptions);
} else {
result = logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId(
newOptions);
}
}
if (result == -1) {
System.err.println(
"Unable to get logs for this container:"
+ containerIdStr + " for the application:"
+ appIdStr + " with the appOwner: " + appOwner);
System.err.println("The application: " + appIdStr
+ " is still running, and we can not get Container report "
+ "for the container: " + containerIdStr + ". Please try later "
+ "or after the application finishes.");
}
return result;
}
}
// If the application is not in the final state,
// we will provide the NodeHttpAddress and get the container logs
// by calling NodeManager webservice.
ContainerLogsRequest newRequest = getMatchedOptionForRunningApp(
request, useRegex, ignoreSizeLimit);
if (newRequest == null) {
return -1;
}
return printContainerLogsFromRunningApplication(getConf(), request,
logCliHelper, useRegex, ignoreSizeLimit);
}
private int fetchApplicationLogs(ContainerLogsRequest options,
LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit)
throws IOException, YarnException {
// If the application has finished, we would fetch the logs
// from HDFS.
// If the application is still running, we would get the full
// list of the containers first, then fetch the logs for each
// container from NM.
int resultCode = -1;
if (options.isAppFinished()) {
ContainerLogsRequest newOptions = getMatchedLogOptions(
options, logCliHelper, useRegex, ignoreSizeLimit);
if (newOptions == null) {
System.err.println("Can not find any log file matching the pattern: "
+ options.getLogTypes() + " for the application: "
+ options.getAppId());
} else {
resultCode =
logCliHelper.dumpAllContainersLogs(newOptions);
}
} else {
List containerLogRequests =
getContainersLogRequestForRunningApplication(options);
// get all matched container log types and check the total log size.
Map matchedLogTypes =
getMatchedLogTypesForRunningApp(containerLogRequests,
useRegex, ignoreSizeLimit);
for (Entry container
: matchedLogTypes.entrySet()) {
int result = printContainerLogsFromRunningApplication(getConf(),
container.getValue(), logCliHelper,
useRegex, ignoreSizeLimit);
if (result == 0) {
resultCode = 0;
}
}
}
if (resultCode == -1) {
System.err.println("Can not find the logs for the application: "
+ options.getAppId() + " with the appOwner: "
+ options.getAppOwner());
}
return resultCode;
}
private String guessAppOwner(ApplicationReport appReport,
ApplicationId appId) throws IOException {
String appOwner = null;
if (appReport != null) {
//always use the app owner from the app report if possible
appOwner = appReport.getUser();
} else {
appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
appOwner = LogCLIHelpers.getOwnerForAppIdOrNull(
appId, appOwner, getConf());
}
return appOwner;
}
private ContainerLogsRequest getMatchedLogOptions(
ContainerLogsRequest request, LogCLIHelpers logCliHelper,
boolean useRegex, boolean ignoreSizeLimit) throws IOException {
ContainerLogsRequest newOptions = new ContainerLogsRequest(request);
Set files = logCliHelper.listContainerLogs(
request);
Set matchedFiles = getMatchedLogFiles(request, files,
useRegex, ignoreSizeLimit);
if (matchedFiles.isEmpty()) {
return null;
} else {
newOptions.setLogTypes(matchedFiles);
return newOptions;
}
}
private Set getMatchedLogFiles(ContainerLogsRequest options,
Collection candidate, boolean useRegex,
boolean ignoreSizeLimit) throws IOException {
Set matchedFiles = new HashSet();
Set filePattern = options.getLogTypes();
long size = options.getBytes();
boolean getAll = options.getLogTypes().contains("ALL");
Iterator iterator = candidate.iterator();
while(iterator.hasNext()) {
boolean matchedFile = false;
ContainerLogFileInfo logInfo = iterator.next();
if (getAll) {
matchedFile = true;
} else if (useRegex) {
if (isFileMatching(logInfo.getFileName(), filePattern)) {
matchedFile = true;
}
} else {
if (filePattern.contains(logInfo.getFileName())) {
matchedFile = true;
}
}
if (matchedFile) {
matchedFiles.add(logInfo.getFileName());
if (!ignoreSizeLimit) {
decrLogSizeLimit(Math.min(
Long.parseLong(logInfo.getFileSize()), size));
if (getLogSizeLimitLeft() < 0) {
throw new RuntimeException("The total log size is too large."
+ "The log size limit is " + specifedLogLimits + "MB. "
+ "Please specify a proper value --size option or if you "
+ "really want to fetch all, please "
+ "specify -1 for --size_limit_mb option.");
}
}
}
}
return matchedFiles;
}
private boolean isFileMatching(String fileType,
Set logTypes) {
for (String logType : logTypes) {
Pattern filterPattern = Pattern.org.apache.hadoop.shaded.com.ile(logType);
boolean match = filterPattern.matcher(fileType).find();
if (match) {
return true;
}
}
return false;
}
private List
getContainersLogRequestForRunningApplication(
ContainerLogsRequest options) throws YarnException, IOException {
List newOptionsList =
new ArrayList();
List reports =
getContainerReportsFromRunningApplication(options);
for (ContainerReport container : reports) {
ContainerLogsRequest newOptions = new ContainerLogsRequest(options);
newOptions.setContainerId(container.getContainerId().toString());
newOptions.setNodeId(container.getAssignedNode().toString());
String httpAddress = container.getNodeHttpAddress();
if (httpAddress != null && !httpAddress.isEmpty()) {
newOptions.setNodeHttpAddress(httpAddress
.replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
}
newOptions.setContainerState(container.getContainerState());
newOptionsList.add(newOptions);
}
return newOptionsList;
}
private List getContainerReportsFromRunningApplication(
ContainerLogsRequest options) throws YarnException, IOException {
List reports = new ArrayList();
List attempts =
yarnClient.getApplicationAttempts(options.getAppId());
Map containerMap = new TreeMap<
ContainerId, ContainerReport>();
for (ApplicationAttemptReport attempt : attempts) {
List containers = yarnClient.getContainers(
attempt.getApplicationAttemptId());
for (ContainerReport container : containers) {
if (!containerMap.containsKey(container.getContainerId())) {
containerMap.put(container.getContainerId(), container);
}
}
}
reports.addAll(containerMap.values());
return reports;
}
// filter the containerReports based on the nodeId and ContainerId
private List filterContainersInfo(
ContainerLogsRequest options, List containers) {
List filterReports = new ArrayList(
containers);
String nodeId = options.getNodeId();
boolean filterBasedOnNodeId = (nodeId != null && !nodeId.isEmpty());
String containerId = options.getContainerId();
boolean filterBasedOnContainerId = (containerId != null
&& !containerId.isEmpty());
if (filterBasedOnNodeId || filterBasedOnContainerId) {
// filter the reports based on the containerId and.or nodeId
for(ContainerReport report : containers) {
if (filterBasedOnContainerId) {
if (!report.getContainerId().toString()
.equalsIgnoreCase(containerId)) {
filterReports.remove(report);
}
}
if (filterBasedOnNodeId) {
if (!report.getAssignedNode().toString().equalsIgnoreCase(nodeId)) {
filterReports.remove(report);
}
}
}
}
return filterReports;
}
private int printContainerInfoFromRunningApplication(
ContainerLogsRequest options, LogCLIHelpers logCliHelper)
throws YarnException, IOException, ClientHandlerException,
UniformInterfaceException, JSONException {
String containerIdStr = options.getContainerId();
String nodeIdStr = options.getNodeId();
List reports =
getContainerReportsFromRunningApplication(options);
List filteredReports = filterContainersInfo(
options, reports);
if (filteredReports.isEmpty()) {
// if we specify the containerId as well as NodeAddress
String nodeHttpAddress = null;
if (options.getContainerId() != null
&& !options.getContainerId().isEmpty()) {
nodeHttpAddress = getNodeHttpAddressFromRMWebString(options);
}
if (nodeHttpAddress != null) {
outputContainerLogMeta(options.getContainerId(), options.getNodeId(),
nodeHttpAddress);
return 0;
} else {
int result = logCliHelper.printAContainerLogMetadata(
options, System.out, System.err);
if (result == -1) {
StringBuilder sb = new StringBuilder();
if (containerIdStr != null && !containerIdStr.isEmpty()) {
sb.append("Trying to get container with ContainerId: "
+ containerIdStr + "\n");
}
if (nodeIdStr != null && !nodeIdStr.isEmpty()) {
sb.append("Trying to get container from NodeManager: "
+ nodeIdStr + "\n");
}
sb.append("Can not find any matched containers for the application: "
+ options.getAppId());
System.err.println(sb.toString());
}
return result;
}
}
for (ContainerReport report : filteredReports) {
String nodeId = report.getAssignedNode().toString();
String nodeHttpAddress = report.getNodeHttpAddress().replaceFirst(
WebAppUtils.getHttpSchemePrefix(getConf()), "");
String containerId = report.getContainerId().toString();
outputContainerLogMeta(containerId, nodeId, nodeHttpAddress);
}
return 0;
}
private void outputContainerLogMeta(String containerId, String nodeId,
String nodeHttpAddress) throws IOException {
String containerString = String.format(
LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerId, nodeId);
outStream.println(containerString);
outStream.println(StringUtils.repeat("=", containerString.length()));
outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN,
"LogFile", "LogLength", "LastModificationTime", "LogAggregationType");
outStream.println(StringUtils.repeat("=", containerString.length() * 2));
List> infos = getContainerLogFiles(
getConf(), containerId, nodeHttpAddress);
for (Pair info : infos) {
outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN,
info.getKey().getFileName(), info.getKey().getFileSize(),
info.getKey().getLastModifiedTime(), info.getValue());
}
}
@VisibleForTesting
public Set getMatchedContainerLogFiles(ContainerLogsRequest request,
boolean useRegex, boolean ignoreSizeLimit) throws IOException {
// fetch all the log files for the container
// filter the log files based on the given -log_files pattern
List> allLogFileInfos=
getContainerLogFiles(getConf(), request.getContainerId(),
request.getNodeHttpAddress());
List fileNames = new ArrayList<
ContainerLogFileInfo>();
for (Pair fileInfo : allLogFileInfos) {
fileNames.add(fileInfo.getKey());
}
return getMatchedLogFiles(request, fileNames,
useRegex, ignoreSizeLimit);
}
@VisibleForTesting
public ClientResponse getResponseFromNMWebService(Configuration conf,
Client webServiceClient, ContainerLogsRequest request, String logFile) {
return LogToolUtils.getResponseFromNMWebService(
conf, webServiceClient, request, logFile);
}
@VisibleForTesting
public String getNodeHttpAddressFromRMWebString(ContainerLogsRequest request)
throws ClientHandlerException, UniformInterfaceException, JSONException {
if (request.getNodeId() == null || request.getNodeId().isEmpty()) {
return null;
}
JSONObject nodeInfo = YarnWebServiceUtils
.getNodeInfoFromRMWebService(getConf(), request.getNodeId())
.getJSONObject("node");
return nodeInfo.has("nodeHTTPAddress") ?
nodeInfo.getString("nodeHTTPAddress") : null;
}
// Class to handle retry
static class ClientConnectionRetry {
// maxRetries < 0 means keep trying
@Private
@VisibleForTesting
public int maxRetries;
@Private
@VisibleForTesting
public long retryInterval;
// Indicates if retries happened last time. Only tests should read it.
// In unit tests, retryOn() calls should _not_ be concurrent.
private boolean retried = false;
@Private
@VisibleForTesting
boolean getRetired() {
return retried;
}
// Constructor with default retry settings
public ClientConnectionRetry(int inputMaxRetries,
long inputRetryInterval) {
this.maxRetries = inputMaxRetries;
this.retryInterval = inputRetryInterval;
}
public Object retryOn(ClientRetryOp op)
throws RuntimeException, IOException {
int leftRetries = maxRetries;
retried = false;
// keep trying
while (true) {
try {
// try perform the op, if fail, keep retrying
return op.run();
} catch (IOException | RuntimeException e) {
// break if there's no retries left
if (leftRetries == 0) {
break;
}
if (op.shouldRetryOn(e)) {
logException(e, leftRetries);
} else {
throw e;
}
}
if (leftRetries > 0) {
leftRetries--;
}
retried = true;
try {
// sleep for the given time interval
Thread.sleep(retryInterval);
} catch (InterruptedException ie) {
System.out.println("Client retry sleep interrupted! ");
}
}
throw new RuntimeException("Connection retries limit exceeded.");
};
private void logException(Exception e, int leftRetries) {
if (leftRetries > 0) {
System.out.println("Exception caught by ClientConnectionRetry,"
+ " will try " + leftRetries + " more time(s).\nMessage: "
+ e.getMessage());
} else {
// note that maxRetries may be -1 at the very beginning
System.out.println("ConnectionException caught by ClientConnectionRetry,"
+ " will keep retrying.\nMessage: "
+ e.getMessage());
}
}
}
private class ClientJerseyRetryFilter extends ClientFilter {
@Override
public ClientResponse handle(final ClientRequest cr)
throws ClientHandlerException {
// Set up the retry operation
ClientRetryOp jerseyRetryOp = new ClientRetryOp() {
@Override
public Object run() {
// Try pass the request, if fail, keep retrying
return getNext().handle(cr);
}
@Override
public boolean shouldRetryOn(Exception e) {
// Only retry on connection exceptions
return (e instanceof ClientHandlerException)
&& (e.getCause() instanceof ConnectException ||
e.getCause() instanceof SocketTimeoutException ||
e.getCause() instanceof SocketException);
}
};
try {
return (ClientResponse) connectionRetry.retryOn(jerseyRetryOp);
} catch (IOException e) {
throw new ClientHandlerException("Jersey retry failed!\nMessage: "
+ e.getMessage());
}
}
}
// Abstract class for an operation that should be retried by client
private static abstract class ClientRetryOp {
// The operation that should be retried
public abstract Object run() throws IOException;
// The method to indicate if we should retry given the incoming exception
public abstract boolean shouldRetryOn(Exception e);
}
private long getLogSizeLimitLeft() {
return this.logSizeLeft;
}
private void decrLogSizeLimit(long used) {
this.logSizeLeft -= used;
}
@Private
@VisibleForTesting
public ContainerLogsRequest getMatchedOptionForRunningApp(
ContainerLogsRequest container, boolean useRegex,
boolean ignoreSizeLimit) throws IOException {
String containerIdStr = container.getContainerId().toString();
String nodeHttpAddress = container.getNodeHttpAddress();
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
System.err.println("Can not get the logs for the container: "
+ containerIdStr);
System.err.println("The node http address is required to get container "
+ "logs for the Running application.");
return null;
}
Set matchedFiles = getMatchedContainerLogFiles(container,
useRegex, ignoreSizeLimit);
if (matchedFiles.isEmpty()) {
System.err.println("Can not find any log file matching the pattern: "
+ container.getLogTypes() + " for the container: " + containerIdStr
+ " within the application: " + container.getAppId());
return null;
}
container.setLogTypes(matchedFiles);
return container;
}
@Private
@VisibleForTesting
public Map getMatchedLogTypesForRunningApp(
List containerLogRequests, boolean useRegex,
boolean ignoreSizeLimit) {
Map containerMatchedLog = new HashMap<>();
for (ContainerLogsRequest container : containerLogRequests) {
try {
ContainerLogsRequest request = getMatchedOptionForRunningApp(
container, useRegex, ignoreSizeLimit);
if (request == null) {
continue;
}
containerMatchedLog.put(container.getContainerId(), request);
} catch(IOException ex) {
System.err.println(ex);
continue;
}
}
return containerMatchedLog;
}
private Map getMatchedLogTypesForFinishedApp(
List containerLogRequests,
LogCLIHelpers logCliHelper, boolean useRegex,
boolean ignoreSizeLimit) {
Map containerMatchedLog = new HashMap<>();
for (ContainerLogsRequest container : containerLogRequests) {
try {
ContainerLogsRequest request = getMatchedLogOptions(container,
logCliHelper, useRegex, ignoreSizeLimit);
if (request == null) {
System.err.println("Can not find any log file matching the pattern: "
+ container.getLogTypes() + " for the container: "
+ container.getContainerId() + " within the application: "
+ container.getAppId());
continue;
}
containerMatchedLog.put(container.getContainerId(), request);
} catch (IOException ex) {
System.err.println(ex);
continue;
}
}
return containerMatchedLog;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy