org.apache.hadoop.yarn.client.cli.SchedConfCLI Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in org.apache.hadoop.shaded.com.liance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org.apache.hadoop.shaded.org.licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.shaded.org.apache.hadoop.yarn.client.cli;
import org.apache.hadoop.shaded.org.apache.hadoop.thirdparty.org.apache.hadoop.shaded.com.google.org.apache.hadoop.shaded.com.on.annotations.VisibleForTesting;
import org.apache.hadoop.shaded.com.sun.jersey.api.client.Client;
import org.apache.hadoop.shaded.com.sun.jersey.api.client.ClientResponse;
import org.apache.hadoop.shaded.com.sun.jersey.api.client.WebResource;
import org.apache.hadoop.shaded.com.sun.jersey.api.client.WebResource.Builder;
import org.apache.hadoop.shaded.com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
import org.apache.hadoop.shaded.com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.cli.CommandLine;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.cli.GnuParser;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.cli.MissingArgumentException;
import org.apache.hadoop.shaded.org.apache.org.apache.hadoop.shaded.com.ons.cli.Options;
import org.apache.hadoop.shaded.org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.shaded.org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.shaded.org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.shaded.org.apache.hadoop.conf.Configured;
import org.apache.hadoop.shaded.org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.shaded.org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.shaded.org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.shaded.org.apache.hadoop.util.Tool;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.webapp.dao.ConfInfo;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.shaded.org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Marshaller;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
import org.apache.hadoop.shaded.javax.ws.rs.core.MediaType;
import org.apache.hadoop.shaded.javax.ws.rs.core.Response.Status;
import java.org.apache.hadoop.shaded.io.StringReader;
import java.org.apache.hadoop.shaded.io.StringWriter;
import java.org.apache.hadoop.shaded.io.IOException;
import java.org.apache.hadoop.shaded.net.HttpURLConnection;
import java.org.apache.hadoop.shaded.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* CLI for modifying scheduler configuration.
*/
@Public
@Unstable
public class SchedConfCLI extends Configured implements Tool {
private static final String ADD_QUEUES_OPTION = "addQueues";
private static final String REMOVE_QUEUES_OPTION = "removeQueues";
private static final String UPDATE_QUEUES_OPTION = "updateQueues";
private static final String GLOBAL_OPTIONS = "globalUpdates";
private static final String GET_SCHEDULER_CONF = "getConf";
private static final String FORMAT_CONF = "formatConfig";
private static final String HELP_CMD = "help";
private static final String SPLIT_BY_SLASH_COMMA = "(? queueConfigInfos = new ArrayList<>();
for (String arg : args.split(";")) {
queueConfigInfos.add(getQueueConfigInfo(arg));
}
updateInfo.setAddQueueInfo(queueConfigInfos);
}
@VisibleForTesting
void removeQueues(String args, SchedConfUpdateInfo updateInfo) {
if (args == null) {
return;
}
List queuesToRemove = Arrays.asList(args.split(";"));
updateInfo.setRemoveQueueInfo(new ArrayList<>(queuesToRemove));
}
@VisibleForTesting
void updateQueues(String args, SchedConfUpdateInfo updateInfo) {
if (args == null) {
return;
}
ArrayList queueConfigInfos = new ArrayList<>();
for (String arg : args.split(";")) {
queueConfigInfos.add(getQueueConfigInfo(arg));
}
updateInfo.setUpdateQueueInfo(queueConfigInfos);
}
@VisibleForTesting
void globalUpdates(String args, SchedConfUpdateInfo updateInfo) {
if (args == null) {
return;
}
HashMap globalUpdates = new HashMap<>();
for (String globalUpdate : args.split(SPLIT_BY_SLASH_COMMA)) {
globalUpdate = globalUpdate.replace("\\", "");
putKeyValuePair(globalUpdates, globalUpdate);
}
updateInfo.setGlobalParams(globalUpdates);
}
private QueueConfigInfo getQueueConfigInfo(String arg) {
String[] args = arg.split(":");
String queuePath = args[0];
Map queueConfigs = new HashMap<>();
if (args.length > 1) {
String[] queueArgs = args[1].split(SPLIT_BY_SLASH_COMMA);
for (int i = 0; i < queueArgs.length; ++i) {
queueArgs[i] = queueArgs[i].replace("\\", "");
putKeyValuePair(queueConfigs, queueArgs[i]);
}
}
return new QueueConfigInfo(queuePath, queueConfigs);
}
private void putKeyValuePair(Map kv, String args) {
String[] argParts = args.split("=");
if (argParts.length == 1) {
if (argParts[0].isEmpty() || !args.contains("=")) {
throw new IllegalArgumentException(CONF_ERR_MSG);
} else {
// key specified, but no value e.g. "confKey="
kv.put(argParts[0], null);
}
} else if (argParts.length > 2) {
throw new IllegalArgumentException(CONF_ERR_MSG);
} else {
if (argParts[0].isEmpty()) {
throw new IllegalArgumentException(CONF_ERR_MSG);
}
kv.put(argParts[0], argParts[1]);
}
}
private void printUsage() {
System.out.println("yarn schedulerconf [-add "
+ "\"queueAddPath1:confKey1=confVal1,confKey2=confVal2;"
+ "queueAddPath2:confKey3=confVal3\"] "
+ "[-remove \"queueRemovePath1;queueRemovePath2\"] "
+ "[-update \"queueUpdatePath1:confKey1=confVal1\"] "
+ "[-global globalConfKey1=globalConfVal1,"
+ "globalConfKey2=globalConfVal2] "
+ "[-format] "
+ "[-getconf]\n"
+ "Example (adding queues): yarn schedulerconf -add "
+ "\"root.a.a1:capacity=100,maximum-capacity=100;root.a.a2:capacity=0,"
+ "maximum-capacity=0\"\n"
+ "Example (adding queues with org.apache.hadoop.shaded.com.a in value): yarn schedulerconf "
+ "-add \"root.default:acl_administer_queue=user1\\,user2 group1\\,"
+ "group2,maximum-capacity=100;root.a.a2:capacity=0\"\n"
+ "Example (removing queues): yarn schedulerconf -remove \"root.a.a1;"
+ "root.a.a2\"\n"
+ "Example (updating queues): yarn schedulerconf -update \"root.a.a1"
+ ":capacity=25,maximum-capacity=25;root.a.a2:capacity=75,"
+ "maximum-capacity=75\"\n"
+ "Example (updating queues with org.apache.hadoop.shaded.com.a in value): yarn schedulerconf "
+ "-update \"root.default:acl_administer_queue=user1\\,user2 group1\\,"
+ "group2,maximum-capacity=25;root.a.a2:capacity=75\"\n"
+ "Example (global scheduler update): yarn schedulerconf "
+ "-global yarn.scheduler.capacity.maximum-applications=10000\n"
+ "Example (global scheduler update with org.apache.hadoop.shaded.com.a in value): yarn "
+ "schedulerconf "
+ "-global \"acl_administer_queue=user1\\,user2 group1\\,group2\"\n"
+ "Example (format scheduler configuration): yarn schedulerconf "
+ "-format\n"
+ "Example (get scheduler configuration): yarn schedulerconf "
+ "-getconf\n"
+ "Note: This is an alpha feature, the syntax/options are subject to "
+ "change, please run at your own risk.");
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy