Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.launcher;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Helper methods for command builders.
*/
class CommandBuilderUtils {
static final String DEFAULT_MEM = "1g";
static final String DEFAULT_PROPERTIES_FILE = "spark-defaults.conf";
static final String ENV_SPARK_HOME = "SPARK_HOME";
/** The set of known JVM vendors. */
enum JavaVendor {
Oracle, IBM, OpenJDK, Unknown
}
/** Returns whether the given string is null or empty. */
static boolean isEmpty(String s) {
return s == null || s.isEmpty();
}
/** Joins a list of strings using the given separator. */
static String join(String sep, String... elements) {
StringBuilder sb = new StringBuilder();
for (String e : elements) {
if (e != null) {
if (sb.length() > 0) {
sb.append(sep);
}
sb.append(e);
}
}
return sb.toString();
}
/** Joins a list of strings using the given separator. */
static String join(String sep, Iterable elements) {
StringBuilder sb = new StringBuilder();
for (String e : elements) {
if (e != null) {
if (sb.length() > 0) {
sb.append(sep);
}
sb.append(e);
}
}
return sb.toString();
}
/**
* Returns the first non-empty value mapped to the given key in the given maps, or null otherwise.
*/
static String firstNonEmptyValue(String key, Map... maps) {
for (Map map : maps) {
String value = (String) map.get(key);
if (!isEmpty(value)) {
return value;
}
}
return null;
}
/** Returns the first non-empty, non-null string in the given list, or null otherwise. */
static String firstNonEmpty(String... candidates) {
for (String s : candidates) {
if (!isEmpty(s)) {
return s;
}
}
return null;
}
/** Returns the name of the env variable that holds the native library path. */
static String getLibPathEnvName() {
if (isWindows()) {
return "PATH";
}
String os = System.getProperty("os.name");
if (os.startsWith("Mac OS X")) {
return "DYLD_LIBRARY_PATH";
} else {
return "LD_LIBRARY_PATH";
}
}
/** Returns whether the OS is Windows. */
static boolean isWindows() {
String os = System.getProperty("os.name");
return os.startsWith("Windows");
}
/** Returns an enum value indicating whose JVM is being used. */
static JavaVendor getJavaVendor() {
String vendorString = System.getProperty("java.vendor");
if (vendorString.contains("Oracle")) {
return JavaVendor.Oracle;
}
if (vendorString.contains("IBM")) {
return JavaVendor.IBM;
}
if (vendorString.contains("OpenJDK")) {
return JavaVendor.OpenJDK;
}
return JavaVendor.Unknown;
}
/**
* Updates the user environment, appending the given pathList to the existing value of the given
* environment variable (or setting it if it hasn't yet been set).
*/
static void mergeEnvPathList(Map userEnv, String envKey, String pathList) {
if (!isEmpty(pathList)) {
String current = firstNonEmpty(userEnv.get(envKey), System.getenv(envKey));
userEnv.put(envKey, join(File.pathSeparator, current, pathList));
}
}
/**
* Parse a string as if it were a list of arguments, following bash semantics.
* For example:
*
* Input: "\"ab cd\" efgh 'i \" j'"
* Output: [ "ab cd", "efgh", "i \" j" ]
*/
static List parseOptionString(String s) {
List opts = new ArrayList<>();
StringBuilder opt = new StringBuilder();
boolean inOpt = false;
boolean inSingleQuote = false;
boolean inDoubleQuote = false;
boolean escapeNext = false;
// This is needed to detect when a quoted empty string is used as an argument ("" or '').
boolean hasData = false;
for (int i = 0; i < s.length(); i++) {
int c = s.codePointAt(i);
if (escapeNext) {
opt.appendCodePoint(c);
escapeNext = false;
} else if (inOpt) {
switch (c) {
case '\\':
if (inSingleQuote) {
opt.appendCodePoint(c);
} else {
escapeNext = true;
}
break;
case '\'':
if (inDoubleQuote) {
opt.appendCodePoint(c);
} else {
inSingleQuote = !inSingleQuote;
}
break;
case '"':
if (inSingleQuote) {
opt.appendCodePoint(c);
} else {
inDoubleQuote = !inDoubleQuote;
}
break;
default:
if (!Character.isWhitespace(c) || inSingleQuote || inDoubleQuote) {
opt.appendCodePoint(c);
} else {
opts.add(opt.toString());
opt.setLength(0);
inOpt = false;
hasData = false;
}
}
} else {
switch (c) {
case '\'':
inSingleQuote = true;
inOpt = true;
hasData = true;
break;
case '"':
inDoubleQuote = true;
inOpt = true;
hasData = true;
break;
case '\\':
escapeNext = true;
inOpt = true;
hasData = true;
break;
default:
if (!Character.isWhitespace(c)) {
inOpt = true;
hasData = true;
opt.appendCodePoint(c);
}
}
}
}
checkArgument(!inSingleQuote && !inDoubleQuote && !escapeNext, "Invalid option string: %s", s);
if (hasData) {
opts.add(opt.toString());
}
return opts;
}
/** Throws IllegalArgumentException if the given object is null. */
static void checkNotNull(Object o, String arg) {
if (o == null) {
throw new IllegalArgumentException(String.format("'%s' must not be null.", arg));
}
}
/** Throws IllegalArgumentException with the given message if the check is false. */
static void checkArgument(boolean check, String msg, Object... args) {
if (!check) {
throw new IllegalArgumentException(String.format(msg, args));
}
}
/** Throws IllegalStateException with the given message if the check is false. */
static void checkState(boolean check, String msg, Object... args) {
if (!check) {
throw new IllegalStateException(String.format(msg, args));
}
}
/**
* Quote a command argument for a command to be run by a Windows batch script, if the argument
* needs quoting. Arguments only seem to need quotes in batch scripts if they have certain
* special characters, some of which need extra (and different) escaping.
*
* For example:
* original single argument: ab="cde fgh"
* quoted: "ab^=""cde fgh"""
*/
static String quoteForBatchScript(String arg) {
boolean needsQuotes = false;
for (int i = 0; i < arg.length(); i++) {
int c = arg.codePointAt(i);
if (Character.isWhitespace(c) || c == '"' || c == '=' || c == ',' || c == ';') {
needsQuotes = true;
break;
}
}
if (!needsQuotes) {
return arg;
}
StringBuilder quoted = new StringBuilder();
quoted.append("\"");
for (int i = 0; i < arg.length(); i++) {
int cp = arg.codePointAt(i);
switch (cp) {
case '"':
quoted.append('"');
break;
default:
break;
}
quoted.appendCodePoint(cp);
}
if (arg.codePointAt(arg.length() - 1) == '\\') {
quoted.append("\\");
}
quoted.append("\"");
return quoted.toString();
}
/**
* Quotes a string so that it can be used in a command string.
* Basically, just add simple escapes. E.g.:
* original single argument : ab "cd" ef
* after: "ab \"cd\" ef"
*
* This can be parsed back into a single argument by python's "shlex.split()" function.
*/
static String quoteForCommandString(String s) {
StringBuilder quoted = new StringBuilder().append('"');
for (int i = 0; i < s.length(); i++) {
int cp = s.codePointAt(i);
if (cp == '"' || cp == '\\') {
quoted.appendCodePoint('\\');
}
quoted.appendCodePoint(cp);
}
return quoted.append('"').toString();
}
/**
* Adds the default perm gen size option for Spark if the VM requires it and the user hasn't
* set it.
*/
static void addPermGenSizeOpt(List cmd) {
// Don't set MaxPermSize for IBM Java, or Oracle Java 8 and later.
if (getJavaVendor() == JavaVendor.IBM) {
return;
}
if (javaMajorVersion(System.getProperty("java.version")) > 7) {
return;
}
for (String arg : cmd) {
if (arg.contains("-XX:MaxPermSize=")) {
return;
}
}
cmd.add("-XX:MaxPermSize=256m");
}
/**
* Get the major version of the java version string supplied. This method
* accepts any JEP-223-compliant strings (9-ea, 9+100), as well as legacy
* version strings such as 1.7.0_79
*/
static int javaMajorVersion(String javaVersion) {
String[] version = javaVersion.split("[+.\\-]+");
int major = Integer.parseInt(version[0]);
// if major > 1, we're using the JEP-223 version string, e.g., 9-ea, 9+120
// otherwise the second number is the major version
if (major > 1) {
return major;
} else {
return Integer.parseInt(version[1]);
}
}
/**
* Find the location of the Spark jars dir, depending on whether we're looking at a build
* or a distribution directory.
*/
static String findJarsDir(String sparkHome, String scalaVersion, boolean failIfNotFound) {
// TODO: change to the correct directory once the assembly build is changed.
File libdir;
if (new File(sparkHome, "jars").isDirectory()) {
libdir = new File(sparkHome, "jars");
checkState(!failIfNotFound || libdir.isDirectory(),
"Library directory '%s' does not exist.",
libdir.getAbsolutePath());
} else {
libdir = new File(sparkHome, String.format("assembly/target/scala-%s/jars", scalaVersion));
if (!libdir.isDirectory()) {
checkState(!failIfNotFound,
"Library directory '%s' does not exist; make sure Spark is built.",
libdir.getAbsolutePath());
libdir = null;
}
}
return libdir != null ? libdir.getAbsolutePath() : null;
}
}
