org.embulk.cli.EmbulkSystemPropertiesBuilder Maven / Gradle / Ivy
package org.embulk.cli;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.jar.Attributes;
import java.util.jar.Manifest;
import java.util.stream.Collectors;
import org.embulk.EmbulkSystemProperties;
import org.slf4j.Logger;
/**
* Builds the eventual Embulk system properties from the command line, environment variables, and the actual file system.
*/
class EmbulkSystemPropertiesBuilder {
private EmbulkSystemPropertiesBuilder(
final Properties commandLineProperties,
final Map env,
final Manifest manifest,
final PathOrException userHomeOrEx,
final PathOrException userDirOrEx,
final Logger logger) {
this.commandLineProperties = commandLineProperties;
this.env = env;
this.manifest = manifest;
this.userHomeOrEx = userHomeOrEx;
this.userDirOrEx = userDirOrEx;
this.logger = logger;
}
static EmbulkSystemPropertiesBuilder from(
final Properties javaSystemProperties,
final Properties commandLineProperties,
final Map env,
final Manifest manifest,
final Logger logger) {
return new EmbulkSystemPropertiesBuilder(
commandLineProperties,
env,
manifest,
getUserHome(javaSystemProperties, logger),
getUserDir(javaSystemProperties, logger),
logger);
}
EmbulkSystemProperties buildProperties() {
final Path embulkHome = findEmbulkHome();
final Properties embulkPropertiesFromFile = loadEmbulkPropertiesFromFile(embulkHome);
final Path m2Repo = findM2Repo(embulkHome, embulkPropertiesFromFile);
final Path gemHome = findGemHome(embulkHome, embulkPropertiesFromFile);
final List gemPath = findGemPath(embulkHome, embulkPropertiesFromFile);
final Properties mergedProperties = new Properties();
// 1) Properties from the "embulk.properties" file are loaded first.
//
// Later sources of properties would overwrite it.
for (final String key : embulkPropertiesFromFile.stringPropertyNames()) {
mergedProperties.setProperty(key, embulkPropertiesFromFile.getProperty(key));
}
// 2) Properties from the command-line are loaded second.
//
// It overwrites the properties from "embulk.properties".
for (final String key : this.commandLineProperties.stringPropertyNames()) {
mergedProperties.setProperty(key, this.commandLineProperties.getProperty(key));
}
// 3) Some specific properties are forcibly overwritten from file/directory lookups.
mergedProperties.setProperty("embulk_home", embulkHome.toString());
mergedProperties.setProperty("m2_repo", m2Repo.toString());
mergedProperties.setProperty("gem_home", gemHome.toString());
mergedProperties.setProperty(
"gem_path", gemPath.stream().map(path -> path.toString()).collect(Collectors.joining(File.pathSeparator)));
if (mergedProperties.getProperty("default_guess_plugins") != null) {
logger.warn("Embulk system property \"default_guess_plugins\" is unexpectedly set. It is reset.");
mergedProperties.remove("default_guess_plugins");
}
if (this.manifest != null) {
final Attributes mainAttributes = this.manifest.getMainAttributes();
if (mainAttributes != null) {
final String defaultGuessPluginsString = mainAttributes.getValue("Embulk-Default-Guess-Plugins");
if (defaultGuessPluginsString != null) {
final String[] defaultGuessPluginsSpecified = defaultGuessPluginsString.split("\\,");
final ArrayList defaultGuessPlugins = new ArrayList<>();
for (final String defaultGuessPlugin : defaultGuessPluginsSpecified) {
if ("gzip".equals(defaultGuessPlugin)) {
final String value = mergedProperties.getProperty("standards.decoder.gzip.disabled");
if (value != null && EmbulkSystemProperties.parseBoolean(value, false)) {
logger.warn(
"Disabling the standard gzip decoder plugin by the Embulk system property "
+ "\"standards.decoder.gzip.disabled\" is deprecated.");
continue;
}
} else if ("bzip2".equals(defaultGuessPlugin)) {
final String value = mergedProperties.getProperty("standards.decoder.bzip2.disabled");
if (value != null && EmbulkSystemProperties.parseBoolean(value, false)) {
logger.warn(
"Disabling the standard bzip2 decoder plugin by the Embulk system property "
+ "\"standards.decoder.bzip2.disabled\" is deprecated.");
continue;
}
} else if ("json".equals(defaultGuessPlugin)) {
final String value = mergedProperties.getProperty("standards.parser.json.disabled");
if (value != null && EmbulkSystemProperties.parseBoolean(value, false)) {
logger.warn(
"Disabling the standard JSON parser plugin by the Embulk system property "
+ "\"standards.parser.json.disabled\" is deprecated.");
continue;
}
} else if ("csv".equals(defaultGuessPlugin)) {
final String value = mergedProperties.getProperty("standards.parser.csv.disabled");
if (value != null && EmbulkSystemProperties.parseBoolean(value, false)) {
logger.warn(
"Disabling the standard CSV parser plugin by the Embulk system property "
+ "\"standards.parser.csv.disabled\" is deprecated.");
continue;
}
}
defaultGuessPlugins.add(defaultGuessPlugin);
}
final String defaultGuessPluginsProperty = String.join(",", defaultGuessPlugins);
logger.debug("Embulk system property \"default_guess_plugin\" is set to: \"{}\"", defaultGuessPluginsProperty);
mergedProperties.setProperty("default_guess_plugins", defaultGuessPluginsProperty);
} else {
logger.warn("Embulk-Default-Guess-Plugins is not found in the JAR Manifest.");
}
} else {
logger.warn("The JAR Manifest unexpectedly has no main attributes.");
}
} else {
logger.warn("The JAR unexpectedly has no Manifest.");
}
return EmbulkSystemProperties.of(mergedProperties);
}
private static PathOrException getUserHome(final Properties javaSystemProperties, final Logger logger) {
return normalizePathInJavaSystemProperty("user.home", javaSystemProperties, logger);
}
private static PathOrException getUserDir(final Properties javaSystemProperties, final Logger logger) {
return normalizePathInJavaSystemProperty("user.dir", javaSystemProperties, logger);
}
/**
* Finds an appropriate "embulk_home" directory based on a rule defined.
*
*
* - 1) If a system config {@code "embulk_home"} is set from the command line, it is the most prioritized.
*
- 2) If an environment variable {@code "EMBULK_HOME"} is set, it is the second prioritized.
*
- 3) If neither (1) nor (2) is set, it iterates up over parent directories from "user.dir" for a directory that:
*
* - is named ".embulk",
*
- has "embulk.properties" just under itself.
*
*
* - 3-1) If "user.dir" (almost equal to the working directory) is under "user.home", it iterates up till "user.home".
*
- 3-2) If "user.dir" is not under "user.home", Embulk iterates until the root directory.
*
*
* - 4) If none of the above does not work, use the traditional predefined directory "~/.embulk".
*
*/
private Path findEmbulkHome() {
// 1) If a system config "embulk_home" is set from the command line, it is the most prioritized.
final Optional ofCommandLine = normalizePathInCommandLineProperties("embulk_home");
if (ofCommandLine.isPresent()) {
logger.info("embulk_home is set from command-line: {}", ofCommandLine.get().toString());
return ofCommandLine.get();
}
// 2) If an environment variable "EMBULK_HOME" is set, it is the second prioritized.
final Optional ofEnv = normalizePathInEnv("EMBULK_HOME");
if (ofEnv.isPresent()) {
logger.info("embulk_home is set from environment variable: {}", ofEnv.get().toString());
return ofEnv.get();
}
// (3) and (4) depend on "user.home" and "user.dir". Exception if they are unavailable.
final Path userHome = userHomeOrEx.orRethrow();
final Path userDir = userDirOrEx.orRethrow();
final Path iterateUpTill;
if (isUnderHome()) {
// 3-1) If "user.dir" (almost equal to the working directory) is under "user.home", it iterates up till "user.home".
iterateUpTill = userHome;
} else {
// 3-2) If "user.dir" is not under "user.home", it iterates up till the root directory.
iterateUpTill = userDir.getRoot();
}
// 3) If neither (1) nor (2) is set, it iterates up over parent directories from "user.dir" for a directory that:
// * is named ".embulk",
// * has a readable file "embulk.properties" just under itself.
if (iterateUpTill != null) {
for (Path pwd = userDir; pwd != null && pwd.startsWith(iterateUpTill); pwd = pwd.getParent()) {
// When checking the actual file/directory, symbolic links are resolved.
final Path dotEmbulk;
try {
dotEmbulk = pwd.resolve(".embulk");
if (Files.notExists(dotEmbulk) || (!Files.isDirectory(dotEmbulk))) {
continue;
}
} catch (final RuntimeException ex) {
logger.debug("Failed to check for \".embulk\" at: " + pwd.toString(), ex);
continue;
}
try {
final Path properties = dotEmbulk.resolve("embulk.properties");
if (Files.notExists(properties) || (!Files.isRegularFile(properties)) || (!Files.isReadable(properties))) {
continue;
}
} catch (final RuntimeException ex) {
logger.debug("Failed to check for \"embulk.properties\" at: " + dotEmbulk.toString(), ex);
continue;
}
logger.info("embulk_home is set by the location of embulk.properties found in: {}", dotEmbulk.toString());
return dotEmbulk;
}
}
// 4) If none of the above does not work, use the traditional predefined directory "~/.embulk".
return userHome.resolve(".embulk");
}
private Properties loadEmbulkPropertiesFromFile(final Path embulkHome) {
final Path path = embulkHome.resolve("embulk.properties");
if (Files.notExists(path)) {
logger.debug(path.toString() + " does not exist. Ignored.");
return new Properties();
}
if (!Files.isRegularFile(path)) {
logger.info(path.toString() + " exists, but not a regular file. Ignored.");
return new Properties();
}
if (!Files.isReadable(path)) {
logger.info(path.toString() + " exists, but not readable. Ignored.");
return new Properties();
}
final Properties properties = new Properties();
try (final InputStream input = Files.newInputStream(path, StandardOpenOption.READ)) {
properties.load(input);
} catch (final IOException ex) {
logger.warn(path.toString() + " exists, but failed to load. Ignored.", ex);
return new Properties();
}
return properties;
}
private Path findM2Repo(final Path embulkHome, final Properties embulkPropertiesFromFile) {
return findSubdirectory(embulkHome, embulkPropertiesFromFile, "m2_repo", "M2_REPO", M2_REPO_RELATIVE);
}
private Path findGemHome(final Path embulkHome, final Properties embulkPropertiesFromFile) {
return findSubdirectory(embulkHome, embulkPropertiesFromFile, "gem_home", "GEM_HOME", GEM_HOME_RELATIVE);
}
private List findGemPath(final Path embulkHome, final Properties embulkPropertiesFromFile) {
return findSubdirectories(embulkHome, embulkPropertiesFromFile, "gem_path", "GEM_PATH");
}
private Path findSubdirectory(
final Path embulkHome,
final Properties embulkPropertiesFromFile,
final String propertyName,
final String envName,
final Path subPath) {
// 1) If a system config is set from the command line, it is the most prioritized.
//
// A path in the command line should be an absolute path, or a relative path from the working directory.
final Optional ofCommandLine = normalizePathInCommandLineProperties(propertyName);
if (ofCommandLine.isPresent()) {
return ofCommandLine.get();
}
// 2) If a system config is set from "embulk.properties", it is the second prioritized.
//
// A path in the "embulk.properties" file should be an absolute path, or a relative path from "embulk_home".
final Optional ofEmbulkPropertiesFile =
normalizePathInEmbulkPropertiesFile(propertyName, embulkPropertiesFromFile, embulkHome);
if (ofEmbulkPropertiesFile.isPresent()) {
return ofEmbulkPropertiesFile.get();
}
// 3) If an environment variable is set, it is the third prioritized.
//
// A path in an environment variable should be an absolute path.
final Optional ofEnv = normalizePathInEnv(envName);
if (ofEnv.isPresent()) {
return ofEnv.get();
}
// 4) If none of the above does not match, use the specific sub directory of "embulk_home".
return embulkHome.resolve(subPath);
}
private List findSubdirectories(
final Path embulkHome,
final Properties embulkPropertiesFromFile,
final String propertyName,
final String envName) {
// 1) If a system config is set from the command line, it is the most prioritized.
final List ofCommandLine = normalizePathsInCommandLineProperties(propertyName, true);
if (!ofCommandLine.isEmpty()) {
return ofCommandLine;
}
// 2) If a system config is set from "embulk.properties", it is the second prioritized.
final List ofEmbulkPropertiesFile =
normalizePathsInEmbulkPropertiesFile(propertyName, embulkPropertiesFromFile, embulkHome, true);
if (!ofEmbulkPropertiesFile.isEmpty()) {
return ofEmbulkPropertiesFile;
}
// 3) If an environment variable is set, it is the third prioritized.
final List ofEnv = normalizePathsInEnv(envName, true);
if (!ofEnv.isEmpty()) {
return ofEnv;
}
// 4) If none of the above does not match, return an empty list.
return Collections.unmodifiableList(new ArrayList<>());
}
/**
* Returns a normalized path in a specified Java system property "user.home" or "user.dir".
*
* Note that a path in a Java system property should be an absolute path.
*/
private static PathOrException normalizePathInJavaSystemProperty(
final String propertyName, final Properties javaSystemProperties, final Logger logger) {
final String property = javaSystemProperties.getProperty(propertyName);
if (property == null || property.isEmpty()) {
final String message = "Java system property \"" + propertyName + "\" is unexpectedly unset.";
final IllegalArgumentException ex = new IllegalArgumentException(message);
logger.error(message, ex);
return new PathOrException(ex);
}
final Path path;
try {
path = Paths.get(property);
} catch (final InvalidPathException ex) {
logger.error("Java system property \"" + propertyName + "\" is unexpectedly invalid: \"" + property + "\"", ex);
return new PathOrException(ex);
}
if (!path.isAbsolute()) {
final String message = "Java system property \"" + propertyName + "\" is unexpectedly not absolute.";
final IllegalArgumentException ex = new IllegalArgumentException(message);
logger.error(message, ex);
return new PathOrException(ex);
}
final Path normalized = path.normalize();
if (!normalized.equals(path)) {
logger.warn("Java system property \"" + propertyName + "\" is unexpectedly not normalized: \"" + property + "\", "
+ "then resolved to: \"" + normalized.toString() + "\"");
}
// Symbolic links are intentionally NOT resolved with Path#toRealPath.
return new PathOrException(normalized);
}
private Optional normalizePathInCommandLineProperties(final String propertyName) {
final List paths = normalizePathsInCommandLineProperties(propertyName, false);
if (paths.size() > 1) {
throw new IllegalStateException("Multiple paths returned for an unsplit path.");
}
if (paths.isEmpty()) {
return Optional.empty();
}
return Optional.of(paths.get(0));
}
/**
* Returns normalized paths in a specified property from the command line.
*
* Note that a path in the command line should be an absolute path, or a relative path from the working directory.
*/
private List normalizePathsInCommandLineProperties(final String propertyName, final boolean multi) {
if (!this.commandLineProperties.containsKey(propertyName)) {
return Collections.unmodifiableList(new ArrayList());
}
final String property = this.commandLineProperties.getProperty(propertyName);
if (property == null || property.isEmpty()) {
return Collections.unmodifiableList(new ArrayList());
}
final List pathStrings = splitPathStrings(property, multi);
final ArrayList paths = new ArrayList<>();
for (final String pathString : pathStrings) {
if (pathString.isEmpty()) {
continue;
}
final Path path;
try {
path = Paths.get(pathString);
} catch (final InvalidPathException ex) {
logger.error("Embulk system property \"" + propertyName + "\" in command-line is invalid: \"" + pathString + "\"", ex);
throw ex;
}
final Path absolute;
if (path.isAbsolute()) {
absolute = path;
} else {
absolute = path.toAbsolutePath();
}
final Path normalized = absolute.normalize();
if (!normalized.equals(path)) {
logger.warn("Embulk system property \"" + propertyName + "\" in command-line is not normalized: "
+ "\"" + pathString + "\", " + "then resolved to: \"" + normalized.toString() + "\"");
}
// Symbolic links are intentionally NOT resolved with Path#toRealPath.
paths.add(normalized);
}
return Collections.unmodifiableList(paths);
}
private Optional normalizePathInEnv(final String envName) {
final List paths = normalizePathsInEnv(envName, false);
if (paths.size() > 1) {
throw new IllegalStateException("Multiple paths returned for an unsplit path.");
}
if (paths.isEmpty()) {
return Optional.empty();
}
return Optional.of(paths.get(0));
}
/**
* Returns normalized paths in an environment variable.
*
* Note that a path in an environment variable should be an absolute path.
*/
private List normalizePathsInEnv(final String envName, final boolean multi) {
if (!this.env.containsKey(envName)) {
return Collections.unmodifiableList(new ArrayList());
}
final String value = this.env.get(envName);
if (value == null || value.isEmpty()) {
return Collections.unmodifiableList(new ArrayList());
}
final List pathStrings = splitPathStrings(value, multi);
final ArrayList paths = new ArrayList<>();
for (final String pathString : pathStrings) {
if (pathString.isEmpty()) {
continue;
}
final Path path;
try {
path = Paths.get(pathString);
} catch (final InvalidPathException ex) {
logger.error("Environment variable \"" + envName + "\" is invalid: \"" + pathString + "\"", ex);
throw ex;
}
if (!path.isAbsolute()) {
final String message = "Environment variable \"" + envName + "\" is not absolute.";
final IllegalArgumentException ex = new IllegalArgumentException(message);
logger.error(message, ex);
throw ex;
}
final Path normalized = path.normalize();
if (!normalized.equals(path)) {
logger.warn("Environment variable \"" + envName + "\" is not normalized: "
+ "\"" + pathString + "\", " + "then resolved to: \"" + normalized.toString() + "\"");
}
// Symbolic links are intentionally NOT resolved with Path#toRealPath.
paths.add(normalized);
}
return Collections.unmodifiableList(paths);
}
private Optional normalizePathInEmbulkPropertiesFile(
final String propertyName,
final Properties embulkPropertiesFromFile,
final Path embulkHome) {
final List paths = normalizePathsInEmbulkPropertiesFile(
propertyName, embulkPropertiesFromFile, embulkHome, false);
if (paths.size() > 1) {
throw new IllegalStateException("Multiple paths returned for an unsplit path.");
}
if (paths.isEmpty()) {
return Optional.empty();
}
return Optional.of(paths.get(0));
}
/**
* Returns normalized paths from the "embulk.properties" file.
*
* Note that a path in the "embulk.properties" file should be an absolute path, or a relative path from "embulk_home".
*/
private List normalizePathsInEmbulkPropertiesFile(
final String propertyName,
final Properties embulkPropertiesFromFile,
final Path embulkHome,
final boolean multi) {
if (!embulkPropertiesFromFile.containsKey(propertyName)) {
return Collections.unmodifiableList(new ArrayList());
}
final String property = embulkPropertiesFromFile.getProperty(propertyName);
if (property == null || property.isEmpty()) {
return Collections.unmodifiableList(new ArrayList());
}
final List pathStrings = splitPathStrings(property, multi);
final ArrayList paths = new ArrayList<>();
for (final String pathString : pathStrings) {
if (pathString.isEmpty()) {
continue;
}
final Path path;
try {
path = Paths.get(pathString);
} catch (final InvalidPathException ex) {
logger.error(
"Embulk system property \"" + propertyName + "\" in embulk.properties is invalid: \"" + pathString + "\"",
ex);
throw ex;
}
final Path absolute;
if (path.isAbsolute()) {
absolute = path;
} else {
absolute = embulkHome.resolve(path);
}
final Path normalized = absolute.normalize();
if (!normalized.equals(path)) {
logger.warn("Embulk system property \"" + propertyName + "\" in embulk.properties is not normalized: "
+ "\"" + pathString + "\", " + "then resolved to: \"" + normalized.toString() + "\"");
}
// Symbolic links are intentionally NOT resolved with Path#toRealPath.
paths.add(normalized);
}
return Collections.unmodifiableList(paths);
}
private List splitPathStrings(final String pathStrings, final boolean multi) {
final ArrayList split = new ArrayList<>();
if (multi) {
for (final String pathString : pathStrings.split(File.pathSeparator)) {
split.add(pathString);
}
} else {
split.add(pathStrings);
}
return Collections.unmodifiableList(split);
}
/**
* Returns {@code true} if {@code userDir} is under {@code userHome}.
*
* Note that the check is performed "literally". It does not take care of the existence of the path.
* It does not resolve a symbolic link.
*/
private boolean isUnderHome() {
return this.userDirOrEx.orRethrow().startsWith(this.userHomeOrEx.orRethrow());
}
/**
* Contains a Path, or an Exception in case the Path is invalid.
*
*
It is used for Java system properties "user.home" and "user.dir" to delay throwing the Exception.
*
*
Even if "user.home" or "user.dir" is invalid, it should be okay when "embulk_home" is configured explicitly.
*/
private static class PathOrException {
PathOrException(final Path path) {
if (path == null) {
this.path = null;
this.exception = new NullPointerException("Path is null.");
} else {
this.path = path;
this.exception = null;
}
}
PathOrException(final RuntimeException exception) {
this.path = null;
this.exception = exception;
}
Path orRethrow() {
if (this.path == null) {
throw this.exception;
}
return this.path;
}
private final Path path;
private final RuntimeException exception;
}
private static final Path M2_REPO_RELATIVE = Paths.get("lib").resolve("m2").resolve("repository");
private static final Path GEM_HOME_RELATIVE = Paths.get("lib").resolve("gems");
private final Properties commandLineProperties;
private final Map env;
private final Manifest manifest;
private final PathOrException userHomeOrEx;
private final PathOrException userDirOrEx;
private final Logger logger;
}