org.apache.hadoop.conf.Configuration Maven / Gradle / Ivy

Go to download
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.conf;

import com.facebook.presto.hadoop.$internal.com.google.common.annotations.VisibleForTesting;

import java.io.BufferedInputStream;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.lang.ref.WeakReference;
import java.net.InetSocketAddress;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.WeakHashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import com.facebook.presto.hadoop.$internal.com.google.common.base.Charsets;
import com.facebook.presto.hadoop.$internal.org.apache.commons.collections.map.UnmodifiableMap;
import com.facebook.presto.hadoop.$internal.org.apache.commons.logging.Log;
import com.facebook.presto.hadoop.$internal.org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
import com.facebook.presto.hadoop.$internal.org.codehaus.jackson.JsonFactory;
import com.facebook.presto.hadoop.$internal.org.codehaus.jackson.JsonGenerator;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.dom.Text;
import org.xml.sax.SAXException;

import com.facebook.presto.hadoop.$internal.com.google.common.base.Preconditions;

/**
 * Provides access to configuration parameters.
 *
 * Resources
 *
 * Configurations are specified by resources. A resource contains a set of
 * name/value pairs as XML data. Each resource is named by either a
 * String or by a {@link Path}. If named by a String,
 * then the classpath is examined for a file with that name.  If named by a
 * Path, then the local filesystem is examined directly, without
 * referring to the classpath.
 *
 * 
Unless explicitly turned off, Hadoop by default specifies two
 * resources, loaded in-order from the classpath: 

 * 
 * 
 * core-default-shaded.xml: Read-only defaults for hadoop.
 * core-site.xml: Site-specific configuration for a given hadoop
 * installation.
 * 
 * Applications may add additional resources, which are loaded
 * subsequent to these resources in the order they are added.
 *
 * Final Parameters
 *
 * Configuration parameters may be declared final.
 * Once a resource declares a value final, no subsequently-loaded
 * resource can alter that value.
 * For example, one might define a final parameter with:
 * 
 *  <property>
 *    <name>dfs.hosts.include</name>
 *    <value>/etc/hadoop/conf/hosts.include</value>
 *    <final>true</final>
 *  </property>
 *
 * Administrators typically define parameters as final in
 * core-site.xml for values that user applications may not alter.
 *
 * Variable Expansion
 *
 * Value strings are first processed for variable expansion. The
 * available properties are:

 * Other properties defined in this Configuration; and, if a name is
 * undefined here,
 * Properties in {@link System#getProperties()}.
 * 
 *
 * For example, if a configuration resource contains the following property
 * definitions:
 * 
 *  <property>
 *    <name>basedir</name>
 *    <value>/user/${user.name}</value>
 *  </property>
 *
 *  <property>
 *    <name>tempdir</name>
 *    <value>${basedir}/tmp</value>
 *  </property>
 *
 * When conf.get("tempdir") is called, then ${basedir}
 * will be resolved to another property in this Configuration, while
 * ${user.name} would then ordinarily be resolved to the value
 * of the System property with that name.
 * By default, warnings will be given to any deprecated configuration
 * parameters and these are suppressible by configuring
 * log4j.logger.org.apache.hadoop.conf.Configuration.deprecation in
 * log4j.properties file.
 */
@InterfaceAudience.Public
@InterfaceStability.Stable
public class Configuration implements Iterable>,
	Writable {
	private static final Log LOG =
		LogFactory.getLog(Configuration.class);

	private static final Log LOG_DEPRECATION =
		LogFactory.getLog("org.apache.hadoop.conf.Configuration.deprecation");

	private boolean quietmode = true;

	private static final String DEFAULT_STRING_CHECK =
		"testingforemptydefaultvalue";

	private boolean allowNullValueProperties = false;

	private static class Resource {
		private final Object resource;
		private final String name;

		public Resource(Object resource) {
			this(resource, resource.toString());
		}

		public Resource(Object resource, String name) {
			this.resource = resource;
			this.name = name;
		}

		public String getName(){
			return name;
		}

		public Object getResource() {
			return resource;
		}

		@Override
		public String toString() {
			return name;
		}
	}

	/**
	 * List of configuration resources.
	 */
	private ArrayList resources = new ArrayList();

	/**
	 * The value reported as the setting resource when a key is set
	 * by code rather than a file resource by dumpConfiguration.
	 */
	static final String UNKNOWN_RESOURCE = "Unknown";


	/**
	 * List of configuration parameters marked final.
	 */
	private Set finalParameters = Collections.newSetFromMap(
		new ConcurrentHashMap());

	private boolean loadDefaults = true;

	/**
	 * Configuration objects
	 */
	private static final WeakHashMap REGISTRY =
		new WeakHashMap();

	/**
	 * List of default Resources. Resources are loaded in the order of the list
	 * entries
	 */
	private static final CopyOnWriteArrayList defaultResources =
		new CopyOnWriteArrayList();

	private static final Map>>>
		CACHE_CLASSES = new WeakHashMap>>>();

	/**
	 * Sentinel value to store negative cache results in {@link #CACHE_CLASSES}.
	 */
	private static final Class NEGATIVE_CACHE_SENTINEL =
		NegativeCacheSentinel.class;

	/**
	 * Stores the mapping of key to the resource which modifies or loads
	 * the key most recently
	 */
	private Map updatingResource;

	/**
	 * Class to keep the information about the keys which replace the deprecated
	 * ones.
	 *
	 * This class stores the new keys which replace the deprecated keys and also
	 * gives a provision to have a custom message for each of the deprecated key
	 * that is being replaced. It also provides method to get the appropriate
	 * warning message which can be logged whenever the deprecated key is used.
	 */
	private static class DeprecatedKeyInfo {
		private final String[] newKeys;
		private final String customMessage;
		private final AtomicBoolean accessed = new AtomicBoolean(false);

		DeprecatedKeyInfo(String[] newKeys, String customMessage) {
			this.newKeys = newKeys;
			this.customMessage = customMessage;
		}

		/**
		 * Method to provide the warning message. It gives the custom message if
		 * non-null, and default message otherwise.
		 * @param key the associated deprecated key.
		 * @return message that is to be logged when a deprecated key is used.
		 */
		private final String getWarningMessage(String key) {
			String warningMessage;
			if(customMessage == null) {
				StringBuilder message = new StringBuilder(key);
				String deprecatedKeySuffix = " is deprecated. Instead, use ";
				message.append(deprecatedKeySuffix);
				for (int i = 0; i < newKeys.length; i++) {
					message.append(newKeys[i]);
					if(i != newKeys.length-1) {
						message.append(", ");
					}
				}
				warningMessage = message.toString();
			}
			else {
				warningMessage = customMessage;
			}
			return warningMessage;
		}

		boolean getAndSetAccessed() {
			return accessed.getAndSet(true);
		}

		public void clearAccessed() {
			accessed.set(false);
		}
	}

	/**
	 * A pending addition to the global set of deprecated keys.
	 */
	public static class DeprecationDelta {
		private final String key;
		private final String[] newKeys;
		private final String customMessage;

		DeprecationDelta(String key, String[] newKeys, String customMessage) {
			Preconditions.checkNotNull(key);
			Preconditions.checkNotNull(newKeys);
			Preconditions.checkArgument(newKeys.length > 0);
			this.key = key;
			this.newKeys = newKeys;
			this.customMessage = customMessage;
		}

		public DeprecationDelta(String key, String newKey, String customMessage) {
			this(key, new String[] { newKey }, customMessage);
		}

		public DeprecationDelta(String key, String newKey) {
			this(key, new String[] { newKey }, null);
		}

		public String getKey() {
			return key;
		}

		public String[] getNewKeys() {
			return newKeys;
		}

		public String getCustomMessage() {
			return customMessage;
		}
	}

	/**
	 * The set of all keys which are deprecated.
	 *
	 * DeprecationContext objects are immutable.
	 */
	private static class DeprecationContext {
		/**
		 * Stores the deprecated keys, the new keys which replace the deprecated keys
		 * and custom message(if any provided).
		 */
		private final Map deprecatedKeyMap;

		/**
		 * Stores a mapping from superseding keys to the keys which they deprecate.
		 */
		private final Map reverseDeprecatedKeyMap;

		/**
		 * Create a new DeprecationContext by copying a previous DeprecationContext
		 * and adding some deltas.
		 *
		 * @param other   The previous deprecation context to copy, or null to start
		 *                from nothing.
		 * @param deltas  The deltas to apply.
		 */
		@SuppressWarnings("unchecked")
		DeprecationContext(DeprecationContext other, DeprecationDelta[] deltas) {
			HashMap newDeprecatedKeyMap =
				new HashMap();
			HashMap newReverseDeprecatedKeyMap =
				new HashMap();
			if (other != null) {
				for (Entry entry :
					other.deprecatedKeyMap.entrySet()) {
					newDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
				}
				for (Entry entry :
					other.reverseDeprecatedKeyMap.entrySet()) {
					newReverseDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
				}
			}
			for (DeprecationDelta delta : deltas) {
				if (!newDeprecatedKeyMap.containsKey(delta.getKey())) {
					DeprecatedKeyInfo newKeyInfo =
						new DeprecatedKeyInfo(delta.getNewKeys(), delta.getCustomMessage());
					newDeprecatedKeyMap.put(delta.key, newKeyInfo);
					for (String newKey : delta.getNewKeys()) {
						newReverseDeprecatedKeyMap.put(newKey, delta.key);
					}
				}
			}
			this.deprecatedKeyMap =
				UnmodifiableMap.decorate(newDeprecatedKeyMap);
			this.reverseDeprecatedKeyMap =
				UnmodifiableMap.decorate(newReverseDeprecatedKeyMap);
		}

		Map getDeprecatedKeyMap() {
			return deprecatedKeyMap;
		}

		Map getReverseDeprecatedKeyMap() {
			return reverseDeprecatedKeyMap;
		}
	}

	private static DeprecationDelta[] defaultDeprecations =
		new DeprecationDelta[] {
			new DeprecationDelta("topology.script.file.name",
				CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY),
			new DeprecationDelta("topology.script.number.args",
				CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY),
			new DeprecationDelta("hadoop.configured.node.mapping",
				CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY),
			new DeprecationDelta("topology.node.switch.mapping.impl",
				CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY),
			new DeprecationDelta("dfs.df.interval",
				CommonConfigurationKeys.FS_DF_INTERVAL_KEY),
			new DeprecationDelta("hadoop.native.lib",
				CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY),
			new DeprecationDelta("fs.default.name",
				CommonConfigurationKeys.FS_DEFAULT_NAME_KEY),
			new DeprecationDelta("dfs.umaskmode",
				CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY),
			new DeprecationDelta("dfs.nfs.exports.allowed.hosts",
				CommonConfigurationKeys.NFS_EXPORTS_ALLOWED_HOSTS_KEY)
		};

	/**
	 * The global DeprecationContext.
	 */
	private static AtomicReference deprecationContext =
		new AtomicReference(
			new DeprecationContext(null, defaultDeprecations));

	/**
	 * Adds a set of deprecated keys to the global deprecations.
	 *
	 * This method is lockless.  It works by means of creating a new
	 * DeprecationContext based on the old one, and then atomically swapping in
	 * the new context.  If someone else updated the context in between us reading
	 * the old context and swapping in the new one, we try again until we win the
	 * race.
	 *
	 * @param deltas   The deprecations to add.
	 */
	public static void addDeprecations(DeprecationDelta[] deltas) {
		DeprecationContext prev, next;
		do {
			prev = deprecationContext.get();
			next = new DeprecationContext(prev, deltas);
		} while (!deprecationContext.compareAndSet(prev, next));
	}

	/**
	 * Adds the deprecated key to the global deprecation map.
	 * It does not override any existing entries in the deprecation map.
	 * This is to be used only by the developers in order to add deprecation of
	 * keys, and attempts to call this method after loading resources once,
	 * would lead to UnsupportedOperationException
	 *
	 * If a key is deprecated in favor of multiple keys, they are all treated as
	 * aliases of each other, and setting any one of them resets all the others
	 * to the new value.
	 *
	 * If you have multiple deprecation entries to add, it is more efficient to
	 * use #addDeprecations(DeprecationDelta[] deltas) instead.
	 *
	 * @param key
	 * @param newKeys
	 * @param customMessage
	 * @deprecated use {@link #addDeprecation(String key, String newKey,
		String customMessage)} instead
	 */
	@Deprecated
	public static void addDeprecation(String key, String[] newKeys,
		String customMessage) {
		addDeprecations(new DeprecationDelta[] {
			new DeprecationDelta(key, newKeys, customMessage)
		});
	}

	/**
	 * Adds the deprecated key to the global deprecation map.
	 * It does not override any existing entries in the deprecation map.
	 * This is to be used only by the developers in order to add deprecation of
	 * keys, and attempts to call this method after loading resources once,
	 * would lead to UnsupportedOperationException
	 *
	 * If you have multiple deprecation entries to add, it is more efficient to
	 * use #addDeprecations(DeprecationDelta[] deltas) instead.
	 *
	 * @param key
	 * @param newKey
	 * @param customMessage
	 */
	public static void addDeprecation(String key, String newKey,
		String customMessage) {
		addDeprecation(key, new String[] {newKey}, customMessage);
	}

	/**
	 * Adds the deprecated key to the global deprecation map when no custom
	 * message is provided.
	 * It does not override any existing entries in the deprecation map.
	 * This is to be used only by the developers in order to add deprecation of
	 * keys, and attempts to call this method after loading resources once,
	 * would lead to UnsupportedOperationException
	 *
	 * If a key is deprecated in favor of multiple keys, they are all treated as
	 * aliases of each other, and setting any one of them resets all the others
	 * to the new value.
	 *
	 * If you have multiple deprecation entries to add, it is more efficient to
	 * use #addDeprecations(DeprecationDelta[] deltas) instead.
	 *
	 * @param key Key that is to be deprecated
	 * @param newKeys list of keys that take up the values of deprecated key
	 * @deprecated use {@link #addDeprecation(String key, String newKey)} instead
	 */
	@Deprecated
	public static void addDeprecation(String key, String[] newKeys) {
		addDeprecation(key, newKeys, null);
	}

	/**
	 * Adds the deprecated key to the global deprecation map when no custom
	 * message is provided.
	 * It does not override any existing entries in the deprecation map.
	 * This is to be used only by the developers in order to add deprecation of
	 * keys, and attempts to call this method after loading resources once,
	 * would lead to UnsupportedOperationException
	 *
	 * If you have multiple deprecation entries to add, it is more efficient to
	 * use #addDeprecations(DeprecationDelta[] deltas) instead.
	 *
	 * @param key Key that is to be deprecated
	 * @param newKey key that takes up the value of deprecated key
	 */
	public static void addDeprecation(String key, String newKey) {
		addDeprecation(key, new String[] {newKey}, null);
	}

	/**
	 * checks whether the given key is deprecated.
	 *
	 * @param key the parameter which is to be checked for deprecation
	 * @return true if the key is deprecated and
	 *         false otherwise.
	 */
	public static boolean isDeprecated(String key) {
		return deprecationContext.get().getDeprecatedKeyMap().containsKey(key);
	}

	/**
	 * Sets all deprecated properties that are not currently set but have a
	 * corresponding new property that is set. Useful for iterating the
	 * properties when all deprecated properties for currently set properties
	 * need to be present.
	 */
	public void setDeprecatedProperties() {
		DeprecationContext deprecations = deprecationContext.get();
		Properties props = getProps();
		Properties overlay = getOverlay();
		for (Map.Entry entry :
			deprecations.getDeprecatedKeyMap().entrySet()) {
			String depKey = entry.getKey();
			if (!overlay.contains(depKey)) {
				for (String newKey : entry.getValue().newKeys) {
					String val = overlay.getProperty(newKey);
					if (val != null) {
						props.setProperty(depKey, val);
						overlay.setProperty(depKey, val);
						break;
					}
				}
			}
		}
	}

	/**
	 * Checks for the presence of the property name in the
	 * deprecation map. Returns the first of the list of new keys if present
	 * in the deprecation map or the name itself. If the property
	 * is not presently set but the property map contains an entry for the
	 * deprecated key, the value of the deprecated key is set as the value for
	 * the provided property name.
	 *
	 * @param name the property name
	 * @return the first property in the list of properties mapping
	 *         the name or the name itself.
	 */
	private String[] handleDeprecation(DeprecationContext deprecations,
		String name) {
		if (null != name) {
			name = name.trim();
		}
		ArrayList names = new ArrayList();
		if (isDeprecated(name)) {
			DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
			warnOnceIfDeprecated(deprecations, name);
			for (String newKey : keyInfo.newKeys) {
				if(newKey != null) {
					names.add(newKey);
				}
			}
		}
		if(names.size() == 0) {
			names.add(name);
		}
		for(String n : names) {
			String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n);
			if (deprecatedKey != null && !getOverlay().containsKey(n) &&
				getOverlay().containsKey(deprecatedKey)) {
				getProps().setProperty(n, getOverlay().getProperty(deprecatedKey));
				getOverlay().setProperty(n, getOverlay().getProperty(deprecatedKey));
			}
		}
		return names.toArray(new String[names.size()]);
	}

	private void handleDeprecation() {
		LOG.debug("Handling deprecation for all properties in config...");
		DeprecationContext deprecations = deprecationContext.get();
		Set