org.apache.hadoop.conf.Configuration Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hadoop-common Show documentation
Apache Hadoop Common
There is a newer version: 3.4.1
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.conf;

import com.ctc.wstx.api.ReaderConfig;
import com.ctc.wstx.io.StreamBootstrapper;
import com.ctc.wstx.io.SystemId;
import com.ctc.wstx.stax.WstxInputFactory;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.google.common.annotations.VisibleForTesting;

import java.io.BufferedInputStream;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.lang.ref.WeakReference;
import java.net.InetSocketAddress;
import java.net.JarURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.WeakHashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import com.google.common.base.Charsets;
import org.apache.commons.collections.map.UnmodifiableMap;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
import org.codehaus.stax2.XMLStreamReader2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;

import com.google.common.base.Preconditions;
import com.google.common.base.Strings;

import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;

/**
 * Provides access to configuration parameters.
 *
 * Resources
 *
 * Configurations are specified by resources. A resource contains a set of
 * name/value pairs as XML data. Each resource is named by either a 
 * String or by a {@link Path}. If named by a String, 
 * then the classpath is examined for a file with that name.  If named by a 
 * Path, then the local filesystem is examined directly, without 
 * referring to the classpath.
 *
 * 
Unless explicitly turned off, Hadoop by default specifies two 
 * resources, loaded in-order from the classpath: 

 * 
 * 
 * core-default.xml: Read-only defaults for hadoop.
 * core-site.xml: Site-specific configuration for a given hadoop
 * installation.
 * 
 * Applications may add additional resources, which are loaded
 * subsequent to these resources in the order they are added.
 * 
 * Final Parameters
 *
 * Configuration parameters may be declared final. 
 * Once a resource declares a value final, no subsequently-loaded 
 * resource can alter that value.  
 * For example, one might define a final parameter with:
 * 
 *  <property>
 *    <name>dfs.hosts.include</name>
 *    <value>/etc/hadoop/conf/hosts.include</value>
 *    <final>true</final>
 *  </property>
 *
 * Administrators typically define parameters as final in 
 * core-site.xml for values that user applications may not alter.
 *
 * Variable Expansion
 *
 * Value strings are first processed for variable expansion. The
 * available properties are:

 * Other properties defined in this Configuration; and, if a name is
 * undefined here,
 * Environment variables in {@link System#getenv()} if a name starts with
 * "env.", or
 * Properties in {@link System#getProperties()}.
 * 
 *
 * For example, if a configuration resource contains the following property
 * definitions: 
 * 
 *  <property>
 *    <name>basedir</name>
 *    <value>/user/${user.name}</value>
 *  </property>
 *  
 *  <property>
 *    <name>tempdir</name>
 *    <value>${basedir}/tmp</value>
 *  </property>
 *
 *  <property>
 *    <name>otherdir</name>
 *    <value>${env.BASE_DIR}/other</value>
 *  </property>
 *  
 *
 * When conf.get("tempdir") is called, then ${basedir}
 * will be resolved to another property in this Configuration, while
 * ${user.name} would then ordinarily be resolved to the value
 * of the System property with that name.
 * 
When conf.get("otherdir") is called, then ${env.BASE_DIR}
 * will be resolved to the value of the ${BASE_DIR} environment variable.
 * It supports ${env.NAME:-default} and ${env.NAME-default} notations.
 * The former is resolved to "default" if ${NAME} environment variable is undefined
 * or its value is empty.
 * The latter behaves the same way only if ${NAME} is undefined.
 * 
By default, warnings will be given to any deprecated configuration 
 * parameters and these are suppressible by configuring
 * log4j.logger.org.apache.hadoop.conf.Configuration.deprecation in
 * log4j.properties file.
 */
@InterfaceAudience.Public
@InterfaceStability.Stable
public class Configuration implements Iterable>,
                                      Writable {
  private static final Logger LOG =
      LoggerFactory.getLogger(Configuration.class);

  private static final Logger LOG_DEPRECATION =
      LoggerFactory.getLogger(
          "org.apache.hadoop.conf.Configuration.deprecation");

  private boolean quietmode = true;

  private static final String DEFAULT_STRING_CHECK =
    "testingforemptydefaultvalue";

  private static boolean restrictSystemPropsDefault = false;
  private boolean restrictSystemProps = restrictSystemPropsDefault;
  private boolean allowNullValueProperties = false;

  private static class Resource {
    private final Object resource;
    private final String name;
    private final boolean restrictParser;
    
    public Resource(Object resource) {
      this(resource, resource.toString());
    }

    public Resource(Object resource, boolean useRestrictedParser) {
      this(resource, resource.toString(), useRestrictedParser);
    }

    public Resource(Object resource, String name) {
      this(resource, name, getRestrictParserDefault(resource));
    }

    public Resource(Object resource, String name, boolean restrictParser) {
      this.resource = resource;
      this.name = name;
      this.restrictParser = restrictParser;
    }
    
    public String getName(){
      return name;
    }
    
    public Object getResource() {
      return resource;
    }

    public boolean isParserRestricted() {
      return restrictParser;
    }

    @Override
    public String toString() {
      return name;
    }

    private static boolean getRestrictParserDefault(Object resource) {
      if (resource instanceof String || !UserGroupInformation.isInitialized()) {
        return false;
      }
      UserGroupInformation user;
      try {
        user = UserGroupInformation.getCurrentUser();
      } catch (IOException e) {
        throw new RuntimeException("Unable to determine current user", e);
      }
      return user.getRealUser() != null;
    }
  }
  
  /**
   * List of configuration resources.
   */
  private ArrayList resources = new ArrayList();
  
  /**
   * The value reported as the setting resource when a key is set
   * by code rather than a file resource by dumpConfiguration.
   */
  static final String UNKNOWN_RESOURCE = "Unknown";


  /**
   * List of configuration parameters marked final. 
   */
  private Set finalParameters = Collections.newSetFromMap(
      new ConcurrentHashMap());
  
  private boolean loadDefaults = true;

  /**
   * Configuration objects
   */
  private static final WeakHashMap REGISTRY = 
    new WeakHashMap();
  
  /**
   * List of default Resources. Resources are loaded in the order of the list 
   * entries
   */
  private static final CopyOnWriteArrayList defaultResources =
    new CopyOnWriteArrayList();

  private static final Map>>>
    CACHE_CLASSES = new WeakHashMap>>>();

  /**
   * Sentinel value to store negative cache results in {@link #CACHE_CLASSES}.
   */
  private static final Class NEGATIVE_CACHE_SENTINEL =
    NegativeCacheSentinel.class;

  /**
   * Stores the mapping of key to the resource which modifies or loads 
   * the key most recently. Created lazily to avoid wasting memory.
   */
  private volatile Map updatingResource;

  /**
   * Specify exact input factory to avoid time finding correct one.
   * Factory is reusable across un-synchronized threads once initialized
   */
  private static final WstxInputFactory XML_INPUT_FACTORY =
      new WstxInputFactory();

  /**
   * Class to keep the information about the keys which replace the deprecated
   * ones.
   * 
   * This class stores the new keys which replace the deprecated keys and also
   * gives a provision to have a custom message for each of the deprecated key
   * that is being replaced. It also provides method to get the appropriate
   * warning message which can be logged whenever the deprecated key is used.
   */
  private static class DeprecatedKeyInfo {
    private final String[] newKeys;
    private final String customMessage;
    private final AtomicBoolean accessed = new AtomicBoolean(false);

    DeprecatedKeyInfo(String[] newKeys, String customMessage) {
      this.newKeys = newKeys;
      this.customMessage = customMessage;
    }

    private final String getWarningMessage(String key) {
      return getWarningMessage(key, null);
    }

    /**
     * Method to provide the warning message. It gives the custom message if
     * non-null, and default message otherwise.
     * @param key the associated deprecated key.
     * @param source the property source.
     * @return message that is to be logged when a deprecated key is used.
     */
    private String getWarningMessage(String key, String source) {
      String warningMessage;
      if(customMessage == null) {
        StringBuilder message = new StringBuilder(key);
        if (source != null) {
          message.append(" in " + source);
        }
        message.append(" is deprecated. Instead, use ");
        for (int i = 0; i < newKeys.length; i++) {
          message.append(newKeys[i]);
          if(i != newKeys.length-1) {
            message.append(", ");
          }
        }
        warningMessage = message.toString();
      }
      else {
        warningMessage = customMessage;
      }
      return warningMessage;
    }

    boolean getAndSetAccessed() {
      return accessed.getAndSet(true);
    }

    public void clearAccessed() {
      accessed.set(false);
    }
  }
  
  /**
   * A pending addition to the global set of deprecated keys.
   */
  public static class DeprecationDelta {
    private final String key;
    private final String[] newKeys;
    private final String customMessage;

    DeprecationDelta(String key, String[] newKeys, String customMessage) {
      Preconditions.checkNotNull(key);
      Preconditions.checkNotNull(newKeys);
      Preconditions.checkArgument(newKeys.length > 0);
      this.key = key;
      this.newKeys = newKeys;
      this.customMessage = customMessage;
    }

    public DeprecationDelta(String key, String newKey, String customMessage) {
      this(key, new String[] { newKey }, customMessage);
    }

    public DeprecationDelta(String key, String newKey) {
      this(key, new String[] { newKey }, null);
    }

    public String getKey() {
      return key;
    }

    public String[] getNewKeys() {
      return newKeys;
    }

    public String getCustomMessage() {
      return customMessage;
    }
  }

  /**
   * The set of all keys which are deprecated.
   *
   * DeprecationContext objects are immutable.
   */
  private static class DeprecationContext {
    /**
     * Stores the deprecated keys, the new keys which replace the deprecated keys
     * and custom message(if any provided).
     */
    private final Map deprecatedKeyMap;

    /**
     * Stores a mapping from superseding keys to the keys which they deprecate.
     */
    private final Map reverseDeprecatedKeyMap;

    /**
     * Create a new DeprecationContext by copying a previous DeprecationContext
     * and adding some deltas.
     *
     * @param other   The previous deprecation context to copy, or null to start
     *                from nothing.
     * @param deltas  The deltas to apply.
     */
    @SuppressWarnings("unchecked")
    DeprecationContext(DeprecationContext other, DeprecationDelta[] deltas) {
      HashMap newDeprecatedKeyMap = 
        new HashMap();
      HashMap newReverseDeprecatedKeyMap =
        new HashMap();
      if (other != null) {
        for (Entry entry :
            other.deprecatedKeyMap.entrySet()) {
          newDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
        }
        for (Entry entry :
            other.reverseDeprecatedKeyMap.entrySet()) {
          newReverseDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
        }
      }
      for (DeprecationDelta delta : deltas) {
        if (!newDeprecatedKeyMap.containsKey(delta.getKey())) {
          DeprecatedKeyInfo newKeyInfo =
            new DeprecatedKeyInfo(delta.getNewKeys(), delta.getCustomMessage());
          newDeprecatedKeyMap.put(delta.key, newKeyInfo);
          for (String newKey : delta.getNewKeys()) {
            newReverseDeprecatedKeyMap.put(newKey, delta.key);
          }
        }
      }
      this.deprecatedKeyMap =
        UnmodifiableMap.decorate(newDeprecatedKeyMap);
      this.reverseDeprecatedKeyMap =
        UnmodifiableMap.decorate(newReverseDeprecatedKeyMap);
    }

    Map getDeprecatedKeyMap() {
      return deprecatedKeyMap;
    }

    Map getReverseDeprecatedKeyMap() {
      return reverseDeprecatedKeyMap;
    }
  }
  
  private static DeprecationDelta[] defaultDeprecations = 
    new DeprecationDelta[] {
      new DeprecationDelta("topology.script.file.name", 
        CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY),
      new DeprecationDelta("topology.script.number.args", 
        CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY),
      new DeprecationDelta("hadoop.configured.node.mapping", 
        CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY),
      new DeprecationDelta("topology.node.switch.mapping.impl", 
        CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY),
      new DeprecationDelta("dfs.df.interval", 
        CommonConfigurationKeys.FS_DF_INTERVAL_KEY),
      new DeprecationDelta("fs.default.name", 
        CommonConfigurationKeys.FS_DEFAULT_NAME_KEY),
      new DeprecationDelta("dfs.umaskmode",
        CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY),
      new DeprecationDelta("dfs.nfs.exports.allowed.hosts",
          CommonConfigurationKeys.NFS_EXPORTS_ALLOWED_HOSTS_KEY)
    };

  /**
   * The global DeprecationContext.
   */
  private static AtomicReference deprecationContext =
      new AtomicReference(
          new DeprecationContext(null, defaultDeprecations));

  /**
   * Adds a set of deprecated keys to the global deprecations.
   *
   * This method is lockless.  It works by means of creating a new
   * DeprecationContext based on the old one, and then atomically swapping in
   * the new context.  If someone else updated the context in between us reading
   * the old context and swapping in the new one, we try again until we win the
   * race.
   *
   * @param deltas   The deprecations to add.
   */
  public static void addDeprecations(DeprecationDelta[] deltas) {
    DeprecationContext prev, next;
    do {
      prev = deprecationContext.get();
      next = new DeprecationContext(prev, deltas);
    } while (!deprecationContext.compareAndSet(prev, next));
  }

  /**
   * Adds the deprecated key to the global deprecation map.
   * It does not override any existing entries in the deprecation map.
   * This is to be used only by the developers in order to add deprecation of
   * keys, and attempts to call this method after loading resources once,
   * would lead to UnsupportedOperationException
   * 
   * If a key is deprecated in favor of multiple keys, they are all treated as 
   * aliases of each other, and setting any one of them resets all the others 
   * to the new value.
   *
   * If you have multiple deprecation entries to add, it is more efficient to
   * use #addDeprecations(DeprecationDelta[] deltas) instead.
   * 
   * @param key
   * @param newKeys
   * @param customMessage
   * @deprecated use {@link #addDeprecation(String key, String newKey,
      String customMessage)} instead
   */
  @Deprecated
  public static void addDeprecation(String key, String[] newKeys,
      String customMessage) {
    addDeprecations(new DeprecationDelta[] {
      new DeprecationDelta(key, newKeys, customMessage)
    });
  }

  /**
   * Adds the deprecated key to the global deprecation map.
   * It does not override any existing entries in the deprecation map.
   * This is to be used only by the developers in order to add deprecation of
   * keys, and attempts to call this method after loading resources once,
   * would lead to UnsupportedOperationException
   * 
   * If you have multiple deprecation entries to add, it is more efficient to
   * use #addDeprecations(DeprecationDelta[] deltas) instead.
   *
   * @param key
   * @param newKey
   * @param customMessage
   */
  public static void addDeprecation(String key, String newKey,
	      String customMessage) {
	  addDeprecation(key, new String[] {newKey}, customMessage);
  }

  /**
   * Adds the deprecated key to the global deprecation map when no custom
   * message is provided.
   * It does not override any existing entries in the deprecation map.
   * This is to be used only by the developers in order to add deprecation of
   * keys, and attempts to call this method after loading resources once,
   * would lead to UnsupportedOperationException
   * 
   * If a key is deprecated in favor of multiple keys, they are all treated as 
   * aliases of each other, and setting any one of them resets all the others 
   * to the new value.
   * 
   * If you have multiple deprecation entries to add, it is more efficient to
   * use #addDeprecations(DeprecationDelta[] deltas) instead.
   *
   * @param key Key that is to be deprecated
   * @param newKeys list of keys that take up the values of deprecated key
   * @deprecated use {@link #addDeprecation(String key, String newKey)} instead
   */
  @Deprecated
  public static void addDeprecation(String key, String[] newKeys) {
    addDeprecation(key, newKeys, null);
  }
  
  /**
   * Adds the deprecated key to the global deprecation map when no custom
   * message is provided.
   * It does not override any existing entries in the deprecation map.
   * This is to be used only by the developers in order to add deprecation of
   * keys, and attempts to call this method after loading resources once,
   * would lead to UnsupportedOperationException
   * 
   * If you have multiple deprecation entries to add, it is more efficient to
   * use #addDeprecations(DeprecationDelta[] deltas) instead.
   *
   * @param key Key that is to be deprecated
   * @param newKey key that takes up the value of deprecated key
   */
  public static void addDeprecation(String key, String newKey) {
    addDeprecation(key, new String[] {newKey}, null);
  }
  
  /**
   * checks whether the given key is deprecated.
   * 
   * @param key the parameter which is to be checked for deprecation
   * @return true if the key is deprecated and 
   *         false otherwise.
   */
  public static boolean isDeprecated(String key) {
    return deprecationContext.get().getDeprecatedKeyMap().containsKey(key);
  }

  private static String getDeprecatedKey(String key) {
    return deprecationContext.get().getReverseDeprecatedKeyMap().get(key);
  }

  private static DeprecatedKeyInfo getDeprecatedKeyInfo(String key) {
    return deprecationContext.get().getDeprecatedKeyMap().get(key);
  }

  /**
   * Sets all deprecated properties that are not currently set but have a
   * corresponding new property that is set. Useful for iterating the
   * properties when all deprecated properties for currently set properties
   * need to be present.
   */
  public void setDeprecatedProperties() {
    DeprecationContext deprecations = deprecationContext.get();
    Properties props = getProps();
    Properties overlay = getOverlay();
    for (Map.Entry entry :
        deprecations.getDeprecatedKeyMap().entrySet()) {
      String depKey = entry.getKey();
      if (!overlay.contains(depKey)) {
        for (String newKey : entry.getValue().newKeys) {
          String val = overlay.getProperty(newKey);
          if (val != null) {
            props.setProperty(depKey, val);
            overlay.setProperty(depKey, val);
            break;
          }
        }
      }
    }
  }

  /**
   * Checks for the presence of the property name in the
   * deprecation map. Returns the first of the list of new keys if present
   * in the deprecation map or the name itself. If the property
   * is not presently set but the property map contains an entry for the
   * deprecated key, the value of the deprecated key is set as the value for
   * the provided property name.
   *
   * @param deprecations deprecation context
   * @param name the property name
   * @return the first property in the list of properties mapping
   *         the name or the name itself.
   */
  private String[] handleDeprecation(DeprecationContext deprecations,
                                     String name) {
    if (null != name) {
      name = name.trim();
    }
    // Initialize the return value with requested name
    String[] names = new String[]{name};
    // Deprecated keys are logged once and an updated names are returned
    DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
    if (keyInfo != null) {
      if (!keyInfo.getAndSetAccessed()) {
        logDeprecation(keyInfo.getWarningMessage(name));
      }
      // Override return value for deprecated keys
      names = keyInfo.newKeys;
    }
    // If there are no overlay values we can return early
    Properties overlayProperties = getOverlay();
    if (overlayProperties.isEmpty()) {
      return names;
    }
    // Update properties and overlays with reverse lookup values
    for (String n : names) {
      String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n);
      if (deprecatedKey != null && !overlayProperties.containsKey(n)) {
        String deprecatedValue = overlayProperties.getProperty(deprecatedKey);
        if (deprecatedValue != null) {
          getProps().setProperty(n, deprecatedValue);
          overlayProperties.setProperty(n, deprecatedValue);
        }
      }
    }
    return names;
  }
 
  private void handleDeprecation() {
    LOG.debug("Handling deprecation for all properties in config...");
    DeprecationContext deprecations = deprecationContext.get();
    Set