org.apache.hadoop.security.SecurityUtil Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of hadoop-apache Show documentation
Show all versions of hadoop-apache Show documentation
Shaded version of Apache Hadoop for Presto
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.security;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_DNS_INTERFACE_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_DNS_NAMESERVER_KEY;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.ServiceLoader;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import javax.security.auth.kerberos.KerberosPrincipal;
import javax.security.auth.kerberos.KerberosTicket;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenInfo;
import org.apache.hadoop.util.StopWatch;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ZKUtil;
import io.prestosql.hadoop.$internal.org.slf4j.Logger;
import io.prestosql.hadoop.$internal.org.slf4j.LoggerFactory;
import org.xbill.DNS.Name;
import org.xbill.DNS.ResolverConfig;
import io.prestosql.hadoop.$internal.com.google.common.annotations.VisibleForTesting;
import io.prestosql.hadoop.$internal.com.google.common.net.InetAddresses;
/**
* Security Utils.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public final class SecurityUtil {
public static final Logger LOG = LoggerFactory.getLogger(SecurityUtil.class);
public static final String HOSTNAME_PATTERN = "_HOST";
public static final String FAILED_TO_GET_UGI_MSG_HEADER =
"Failed to obtain user group information:";
private SecurityUtil() {
}
// controls whether buildTokenService will use an ip or host/ip as given
// by the user
@VisibleForTesting
static boolean useIpForTokenService;
@VisibleForTesting
static HostResolver hostResolver;
private static boolean logSlowLookups;
private static int slowLookupThresholdMs;
static {
setConfigurationInternal(new Configuration());
}
@InterfaceAudience.Public
@InterfaceStability.Evolving
public static void setConfiguration(Configuration conf) {
LOG.info("Updating Configuration");
setConfigurationInternal(conf);
}
private static void setConfigurationInternal(Configuration conf) {
boolean useIp = conf.getBoolean(
CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP,
CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP_DEFAULT);
setTokenServiceUseIp(useIp);
logSlowLookups = conf.getBoolean(
CommonConfigurationKeys
.HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_KEY,
CommonConfigurationKeys
.HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_ENABLED_DEFAULT);
slowLookupThresholdMs = conf.getInt(
CommonConfigurationKeys
.HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_KEY,
CommonConfigurationKeys
.HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_DEFAULT);
}
/**
* For use only by tests and initialization
*/
@InterfaceAudience.Private
@VisibleForTesting
public static void setTokenServiceUseIp(boolean flag) {
if (LOG.isDebugEnabled()) {
LOG.debug("Setting "
+ CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP
+ " to " + flag);
}
useIpForTokenService = flag;
hostResolver = !useIpForTokenService
? new QualifiedHostResolver()
: new StandardHostResolver();
}
/**
* TGS must have the server principal of the form "krbtgt/FOO@FOO".
* @param principal
* @return true or false
*/
static boolean
isTGSPrincipal(KerberosPrincipal principal) {
if (principal == null)
return false;
if (principal.getName().equals("krbtgt/" + principal.getRealm() +
"@" + principal.getRealm())) {
return true;
}
return false;
}
/**
* Check whether the server principal is the TGS's principal
* @param ticket the original TGT (the ticket that is obtained when a
* kinit is done)
* @return true or false
*/
protected static boolean isOriginalTGT(KerberosTicket ticket) {
return isTGSPrincipal(ticket.getServer());
}
/**
* Convert Kerberos principal name pattern to valid Kerberos principal
* names. It replaces hostname pattern with hostname, which should be
* fully-qualified domain name. If hostname is null or "0.0.0.0", it uses
* dynamically looked-up fqdn of the current host instead.
*
* @param principalConfig
* the Kerberos principal name conf value to convert
* @param hostname
* the fully-qualified domain name used for substitution
* @return converted Kerberos principal name
* @throws IOException if the client address cannot be determined
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public static String getServerPrincipal(String principalConfig,
String hostname) throws IOException {
String[] components = getComponents(principalConfig);
if (components == null || components.length != 3
|| !components[1].equals(HOSTNAME_PATTERN)) {
return principalConfig;
} else {
return replacePattern(components, hostname);
}
}
/**
* Convert Kerberos principal name pattern to valid Kerberos principal names.
* This method is similar to {@link #getServerPrincipal(String, String)},
* except 1) the reverse DNS lookup from addr to hostname is done only when
* necessary, 2) param addr can't be null (no default behavior of using local
* hostname when addr is null).
*
* @param principalConfig
* Kerberos principal name pattern to convert
* @param addr
* InetAddress of the host used for substitution
* @return converted Kerberos principal name
* @throws IOException if the client address cannot be determined
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public static String getServerPrincipal(String principalConfig,
InetAddress addr) throws IOException {
String[] components = getComponents(principalConfig);
if (components == null || components.length != 3
|| !components[1].equals(HOSTNAME_PATTERN)) {
return principalConfig;
} else {
if (addr == null) {
throw new IOException("Can't replace " + HOSTNAME_PATTERN
+ " pattern since client address is null");
}
return replacePattern(components, addr.getCanonicalHostName());
}
}
private static String[] getComponents(String principalConfig) {
if (principalConfig == null)
return null;
return principalConfig.split("[/@]");
}
private static String replacePattern(String[] components, String hostname)
throws IOException {
String fqdn = hostname;
if (fqdn == null || fqdn.isEmpty() || fqdn.equals("0.0.0.0")) {
fqdn = getLocalHostName(null);
}
return components[0] + "/" +
StringUtils.toLowerCase(fqdn) + "@" + components[2];
}
/**
* Retrieve the name of the current host. Multihomed hosts may restrict the
* hostname lookup to a specific interface and nameserver with {@link
* org.apache.hadoop.fs.CommonConfigurationKeysPublic#HADOOP_SECURITY_DNS_INTERFACE_KEY}
* and {@link org.apache.hadoop.fs.CommonConfigurationKeysPublic#HADOOP_SECURITY_DNS_NAMESERVER_KEY}
*
* @param conf Configuration object. May be null.
* @return
* @throws UnknownHostException
*/
static String getLocalHostName(@Nullable Configuration conf)
throws UnknownHostException {
if (conf != null) {
String dnsInterface = conf.get(HADOOP_SECURITY_DNS_INTERFACE_KEY);
String nameServer = conf.get(HADOOP_SECURITY_DNS_NAMESERVER_KEY);
if (dnsInterface != null) {
return DNS.getDefaultHost(dnsInterface, nameServer, true);
} else if (nameServer != null) {
throw new IllegalArgumentException(HADOOP_SECURITY_DNS_NAMESERVER_KEY +
" requires " + HADOOP_SECURITY_DNS_INTERFACE_KEY + ". Check your" +
"configuration.");
}
}
// Fallback to querying the default hostname as we did before.
return InetAddress.getLocalHost().getCanonicalHostName();
}
/**
* Login as a principal specified in config. Substitute $host in
* user's Kerberos principal name with a dynamically looked-up fully-qualified
* domain name of the current host.
*
* @param conf
* conf to use
* @param keytabFileKey
* the key to look for keytab file in conf
* @param userNameKey
* the key to look for user's Kerberos principal name in conf
* @throws IOException if login fails
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public static void login(final Configuration conf,
final String keytabFileKey, final String userNameKey) throws IOException {
login(conf, keytabFileKey, userNameKey, getLocalHostName(conf));
}
/**
* Login as a principal specified in config. Substitute $host in user's Kerberos principal
* name with hostname. If non-secure mode - return. If no keytab available -
* bail out with an exception
*
* @param conf
* conf to use
* @param keytabFileKey
* the key to look for keytab file in conf
* @param userNameKey
* the key to look for user's Kerberos principal name in conf
* @param hostname
* hostname to use for substitution
* @throws IOException if the config doesn't specify a keytab
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public static void login(final Configuration conf,
final String keytabFileKey, final String userNameKey, String hostname)
throws IOException {
if(! UserGroupInformation.isSecurityEnabled())
return;
String keytabFilename = conf.get(keytabFileKey);
if (keytabFilename == null || keytabFilename.length() == 0) {
throw new IOException("Running in secure mode, but config doesn't have a keytab");
}
String principalConfig = conf.get(userNameKey, System
.getProperty("user.name"));
String principalName = SecurityUtil.getServerPrincipal(principalConfig,
hostname);
UserGroupInformation.loginUserFromKeytab(principalName, keytabFilename);
}
/**
* create the service name for a Delegation token
* @param uri of the service
* @param defPort is used if the uri lacks a port
* @return the token service, or null if no authority
* @see #buildTokenService(InetSocketAddress)
*/
public static String buildDTServiceName(URI uri, int defPort) {
String authority = uri.getAuthority();
if (authority == null) {
return null;
}
InetSocketAddress addr = NetUtils.createSocketAddr(authority, defPort);
return buildTokenService(addr).toString();
}
/**
* Get the host name from the principal name of format /host@realm.
* @param principalName principal name of format as described above
* @return host name if the the string conforms to the above format, else null
*/
public static String getHostFromPrincipal(String principalName) {
return new HadoopKerberosName(principalName).getHostName();
}
private static ServiceLoader securityInfoProviders =
ServiceLoader.load(SecurityInfo.class);
private static SecurityInfo[] testProviders = new SecurityInfo[0];
/**
* Test setup method to register additional providers.
* @param providers a list of high priority providers to use
*/
@InterfaceAudience.Private
public static void setSecurityInfoProviders(SecurityInfo... providers) {
testProviders = providers;
}
/**
* Look up the KerberosInfo for a given protocol. It searches all known
* SecurityInfo providers.
* @param protocol the protocol class to get the information for
* @param conf configuration object
* @return the KerberosInfo or null if it has no KerberosInfo defined
*/
public static KerberosInfo
getKerberosInfo(Class protocol, Configuration conf) {
for(SecurityInfo provider: testProviders) {
KerberosInfo result = provider.getKerberosInfo(protocol, conf);
if (result != null) {
return result;
}
}
synchronized (securityInfoProviders) {
for(SecurityInfo provider: securityInfoProviders) {
KerberosInfo result = provider.getKerberosInfo(protocol, conf);
if (result != null) {
return result;
}
}
}
return null;
}
/**
* Look up the TokenInfo for a given protocol. It searches all known
* SecurityInfo providers.
* @param protocol The protocol class to get the information for.
* @param conf Configuration object
* @return the TokenInfo or null if it has no KerberosInfo defined
*/
public static TokenInfo getTokenInfo(Class protocol, Configuration conf) {
for(SecurityInfo provider: testProviders) {
TokenInfo result = provider.getTokenInfo(protocol, conf);
if (result != null) {
return result;
}
}
synchronized (securityInfoProviders) {
for(SecurityInfo provider: securityInfoProviders) {
TokenInfo result = provider.getTokenInfo(protocol, conf);
if (result != null) {
return result;
}
}
}
return null;
}
/**
* Decode the given token's service field into an InetAddress
* @param token from which to obtain the service
* @return InetAddress for the service
*/
public static InetSocketAddress getTokenServiceAddr(Token token) {
return NetUtils.createSocketAddr(token.getService().toString());
}
/**
* Set the given token's service to the format expected by the RPC client
* @param token a delegation token
* @param addr the socket for the rpc connection
*/
public static void setTokenService(Token token, InetSocketAddress addr) {
Text service = buildTokenService(addr);
if (token != null) {
token.setService(service);
if (LOG.isDebugEnabled()) {
LOG.debug("Acquired token "+token); // Token#toString() prints service
}
} else {
LOG.warn("Failed to get token for service "+service);
}
}
/**
* Construct the service key for a token
* @param addr InetSocketAddress of remote connection with a token
* @return "ip:port" or "host:port" depending on the value of
* hadoop.security.token.service.use_ip
*/
public static Text buildTokenService(InetSocketAddress addr) {
String host = null;
if (useIpForTokenService) {
if (addr.isUnresolved()) { // host has no ip address
throw new IllegalArgumentException(
new UnknownHostException(addr.getHostName())
);
}
host = addr.getAddress().getHostAddress();
} else {
host = StringUtils.toLowerCase(addr.getHostName());
}
return new Text(host + ":" + addr.getPort());
}
/**
* Construct the service key for a token
* @param uri of remote connection with a token
* @return "ip:port" or "host:port" depending on the value of
* hadoop.security.token.service.use_ip
*/
public static Text buildTokenService(URI uri) {
return buildTokenService(NetUtils.createSocketAddr(uri.getAuthority()));
}
/**
* Perform the given action as the daemon's login user. If the login
* user cannot be determined, this will log a FATAL error and exit
* the whole JVM.
*/
public static T doAsLoginUserOrFatal(PrivilegedAction action) {
if (UserGroupInformation.isSecurityEnabled()) {
UserGroupInformation ugi = null;
try {
ugi = UserGroupInformation.getLoginUser();
} catch (IOException e) {
LOG.error("Exception while getting login user", e);
e.printStackTrace();
Runtime.getRuntime().exit(-1);
}
return ugi.doAs(action);
} else {
return action.run();
}
}
/**
* Perform the given action as the daemon's login user. If an
* InterruptedException is thrown, it is converted to an IOException.
*
* @param action the action to perform
* @return the result of the action
* @throws IOException in the event of error
*/
public static T doAsLoginUser(PrivilegedExceptionAction action)
throws IOException {
return doAsUser(UserGroupInformation.getLoginUser(), action);
}
/**
* Perform the given action as the daemon's current user. If an
* InterruptedException is thrown, it is converted to an IOException.
*
* @param action the action to perform
* @return the result of the action
* @throws IOException in the event of error
*/
public static T doAsCurrentUser(PrivilegedExceptionAction action)
throws IOException {
return doAsUser(UserGroupInformation.getCurrentUser(), action);
}
private static T doAsUser(UserGroupInformation ugi,
PrivilegedExceptionAction action) throws IOException {
try {
return ugi.doAs(action);
} catch (InterruptedException ie) {
throw new IOException(ie);
}
}
/**
* Resolves a host subject to the security requirements determined by
* hadoop.security.token.service.use_ip. Optionally logs slow resolutions.
*
* @param hostname host or ip to resolve
* @return a resolved host
* @throws UnknownHostException if the host doesn't exist
*/
@InterfaceAudience.Private
public static
InetAddress getByName(String hostname) throws UnknownHostException {
if (logSlowLookups || LOG.isTraceEnabled()) {
StopWatch lookupTimer = new StopWatch().start();
InetAddress result = hostResolver.getByName(hostname);
long elapsedMs = lookupTimer.stop().now(TimeUnit.MILLISECONDS);
if (elapsedMs >= slowLookupThresholdMs) {
LOG.warn("Slow name lookup for " + hostname + ". Took " + elapsedMs +
" ms.");
} else if (LOG.isTraceEnabled()) {
LOG.trace("Name lookup for " + hostname + " took " + elapsedMs +
" ms.");
}
return result;
} else {
return hostResolver.getByName(hostname);
}
}
interface HostResolver {
InetAddress getByName(String host) throws UnknownHostException;
}
/**
* Uses standard java host resolution
*/
static class StandardHostResolver implements HostResolver {
@Override
public InetAddress getByName(String host) throws UnknownHostException {
return InetAddress.getByName(host);
}
}
/**
* This an alternate resolver with important properties that the standard
* java resolver lacks:
* 1) The hostname is fully qualified. This avoids security issues if not
* all hosts in the cluster do not share the same search domains. It
* also prevents other hosts from performing unnecessary dns searches.
* In contrast, InetAddress simply returns the host as given.
* 2) The InetAddress is instantiated with an exact host and IP to prevent
* further unnecessary lookups. InetAddress may perform an unnecessary
* reverse lookup for an IP.
* 3) A call to getHostName() will always return the qualified hostname, or
* more importantly, the IP if instantiated with an IP. This avoids
* unnecessary dns timeouts if the host is not resolvable.
* 4) Point 3 also ensures that if the host is re-resolved, ex. during a
* connection re-attempt, that a reverse lookup to host and forward
* lookup to IP is not performed since the reverse/forward mappings may
* not always return the same IP. If the client initiated a connection
* with an IP, then that IP is all that should ever be contacted.
*
* NOTE: this resolver is only used if:
* hadoop.security.token.service.use_ip=false
*/
protected static class QualifiedHostResolver implements HostResolver {
private List searchDomains = new ArrayList<>();
{
ResolverConfig resolverConfig = ResolverConfig.getCurrentConfig();
Name[] names = resolverConfig.searchPath();
if (names != null) {
for (Name name : names) {
searchDomains.add(name.toString());
}
}
}
/**
* Create an InetAddress with a fully qualified hostname of the given
* hostname. InetAddress does not qualify an incomplete hostname that
* is resolved via the domain search list.
* {@link InetAddress#getCanonicalHostName()} will fully qualify the
* hostname, but it always return the A record whereas the given hostname
* may be a CNAME.
*
* @param host a hostname or ip address
* @return InetAddress with the fully qualified hostname or ip
* @throws UnknownHostException if host does not exist
*/
@Override
public InetAddress getByName(String host) throws UnknownHostException {
InetAddress addr = null;
if (InetAddresses.isInetAddress(host)) {
// valid ip address. use it as-is
addr = InetAddresses.forString(host);
// set hostname
addr = InetAddress.getByAddress(host, addr.getAddress());
} else if (host.endsWith(".")) {
// a rooted host ends with a dot, ex. "host."
// rooted hosts never use the search path, so only try an exact lookup
addr = getByExactName(host);
} else if (host.contains(".")) {
// the host contains a dot (domain), ex. "host.domain"
// try an exact host lookup, then fallback to search list
addr = getByExactName(host);
if (addr == null) {
addr = getByNameWithSearch(host);
}
} else {
// it's a simple host with no dots, ex. "host"
// try the search list, then fallback to exact host
InetAddress loopback = InetAddress.getByName(null);
if (host.equalsIgnoreCase(loopback.getHostName())) {
addr = InetAddress.getByAddress(host, loopback.getAddress());
} else {
addr = getByNameWithSearch(host);
if (addr == null) {
addr = getByExactName(host);
}
}
}
// unresolvable!
if (addr == null) {
throw new UnknownHostException(host);
}
return addr;
}
InetAddress getByExactName(String host) {
InetAddress addr = null;
// InetAddress will use the search list unless the host is rooted
// with a trailing dot. The trailing dot will disable any use of the
// search path in a lower level resolver. See RFC 1535.
String fqHost = host;
if (!fqHost.endsWith(".")) fqHost += ".";
try {
addr = getInetAddressByName(fqHost);
// can't leave the hostname as rooted or other parts of the system
// malfunction, ex. kerberos principals are lacking proper host
// equivalence for rooted/non-rooted hostnames
addr = InetAddress.getByAddress(host, addr.getAddress());
} catch (UnknownHostException e) {
// ignore, caller will throw if necessary
}
return addr;
}
InetAddress getByNameWithSearch(String host) {
InetAddress addr = null;
if (host.endsWith(".")) { // already qualified?
addr = getByExactName(host);
} else {
for (String domain : searchDomains) {
String dot = !domain.startsWith(".") ? "." : "";
addr = getByExactName(host + dot + domain);
if (addr != null) break;
}
}
return addr;
}
// implemented as a separate method to facilitate unit testing
InetAddress getInetAddressByName(String host) throws UnknownHostException {
return InetAddress.getByName(host);
}
void setSearchDomains(String ... domains) {
searchDomains = Arrays.asList(domains);
}
}
public static AuthenticationMethod getAuthenticationMethod(Configuration conf) {
String value = conf.get(HADOOP_SECURITY_AUTHENTICATION, "simple");
try {
return Enum.valueOf(AuthenticationMethod.class,
StringUtils.toUpperCase(value));
} catch (IllegalArgumentException iae) {
throw new IllegalArgumentException("Invalid attribute value for " +
HADOOP_SECURITY_AUTHENTICATION + " of " + value);
}
}
public static void setAuthenticationMethod(
AuthenticationMethod authenticationMethod, Configuration conf) {
if (authenticationMethod == null) {
authenticationMethod = AuthenticationMethod.SIMPLE;
}
conf.set(HADOOP_SECURITY_AUTHENTICATION,
StringUtils.toLowerCase(authenticationMethod.toString()));
}
/*
* Check if a given port is privileged.
* The ports with number smaller than 1024 are treated as privileged ports in
* unix/linux system. For other operating systems, use this method with care.
* For example, Windows doesn't have the concept of privileged ports.
* However, it may be used at Windows client to check port of linux server.
*
* @param port the port number
* @return true for privileged ports, false otherwise
*
*/
public static boolean isPrivilegedPort(final int port) {
return port < 1024;
}
/**
* Utility method to fetch ZK auth info from the configuration.
* @throws java.io.IOException if the Zookeeper ACLs configuration file
* cannot be read
* @throws ZKUtil.BadAuthFormatException if the auth format is invalid
*/
public static List getZKAuthInfos(Configuration conf,
String configKey) throws IOException {
char[] zkAuthChars = conf.getPassword(configKey);
String zkAuthConf =
zkAuthChars != null ? String.valueOf(zkAuthChars) : null;
try {
zkAuthConf = ZKUtil.resolveConfIndirection(zkAuthConf);
if (zkAuthConf != null) {
return ZKUtil.parseAuth(zkAuthConf);
} else {
return Collections.emptyList();
}
} catch (IOException | ZKUtil.BadAuthFormatException e) {
LOG.error("Couldn't read Auth based on {}", configKey);
throw e;
}
}
}