org.apache.hadoop.fs.s3a.s3guard.S3Guard Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of hadoop-aws Show documentation
Show all versions of hadoop-aws Show documentation
This module contains code to support integration with Amazon Web Services.
It also declares the dependencies needed to work with AWS services.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.s3a.s3guard;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.function.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH;
import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_AUTHORITATIVE_PATH;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_DYNAMO;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_LOCAL;
import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
/**
* Logic for integrating MetadataStore with S3A.
* Most of the methods here were deleted when the S3Guard feature was removed.
*/
@SuppressWarnings("deprecation")
@InterfaceAudience.Private
@InterfaceStability.Unstable
public final class S3Guard {
private static final Logger LOG = LoggerFactory.getLogger(S3Guard.class);
static final String NULL_METADATA_STORE
= "org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore";
// Utility class. All static functions.
private S3Guard() {
}
/**
* Assert that the FS is not configured to use an unsupported S3Guard
* option.
* the empty/null option is preferred.
* if the local store or null store is requested, the configuration is
* allowed.
* request for a DynamoDB or other store class will raise an exception.
* @param fsURI FileSystem URI
* @param conf configuration
* @return true if an option was set but ignored
* @throws PathIOException if an unsupported metastore was found.
*/
public static boolean checkNoS3Guard(URI fsURI, Configuration conf) throws PathIOException {
final String classname = conf.getTrimmed(S3_METADATA_STORE_IMPL, "");
if (classname.isEmpty()) {
// all good. declare nothing was found.
return false;
}
// there is a s3guard configuration option
// ignore if harmless; reject if DDB or unknown
final String[] sources = conf.getPropertySources(S3_METADATA_STORE_IMPL);
final String origin = sources == null
? "unknown"
: sources[0];
final String fsPath = fsURI.toString();
switch (classname) {
case NULL_METADATA_STORE:
// harmless
LOG.debug("Ignoring S3Guard store option of {} -no longer needed " +
"Origin {}",
NULL_METADATA_STORE, origin);
break;
case S3GUARD_METASTORE_LOCAL:
// used in some libraries (e.g. hboss) to force a consistent s3 in a test
// run.
// print a message and continue
LOG.warn("Ignoring S3Guard store option of {} -no longer needed or supported. "
+ "Origin {}",
S3GUARD_METASTORE_LOCAL, origin);
break;
case S3GUARD_METASTORE_DYNAMO:
// this is the dangerous one, as it is a sign that a config is in use where
// older releases will use DDB for listing metadata, yet this
// client will not update it.
final String message = String.format("S3Guard is no longer needed/supported,"
+ " yet %s is configured to use DynamoDB as the S3Guard metadata store."
+ " This is no longer needed or supported. " +
"Origin of setting is %s",
fsPath, origin);
LOG.error(message);
throw new PathIOException(fsPath, message);
default:
// an unknown store entirely.
throw new PathIOException(fsPath,
"Filesystem is configured to use unknown S3Guard store " + classname
+ " origin " + origin);
}
// an option was set, but it was harmless
return true;
}
public static Collection getAuthoritativePaths(S3AFileSystem fs) {
return getAuthoritativePaths(
fs.getUri(),
fs.getConf(),
p -> fs.maybeAddTrailingSlash(fs.qualify(p).toString()));
}
/**
* Get the authoritative paths of a filesystem.
*
* @param uri FS URI
* @param conf configuration
* @param qualifyToDir a qualification operation
* @return list of URIs valid for this FS.
*/
@VisibleForTesting
static Collection getAuthoritativePaths(
final URI uri,
final Configuration conf,
final Function qualifyToDir) {
String[] rawAuthoritativePaths =
conf.getTrimmedStrings(AUTHORITATIVE_PATH, DEFAULT_AUTHORITATIVE_PATH);
Collection authoritativePaths = new ArrayList<>();
if (rawAuthoritativePaths.length > 0) {
for (int i = 0; i < rawAuthoritativePaths.length; i++) {
Path path = new Path(rawAuthoritativePaths[i]);
URI pathURI = path.toUri();
if (pathURI.getAuthority() != null &&
!pathURI.getAuthority().equals(uri.getAuthority())) {
// skip on auth
continue;
}
if (pathURI.getScheme() != null &&
!pathURI.getScheme().equals(uri.getScheme())) {
// skip on auth
continue;
}
authoritativePaths.add(qualifyToDir.apply(path));
}
}
return authoritativePaths;
}
/**
* Is the path for the given FS instance authoritative?
* @param p path
* @param fs filesystem
* @param authPaths possibly empty list of authoritative paths
* @return true iff the path is authoritative
*/
public static boolean allowAuthoritative(Path p, S3AFileSystem fs,
Collection authPaths) {
String haystack = fs.maybeAddTrailingSlash(fs.qualify(p).toString());
if (!authPaths.isEmpty()) {
for (String needle : authPaths) {
if (haystack.startsWith(needle)) {
return true;
}
}
}
return false;
}
}