org.apache.hadoop.fs.s3a.impl.DirectoryPolicyImpl Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.s3a.impl;
import java.util.EnumSet;
import java.util.Locale;
import java.util.Set;
import java.util.function.Predicate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_DIRECTORY_MARKER_POLICY;
import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY;
import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_AUTHORITATIVE;
import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE;
import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_KEEP;
import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_AWARE;
import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE;
import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP;
import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_AUTHORITATIVE;
import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_DELETE;
import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP;
/**
* Implementation of directory policy.
*/
public final class DirectoryPolicyImpl
implements DirectoryPolicy {
private static final Logger LOG = LoggerFactory.getLogger(
DirectoryPolicyImpl.class);
/**
* Error string when unable to parse the marker policy option.
*/
public static final String UNKNOWN_MARKER_POLICY =
"Unknown policy in "
+ DIRECTORY_MARKER_POLICY + ": ";
/**
* All available policies.
*/
private static final Set AVAILABLE_POLICIES =
EnumSet.allOf(MarkerPolicy.class);
/**
* Keep all markers.
*/
public static final DirectoryPolicy KEEP = new DirectoryPolicyImpl(
MarkerPolicy.Keep, (p) -> false);
/**
* Delete all markers.
*/
public static final DirectoryPolicy DELETE = new DirectoryPolicyImpl(
MarkerPolicy.Delete, (p) -> false);
/**
* Chosen marker policy.
*/
private final MarkerPolicy markerPolicy;
/**
* Callback to evaluate authoritativeness of a
* path.
*/
private final Predicate authoritativeness;
/**
* Constructor.
* @param markerPolicy marker policy
* @param authoritativeness function for authoritativeness
*/
public DirectoryPolicyImpl(final MarkerPolicy markerPolicy,
final Predicate authoritativeness) {
this.markerPolicy = markerPolicy;
this.authoritativeness = authoritativeness;
}
@Override
public boolean keepDirectoryMarkers(final Path path) {
switch (markerPolicy) {
case Keep:
return true;
case Authoritative:
return authoritativeness.test(path);
case Delete:
default: // which cannot happen
return false;
}
}
@Override
public MarkerPolicy getMarkerPolicy() {
return markerPolicy;
}
@Override
public String describe() {
return markerPolicy.getOptionName();
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder(
"DirectoryMarkerRetention{");
sb.append("policy='").append(markerPolicy.getOptionName()).append('\'');
sb.append('}');
return sb.toString();
}
/**
* Return path policy for store and paths.
* @param path path
* @param capability capability
* @return true if a capability is active
*/
@Override
public boolean hasPathCapability(final Path path, final String capability) {
switch (capability) {
/*
* Marker policy is dynamically determined for the given path.
*/
case STORE_CAPABILITY_DIRECTORY_MARKER_AWARE:
return true;
case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP:
return markerPolicy == MarkerPolicy.Keep;
case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_DELETE:
return markerPolicy == MarkerPolicy.Delete;
case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_AUTHORITATIVE:
return markerPolicy == MarkerPolicy.Authoritative;
case STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP:
return keepDirectoryMarkers(path);
case STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE:
return !keepDirectoryMarkers(path);
default:
throw new IllegalArgumentException("Unknown capability " + capability);
}
}
/**
* Create/Get the policy for this configuration.
* @param conf config
* @param authoritativeness Callback to evaluate authoritativeness of a
* path.
* @return a policy
*/
public static DirectoryPolicy getDirectoryPolicy(
final Configuration conf,
final Predicate authoritativeness) {
DirectoryPolicy policy;
String option = conf.getTrimmed(DIRECTORY_MARKER_POLICY,
DEFAULT_DIRECTORY_MARKER_POLICY);
switch (option.toLowerCase(Locale.ENGLISH)) {
case DIRECTORY_MARKER_POLICY_DELETE:
// backwards compatible.
LOG.debug("Directory markers will be deleted");
policy = DELETE;
break;
case DIRECTORY_MARKER_POLICY_KEEP:
LOG.info("Directory markers will be kept");
policy = KEEP;
break;
case DIRECTORY_MARKER_POLICY_AUTHORITATIVE:
LOG.info("Directory markers will be kept on authoritative"
+ " paths");
policy = new DirectoryPolicyImpl(MarkerPolicy.Authoritative,
authoritativeness);
break;
default:
throw new IllegalArgumentException(UNKNOWN_MARKER_POLICY + option);
}
return policy;
}
/**
* Enumerate all available policies.
* @return set of the policies.
*/
public static Set availablePolicies() {
return AVAILABLE_POLICIES;
}
}