alluxio.underfs.BaseUnderFileSystem Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of alluxio-core-common Show documentation
Show all versions of alluxio-core-common Show documentation
Common utilities shared in Alluxio core modules
/*
* The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
* (the "License"). You may not use this work except in compliance with the License, which is
* available at www.apache.org/licenses/LICENSE-2.0
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied, as more fully set forth in the License.
*
* See the NOTICE file distributed with this work for information regarding copyright ownership.
*/
package alluxio.underfs;
import alluxio.AlluxioURI;
import alluxio.Constants;
import alluxio.collections.Pair;
import alluxio.conf.AlluxioConfiguration;
import alluxio.file.options.DescendantType;
import alluxio.security.authorization.AccessControlList;
import alluxio.security.authorization.AclEntry;
import alluxio.security.authorization.DefaultAccessControlList;
import alluxio.underfs.options.CreateOptions;
import alluxio.underfs.options.DeleteOptions;
import alluxio.underfs.options.ListOptions;
import alluxio.underfs.options.MkdirsOptions;
import alluxio.underfs.options.OpenOptions;
import alluxio.util.ThreadFactoryUtils;
import alluxio.util.io.PathUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
import com.google.common.io.Closer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import javax.annotation.concurrent.ThreadSafe;
/**
* A base abstract {@link UnderFileSystem}.
*/
@ThreadSafe
public abstract class BaseUnderFileSystem implements UnderFileSystem {
private static final Logger LOG = LoggerFactory.getLogger(BaseUnderFileSystem.class);
public static final Pair EMPTY_ACL =
new Pair<>(null, null);
/** The UFS {@link AlluxioURI} used to create this {@link BaseUnderFileSystem}. */
protected final AlluxioURI mUri;
/** UFS Configuration options. */
protected final UnderFileSystemConfiguration mUfsConf;
private final ExecutorService mAsyncIOExecutor;
/**
* Constructs an {@link BaseUnderFileSystem}.
*
* @param uri the {@link AlluxioURI} used to create this ufs
* @param ufsConf UFS configuration
*/
protected BaseUnderFileSystem(AlluxioURI uri, UnderFileSystemConfiguration ufsConf) {
mUri = Preconditions.checkNotNull(uri, "uri");
mUfsConf = Preconditions.checkNotNull(ufsConf, "ufsConf");
mAsyncIOExecutor = Executors.newCachedThreadPool(
ThreadFactoryUtils.build(uri.getPath() + "IOThread", true));
}
@Override
public void close() throws IOException {
try (Closer closer = Closer.create()) {
closer.register(() -> {
if (mAsyncIOExecutor != null) {
mAsyncIOExecutor.shutdown();
}
});
}
}
@Override
public OutputStream create(String path) throws IOException {
return create(path, CreateOptions.defaults(mUfsConf).setCreateParent(true));
}
@Override
public boolean deleteDirectory(String path) throws IOException {
return deleteDirectory(path, DeleteOptions.defaults());
}
@Override
public boolean exists(String path) throws IOException {
return isFile(path) || isDirectory(path);
}
@Override
public Pair getAclPair(String path)
throws IOException {
return EMPTY_ACL;
}
@Override
public void setAclEntries(String path, List aclEntries) throws IOException {
// Noop here by default
}
@Override
public AlluxioConfiguration getConfiguration() {
return mUfsConf;
}
@Override
public String getFingerprint(String path) {
// TODO(yuzhu): include default ACL in the fingerprint
try {
UfsStatus status = getStatus(path);
Pair aclPair = getAclPair(path);
if (aclPair == null || aclPair.getFirst() == null || !aclPair.getFirst().hasExtended()) {
return Fingerprint.create(getUnderFSType(), status).serialize();
} else {
return Fingerprint.create(getUnderFSType(), status, null, aclPair.getFirst()).serialize();
}
} catch (Exception e) {
// In certain scenarios, it is expected that the UFS path does not exist.
LOG.debug("Failed fingerprint. path: {} error: {}", path, e.toString());
return Constants.INVALID_UFS_FINGERPRINT;
}
}
@Override
public Fingerprint getParsedFingerprint(String path) {
return getParsedFingerprint(path, null);
}
@Override
public Fingerprint getParsedFingerprint(String path, @Nullable String contentHash) {
try {
UfsStatus status = getStatus(path);
Pair aclPair = getAclPair(path);
if (aclPair == null || aclPair.getFirst() == null || !aclPair.getFirst().hasExtended()) {
return Fingerprint.create(getUnderFSType(), status, contentHash);
} else {
return Fingerprint.create(getUnderFSType(), status, contentHash, aclPair.getFirst());
}
} catch (IOException e) {
return Fingerprint.INVALID_FINGERPRINT;
}
}
@Override
public UfsMode getOperationMode(Map physicalUfsState) {
UfsMode ufsMode = physicalUfsState.get(mUri.getRootPath());
if (ufsMode != null) {
return ufsMode;
}
return UfsMode.READ_WRITE;
}
@Override
public List getPhysicalStores() {
return new ArrayList<>(Arrays.asList(mUri.getRootPath()));
}
@Override
public boolean isObjectStorage() {
return false;
}
@Override
public boolean isSeekable() {
return false;
}
@Nullable
@Override
public Iterator listStatusIterable(
String path, ListOptions options, String startAfter, int batchSize) throws IOException {
// Calling this method on non s3 UFS might result in OOM because batch based fetching
// is not supported and this method essentially fetches all ufs status and converts it to
// an iterator.
UfsStatus[] result = listStatus(path, options);
if (result == null) {
return null;
}
return Iterators.forArray(result);
}
@Override
public void performListingAsync(
String path, @Nullable String continuationToken, @Nullable String startAfter,
DescendantType descendantType, boolean checkStatus, Consumer onComplete,
Consumer onError) {
mAsyncIOExecutor.submit(() -> {
try {
UfsStatus baseStatus = null;
if (checkStatus) {
try {
baseStatus = getStatus(path);
if (baseStatus == null && !isObjectStorage()) {
onComplete.accept(new UfsLoadResult(Stream.empty(), 0,
null, null, false, false, false));
return;
}
if (baseStatus != null && (descendantType == DescendantType.NONE
|| baseStatus.isFile())) {
onComplete.accept(new UfsLoadResult(Stream.of(baseStatus), 1,
null, new AlluxioURI(baseStatus.getName()), false,
baseStatus.isFile(), isObjectStorage()));
return;
}
} catch (FileNotFoundException e) {
// if we are not using object storage we know nothing exists at the path,
// so just return an empty result
if (!isObjectStorage()) {
onComplete.accept(new UfsLoadResult(Stream.empty(), 0,
null, null, false, false, false));
return;
}
}
}
UfsStatus[] items = listStatus(path, ListOptions.defaults()
.setRecursive(descendantType == DescendantType.ALL));
if (items != null) {
if (descendantType == DescendantType.NONE && items.length > 0) {
assert isObjectStorage() && this instanceof ObjectUnderFileSystem;
ObjectUnderFileSystem.ObjectPermissions permissions =
((ObjectUnderFileSystem) this).getPermissions();
items = new UfsStatus[] {
new UfsDirectoryStatus("", permissions.getOwner(), permissions.getGroup(),
permissions.getMode())};
}
Arrays.sort(items, Comparator.comparing(UfsStatus::getName));
for (UfsStatus item: items) {
// performListingAsync is used by metadata sync v2
// which expects the name of an item to be a full path
item.setName(PathUtils.concatPath(path, item.getName()));
}
}
if (items != null && items.length == 0) {
items = null;
}
UfsStatus firstItem = baseStatus != null ? baseStatus
: items != null ? items[0] : null;
UfsStatus lastItem = items == null ? firstItem
: items[items.length - 1];
Stream itemStream = items == null ? Stream.empty() : Arrays.stream(items);
int itemCount = items == null ? 0 : items.length;
if (baseStatus != null) {
itemStream = Stream.concat(Stream.of(baseStatus), itemStream);
itemCount++;
}
onComplete.accept(new UfsLoadResult(itemStream, itemCount,
null, lastItem == null ? null : new AlluxioURI(lastItem.getName()), false,
firstItem != null && firstItem.isFile(), isObjectStorage()));
} catch (Throwable t) {
onError.accept(t);
}
});
}
@Override
@Nullable
public UfsStatus[] listStatus(String path, ListOptions options) throws IOException {
if (!options.isRecursive()) {
return listStatus(path);
}
path = validatePath(path);
List returnPaths = new ArrayList<>();
// Each element is a pair of (full path, UfsStatus)
Queue> pathsToProcess = new ArrayDeque<>();
// We call list initially, so we can return null if the path doesn't denote a directory
UfsStatus[] statuses = listStatus(path);
if (statuses == null) {
return null;
} else {
for (UfsStatus status : statuses) {
pathsToProcess.add(new Pair<>(PathUtils.concatPath(path, status.getName()), status));
}
}
while (!pathsToProcess.isEmpty()) {
final Pair pathToProcessPair = pathsToProcess.remove();
final String pathToProcess = pathToProcessPair.getFirst();
UfsStatus pathStatus = pathToProcessPair.getSecond();
int beginIndex = path.endsWith(AlluxioURI.SEPARATOR) ? path.length() : path.length() + 1;
returnPaths.add(pathStatus.setName(pathToProcess.substring(beginIndex)));
if (pathStatus.isDirectory()) {
// Add all of its subpaths
UfsStatus[] children = listStatus(pathToProcess);
if (children != null) {
for (UfsStatus child : children) {
pathsToProcess.add(
new Pair<>(PathUtils.concatPath(pathToProcess, child.getName()), child));
}
}
}
}
return returnPaths.toArray(new UfsStatus[0]);
}
@Override
public InputStream open(String path) throws IOException {
return open(path, OpenOptions.defaults());
}
@Override
public boolean mkdirs(String path) throws IOException {
return mkdirs(path, MkdirsOptions.defaults(mUfsConf));
}
@Override
public AlluxioURI resolveUri(AlluxioURI ufsBaseUri, String alluxioPath) {
return new AlluxioURI(ufsBaseUri, PathUtils.concatPath(ufsBaseUri.getPath(), alluxioPath),
false);
}
/**
* Clean the path by creating a URI and turning it back to a string.
*
* @param path the path to validate
* @return validated path
*/
protected static String validatePath(String path) {
return new AlluxioURI(path).toString();
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy