software.amazon.nio.spi.s3.S3DirectoryStream Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of aws-java-nio-spi-for-s3 Show documentation

A Java NIO.2 service provider for S3, allowing Java NIO operations to be performed on paths using the `s3` scheme. This package implements the service provider interface (SPI) defined for Java NIO.2 in JDK 1.7 providing "plug-in" non-blocking access to S3 objects for Java applications using Java NIO.2 for file access.

The newest version!

/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0
 */

package software.amazon.nio.spi.s3;

import static software.amazon.nio.spi.s3.Constants.PATH_SEPARATOR;

import io.reactivex.rxjava3.core.Flowable;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystem;
import java.nio.file.Path;
import java.util.Iterator;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.reactivestreams.Publisher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.s3.model.CommonPrefix;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Publisher;

class S3DirectoryStream implements DirectoryStream {
    private final Logger logger = LoggerFactory.getLogger(this.getClass().getName());
    private final Iterator iterator;

    S3DirectoryStream(S3FileSystem fs, String bucketName, String finalDirName, Filter filter) {
        final var listObjectsV2Publisher = fs.client().listObjectsV2Paginator(req -> req
            .bucket(bucketName)
            .prefix(finalDirName)
            .delimiter(PATH_SEPARATOR));

        iterator = pathIteratorForPublisher(filter, fs, finalDirName, listObjectsV2Publisher);
        //noinspection ResultOfMethodCallIgnored
        iterator.hasNext();
    }

    @Override
    @NonNull
    public Iterator iterator() {
        return iterator;
    }

    @Override
    public void close() {
    }

    /**
     * Get an iterator for a {@code ListObjectsV2Publisher}
     *
     * @param filter                 a filter to apply to returned Paths. Only accepted paths will be included.
     * @param fs                     the Filesystem.
     * @param finalDirName           the directory name that will be streamed.
     * @param listObjectsV2Publisher the publisher that returns objects and common prefixes that are iterated on.
     * @return an iterator for {@code Path}s constructed from the {@code ListObjectsV2Publisher}s responses.
     * @throws SdkException          if there is an error with S3 access. This is an unchecked Exception
     */
    private Iterator pathIteratorForPublisher(
        final DirectoryStream.Filter filter,
        final FileSystem fs, String finalDirName,
        final ListObjectsV2Publisher listObjectsV2Publisher) throws SdkException {

        final Publisher prefixPublisher =
                listObjectsV2Publisher.commonPrefixes().map(CommonPrefix::prefix);
        final Publisher keysPublisher =
                listObjectsV2Publisher.contents().map(S3Object::key);

        return Flowable.concat(prefixPublisher, keysPublisher)
                .map(fs::getPath)
                .filter(path -> !isEqualToParent(finalDirName, path))  // including the parent will induce loops
                .filter(path -> tryAccept(filter, path))
                .blockingIterable()
                .iterator();
    }


    private static boolean isEqualToParent(String finalDirName, Path p) {
        return ((S3Path) p).getKey().equals(finalDirName);
    }

    private boolean tryAccept(DirectoryStream.Filter filter, Path path) {
        try {
            return filter.accept(path);
        } catch (IOException e) {
            logger.warn("An IOException was thrown while filtering the path: {}." +
                " Set log level to debug to show stack trace", path);
            logger.debug(e.getMessage(), e);
            return false;
        }
    }
}