com.markosindustries.distroboy.aws.s3.S3ObjectsSource Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of aws-s3 Show documentation

DistroBoy support library for interacting with AWS S3 using the AWS SDK

The newest version!

package com.markosindustries.distroboy.aws.s3;

import com.markosindustries.distroboy.core.Cluster;
import com.markosindustries.distroboy.core.iterators.IteratorWithResources;
import com.markosindustries.distroboy.core.operations.InterleavedDataSource;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.S3Object;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;

public class S3ObjectsSource extends InterleavedDataSource {
  private final S3Client s3Client;
  private final String bucket;
  private final String keyPrefix;

  public S3ObjectsSource(Cluster cluster, S3Client s3Client, String bucket, String keyPrefix) {
    super(cluster);
    this.s3Client = s3Client;
    this.bucket = bucket;
    this.keyPrefix = keyPrefix;
  }

  @Override
  public IteratorWithResources enumerateFullSet() {
    ListObjectsV2Iterable responses =
        s3Client.listObjectsV2Paginator(req -> req.bucket(bucket).prefix(keyPrefix));
    return IteratorWithResources.from(
        responses.stream().flatMap(response -> response.contents().stream()).iterator());
  }
}