com.expediagroup.beekeeper.cleanup.aws.S3Client Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of beekeeper-cleanup Show documentation
The newest version!
/**
 * Copyright (C) ${license.git.copyrightYears} Expedia, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.expediagroup.beekeeper.cleanup.aws;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.DeleteObjectsResult;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;

public class S3Client {

  private static final int REQUEST_CHUNK_SIZE = 1000;
  private static final Logger log = LoggerFactory.getLogger(S3Client.class);
  private final AmazonS3 amazonS3;
  private final boolean dryRunEnabled;

  public S3Client(AmazonS3 amazonS3, boolean dryRunEnabled) {
    this.amazonS3 = amazonS3;
    this.dryRunEnabled = dryRunEnabled;
  }

  void deleteObject(String bucket, String key) {
    if (dryRunEnabled) {
      log.info("Dry run - deleting: \"{}/{}\"", bucket, key);
    } else {
      log.info("Deleting \"{}/{}\"", bucket, key);
      amazonS3.deleteObject(bucket, key);
    }
  }

  List listObjects(String bucket, String key) {
    List objectSummaries = new ArrayList<>();
    ListObjectsV2Result listObjectsV2Result;
    String continuationToken = null;
    do {
      ListObjectsV2Request request = new ListObjectsV2Request()
          .withBucketName(bucket)
          .withPrefix(key)
          .withEncodingType("url")
          .withContinuationToken(continuationToken);
      listObjectsV2Result = amazonS3.listObjectsV2(request);
      objectSummaries.addAll(listObjectsV2Result.getObjectSummaries());
      continuationToken = listObjectsV2Result.getNextContinuationToken();
    } while (listObjectsV2Result.isTruncated());
    return objectSummaries;
  }

  List deleteObjects(String bucket, List keys) {
    if (keys.isEmpty()) {
      return Collections.emptyList();
    }
    if (!dryRunEnabled) {
      log.info("Attempting to delete a total of {} objects, from [{}] to [{}]", keys.size(), keys.get(0),
          keys.get(keys.size() - 1));
      DeleteObjectsResult deleteObjectsResult = new DeleteObjectsResult(new ArrayList<>());
      DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket);
      int totalKeys = keys.size();
      int indexStart;
      int indexEnd = 0;
      while (indexEnd < totalKeys) {
        indexStart = indexEnd;
        indexEnd = nextIndexEnd(indexStart, REQUEST_CHUNK_SIZE, totalKeys);
        deleteObjectsRequest.withKeys(keys.subList(indexStart, indexEnd).toArray(String[]::new));
        deleteObjectsResult.getDeletedObjects().addAll(
            amazonS3.deleteObjects(deleteObjectsRequest).getDeletedObjects());
      }
      log.info("Successfully deleted {} objects", keys.size());
      return deleteObjectsResult.getDeletedObjects()
          .stream()
          .map(DeleteObjectsResult.DeletedObject::getKey)
          .collect(Collectors.toList());
    } else {
      return keys.stream()
          .peek(key -> log.info("Dry run - deleting: \"{}/{}\"", bucket, key))
          .collect(Collectors.toList());
    }
  }

  boolean doesObjectExist(String bucket, String key) {
    return amazonS3.doesObjectExist(bucket, key);
  }

  ObjectMetadata getObjectMetadata(String bucket, String key) {
    return amazonS3.getObjectMetadata(bucket, key);
  }

  boolean isEmpty(String bucket, String key, String leafKey) {
    List objectsLeftAtPath = amazonS3.listObjectsV2(bucket, key + "/").getObjectSummaries();
    if (!dryRunEnabled) {
      return objectsLeftAtPath.size() == 0;
    } else {
      String leafKeySentinel = leafKey + S3SentinelFilesCleaner.SENTINEL_SUFFIX;

      for (S3ObjectSummary s3ObjectSummary : objectsLeftAtPath) {
        String currentKey = s3ObjectSummary.getKey();
        if (!currentKey.startsWith(leafKey + "/") && !currentKey.equals(leafKeySentinel)) {
          return false;
        }
      }
      return true;
    }
  }

  private int nextIndexEnd(final int indexStart, final int chunkSize, final int totalKeys) {
    int calculatedNextIndexEnd = indexStart + chunkSize;
    return Math.min(calculatedNextIndexEnd, totalKeys);
  }
}