com.hubspot.singularity.runner.base.shared.S3UploadMetadata Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of SingularityRunnerBase Show documentation
The newest version!
package com.hubspot.singularity.runner.base.shared;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;

/**
 *
 * directory - the directory to watch for files inside of
 * fileGlob - only files matching this glob will be uploaded to S3.
 * s3Bucket - the name of the bucket to upload to in S3
 *
 * s3KeyFormat - the format for the actual key name for the object in S3 corresponding to each file uploaded. This can be dynamically
 * formatted with the following variables:
 *
 * %filename - adds the original file's filename
 * %fileext - adds the original file's file ext
 * %Y - adds year
 * %m - adds month
 * %d - adds day of the month
 * %s - adds milliseconds
 * %guid - adds a guid
 * %host - adds the hostname
 * %index - adds the index of the file uploaded at this moment (to preserve uniqueness)
 *
 * For example, if the s3KeyFormat was: %filename-%Y and the file name on local disk was "file1.txt" the S3 key would be : s3Bucket/file1.txt-2015 (assuming current year is 2015)
 *
 * finished - set this to true if you wish *this s3 upload metadata configuration* file to be deleted and no more files uploaded after the last matching file is uploaded to S3 successfully (think of it as safe delete.)
 * onFinishGlob - a glob to match files which should be uploaded *only* after finished is set to true OR the pid is no longer active
 * pid - the pid of the process to watch, such that when that pid is no longer running, finished is set to true (stop uploading files / watching directory once all files are successfully uploaded.)
 * s3AccessKey - the access key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key)
 * s3SecretKey - the secret key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key)
 *
 * finishedAfterMillisWithoutNewFile - after millis without a new file, set finished to true (see above for result.) - (-1 never expire) - absent - uses system default.
 * uploadImmediately - When detected, immediately upload to S3 rather than waiting for polling to upload
 *
 */
public class S3UploadMetadata {
  private final String directory;
  private final String fileGlob;
  private final String s3Bucket;
  private final String s3KeyFormat;
  private final boolean finished;
  private final Optional onFinishGlob;
  private final Optional pid;
  private final Optional s3AccessKey;
  private final Optional s3SecretKey;
  private final Optional finishedAfterMillisWithoutNewFile;
  private final Optional s3StorageClass;
  private final Optional applyStorageClassIfOverBytes;
  private final Optional uploadImmediately;
  private final boolean checkSubdirectories;
  private final boolean compressBeforeUpload;
  private final boolean checkIfOpen;
  private final SingularityUploaderType uploaderType;
  private final Map gcsCredentials;
  private final Optional gcsStorageClass;
  private final Optional encryptionKey;
  private final Optional useS3ServerSideEncryption;

  @JsonCreator
  public S3UploadMetadata(
    @JsonProperty("directory") String directory,
    @JsonProperty("fileGlob") String fileGlob,
    @JsonProperty("s3Bucket") String s3Bucket,
    @JsonProperty("s3KeyFormat") String s3KeyFormat,
    @JsonProperty("finished") boolean finished,
    @JsonProperty("onFinishGlob") Optional onFinishGlob,
    @JsonProperty("pid") Optional pid,
    @JsonProperty("s3AccessKey") Optional s3AccessKey,
    @JsonProperty("s3SecretKey") Optional s3SecretKey,
    @JsonProperty(
      "finishedAfterMillisWithoutNewFile"
    ) Optional finishedAfterMillisWithoutNewFile,
    @JsonProperty("storageClass") Optional s3StorageClass,
    @JsonProperty(
      "applyStorageClassIfOverBytes"
    ) Optional applyStorageClassIfOverBytes,
    @JsonProperty("uploadImmediately") Optional uploadImmediately,
    @JsonProperty("checkSubdirectories") Optional checkSubdirectories,
    @JsonProperty("compressBeforeUpload") Optional compressBeforeUpload,
    @JsonProperty("checkIfOpen") Optional checkIfOpen,
    @JsonProperty("uploaderType") Optional uploaderType,
    @JsonProperty("gcsCredentials") Map gcsCredentials,
    @JsonProperty("gcsStorageClass") Optional gcsStorageClass,
    @JsonProperty("encryptionKey") Optional encryptionKey,
    @JsonProperty("useS3ServerSideEncryption") Optional useS3ServerSideEncryption
  ) {
    Preconditions.checkNotNull(directory);
    Preconditions.checkNotNull(fileGlob);
    Preconditions.checkNotNull(s3Bucket);
    Preconditions.checkNotNull(s3KeyFormat);

    this.directory = directory;
    this.fileGlob = fileGlob;
    this.s3Bucket = s3Bucket;
    this.s3KeyFormat = s3KeyFormat;
    this.finished = finished;
    this.pid = pid;
    this.s3AccessKey = s3AccessKey;
    this.s3SecretKey = s3SecretKey;
    this.onFinishGlob = onFinishGlob;
    this.s3StorageClass = s3StorageClass;
    this.finishedAfterMillisWithoutNewFile = finishedAfterMillisWithoutNewFile;
    this.applyStorageClassIfOverBytes = applyStorageClassIfOverBytes;
    this.uploadImmediately = uploadImmediately;
    this.checkSubdirectories = checkSubdirectories.orElse(false);
    this.compressBeforeUpload = compressBeforeUpload.orElse(false);
    this.checkIfOpen = checkIfOpen.orElse(true);
    this.uploaderType = uploaderType.orElse(SingularityUploaderType.S3);
    this.gcsCredentials =
      gcsCredentials != null ? gcsCredentials : Collections.emptyMap();
    this.gcsStorageClass = gcsStorageClass;
    this.encryptionKey = encryptionKey;
    this.useS3ServerSideEncryption = useS3ServerSideEncryption;
  }

  @Override
  public int hashCode() {
    final int prime = 31;
    int result = 1;
    result = prime * result + ((directory == null) ? 0 : directory.hashCode());
    result = prime * result + ((fileGlob == null) ? 0 : fileGlob.hashCode());
    return result;
  }

  @Override
  public boolean equals(Object obj) {
    if (this == obj) {
      return true;
    }
    if (obj == null) {
      return false;
    }
    if (getClass() != obj.getClass()) {
      return false;
    }
    S3UploadMetadata other = (S3UploadMetadata) obj;
    if (directory == null) {
      if (other.directory != null) {
        return false;
      }
    } else if (!directory.equals(other.directory)) {
      return false;
    }
    if (fileGlob == null) {
      if (other.fileGlob != null) {
        return false;
      }
    } else if (!fileGlob.equals(other.fileGlob)) {
      return false;
    }
    return true;
  }

  public String getDirectory() {
    return directory;
  }

  public String getFileGlob() {
    return fileGlob;
  }

  public String getS3Bucket() {
    return s3Bucket;
  }

  public String getS3KeyFormat() {
    return s3KeyFormat;
  }

  public boolean isFinished() {
    return finished;
  }

  public Optional getPid() {
    return pid;
  }

  public Optional getS3AccessKey() {
    return s3AccessKey;
  }

  public Optional getS3SecretKey() {
    return s3SecretKey;
  }

  public Optional getOnFinishGlob() {
    return onFinishGlob;
  }

  public Optional getFinishedAfterMillisWithoutNewFile() {
    return finishedAfterMillisWithoutNewFile;
  }

  public Optional getS3StorageClass() {
    return s3StorageClass;
  }

  public Optional getGcsStorageClass() {
    return gcsStorageClass;
  }

  public Optional getApplyStorageClassIfOverBytes() {
    return applyStorageClassIfOverBytes;
  }

  public Optional getUploadImmediately() {
    return uploadImmediately;
  }

  public boolean isCheckSubdirectories() {
    return checkSubdirectories;
  }

  public boolean isCompressBeforeUpload() {
    return compressBeforeUpload;
  }

  public boolean isCheckIfOpen() {
    return checkIfOpen;
  }

  public SingularityUploaderType getUploaderType() {
    return uploaderType;
  }

  public Map getGcsCredentials() {
    return gcsCredentials;
  }

  public Optional getEncryptionKey() {
    return encryptionKey;
  }

  public boolean isUseS3ServerSideEncryption() {
    return useS3ServerSideEncryption.orElse(false);
  }

  @JsonIgnore
  public boolean isImmediate() {
    return uploadImmediately.orElse(false);
  }

  @Override
  public String toString() {
    return (
      "S3UploadMetadata{" +
      "directory='" +
      directory +
      '\'' +
      ", fileGlob='" +
      fileGlob +
      '\'' +
      ", s3Bucket='" +
      s3Bucket +
      '\'' +
      ", s3KeyFormat='" +
      s3KeyFormat +
      '\'' +
      ", finished=" +
      finished +
      ", onFinishGlob=" +
      onFinishGlob +
      ", pid=" +
      pid +
      ", s3AccessKey=" +
      s3AccessKey +
      ", s3SecretKey=" +
      s3SecretKey +
      ", finishedAfterMillisWithoutNewFile=" +
      finishedAfterMillisWithoutNewFile +
      ", s3StorageClass=" +
      s3StorageClass +
      ", applyStorageClassIfOverBytes=" +
      applyStorageClassIfOverBytes +
      ", uploadImmediately=" +
      uploadImmediately +
      ", checkSubdirectories=" +
      checkSubdirectories +
      ", compressBeforeUpload=" +
      compressBeforeUpload +
      ", checkIfOpen=" +
      checkIfOpen +
      ", uploaderType=" +
      uploaderType +
      ", gcsStorageClass=" +
      gcsStorageClass +
      ", encryptionKey=" +
      encryptionKey +
      ", useS3ServerSideEncryption=" +
      useS3ServerSideEncryption +
      '}'
    );
  }
}