com.hubspot.singularity.runner.base.shared.S3UploadMetadata Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of SingularityRunnerBase Show documentation
There is a newer version: 1.5.0
Show newest version
package com.hubspot.singularity.runner.base.shared;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;

/**
 *
 * directory - the directory to watch for files inside of
 * fileGlob - only files matching this glob will be uploaded to S3.
 * s3Bucket - the name of the bucket to upload to in S3
 *
 * s3KeyFormat - the format for the actual key name for the object in S3 corresponding to each file uploaded. This can be dynamically
 * formatted with the following variables:
 *
 * %filename - adds the original file's filename
 * %fileext - adds the original file's file ext
 * %Y - adds year
 * %m - adds month
 * %d - adds day of the month
 * %s - adds milliseconds
 * %guid - adds a guid
 * %host - adds the hostname
 * %index - adds the index of the file uploaded at this moment (to preserve uniqueness)
 *
 * For example, if the s3KeyFormat was: %filename-%Y and the file name on local disk was "file1.txt" the S3 key would be : s3Bucket/file1.txt-2015 (assuming current year is 2015)
 *
 * finished - set this to true if you wish *this s3 upload metadata configuration* file to be deleted and no more files uploaded after the last matching file is uploaded to S3 successfully (think of it as safe delete.)
 * onFinishGlob - a glob to match files which should be uploaded *only* after finished is set to true OR the pid is no longer active
 * pid - the pid of the process to watch, such that when that pid is no longer running, finished is set to true (stop uploading files / watching directory once all files are successfully uploaded.)
 * s3AccessKey - the access key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key)
 * s3SecretKey - the secret key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key)
 *
 * finishedAfterMillisWithoutNewFile - after millis without a new file, set finished to true (see above for result.) - (-1 never expire) - absent - uses system default.
 * uploadImmediately - When detected, immediately upload to S3 rather than waiting for polling to upload
 *
 */
public class S3UploadMetadata {

  private final String directory;
  private final String fileGlob;
  private final String s3Bucket;
  private final String s3KeyFormat;
  private final boolean finished;
  private final Optional onFinishGlob;
  private final Optional pid;
  private final Optional s3AccessKey;
  private final Optional s3SecretKey;
  private final Optional finishedAfterMillisWithoutNewFile;
  private final Optional s3StorageClass;
  private final Optional applyStorageClassIfOverBytes;
  private final Optional uploadImmediately;

  @JsonCreator
  public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProperty("fileGlob") String fileGlob, @JsonProperty("s3Bucket") String s3Bucket, @JsonProperty("s3KeyFormat") String s3KeyFormat,
      @JsonProperty("finished") boolean finished, @JsonProperty("onFinishGlob") Optional onFinishGlob, @JsonProperty("pid") Optional pid, @JsonProperty("s3AccessKey") Optional s3AccessKey,
      @JsonProperty("s3SecretKey") Optional s3SecretKey, @JsonProperty("finishedAfterMillisWithoutNewFile") Optional finishedAfterMillisWithoutNewFile,
      @JsonProperty("storageClass") Optional s3StorageClass, @JsonProperty("applyStorageClassIfOverBytes") Optional applyStorageClassIfOverBytes,
                          @JsonProperty("uploadImmediately") Optional uploadImmediately) {
    Preconditions.checkNotNull(directory);
    Preconditions.checkNotNull(fileGlob);
    Preconditions.checkNotNull(s3Bucket);
    Preconditions.checkNotNull(s3KeyFormat);

    this.directory = directory;
    this.fileGlob = fileGlob;
    this.s3Bucket = s3Bucket;
    this.s3KeyFormat = s3KeyFormat;
    this.finished = finished;
    this.pid = pid;
    this.s3AccessKey = s3AccessKey;
    this.s3SecretKey = s3SecretKey;
    this.onFinishGlob = onFinishGlob;
    this.s3StorageClass = s3StorageClass;
    this.finishedAfterMillisWithoutNewFile = finishedAfterMillisWithoutNewFile;
    this.applyStorageClassIfOverBytes = applyStorageClassIfOverBytes;
    this.uploadImmediately = uploadImmediately;
  }

  @Override
  public int hashCode() {
    final int prime = 31;
    int result = 1;
    result = prime * result + ((directory == null) ? 0 : directory.hashCode());
    result = prime * result + ((fileGlob == null) ? 0 : fileGlob.hashCode());
    return result;
  }

  @Override
  public boolean equals(Object obj) {
    if (this == obj) {
      return true;
    }
    if (obj == null) {
      return false;
    }
    if (getClass() != obj.getClass()) {
      return false;
    }
    S3UploadMetadata other = (S3UploadMetadata) obj;
    if (directory == null) {
      if (other.directory != null) {
        return false;
      }
    } else if (!directory.equals(other.directory)) {
      return false;
    }
    if (fileGlob == null) {
      if (other.fileGlob != null) {
        return false;
      }
    } else if (!fileGlob.equals(other.fileGlob)) {
      return false;
    }
    return true;
  }

  public String getDirectory() {
    return directory;
  }

  public String getFileGlob() {
    return fileGlob;
  }

  public String getS3Bucket() {
    return s3Bucket;
  }

  public String getS3KeyFormat() {
    return s3KeyFormat;
  }

  public boolean isFinished() {
    return finished;
  }

  public Optional getPid() {
    return pid;
  }

  public Optional getS3AccessKey() {
    return s3AccessKey;
  }

  public Optional getS3SecretKey() {
    return s3SecretKey;
  }

  public Optional getOnFinishGlob() {
    return onFinishGlob;
  }

  public Optional getFinishedAfterMillisWithoutNewFile() {
    return finishedAfterMillisWithoutNewFile;
  }

  public Optional getS3StorageClass() {
    return s3StorageClass;
  }

  public Optional getApplyStorageClassIfOverBytes() {
    return applyStorageClassIfOverBytes;
  }

  public Optional getUploadImmediately() {
    return uploadImmediately;
  }

  @Override
  public String toString() {
    return "S3UploadMetadata [directory=" + directory + ", fileGlob=" + fileGlob + ", s3Bucket=" + s3Bucket + ", s3KeyFormat=" + s3KeyFormat + ", finished=" + finished + ", onFinishGlob="
        + onFinishGlob + ", pid=" + pid + ", s3AccessKey=" + s3AccessKey + ", s3SecretKey=" + s3SecretKey + ", finishedAfterMillisWithoutNewFile="
        + finishedAfterMillisWithoutNewFile
        + ", s3StorageClass=" + s3StorageClass + ", applyStorageClassIfOverBytes=" + applyStorageClassIfOverBytes + ", uploadImmediately=" + uploadImmediately + "]";
  }

}