org.apache.iceberg.ksyun.KsyunProperties Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of iceberg-ksyun Show documentation
apache iceberg support ks3
The newest version!
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.iceberg.ksyun;

import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.util.PropertyUtil;

import java.io.Serializable;
import java.util.Map;

public class KsyunProperties implements Serializable {
  /**
   * S3 Access key id of Ksyun KS3
   */
  public static final String KS3_ACCESS_KEY_ID = "ks3.AccessKey";

  /**
   * S3 Secret access key of Ksyun KS3
   */
  public static final String KS3_SECRET_ACCESS_KEY = "ks3.AccessSecret";

  /**
   * S3 endpoint of Ksyun KS3
   */
  public static final String KS3_ENDPOINT = "ks3.endpoint";

  /**
   * Location to put staging files for upload to KS3, default to temp directory set in java.io.tmpdir.
   */
  public static final String KS3_FILEIO_STAGING_DIRECTORY = "ks3.staging-dir";

  /**
   * Number of threads to use for uploading parts to KS3 (shared pool across all output streams),
   * default to {@link Runtime#availableProcessors()}
   */
  public static final String KS3_FILEIO_MULTIPART_UPLOAD_THREADS  = "ks3.multipart.num-threads";
  public static final int KS3_FILEIO_MULTIPART_UPLOAD_THREADS_DEFAULT  = 10;

  public static final String KS3_CONNECTION_MAX  = "ks3.connection.max";
  public static final int KS3_CONNECTION_MAX_DEFAULT = 100;

  public static final String KS3_CONNECTION_TIMEOUT  = "ks3.connection.timeout";
  public static final int KS3_CONNECTION_TIMEOUT_DEFAULT = 50000;

  public static final String KS3_CONNECTION_TTL  = "ks3.connection.ttl";
  public static final int KS3_CONNECTION_TTL_DEFAULT = -1;

  public static final String KS3_CONNECTION_MAX_PER_ROUTE  = "ks3.connection.max.per.route";

  /**
   * The size of a single part for multipart upload requests in bytes (default: 32MB).
   * based on S3 requirement, the part size must be at least 5MB.
   * Too ensure performance of the reader and writer, the part size must be less than 2GB.
   */
  public static final String S3FILEIO_MULTIPART_SIZE = "ks3.multipart.part-size-bytes";
//  public static final int S3FILEIO_MULTIPART_SIZE_DEFAULT = 32 * 1024 * 1024;
  public static final int S3FILEIO_MULTIPART_SIZE_DEFAULT = 10 * 1024 * 1024;
  public static final int S3FILEIO_MULTIPART_SIZE_MIN = 5 * 1024 * 1024;

  /**
   * The threshold expressed as a factor times the multipart size at which to
   * switch from uploading using a single put object request to uploading using multipart upload
   * (default: 1.5).
   */
  public static final String S3FILEIO_MULTIPART_THRESHOLD_FACTOR = "ks3.multipart.threshold";
  public static final double S3FILEIO_MULTIPART_THRESHOLD_FACTOR_DEFAULT = 1.5;

  /**
   * Enables eTag checks for S3 PUT and MULTIPART upload requests.
   */
  public static final String S3_CHECKSUM_ENABLED = "ks3.checksum-enabled";
  public static final boolean S3_CHECKSUM_ENABLED_DEFAULT = false;

  /**
   * Number of threads to use for adding delete tags to S3 objects, default to {@link
   * Runtime#availableProcessors()}
   */
  public static final String S3FILEIO_DELETE_THREADS = "ks3.delete.num-threads";

  /**
   * Configure the batch size used when deleting multiple files from a given S3 bucket
   */
  public static final String S3FILEIO_DELETE_BATCH_SIZE = "ks3.delete.batch-size";
  /**
   * Default batch size used when deleting files.
   */
  public static final int S3FILEIO_DELETE_BATCH_SIZE_DEFAULT = 250;

  /**
   * Max possible batch size for deletion. Currently, a max of 1000 keys can be deleted in one batch.
   */
  public static final int S3FILEIO_DELETE_BATCH_SIZE_MAX = 1000;
  /**
   * The implementation class of {@link KsyunClientFactory} to customize Ksyun client configurations.
   * If set, all Ksyun clients will be initialized by the specified factory.
   * If not set, {@link KsyunClientFactories.DefaultKsyunClientFactory} is used as default factory.
   */
  public static final String CLIENT_FACTORY = "client.factory";

  public static final String OUTPUT_FILE_TYPE = "ks3.output.file.type";
//  private static final String DEFAULT_OUTPUT_FILE_IMPL = "org.apache.iceberg.ksyun.ks3.Ks3MultipartOutputFile";


  public static final String OUTPUT_BUFFER_SIZE = "ks3.output.buffer.size";
  public static final int OUTPUT_BUFFER_SIZE_DEFAULT = 8096;

  /**
   * What buffer to use.
   * Default is {@link #FAST_UPLOAD_BUFFER_DISK}
   * Value: {@value}
   */
  public static final String FAST_UPLOAD_BUFFER =
      "ks3.upload.buffer.type";

  /**
   * Buffer blocks to disk: {@value}.
   * Capacity is limited to available disk space.
   */

  public static final String FAST_UPLOAD_BUFFER_DISK = "disk";

  /**
   * Use an in-memory array. Fast but will run of heap rapidly: {@value}.
   */
  public static final String FAST_UPLOAD_BUFFER_ARRAY = "array";

  /**
   * Use a byte buffer. May be more memory efficient than the
   * {@link #FAST_UPLOAD_BUFFER_ARRAY}: {@value}.
   */
  public static final String FAST_UPLOAD_BYTEBUFFER = "bytebuffer";

  /**
   * Default buffer option: {@value}.
   */
  public static final String DEFAULT_FAST_UPLOAD_BUFFER =
      FAST_UPLOAD_BUFFER_DISK;


  private String KS3Endpoint;
  private String KS3AccessKeyId;
  private String KS3SecretAccessKey;
  private String s3fileIoStagingDirectory;
  private int multipartUploadThreads;

  private int s3FileIoMultiPartSize;
  private int s3FileIoDeleteBatchSize;
  private double s3FileIoMultipartThresholdFactor;
  private boolean isS3ChecksumEnabled;
  private int s3FileIoDeleteThreads;
  private boolean isS3DeleteEnabled;

  private String outputFileType;
  private int outputBufferSize;

  private int ks3ConnectionMax;
  private int ks3ConnectionTimeout;
  private int ks3ConnectionTTL;
  private int ks3ConnectionMaxPerRoute;

  public KsyunProperties() {
//    this.multipartUploadThreads = Runtime.getRuntime().availableProcessors();
    this.multipartUploadThreads = KS3_FILEIO_MULTIPART_UPLOAD_THREADS_DEFAULT;
    this.s3FileIoMultiPartSize = S3FILEIO_MULTIPART_SIZE_DEFAULT;
    this.s3FileIoMultipartThresholdFactor = S3FILEIO_MULTIPART_THRESHOLD_FACTOR_DEFAULT;
    this.s3FileIoDeleteBatchSize = S3FILEIO_DELETE_BATCH_SIZE_DEFAULT;
    this.s3fileIoStagingDirectory = System.getProperty("java.io.tmpdir");
    this.isS3ChecksumEnabled = S3_CHECKSUM_ENABLED_DEFAULT;
    this.outputBufferSize = OUTPUT_BUFFER_SIZE_DEFAULT;
    this.ks3ConnectionMax = KS3_CONNECTION_MAX_DEFAULT;
    this.ks3ConnectionTTL = KS3_CONNECTION_TTL_DEFAULT;
    this.ks3ConnectionTimeout = KS3_CONNECTION_TIMEOUT_DEFAULT;
    this.ks3ConnectionMaxPerRoute = KS3_CONNECTION_MAX_DEFAULT;
  }

  public KsyunProperties(Map properties) {
    this.KS3AccessKeyId = properties.get(KsyunProperties.KS3_ACCESS_KEY_ID);
    this.KS3SecretAccessKey = properties.get(KsyunProperties.KS3_SECRET_ACCESS_KEY);
    this.KS3Endpoint = properties.get(KsyunProperties.KS3_ENDPOINT);
    this.s3fileIoStagingDirectory = PropertyUtil.propertyAsString(properties, KS3_FILEIO_STAGING_DIRECTORY,
        System.getProperty("java.io.tmpdir"));
    this.multipartUploadThreads = PropertyUtil.propertyAsInt(properties, KS3_FILEIO_MULTIPART_UPLOAD_THREADS,
        KS3_FILEIO_MULTIPART_UPLOAD_THREADS_DEFAULT);

    try {
      this.s3FileIoMultiPartSize = PropertyUtil.propertyAsInt(properties, S3FILEIO_MULTIPART_SIZE,
          S3FILEIO_MULTIPART_SIZE_DEFAULT);
    } catch (NumberFormatException e) {
      throw new IllegalArgumentException("Input malformed or exceeded maximum multipart upload size 5GB: %s" +
          properties.get(S3FILEIO_MULTIPART_SIZE));
    }

    this.s3FileIoMultipartThresholdFactor = PropertyUtil.propertyAsDouble(properties,
        S3FILEIO_MULTIPART_THRESHOLD_FACTOR, S3FILEIO_MULTIPART_THRESHOLD_FACTOR_DEFAULT);

    Preconditions.checkArgument(s3FileIoMultipartThresholdFactor >= 1.0,
        "Multipart threshold factor must be >= to 1.0");

    Preconditions.checkArgument(s3FileIoMultiPartSize >= S3FILEIO_MULTIPART_SIZE_MIN,
        "Minimum multipart upload object size must be larger than 5 MB.");

    this.isS3ChecksumEnabled = PropertyUtil.propertyAsBoolean(properties, S3_CHECKSUM_ENABLED,
        S3_CHECKSUM_ENABLED_DEFAULT);

    this.s3FileIoDeleteBatchSize = PropertyUtil.propertyAsInt(properties, S3FILEIO_DELETE_BATCH_SIZE,
        S3FILEIO_DELETE_BATCH_SIZE_DEFAULT);
    Preconditions.checkArgument(s3FileIoDeleteBatchSize > 0 &&
            s3FileIoDeleteBatchSize <= S3FILEIO_DELETE_BATCH_SIZE_MAX,
        String.format("Deletion batch size must be between 1 and %s", S3FILEIO_DELETE_BATCH_SIZE_MAX));

    this.outputFileType = properties.get(OUTPUT_FILE_TYPE);
    this.outputBufferSize = PropertyUtil.propertyAsInt(properties, OUTPUT_BUFFER_SIZE,
        OUTPUT_BUFFER_SIZE_DEFAULT);
    Preconditions.checkArgument(outputBufferSize >= 1,
        "Output buffer size must be >= to 1 ");

    this.ks3ConnectionMax = PropertyUtil.propertyAsInt(properties, KS3_CONNECTION_MAX,
        KS3_CONNECTION_MAX_DEFAULT);
    this.ks3ConnectionTimeout = PropertyUtil.propertyAsInt(properties, KS3_CONNECTION_TIMEOUT,
        KS3_CONNECTION_TIMEOUT_DEFAULT);
    this.ks3ConnectionTTL = PropertyUtil.propertyAsInt(properties, KS3_CONNECTION_TTL,
        KS3_CONNECTION_TTL_DEFAULT);
    this.ks3ConnectionMaxPerRoute = PropertyUtil.propertyAsInt(
        properties,
        KS3_CONNECTION_MAX_PER_ROUTE,
        PropertyUtil.propertyAsInt(properties, KS3_CONNECTION_MAX, KS3_CONNECTION_MAX_DEFAULT));

  }
  public int ks3ConnectionTimeout() {
    return ks3ConnectionTimeout;
  }
  public void setKs3ConnectionTimeout(int ks3ConnectionTimeout) {
    this.ks3ConnectionTimeout = ks3ConnectionTimeout;
  }

  public int ks3ConnectionTTL() {
    return ks3ConnectionTTL;
  }
  public void setKs3ConnectionTTL(int ks3ConnectionTTL) {
    this.ks3ConnectionTTL = ks3ConnectionTTL;
  }

  public int ks3ConnectionMax() {
    return ks3ConnectionMax;
  }
  public void setKs3ConnectionMax(int ks3ConnectionMax) {
    this.ks3ConnectionMax = ks3ConnectionMax;
  }

  public int ks3ConnectionMaxPerRoute() {
    return ks3ConnectionMaxPerRoute;
  }
  public void setKs3ConnectionMaxPerRoute(int ks3ConnectionMaxPerRoute) {
    this.ks3ConnectionMaxPerRoute = ks3ConnectionMaxPerRoute;
  }

  public String outputFileType() {
    return outputFileType;
  }

  public void setOutputFileType(String outputFileType) {
    this.outputFileType = outputFileType;
  }

  public int outputBufferSize() {
    return outputBufferSize;
  }

  public void setOutputBufferSize(int outputBufferSize) {
    this.outputBufferSize = outputBufferSize;
  }

  public String ks3Endpoint() {
    return KS3Endpoint;
  }

  public void setKS3S3Endpoint(String KS3S3Endpoint) {
    this.KS3Endpoint = KS3S3Endpoint;
  }

  public String ks3AccessKeyId() {
    return KS3AccessKeyId;
  }

  public void setKS3S3AccessKeyId(String KS3S3AccessKeyId) {
    this.KS3AccessKeyId = KS3S3AccessKeyId;
  }

  public String ks3SecretAccessKey() {
    return KS3SecretAccessKey;
  }

  public void setKs3SecretAccessKey(String KS3S3SecretAccessKey) {
    this.KS3SecretAccessKey = KS3S3SecretAccessKey;
  }

  public String s3fileIoStagingDirectory() {
    return s3fileIoStagingDirectory;
  }

  public void setS3fileIoStagingDirectory(String directory) {
    this.s3fileIoStagingDirectory = directory;
  }
  public int multipartUploadThreads() {
    return multipartUploadThreads;
  }

  public void setMultipartUploadThreads(int threads) {
    this.multipartUploadThreads = threads;
  }

  public int s3FileIoMultiPartSize() {
    return s3FileIoMultiPartSize;
  }

  public void setS3FileIoMultiPartSize(int size) {
    this.s3FileIoMultiPartSize = size;
  }

  public double s3FileIOMultipartThresholdFactor() {
    return s3FileIoMultipartThresholdFactor;
  }

  public void setS3FileIoMultipartThresholdFactor(double factor) {
    this.s3FileIoMultipartThresholdFactor = factor;
  }

  public boolean isS3ChecksumEnabled() {
    return this.isS3ChecksumEnabled;
  }

  public void setS3ChecksumEnabled(boolean eTagCheckEnabled) {
    this.isS3ChecksumEnabled = eTagCheckEnabled;
  }

  public int s3FileIoDeleteThreads() {
    return s3FileIoDeleteThreads;
  }

  public void setS3FileIoDeleteThreads(int threads) {
    this.s3FileIoDeleteThreads = threads;
  }

  public boolean isS3DeleteEnabled() {
    return isS3DeleteEnabled;
  }

  public void setS3DeleteEnabled(boolean s3DeleteEnabled) {
    this.isS3DeleteEnabled = s3DeleteEnabled;
  }
}