org.apache.iceberg.ksyun.ks3.Ks3FileIO Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of iceberg-ksyun Show documentation
apache iceberg support ks3
The newest version!
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.iceberg.ksyun.ks3;

import com.ksyun.ks3.service.Ks3;
import org.apache.iceberg.common.DynConstructors;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.io.OutputFile;
import org.apache.iceberg.ksyun.KsyunClientFactories;
import org.apache.iceberg.ksyun.KsyunClientFactory;
import org.apache.iceberg.ksyun.KsyunProperties;
import org.apache.iceberg.util.SerializableSupplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * FileIO implementation backed by Ksyun KS3.
 * 
 * Locations used must follow the conventions for KS3 URIs (e.g. ks3://bucket/path...).
 * URIs with schemes ks3 https are also treated as KS3 object paths.
 * Using this FileIO with other schemes will result in {@link org.apache.iceberg.exceptions.ValidationException}.
 */
public class Ks3FileIO implements FileIO {

  private static final Logger LOG = LoggerFactory.getLogger(Ks3FileIO.class);
  //  private static final String DEFAULT_OUTPUT_FILE_IMPL = "org.apache.iceberg.ksyun.ks3.Ks3MultipartOutputFile";
  public static final String OUTPUT_FILE_TYPE_APPEND = "append";
  public static final String OUTPUT_FILE_TYPE_MULTIPART = "multipart";

  private SerializableSupplier ks3;
  private KsyunProperties ksyunProperties;
  private KsyunClientFactory ksyunClientFactory;
  private transient volatile Ks3 client;
  private final AtomicBoolean isResourceClosed = new AtomicBoolean(false);

  @Override
  public InputFile newInputFile(String path) {
    return Ks3InputFile.fromLocation(path, client(), ksyunProperties);
  }

  @Override
  public OutputFile newOutputFile(String path) {
    String outputFileType = ksyunProperties.outputFileType();
    if (OUTPUT_FILE_TYPE_APPEND.equals(outputFileType)) {
      return Ks3AppendOutputFile.fromLocation(path, client(), ksyunProperties);
    }
    return Ks3MultipartOutputFile.fromLocation(path, client(), ksyunProperties);
  }

  @Override
  public void deleteFile(String path) {
    Ks3URI uri = new Ks3URI(path);

    client().deleteObject(uri.bucket(), uri.name());
  }

  private Ks3 client() {
    if (client == null) {
      synchronized (this) {
        if (client == null) {
          client = ks3.get();
        }
      }
    }
    return client;
  }

  @Override
  public void initialize(Map properties) {
    this.ksyunProperties = new KsyunProperties(properties);
    this.ksyunClientFactory = KsyunClientFactories.from(properties);
    this.ks3 = ksyunClientFactory::ks3Client;
  }

  @Override
  public void close() {
    // handles concurrent calls to close()
    if (isResourceClosed.compareAndSet(false, true)) {
//      client.destroy();
    }
  }
}