com.sap.hana.datalake.files.directaccess.wasb.WasbFileSystem Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of sap-hdlfs Show documentation
An implementation of org.apache.hadoop.fs.FileSystem targeting SAP HANA Data Lake Files.
There is a newer version: 3.0.27
// © 2023-2024 SAP SE or an SAP affiliate company. All rights reserved.
package com.sap.hana.datalake.files.directaccess.wasb;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import com.sap.hana.datalake.files.shaded.org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.util.Progressable;
import org.apache.http.client.HttpClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URI;
import java.util.concurrent.ExecutorService;

import com.sap.hana.datalake.files.HdlfsBaseFileSystem;
import com.sap.hana.datalake.files.HdlfsConstants;
import com.sap.hana.datalake.files.classification.InterfaceAudience;

@InterfaceAudience.Private
public class WasbFileSystem extends HdlfsBaseFileSystem {

  private static final Logger LOG = LoggerFactory.getLogger(WasbFileSystem.class);

  private final HttpClient httpClient;
  private final ExecutorService threadPool;

  private int openChunkSize;
  private int openSignedUrlExpirationSafetyMargin;
  private int createChunkSize;
  private int createMaxActiveChunks;
  private int createSignedUrlExpirationSafetyMargin;

  public WasbFileSystem(final WebHdfsFileSystem webHdfsFileSystem, final HttpClient httpClient, final ExecutorService threadPool) {
    super(webHdfsFileSystem);
    this.httpClient = httpClient;
    this.threadPool = threadPool;
  }

  @Override
  public void initialize(final URI fsUri, final Configuration conf) throws IOException {
    super.initialize(fsUri, conf);
    final Configuration config = this.getConf();

    this.openChunkSize = config.getInt(HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_OPEN_CHUNK_SIZE_BYTES_KEY,
        HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_OPEN_CHUNK_SIZE_BYTES_DEFAULT);
    this.openSignedUrlExpirationSafetyMargin = config.getInt(HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_OPEN_SIGNED_URL_EXPIRATION_SAFETY_MARGIN_SECONDS_KEY,
        HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_OPEN_SIGNED_URL_EXPIRATION_SAFETY_MARGIN_SECONDS_DEFAULT);

    this.createChunkSize = config.getInt(HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_WASB_CREATE_CHUNK_SIZE_BYTES_KEY,
            HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_WASB_CREATE_CHUNK_SIZE_BYTES_DEFAULT);
    this.createMaxActiveChunks = config.getInt(HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_MULTIPART_UPLOAD_MAX_ACTIVE_CHUNKS_KEY,
            HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_MULTIPART_UPLOAD_MAX_ACTIVE_CHUNKS_DEFAULT);
    this.createSignedUrlExpirationSafetyMargin = config.getInt(HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_CREATE_SIGNED_URL_EXPIRATION_SAFETY_MARGIN_SECONDS_KEY,
            HdlfsConstants.FS_HDLFS_DIRECT_ACCESS_CREATE_SIGNED_URL_EXPIRATION_SAFETY_MARGIN_SECONDS_DEFAULT);
  }

  @Override
  public FSDataInputStream open(final Path path, final int bufferSize) throws IOException {
    LOG.debug("Calling open implementation from {}", WasbFileSystem.class);

    final Path delegateFsPath = this.rewritePathToDelegateFs(path);
    this.assertObjectExistsIfNeeded(delegateFsPath);

    return new FSDataInputStream(new WasbDirectAccessInputStream(delegateFsPath, this.openChunkSize,
        this.openSignedUrlExpirationSafetyMargin, this.getWebHdfsFileSystem(), this.httpClient));
  }

  @Override
  public FSDataOutputStream create(final Path path, final FsPermission fsPermission, final boolean overwrite,
          final int bufferSize, final short replication, final long blockSize, final Progressable progress) throws IOException {
    LOG.debug("Calling create implementation from {}", WasbFileSystem.class);

    this.checkCreateOperation(path, overwrite);

    final Path delegateFsPath = this.rewritePathToDelegateFs(path);

    final WasbDirectAccessOutputStream wasbDirectAccessOutputStream = WasbDirectAccessOutputStream.newInstance(delegateFsPath,
            overwrite, this.createChunkSize, this.getWebHdfsFileSystem(), this.httpClient, this.threadPool,
            this.createMaxActiveChunks, this.createSignedUrlExpirationSafetyMargin);

    return new FSDataOutputStream(wasbDirectAccessOutputStream, /* stats */ null);
  }

  @Override
  protected void setupMultipartUpload(final Configuration conf) {
    // Multipart upload is not needed for the ManifestCommitter, which is used when Direct Access is enabled
    // As WasbFileSystem will only be used with Direct Access, we can skip this setup
  }
}

// © 2023-2024 SAP SE or an SAP affiliate company. All rights reserved.