com.sap.hana.datalake.files.HdlfsCommitterFactory Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of sap-hdlfs Show documentation

An implementation of org.apache.hadoop.fs.FileSystem targeting SAP HANA Data Lake Files.

There is a newer version: 3.0.27

// © 2022-2024 SAP SE or an SAP affiliate company. All rights reserved.
package com.sap.hana.datalake.files;

import com.sap.hana.datalake.files.committers.manifest.ManifestCommitter;
import com.sap.hana.datalake.files.enumeration.DirectAccessType;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory;
import org.apache.hadoop.fs.FileSystem;

import java.io.IOException;

public class HdlfsCommitterFactory extends PathOutputCommitterFactory {

  @Override
  public PathOutputCommitter createOutputCommitter(final Path outputPath, final TaskAttemptContext context) throws IOException {
    final DirectAccessType directAccessType = this.getHdlfsDirectAccessType(outputPath, context.getConfiguration());

    if (directAccessType == DirectAccessType.GCS || directAccessType == DirectAccessType.S3 || directAccessType == DirectAccessType.WASB) {
      return new ManifestCommitter(outputPath, context);
    } else {
      return new HdlfsOutputCommitter(outputPath, context);
    }
  }

  private DirectAccessType getHdlfsDirectAccessType(final Path outputPath, final Configuration conf) throws IOException {
    final FileSystem fs = outputPath.getFileSystem(conf);

    if (!(fs instanceof HdlfsFileSystem)) {
      throw new IllegalArgumentException("HdlfsCommitterFactory should only be used when HdlfsFileSystem is used");
    }

    final HdlfsFileSystem hdlfsFileSystem = (HdlfsFileSystem) fs;

    return hdlfsFileSystem.getDirectAccessType();
  }

}

// © 2022-2024 SAP SE or an SAP affiliate company. All rights reserved.