• Home
• com.sap.hana.datalake.files
• sap-hdlfs
• 2.0.43
• source code
• HdlfsRetryPolicies.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.sap.hana.datalake.files.HdlfsRetryPolicies Maven / Gradle / Ivy

// © 2022-2023 SAP SE or an SAP affiliate company. All rights reserved.
package com.sap.hana.datalake.files;

import com.sap.hana.datalake.files.exception.HdlfsOperationException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import com.sap.hana.datalake.files.shaded.org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;

import java.io.FileNotFoundException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;

public class HdlfsRetryPolicies {

  public final static int MERGE_RETRY_FILE_NOT_FOUND_LIMIT_DEFAULT = 50;
  public final static int MERGE_RETRY_FILE_NOT_FOUND_INTERVAL_MS_DEFAULT = 500;
  public final static int MIN_MERGE_RETRY_FILE_NOT_FOUND_LIMIT = 1;
  public final static int MIN_MERGE_RETRY_FILE_NOT_FOUND_INTERVAL_MS = 0;
  public final static int OPEN_RETRY_FILE_NOT_FOUND_LIMIT_DEFAULT = 50;
  public final static int OPEN_RETRY_FILE_NOT_FOUND_INTERVAL_MS_DEFAULT = 500;
  public final static int MIN_OPEN_RETRY_FILE_NOT_FOUND_LIMIT = 1;
  public final static int MIN_OPEN_RETRY_FILE_NOT_FOUND_INTERVAL_MS = 0;
  public final static int DELETE_TEMP_DIR_RETRY_DIR_NOT_EMPTY_LIMIT_DEFAULT = 50;
  public final static int DELETE_TEMP_DIR_RETRY_DIR_NOT_EMPTY_INTERVAL_MS_DEFAULT = 500;
  public final static int MIN_DELETE_TEMP_DIR_RETRY_DIR_NOT_EMPTY_LIMIT = 1;
  public final static int MIN_DELETE_TEMP_DIR_RETRY_DIR_NOT_EMPTY_INTERVAL_MS = 0;

  public static RetryPolicy createDefaultRetryPolicy(final Configuration conf) {
    final boolean retryPolicyEnabled = conf.getBoolean(HdlfsConstants.FS_HDLFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY, HdlfsConstants.FS_HDLFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_DEFAULT);
    final String retryPolicySpec = conf.get(HdlfsConstants.FS_HDLFS_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY, HdlfsConstants.FS_HDLFS_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT);

    final RetryPolicy retryByExceptionDefaultPolicy = RetryUtils.getDefaultRetryPolicy(conf, HdfsClientConfigKeys.HttpClient.RETRY_POLICY_ENABLED_KEY, retryPolicyEnabled,
        HdfsClientConfigKeys.HttpClient.RETRY_POLICY_SPEC_KEY, retryPolicySpec,
        /* remoteExceptionToRetry */ "org.apache.hadoop.hdfs.server.namenode.SafeModeException");

    return new HdlfsRetryPolicies.RetryByExceptionBuilder()
        .withDefaultPolicy(retryByExceptionDefaultPolicy)
        .withExceptionToPolicyMapping(AccessControlException.class, RetryPolicies.TRY_ONCE_THEN_FAIL)
        .build();
  }

  public static RetryPolicy createMergeFileNotFoundRetryPolicy(final Configuration conf) {
    final boolean fsCacheEnabled = conf.getBoolean(HdlfsConstants.FSCACHE_ENABLED, HdlfsConstants.FSCACHE_ENABLED_DEFAULT);
    final boolean mergeFileNotFoundRetryEnabled = conf.getBoolean(HdlfsConstants.CONFIG_MERGE_RETRY_FILE_NOT_FOUND_ENABLED_KEY, fsCacheEnabled);
    // Should be enabled by default in eventual consistency scenarios, so we set the default to be the same as FsCache

    if (mergeFileNotFoundRetryEnabled) {
      final RetryPolicy fileNotFoundPolicy = new FixedSleepPolicyBuilder(conf)
          .maxRetries(HdlfsConstants.CONFIG_MERGE_RETRY_FILE_NOT_FOUND_LIMIT_KEY, HdlfsRetryPolicies.MERGE_RETRY_FILE_NOT_FOUND_LIMIT_DEFAULT, HdlfsRetryPolicies.MIN_MERGE_RETRY_FILE_NOT_FOUND_LIMIT)
          .sleepTimeMs(HdlfsConstants.CONFIG_MERGE_RETRY_FILE_NOT_FOUND_INTERVAL_KEY, HdlfsRetryPolicies.MERGE_RETRY_FILE_NOT_FOUND_INTERVAL_MS_DEFAULT, HdlfsRetryPolicies.MIN_MERGE_RETRY_FILE_NOT_FOUND_INTERVAL_MS)
          .build();

      return new HdlfsRetryPolicies.RetryByExceptionBuilder()
              .withDefaultPolicy(RetryPolicies.TRY_ONCE_THEN_FAIL)
              .withExceptionToPolicyMapping(FileNotFoundException.class, fileNotFoundPolicy)
              .withExceptionProcessor(ex -> ex instanceof HdlfsOperationException ? (Exception) ex.getCause() : ex)
              .build();
    }

    return RetryPolicies.TRY_ONCE_THEN_FAIL;
  }

  public static RetryPolicy createOpenFileNotFoundRetryPolicy(final Configuration conf) {
    final boolean fsCacheEnabled = conf.getBoolean(HdlfsConstants.FSCACHE_ENABLED, HdlfsConstants.FSCACHE_ENABLED_DEFAULT);
    final boolean openFileNotFoundRetryEnabled = conf.getBoolean(HdlfsConstants.CONFIG_OPEN_RETRY_FILE_NOT_FOUND_ENABLED_KEY, fsCacheEnabled);
    // Should be enabled by default in eventual consistency scenarios, so we set the default to be the same as FsCache

    if (openFileNotFoundRetryEnabled) {
      final RetryPolicy fileNotFoundPolicy = new FixedSleepPolicyBuilder(conf)
          .maxRetries(HdlfsConstants.CONFIG_OPEN_RETRY_FILE_NOT_FOUND_LIMIT_KEY, HdlfsRetryPolicies.OPEN_RETRY_FILE_NOT_FOUND_LIMIT_DEFAULT, HdlfsRetryPolicies.MIN_OPEN_RETRY_FILE_NOT_FOUND_LIMIT)
          .sleepTimeMs(HdlfsConstants.CONFIG_OPEN_RETRY_FILE_NOT_FOUND_INTERVAL_KEY, HdlfsRetryPolicies.OPEN_RETRY_FILE_NOT_FOUND_INTERVAL_MS_DEFAULT, HdlfsRetryPolicies.MIN_OPEN_RETRY_FILE_NOT_FOUND_INTERVAL_MS)
          .build();

      return new HdlfsRetryPolicies.RetryByExceptionBuilder()
          .withDefaultPolicy(RetryPolicies.TRY_ONCE_THEN_FAIL)
          .withExceptionToPolicyMapping(FileNotFoundException.class, fileNotFoundPolicy)
          .build();
    }

    return RetryPolicies.TRY_ONCE_THEN_FAIL;
  }

  public static RetryPolicy createMPUDeleteTempDirRetryPolicy(final Configuration conf) {
    final boolean createDeleteTempDirRetryEnabled = conf.getBoolean(HdlfsConstants.CONFIG_DELETE_MPU_TEMP_DIR_RETRY_DIR_NOT_EMPTY_ENABLED_KEY, true);

    if (createDeleteTempDirRetryEnabled) {
      return new FixedSleepPolicyBuilder(conf)
          .maxRetries(HdlfsConstants.CONFIG_DELETE_MPU_TEMP_DIR_RETRY_DIR_NOT_EMPTY_LIMIT_KEY, HdlfsRetryPolicies.DELETE_TEMP_DIR_RETRY_DIR_NOT_EMPTY_LIMIT_DEFAULT, HdlfsRetryPolicies.MIN_DELETE_TEMP_DIR_RETRY_DIR_NOT_EMPTY_LIMIT)
          .sleepTimeMs(HdlfsConstants.CONFIG_DELETE_MPU_TEMP_DIR_RETRY_DIR_NOT_EMPTY_INTERVAL_KEY, HdlfsRetryPolicies.DELETE_TEMP_DIR_RETRY_DIR_NOT_EMPTY_INTERVAL_MS_DEFAULT, HdlfsRetryPolicies.MIN_DELETE_TEMP_DIR_RETRY_DIR_NOT_EMPTY_INTERVAL_MS)
          .build();
    }

    return RetryPolicies.TRY_ONCE_THEN_FAIL;
  }

  private HdlfsRetryPolicies() {
    // no-op
  }

  public static class RetryByExceptionBuilder {

    private final Map, RetryPolicy> exceptionToPolicyMap = new HashMap<>();

    private Function exceptionProcessor;
    private RetryPolicy defaultRetryPolicy;

    public RetryByExceptionBuilder() {
      // no-op
    }

    public RetryByExceptionBuilder withDefaultPolicy(final RetryPolicy defaultRetryPolicy) {
      this.defaultRetryPolicy = defaultRetryPolicy;

      return this;
    }

    public RetryByExceptionBuilder withExceptionToPolicyMapping(final Class exception, final RetryPolicy retryPolicy) {
      this.exceptionToPolicyMap.put(exception, retryPolicy);

      return this;
    }

    public RetryByExceptionBuilder withExceptionToPolicyMap(Map, RetryPolicy> exceptionToPolicyMap) {
      exceptionToPolicyMap.keySet().forEach(key -> this.exceptionToPolicyMap.put(key, exceptionToPolicyMap.get(key)));

      return this;
    }

    public RetryByExceptionBuilder withExceptionProcessor(final Function exceptionProcessor) {
      this.exceptionProcessor = exceptionProcessor;

      return this;
    }

    public RetryPolicy build() {
      this.defaultRetryPolicy = this.defaultRetryPolicy == null ? RetryPolicies.TRY_ONCE_THEN_FAIL : this.defaultRetryPolicy;
      this.exceptionProcessor = this.exceptionProcessor == null ? ex -> ex : this.exceptionProcessor;

      return RetryPolicies.retryByException(this.defaultRetryPolicy, this.exceptionToPolicyMap, this.exceptionProcessor);
    }
  }

  public static class FixedSleepPolicyBuilder {

    private final Configuration conf;

    private Integer maxRetries;
    private Integer sleepTimeMs;

    public FixedSleepPolicyBuilder(final Configuration conf) {
      this.conf = conf;
    }

    public FixedSleepPolicyBuilder maxRetries(final String maxRetriesKey, final int maxRetriesDefault, final int minRetries) {
      this.maxRetries = this.conf.getInt(maxRetriesKey, maxRetriesDefault);

      Preconditions.checkArgument(this.maxRetries >= minRetries,
          String.format("Config [%s] too small: [%d]; must be at least [%d]", maxRetriesKey, this.maxRetries, minRetries));

      return this;
    }

    public FixedSleepPolicyBuilder sleepTimeMs(final String sleepTimeKey, final int sleepTimeMsDefault, final int minSleepTimeMs) {
      this.sleepTimeMs = this.conf.getInt(sleepTimeKey, sleepTimeMsDefault);

      Preconditions.checkArgument(this.sleepTimeMs >= minSleepTimeMs,
          String.format("Config [%s] too small: [%d]; must be at least [%d]", sleepTimeKey, this.sleepTimeMs, minSleepTimeMs));

      return this;
    }

    public RetryPolicy build() {
      Preconditions.checkArgument(this.maxRetries != null, "maxRetries must not be null");
      Preconditions.checkArgument(this.sleepTimeMs != null, "sleepTimeMs must not be null");

      return RetryPolicies.retryUpToMaximumCountWithFixedSleep(this.maxRetries, this.sleepTimeMs, TimeUnit.MILLISECONDS);
    }
  }
}

// © 2022-2023 SAP SE or an SAP affiliate company. All rights reserved.