All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.hudi.common.util.ConfigUtils Maven / Gradle / Ivy

Go to download

The newest version!
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hudi.common.util;

import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.config.PropertiesConfig;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.model.HoodiePayloadProps;
import org.apache.hudi.common.model.RecordPayloadType;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieNotSupportedException;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;

import static org.apache.hudi.common.config.HoodieReaderConfig.USE_NATIVE_HFILE_READER;
import static org.apache.hudi.common.table.HoodieTableConfig.TABLE_CHECKSUM;

public class ConfigUtils {
  public static final String STREAMER_CONFIG_PREFIX = "hoodie.streamer.";
  @Deprecated
  public static final String DELTA_STREAMER_CONFIG_PREFIX = "hoodie.deltastreamer.";
  public static final String SCHEMAPROVIDER_CONFIG_PREFIX = STREAMER_CONFIG_PREFIX + "schemaprovider.";
  @Deprecated
  public static final String OLD_SCHEMAPROVIDER_CONFIG_PREFIX = DELTA_STREAMER_CONFIG_PREFIX + "schemaprovider.";
  /**
   * Config stored in hive serde properties to tell query engine (spark/flink) to
   * read the table as a read-optimized table when this config is true.
   */
  public static final String IS_QUERY_AS_RO_TABLE = "hoodie.query.as.ro.table";

  /**
   * Config stored in hive serde properties to tell query engine (spark) the
   * location to read.
   */
  public static final String TABLE_SERDE_PATH = "path";

  public static final HoodieConfig DEFAULT_HUDI_CONFIG_FOR_READER = new HoodieConfig();

  private static final Logger LOG = LoggerFactory.getLogger(ConfigUtils.class);

  /**
   * Get ordering field.
   */
  public static String getOrderingField(Properties properties) {
    String orderField = null;
    if (properties.containsKey(HoodiePayloadProps.PAYLOAD_ORDERING_FIELD_PROP_KEY)) {
      orderField = properties.getProperty(HoodiePayloadProps.PAYLOAD_ORDERING_FIELD_PROP_KEY);
    } else if (properties.containsKey("hoodie.datasource.write.precombine.field")) {
      orderField = properties.getProperty("hoodie.datasource.write.precombine.field");
    } else if (properties.containsKey(HoodieTableConfig.PRECOMBINE_FIELD.key())) {
      orderField = properties.getProperty(HoodieTableConfig.PRECOMBINE_FIELD.key());
    }
    return orderField;
  }

  /**
   * Get payload class.
   */
  public static String getPayloadClass(Properties properties) {
    return RecordPayloadType.getPayloadClassName(new HoodieConfig(properties));
  }

  public static List split2List(String param) {
    return Arrays.stream(param.split(","))
        .map(String::trim).distinct().collect(Collectors.toList());
  }

  /**
   * Convert the key-value config to a map.  The format of the config
   * is a key-value pair just like "k1=v1\nk2=v2\nk3=v3".
   *
   * @param keyValueConfig Key-value configs in properties format, i.e., multiple lines of
   *                       `key=value`.
   * @return A {@link Map} of key-value configs.
   */
  public static Map toMap(String keyValueConfig) {
    return toMap(keyValueConfig, "\n");
  }

  /**
   * Convert the key-value config to a map. The format of the config is a key-value pair
   * with defined separator.  For example, if the separator is a comma, the input is
   * "k1=v1,k2=v2,k3=v3".
   *
   * @param keyValueConfig key-value configs in properties format, with defined separator.
   * @param separator      the separator.
   * @return A {@link Map} of key-value configs.
   */
  public static Map toMap(String keyValueConfig, String separator) {
    if (StringUtils.isNullOrEmpty(keyValueConfig)) {
      return new HashMap<>();
    }
    String[] keyvalues = keyValueConfig.split(separator);
    Map tableProperties = new HashMap<>();
    for (String keyValue : keyvalues) {
      // Handle multiple new lines and lines that contain only spaces after splitting
      if (keyValue.trim().isEmpty()) {
        continue;
      }
      String[] keyValueArray = keyValue.split("=");
      if (keyValueArray.length == 1 || keyValueArray.length == 2) {
        String key = keyValueArray[0].trim();
        String value = keyValueArray.length == 2 ? keyValueArray[1].trim() : "";
        tableProperties.put(key, value);
      } else {
        throw new IllegalArgumentException("Bad key-value config: " + keyValue + ", must be the"
            + " format 'key = value'");
      }
    }
    return tableProperties;
  }

  /**
   * Convert map config to key-value string.The format of the config
   * is a key-value pair just like "k1=v1\nk2=v2\nk3=v3".
   *
   * @param config A {@link Map} of key-value configs.
   * @return Key-value configs in properties format, i.e., multiple lines of `key=value`.
   */
  public static String configToString(Map config) {
    if (config == null) {
      return null;
    }
    StringBuilder sb = new StringBuilder();
    for (Map.Entry entry : config.entrySet()) {
      if (sb.length() > 0) {
        sb.append("\n");
      }
      sb.append(entry.getKey()).append("=").append(entry.getValue());
    }
    return sb.toString();
  }

  /**
   * Case-insensitive resolution of input enum name to the enum type
   */
  public static > T resolveEnum(Class enumType,
                                                  String name) {
    T[] enumConstants = enumType.getEnumConstants();
    for (T constant : enumConstants) {
      if (constant.name().equalsIgnoreCase(name)) {
        return constant;
      }
    }

    throw new IllegalArgumentException("No enum constant found " + enumType.getName() + "." + name);
  }

  public static > String[] enumNames(Class enumType) {
    T[] enumConstants = enumType.getEnumConstants();
    return Arrays.stream(enumConstants).map(Enum::name).toArray(String[]::new);
  }

  /**
   * Strips the prefix from a config key. The prefix is defined by a {@link ConfigProperty}
   * which can have alternatives.  The method strips any matching prefix.
   *
   * @param prop         The config key for stripping
   * @param prefixConfig The prefix.
   * @return An {@link Option} of the config key after stripping, if any prefix matches the key;
   * empty {@link Option} otherwise.
   */
  public static Option stripPrefix(String prop, ConfigProperty prefixConfig) {
    if (prop.startsWith(prefixConfig.key())) {
      return Option.of(String.join("", prop.split(prefixConfig.key())));
    }
    for (String altPrefix : prefixConfig.getAlternatives()) {
      if (prop.startsWith(altPrefix)) {
        return Option.of(String.join("", prop.split(altPrefix)));
      }
    }
    return Option.empty();
  }

  /**
   * Whether the properties contain a config. If any of the key or alternative keys of the
   * {@link ConfigProperty} exists in the properties, this method returns {@code true}.
   *
   * @param props          Configs in {@link TypedProperties}
   * @param configProperty Config to look up.
   * @return {@code true} if exists; {@code false} otherwise.
   */
  public static boolean containsConfigProperty(TypedProperties props,
                                               ConfigProperty configProperty) {
    if (!props.containsKey(configProperty.key())) {
      for (String alternative : configProperty.getAlternatives()) {
        if (props.containsKey(alternative)) {
          return true;
        }
      }
      return false;
    }
    return true;
  }

  /**
   * Whether the properties contain a config. If any of the key or alternative keys of the
   * {@link ConfigProperty} exists in the properties, this method returns {@code true}.
   *
   * @param props          Configs in {@link Map}
   * @param configProperty Config to look up.
   * @return {@code true} if exists; {@code false} otherwise.
   */
  public static boolean containsConfigProperty(Map props,
                                               ConfigProperty configProperty) {
    if (!props.containsKey(configProperty.key())) {
      for (String alternative : configProperty.getAlternatives()) {
        if (props.containsKey(alternative)) {
          return true;
        }
      }
      return false;
    }
    return true;
  }

  /**
   * Validates that config String keys exist in the properties.
   *
   * @param props          Configs in {@link TypedProperties} to validate.
   * @param checkPropNames List of String keys that must exist.
   */
  public static void checkRequiredProperties(TypedProperties props, List checkPropNames) {
    checkPropNames.forEach(prop -> {
      if (!props.containsKey(prop)) {
        throw new HoodieNotSupportedException("Required property " + prop + " is missing");
      }
    });
  }

  /**
   * Validates that all {@link ConfigProperty} configs exist in the properties. For each
   * {@link ConfigProperty} config, if any of the key or alternative keys of the
   * {@link ConfigProperty} exists in the properties, the validation of this config passes.
   *
   * @param props              Configs in {@link TypedProperties} to validate.
   * @param configPropertyList List of {@link ConfigProperty} configs that must exist.
   */
  public static void checkRequiredConfigProperties(TypedProperties props,
                                                   List> configPropertyList) {
    configPropertyList.forEach(configProperty -> {
      if (!containsConfigProperty(props, configProperty)) {
        throw new HoodieNotSupportedException("Required property " + configProperty.key() + " is missing");
      }
    });
  }

  /**
   * Gets the raw value for a {@link ConfigProperty} config from properties. The key and
   * alternative keys are used to fetch the config.
   *
   * @param props          Configs in {@link Properties}.
   * @param configProperty {@link ConfigProperty} config to fetch.
   * @return {@link Option} of value if the config exists; empty {@link Option} otherwise.
   */
  public static Option getRawValueWithAltKeys(Properties props,
                                                      ConfigProperty configProperty) {
    if (props.containsKey(configProperty.key())) {
      return Option.ofNullable(props.get(configProperty.key()));
    }
    for (String alternative : configProperty.getAlternatives()) {
      if (props.containsKey(alternative)) {
        LOG.warn(String.format("The configuration key '%s' has been deprecated "
                + "and may be removed in the future. Please use the new key '%s' instead.",
            alternative, configProperty.key()));
        return Option.ofNullable(props.get(alternative));
      }
    }
    return Option.empty();
  }

  /**
   * Gets the String value for a {@link ConfigProperty} config from properties. The key and
   * alternative keys are used to fetch the config. If the config is not found, an
   * {@link IllegalArgumentException} is thrown.
   *
   * @param props          Configs in {@link Properties}.
   * @param configProperty {@link ConfigProperty} config of String type to fetch.
   * @return String value if the config exists.
   */
  public static String getStringWithAltKeys(Properties props,
                                            ConfigProperty configProperty) {
    return getStringWithAltKeys(props, configProperty, false);
  }

  /**
   * Gets the String value for a {@link ConfigProperty} config from properties. The key and
   * alternative keys are used to fetch the config. If using default value, the default value
   * of {@link ConfigProperty} config, if exists, is returned if the config is not found in
   * the properties. If not using default value, if the config is not found, an
   * {@link IllegalArgumentException} is thrown.
   *
   * @param props           Configs in {@link Properties}.
   * @param configProperty  {@link ConfigProperty} config of String type to fetch.
   * @param useDefaultValue Whether to use default value from {@link ConfigProperty}.
   * @return String value if the config exists; otherwise, if the config does not exist and
   * {@code useDefaultValue} is true, returns default String value if there is default value
   * defined in the {@link ConfigProperty} config and {@code null} otherwise.
   */
  public static String getStringWithAltKeys(Properties props,
                                            ConfigProperty configProperty,
                                            boolean useDefaultValue) {
    if (useDefaultValue) {
      return getStringWithAltKeys(
          props, configProperty, configProperty.hasDefaultValue() ? configProperty.defaultValue() : null);
    }
    Option rawValue = getRawValueWithAltKeys(props, configProperty);
    if (!rawValue.isPresent()) {
      throw new IllegalArgumentException("Property " + configProperty.key() + " not found");
    }
    return rawValue.get().toString();
  }

  /**
   * Gets the String value for a {@link ConfigProperty} config from properties. The key and
   * alternative keys are used to fetch the config. The default value as the input of the method
   * is returned if the config is not found in the properties.
   *
   * @param props          Configs in {@link Properties}.
   * @param configProperty {@link ConfigProperty} config of String type to fetch.
   * @param defaultValue   Default value.
   * @return String value if the config exists; default value otherwise.
   */
  public static String getStringWithAltKeys(Properties props,
                                            ConfigProperty configProperty,
                                            String defaultValue) {
    Option rawValue = getRawValueWithAltKeys(props, configProperty);
    return rawValue.map(Object::toString).orElse(defaultValue);
  }

  /**
   * Gets the String value for a {@link ConfigProperty} config from a {@link Map}. The key
   * and alternative keys are used to fetch the config. The default value of {@link ConfigProperty}
   * config, if exists, is returned if the config is not found in the properties.
   *
   * @param props          Configs in {@link Map}.
   * @param configProperty {@link ConfigProperty} config of String type to fetch.
   * @return String value if the config exists; default String value if the config does not exist
   * and there is default value defined in the {@link ConfigProperty} config; {@code null} otherwise.
   */
  public static String getStringWithAltKeys(Map props,
                                            ConfigProperty configProperty) {
    Object value = props.get(configProperty.key());
    if (value != null) {
      return value.toString();
    }
    for (String alternative : configProperty.getAlternatives()) {
      value = props.get(alternative);
      if (value != null) {
        LOG.warn(String.format("The configuration key '%s' has been deprecated "
                + "and may be removed in the future. Please use the new key '%s' instead.",
            alternative, configProperty.key()));
        return value.toString();
      }
    }
    return configProperty.hasDefaultValue() ? configProperty.defaultValue() : null;
  }

  /**
   * Gets String value from properties with alternative keys.
   *
   * @param props        Configs in {@link TypedProperties}.
   * @param key          String key.
   * @param altKey       Alternative String key.
   * @param defaultValue Default String value.
   * @return String value if the config exists; default value otherwise.
   */
  public static String getStringWithAltKeys(TypedProperties props,
                                            String key, String altKey,
                                            String defaultValue) {
    if (props.containsKey(altKey)) {
      return props.getString(altKey);
    }
    if (props.containsKey(key)) {
      return props.getString(key);
    }
    return defaultValue;
  }

  /**
   * Gets the boolean value for a {@link ConfigProperty} config from properties. The key and
   * alternative keys are used to fetch the config. The default value of {@link ConfigProperty}
   * config, if exists, is returned if the config is not found in the properties.
   *
   * @param props          Configs in {@link Properties}.
   * @param configProperty {@link ConfigProperty} config to fetch.
   * @return boolean value if the config exists; default boolean value if the config does not exist
   * and there is default value defined in the {@link ConfigProperty} config; {@code false} otherwise.
   */
  public static boolean getBooleanWithAltKeys(Properties props,
                                              ConfigProperty configProperty) {
    Option rawValue = getRawValueWithAltKeys(props, configProperty);
    boolean defaultValue = configProperty.hasDefaultValue()
        ? Boolean.parseBoolean(configProperty.defaultValue().toString()) : false;
    return rawValue.map(v -> Boolean.parseBoolean(v.toString())).orElse(defaultValue);
  }

  /**
   * Gets the integer value for a {@link ConfigProperty} config from properties. The key and
   * alternative keys are used to fetch the config. The default value of {@link ConfigProperty}
   * config, if exists, is returned if the config is not found in the properties.
   *
   * @param props          Configs in {@link TypedProperties}.
   * @param configProperty {@link ConfigProperty} config to fetch.
   * @return integer value if the config exists; default integer value if the config does not exist
   * and there is default value defined in the {@link ConfigProperty} config; {@code 0} otherwise.
   */
  public static int getIntWithAltKeys(TypedProperties props,
                                      ConfigProperty configProperty) {
    Option rawValue = getRawValueWithAltKeys(props, configProperty);
    int defaultValue = configProperty.hasDefaultValue()
        ? Integer.parseInt(configProperty.defaultValue().toString()) : 0;
    return rawValue.map(v -> Integer.parseInt(v.toString())).orElse(defaultValue);
  }

  /**
   * Gets the long value for a {@link ConfigProperty} config from properties. The key and
   * alternative keys are used to fetch the config. The default value of {@link ConfigProperty}
   * config, if exists, is returned if the config is not found in the properties.
   *
   * @param props          Configs in {@link TypedProperties}.
   * @param configProperty {@link ConfigProperty} config to fetch.
   * @return long value if the config exists; default long value if the config does not exist
   * and there is default value defined in the {@link ConfigProperty} config; {@code 0} otherwise.
   */
  public static long getLongWithAltKeys(TypedProperties props,
                                        ConfigProperty configProperty) {
    Option rawValue = getRawValueWithAltKeys(props, configProperty);
    long defaultValue = configProperty.hasDefaultValue()
        ? configProperty.defaultValue() : 0L;
    return rawValue.map(v -> Long.parseLong(v.toString())).orElse(defaultValue);
  }

  /**
   * Removes a {@link ConfigProperty} config from properties. This removes all possible keys
   * including the alternatives from the properties.
   *
   * @param props          Configs in {@link TypedProperties}.
   * @param configProperty {@link ConfigProperty} config to remove.
   */
  public static void removeConfigFromProps(TypedProperties props,
                                           ConfigProperty configProperty) {
    props.remove(configProperty.key());
    for (String alternative : configProperty.getAlternatives()) {
      props.remove(alternative);
    }
  }

  /**
   * Returns filtered properties based on the given {@link ConfigProperty} config list to keep.
   *
   * @param props              Configs in {@link TypedProperties}.
   * @param configPropertyList List of {@link ConfigProperty} configs to keep.
   * @return Filtered configs in {@link Map} with {@link ConfigProperty} configs to keep.
   */
  public static Map filterProperties(TypedProperties props,
                                                     List> configPropertyList) {
    Set persistedKeys = getAllConfigKeys(configPropertyList);
    return props.entrySet().stream()
        .filter(p -> persistedKeys.contains(String.valueOf(p.getKey())))
        .collect(Collectors.toMap(e -> String.valueOf(e.getKey()), e -> String.valueOf(e.getValue())));
  }

  /**
   * @param configPropertyList List of {@link ConfigProperty} configs to keep.
   * @return all String keys of {@link ConfigProperty} configs.
   */
  public static Set getAllConfigKeys(List> configPropertyList) {
    return configPropertyList.stream().flatMap(configProperty -> {
      List keys = new ArrayList<>();
      keys.add(configProperty.key());
      keys.addAll(configProperty.getAlternatives());
      return keys.stream();
    }).collect(Collectors.toSet());
  }

  public static TypedProperties fetchConfigs(
      HoodieStorage storage,
      StoragePath metaPath,
      String propertiesFile,
      String propertiesBackupFile,
      int maxReadRetries,
      int maxReadRetryDelayInMs) throws IOException {
    StoragePath cfgPath = new StoragePath(metaPath, propertiesFile);
    StoragePath backupCfgPath = new StoragePath(metaPath, propertiesBackupFile);
    int readRetryCount = 0;
    boolean found = false;

    TypedProperties props = new TypedProperties();
    while (readRetryCount++ < maxReadRetries) {
      for (StoragePath path : Arrays.asList(cfgPath, backupCfgPath)) {
        // Read the properties and validate that it is a valid file
        try (InputStream is = storage.open(path)) {
          props.clear();
          props.load(is);
          found = true;
          if (props.containsKey(TABLE_CHECKSUM.key())) {
            ValidationUtils.checkArgument(HoodieTableConfig.validateChecksum(props));
          }
          return props;
        } catch (IOException e) {
          LOG.warn(String.format("Could not read properties from %s: %s", path, e));
        } catch (IllegalArgumentException e) {
          LOG.warn(String.format("Invalid properties file %s: %s", path, props));
        }
      }

      // Failed to read all files so wait before retrying. This can happen in cases of parallel updates to the properties.
      try {
        Thread.sleep(maxReadRetryDelayInMs);
      } catch (InterruptedException e) {
        LOG.warn("Interrupted while waiting");
      }
    }

    // If we are here then after all retries either no properties file was found or only an invalid file was found.
    if (found) {
      throw new IllegalArgumentException(
          "hoodie.properties file seems invalid. Please check for left over `.updated` files if any, manually copy it to hoodie.properties and retry");
    } else {
      throw new HoodieIOException("Could not load Hoodie properties from " + cfgPath);
    }
  }

  public static void recoverIfNeeded(HoodieStorage storage, StoragePath cfgPath,
                                     StoragePath backupCfgPath) throws IOException {
    if (!storage.exists(cfgPath)) {
      // copy over from backup
      try (InputStream in = storage.open(backupCfgPath);
           OutputStream out = storage.create(cfgPath, false)) {
        FileIOUtils.copy(in, out);
      }
    }
    // regardless, we don't need the backup anymore.
    storage.deleteFile(backupCfgPath);
  }

  public static void upsertProperties(Properties current, Properties updated) {
    updated.forEach((k, v) -> current.setProperty(k.toString(), v.toString()));
  }

  public static void deleteProperties(Properties current, Properties deleted) {
    deleted.forEach((k, v) -> current.remove(k.toString()));
  }

  public static HoodieConfig getReaderConfigs(StorageConfiguration storageConf) {
    HoodieConfig config = new HoodieConfig();
    config.setAll(DEFAULT_HUDI_CONFIG_FOR_READER.getProps());
    config.setValue(USE_NATIVE_HFILE_READER,
        Boolean.toString(storageConf.getBoolean(USE_NATIVE_HFILE_READER.key(), USE_NATIVE_HFILE_READER.defaultValue())));
    return config;
  }

  public static TypedProperties loadGlobalProperties() {
    return ((PropertiesConfig) ReflectionUtils.loadClass("org.apache.hudi.common.config.DFSPropertiesConfiguration")).getGlobalProperties();
  }
}
    

    

    

            

    

            



    
    






    
© 2015 - 2024 Weber Informatics LLC | Privacy Policy