org.apache.hudi.sync.common.HoodieSyncTool Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hudi-flink1.17-bundle Show documentation
The newest version!
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hudi.sync.common;

import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.HadoopConfigUtils;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.sync.common.metrics.HoodieMetaSyncMetrics;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;

import java.util.Properties;

import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_BASE_PATH;

/**
 * Base class to sync metadata with metastores to make
 * Hudi table queryable through external systems.
 */
public abstract class HoodieSyncTool implements AutoCloseable {

  protected Properties props;
  protected Configuration hadoopConf;
  protected HoodieMetaSyncMetrics metrics;

  public HoodieSyncTool(Properties props) {
    this(props, HadoopConfigUtils.createHadoopConf(props));
  }

  public HoodieSyncTool(Properties props, Configuration hadoopConf) {
    this.props = props;
    this.hadoopConf = hadoopConf;
    this.metrics = new HoodieMetaSyncMetrics(new HoodieSyncConfig(props, hadoopConf), getClass().getSimpleName());
  }

  @Deprecated
  public HoodieSyncTool(TypedProperties props, Configuration conf, FileSystem fs) {
    this(props, conf);
  }

  @Deprecated
  public HoodieSyncTool(Properties props, FileSystem fileSystem) {
    this(props, fileSystem.getConf());
  }

  public abstract void syncHoodieTable();

  @Override
  public void close() throws Exception {
    // no op
  }

  protected static HoodieTableMetaClient buildMetaClient(HoodieSyncConfig config) {
    return HoodieTableMetaClient.builder()
        .setConf(HadoopFSUtils.getStorageConfWithCopy(config.getHadoopConf()))
        .setBasePath(config.getString(META_SYNC_BASE_PATH))
        .setLoadActiveTimelineOnLoad(true)
        .build();
  }
}