io.delta.flink.source.internal.builder.ContinuousDeltaSourceBuilder Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of delta-flink Show documentation
delta-flink
There is a newer version: 3.2.1
package io.delta.flink.source.internal.builder;

import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

import io.delta.flink.source.internal.enumerator.ContinuousSplitEnumeratorProvider;
import io.delta.flink.source.internal.enumerator.supplier.ContinuousSnapshotSupplierFactory;
import org.apache.flink.core.fs.Path;
import org.apache.hadoop.conf.Configuration;
import static io.delta.flink.source.internal.DeltaSourceOptions.IGNORE_CHANGES;
import static io.delta.flink.source.internal.DeltaSourceOptions.IGNORE_DELETES;
import static io.delta.flink.source.internal.DeltaSourceOptions.PARQUET_BATCH_SIZE;
import static io.delta.flink.source.internal.DeltaSourceOptions.STARTING_TIMESTAMP;
import static io.delta.flink.source.internal.DeltaSourceOptions.STARTING_VERSION;
import static io.delta.flink.source.internal.DeltaSourceOptions.UPDATE_CHECK_INITIAL_DELAY;
import static io.delta.flink.source.internal.DeltaSourceOptions.UPDATE_CHECK_INTERVAL;

/**
 * A base class for Delta source builders that should create Delta source instance for {@link
 * org.apache.flink.api.connector.source.Boundedness#CONTINUOUS_UNBOUNDED} mode. This implementation
 * contains methods from {@link DeltaSourceBuilderBase} base class and methods applicable only for
 * Continuous mode.
 *
 * @param  Type of element produced by created source.
 * @param  This builder carries a SELF type to make it convenient to extend this for
 *               subclasses. Please, see {@link DeltaSourceBuilderBase} for details.
 */
public abstract class ContinuousDeltaSourceBuilder
    extends DeltaSourceBuilderBase {

    /**
     * The provider for {@link org.apache.flink.api.connector.source.SplitEnumerator} in {@link
     * org.apache.flink.api.connector.source.Boundedness#CONTINUOUS_UNBOUNDED} mode.
     */
    protected static final ContinuousSplitEnumeratorProvider
        DEFAULT_CONTINUOUS_SPLIT_ENUMERATOR_PROVIDER =
        new ContinuousSplitEnumeratorProvider(DEFAULT_SPLIT_ASSIGNER,
            DEFAULT_SPLITTABLE_FILE_ENUMERATOR);

    protected static final List APPLICABLE_OPTIONS = Collections.unmodifiableList(
        Arrays.asList(
            STARTING_VERSION.key(),
            STARTING_TIMESTAMP.key(),
            IGNORE_CHANGES.key(),
            IGNORE_DELETES.key(),
            UPDATE_CHECK_INTERVAL.key(),
            UPDATE_CHECK_INITIAL_DELAY.key(),
            PARQUET_BATCH_SIZE.key()
        )
    );

    public ContinuousDeltaSourceBuilder(
            Path tablePath,
            Configuration hadoopConfiguration,
            ContinuousSnapshotSupplierFactory snapshotSupplierFactory) {
        super(tablePath, hadoopConfiguration, snapshotSupplierFactory);
    }

    public SELF startingVersion(String startingVersion) {
        this.option(STARTING_VERSION.key(), startingVersion);
        return self();
    }

    public SELF startingVersion(long startingVersion) {
        this.option(STARTING_VERSION.key(), startingVersion);
        return self();
    }

    public SELF startingTimestamp(String startingTimestamp) {
        this.option(STARTING_TIMESTAMP.key(), startingTimestamp);
        return self();
    }

    public SELF updateCheckIntervalMillis(long updateCheckInterval) {
        this.option(UPDATE_CHECK_INTERVAL.key(), updateCheckInterval);
        return self();
    }

    public SELF ignoreDeletes(boolean ignoreDeletes) {
        this.option(IGNORE_DELETES.key(), ignoreDeletes);
        return self();
    }

    public SELF ignoreChanges(boolean ignoreChanges) {
        this.option(IGNORE_CHANGES.key(), ignoreChanges);
        return self();
    }

    @Override
    protected Validator validateOptionExclusions() {

        // mutually exclusive check for STARTING_VERSION and STARTING_TIMESTAMP in Streaming
        // mode.
        return new Validator()
            .checkArgument(
                !sourceConfiguration.hasOption(STARTING_TIMESTAMP)
                    || !sourceConfiguration.hasOption(STARTING_VERSION),
                prepareOptionExclusionMessage(STARTING_VERSION.key(), STARTING_TIMESTAMP.key()));
    }

    @Override
    protected Collection getApplicableOptions() {
        return APPLICABLE_OPTIONS;
    }
}