org.dinky.shaded.paimon.table.source.AbstractInnerTableScan Maven / Gradle / Ivy
The newest version!
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dinky.shaded.paimon.table.source;
import org.dinky.shaded.paimon.CoreOptions;
import org.dinky.shaded.paimon.CoreOptions.ChangelogProducer;
import org.dinky.shaded.paimon.annotation.VisibleForTesting;
import org.dinky.shaded.paimon.consumer.Consumer;
import org.dinky.shaded.paimon.consumer.ConsumerManager;
import org.dinky.shaded.paimon.data.BinaryRow;
import org.dinky.shaded.paimon.metrics.MetricRegistry;
import org.dinky.shaded.paimon.operation.FileStoreScan;
import org.dinky.shaded.paimon.table.source.snapshot.CompactedStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.ContinuousCompactorStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.ContinuousFromSnapshotFullStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.ContinuousFromSnapshotStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.ContinuousFromTimestampStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.ContinuousLatestStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.FullCompactedStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.FullStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.IncrementalStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.IncrementalTagStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.IncrementalTimeStampStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.SnapshotReader;
import org.dinky.shaded.paimon.table.source.snapshot.StartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.StaticFromSnapshotStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.StaticFromTagStartingScanner;
import org.dinky.shaded.paimon.table.source.snapshot.StaticFromTimestampStartingScanner;
import org.dinky.shaded.paimon.utils.Filter;
import org.dinky.shaded.paimon.utils.Pair;
import org.dinky.shaded.paimon.utils.SnapshotManager;
import java.util.List;
import java.util.Optional;
import static org.dinky.shaded.paimon.CoreOptions.FULL_COMPACTION_DELTA_COMMITS;
import static org.dinky.shaded.paimon.utils.Preconditions.checkArgument;
/** An abstraction layer above {@link FileStoreScan} to provide input split generation. */
public abstract class AbstractInnerTableScan implements InnerTableScan {
private final CoreOptions options;
protected final SnapshotReader snapshotReader;
protected AbstractInnerTableScan(CoreOptions options, SnapshotReader snapshotReader) {
this.options = options;
this.snapshotReader = snapshotReader;
}
@VisibleForTesting
public AbstractInnerTableScan withBucket(int bucket) {
snapshotReader.withBucket(bucket);
return this;
}
public AbstractInnerTableScan withBucketFilter(Filter bucketFilter) {
snapshotReader.withBucketFilter(bucketFilter);
return this;
}
public AbstractInnerTableScan withMetricsRegistry(MetricRegistry metricsRegistry) {
snapshotReader.withMetricRegistry(metricsRegistry);
return this;
}
public CoreOptions options() {
return options;
}
protected StartingScanner createStartingScanner(boolean isStreaming) {
SnapshotManager snapshotManager = snapshotReader.snapshotManager();
CoreOptions.StreamScanMode type =
options.toConfiguration().get(CoreOptions.STREAM_SCAN_MODE);
switch (type) {
case COMPACT_BUCKET_TABLE:
checkArgument(
isStreaming, "Set 'streaming-compact' in batch mode. This is unexpected.");
return new ContinuousCompactorStartingScanner(snapshotManager);
case COMPACT_APPEND_NO_BUCKET:
case FILE_MONITOR:
return new FullStartingScanner(snapshotManager);
}
// read from consumer id
String consumerId = options.consumerId();
if (consumerId != null) {
ConsumerManager consumerManager = snapshotReader.consumerManager();
Optional consumer = consumerManager.consumer(consumerId);
if (consumer.isPresent()) {
return new ContinuousFromSnapshotStartingScanner(
snapshotManager, consumer.get().nextSnapshot());
}
}
CoreOptions.StartupMode startupMode = options.startupMode();
switch (startupMode) {
case LATEST_FULL:
return new FullStartingScanner(snapshotManager);
case LATEST:
return isStreaming
? new ContinuousLatestStartingScanner(snapshotManager)
: new FullStartingScanner(snapshotManager);
case COMPACTED_FULL:
if (options.changelogProducer() == ChangelogProducer.FULL_COMPACTION
|| options.toConfiguration().contains(FULL_COMPACTION_DELTA_COMMITS)) {
int deltaCommits =
options.toConfiguration()
.getOptional(FULL_COMPACTION_DELTA_COMMITS)
.orElse(1);
return new FullCompactedStartingScanner(snapshotManager, deltaCommits);
} else {
return new CompactedStartingScanner(snapshotManager);
}
case FROM_TIMESTAMP:
Long startupMillis = options.scanTimestampMills();
return isStreaming
? new ContinuousFromTimestampStartingScanner(snapshotManager, startupMillis)
: new StaticFromTimestampStartingScanner(snapshotManager, startupMillis);
case FROM_SNAPSHOT:
if (options.scanSnapshotId() != null) {
return isStreaming
? new ContinuousFromSnapshotStartingScanner(
snapshotManager, options.scanSnapshotId())
: new StaticFromSnapshotStartingScanner(
snapshotManager, options.scanSnapshotId());
} else {
checkArgument(!isStreaming, "Cannot scan from tag in streaming mode.");
return new StaticFromTagStartingScanner(
snapshotManager, options().scanTagName());
}
case FROM_SNAPSHOT_FULL:
return isStreaming
? new ContinuousFromSnapshotFullStartingScanner(
snapshotManager, options.scanSnapshotId())
: new StaticFromSnapshotStartingScanner(
snapshotManager, options.scanSnapshotId());
case INCREMENTAL:
checkArgument(!isStreaming, "Cannot read incremental in streaming mode.");
Pair incrementalBetween = options.incrementalBetween();
CoreOptions.IncrementalBetweenScanMode scanType =
options.incrementalBetweenScanMode();
ScanMode scanMode;
switch (scanType) {
case DELTA:
scanMode = ScanMode.DELTA;
break;
case CHANGELOG:
scanMode = ScanMode.CHANGELOG;
break;
default:
throw new UnsupportedOperationException(
"Unknown incremental scan type " + scanType.name());
}
if (options.toMap().get(CoreOptions.INCREMENTAL_BETWEEN.key()) != null) {
try {
return new IncrementalStartingScanner(
snapshotManager,
Long.parseLong(incrementalBetween.getLeft()),
Long.parseLong(incrementalBetween.getRight()),
scanMode);
} catch (NumberFormatException e) {
return new IncrementalTagStartingScanner(
snapshotManager,
incrementalBetween.getLeft(),
incrementalBetween.getRight());
}
} else {
return new IncrementalTimeStampStartingScanner(
snapshotManager,
Long.parseLong(incrementalBetween.getLeft()),
Long.parseLong(incrementalBetween.getRight()),
scanMode);
}
default:
throw new UnsupportedOperationException(
"Unknown startup mode " + startupMode.name());
}
}
@Override
public List listPartitions() {
return snapshotReader.partitions();
}
}