org.apache.iceberg.flink.source.StreamingReaderOperator Maven / Gradle / Ivy
Show all versions of iceberg-flink-1.18 Show documentation
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iceberg.flink.source;
import java.io.IOException;
import java.util.Queue;
import org.apache.flink.api.common.operators.MailboxExecutor;
import org.apache.flink.api.common.state.ListState;
import org.apache.flink.api.common.state.ListStateDescriptor;
import org.apache.flink.runtime.state.JavaSerializer;
import org.apache.flink.runtime.state.StateInitializationContext;
import org.apache.flink.runtime.state.StateSnapshotContext;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory;
import org.apache.flink.streaming.api.operators.StreamOperator;
import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
import org.apache.flink.streaming.api.operators.StreamSourceContexts;
import org.apache.flink.streaming.api.operators.YieldingOperatorFactory;
import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
import org.apache.flink.table.data.RowData;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The operator that reads the {@link FlinkInputSplit splits} received from the preceding {@link
* StreamingMonitorFunction}. Contrary to the {@link StreamingMonitorFunction} which has a
* parallelism of 1, this operator can have multiple parallelism.
*
* As soon as a split descriptor is received, it is put in a queue, and use {@link
* MailboxExecutor} read the actual data of the split. This architecture allows the separation of
* the reading thread from the one split processing the checkpoint barriers, thus removing any
* potential back-pressure.
*/
public class StreamingReaderOperator extends AbstractStreamOperator
implements OneInputStreamOperator {
private static final Logger LOG = LoggerFactory.getLogger(StreamingReaderOperator.class);
// It's the same thread that is running this operator and checkpoint actions. we use this executor
// to schedule only
// one split for future reading, so that a new checkpoint could be triggered without blocking long
// time for exhausting
// all scheduled splits.
private final MailboxExecutor executor;
private FlinkInputFormat format;
private transient SourceFunction.SourceContext sourceContext;
private transient ListState inputSplitsState;
private transient Queue splits;
// Splits are read by the same thread that calls processElement. Each read task is submitted to
// that thread by adding
// them to the executor. This state is used to ensure that only one read task is in that queue at
// a time, so that read
// tasks do not accumulate ahead of checkpoint tasks. When there is a read task in the queue, this
// is set to RUNNING.
// When there are no more files to read, this will be set to IDLE.
private transient SplitState currentSplitState;
private StreamingReaderOperator(
FlinkInputFormat format, ProcessingTimeService timeService, MailboxExecutor mailboxExecutor) {
this.format = Preconditions.checkNotNull(format, "The InputFormat should not be null.");
this.processingTimeService = timeService;
this.executor =
Preconditions.checkNotNull(mailboxExecutor, "The mailboxExecutor should not be null.");
}
@Override
public void initializeState(StateInitializationContext context) throws Exception {
super.initializeState(context);
// TODO Replace Java serialization with Avro approach to keep state compatibility.
// See issue: https://github.com/apache/iceberg/issues/1698
inputSplitsState =
context
.getOperatorStateStore()
.getListState(new ListStateDescriptor<>("splits", new JavaSerializer<>()));
// Initialize the current split state to IDLE.
currentSplitState = SplitState.IDLE;
// Recover splits state from flink state backend if possible.
splits = Lists.newLinkedList();
if (context.isRestored()) {
int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();
LOG.info("Restoring state for the {} (taskIdx: {}).", getClass().getSimpleName(), subtaskIdx);
for (FlinkInputSplit split : inputSplitsState.get()) {
splits.add(split);
}
}
this.sourceContext =
StreamSourceContexts.getSourceContext(
getOperatorConfig().getTimeCharacteristic(),
getProcessingTimeService(),
new Object(), // no actual locking needed
output,
getRuntimeContext().getExecutionConfig().getAutoWatermarkInterval(),
-1,
true);
// Enqueue to process the recovered input splits.
enqueueProcessSplits();
}
@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
super.snapshotState(context);
inputSplitsState.clear();
inputSplitsState.addAll(Lists.newArrayList(splits));
}
@Override
public void processElement(StreamRecord element) {
splits.add(element.getValue());
enqueueProcessSplits();
}
private void enqueueProcessSplits() {
if (currentSplitState == SplitState.IDLE && !splits.isEmpty()) {
currentSplitState = SplitState.RUNNING;
executor.execute(this::processSplits, this.getClass().getSimpleName());
}
}
private void processSplits() throws IOException {
FlinkInputSplit split = splits.poll();
if (split == null) {
currentSplitState = SplitState.IDLE;
return;
}
format.open(split);
try {
RowData nextElement = null;
while (!format.reachedEnd()) {
nextElement = format.nextRecord(nextElement);
sourceContext.collect(nextElement);
}
} finally {
currentSplitState = SplitState.IDLE;
format.close();
}
// Re-schedule to process the next split.
enqueueProcessSplits();
}
@Override
public void processWatermark(Watermark mark) {
// we do nothing because we emit our own watermarks if needed.
}
@Override
public void close() throws Exception {
super.close();
if (format != null) {
format.close();
format.closeInputFormat();
format = null;
}
sourceContext = null;
}
@Override
public void finish() throws Exception {
super.finish();
output.close();
if (sourceContext != null) {
sourceContext.emitWatermark(Watermark.MAX_WATERMARK);
sourceContext.close();
sourceContext = null;
}
}
static OneInputStreamOperatorFactory factory(FlinkInputFormat format) {
return new OperatorFactory(format);
}
private enum SplitState {
IDLE,
RUNNING
}
private static class OperatorFactory extends AbstractStreamOperatorFactory
implements YieldingOperatorFactory,
OneInputStreamOperatorFactory {
private final FlinkInputFormat format;
private transient MailboxExecutor mailboxExecutor;
private OperatorFactory(FlinkInputFormat format) {
this.format = format;
}
@Override
public void setMailboxExecutor(MailboxExecutor mailboxExecutor) {
this.mailboxExecutor = mailboxExecutor;
}
@SuppressWarnings("unchecked")
@Override
public > O createStreamOperator(
StreamOperatorParameters parameters) {
StreamingReaderOperator operator =
new StreamingReaderOperator(format, processingTimeService, mailboxExecutor);
operator.setup(
parameters.getContainingTask(), parameters.getStreamConfig(), parameters.getOutput());
return (O) operator;
}
@Override
public Class getStreamOperatorClass(ClassLoader classLoader) {
return StreamingReaderOperator.class;
}
}
}