org.apache.beam.sdk.io.kafka.KafkaCheckpointMark Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of beam-sdks-java-io-kafka Show documentation
Show all versions of beam-sdks-java-io-kafka Show documentation
Library to read Kafka topics.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.beam.sdk.io.kafka;
import com.google.common.base.Joiner;
import java.io.IOException;
import java.io.Serializable;
import java.util.List;
import org.apache.beam.sdk.coders.AvroCoder;
import org.apache.beam.sdk.coders.DefaultCoder;
import org.apache.beam.sdk.io.UnboundedSource;
/**
* Checkpoint for an unbounded KafkaIO.Read. Consists of Kafka topic name, partition id,
* and the latest offset consumed so far.
*/
@DefaultCoder(AvroCoder.class)
public class KafkaCheckpointMark implements UnboundedSource.CheckpointMark {
private List partitions;
private KafkaCheckpointMark() {} // for Avro
public KafkaCheckpointMark(List partitions) {
this.partitions = partitions;
}
public List getPartitions() {
return partitions;
}
@Override
public void finalizeCheckpoint() throws IOException {
/* nothing to do */
// We might want to support committing offset in Kafka for better resume point when the job
// is restarted (checkpoint is not available for job restarts).
}
@Override
public String toString() {
return "KafkaCheckpointMark{partitions=" + Joiner.on(",").join(partitions) + '}';
}
/**
* A tuple to hold topic, partition, and offset that comprise the checkpoint
* for a single partition.
*/
public static class PartitionMark implements Serializable {
private String topic;
private int partition;
private long nextOffset;
private PartitionMark() {} // for Avro
public PartitionMark(String topic, int partition, long offset) {
this.topic = topic;
this.partition = partition;
this.nextOffset = offset;
}
public String getTopic() {
return topic;
}
public int getPartition() {
return partition;
}
public long getNextOffset() {
return nextOffset;
}
@Override
public String toString() {
return "PartitionMark{"
+ "topic='" + topic + '\''
+ ", partition=" + partition
+ ", nextOffset=" + nextOffset
+ '}';
}
}
}