io.streamnative.pulsar.handlers.kop.storage.ProducerAppendInfo Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of pulsar-protocol-handler-kafka Show documentation
Show all versions of pulsar-protocol-handler-kafka Show documentation
Kafka on Pulsar implemented using Pulsar Protocol Handler
/**
* Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
*/
/**
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.streamnative.pulsar.handlers.kop.storage;
import com.google.common.annotations.VisibleForTesting;
import java.util.List;
import java.util.Optional;
import javax.annotation.Nullable;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.mledger.Position;
import org.apache.commons.compress.utils.Lists;
import org.apache.kafka.common.errors.InvalidProducerEpochException;
import org.apache.kafka.common.errors.InvalidTxnStateException;
import org.apache.kafka.common.record.ControlRecordType;
import org.apache.kafka.common.record.EndTransactionMarker;
import org.apache.kafka.common.record.RecordBatch;
/**
* This class is used to validate the records appended by a given producer before they are written to the log.
* It is initialized with the producer's state after the last successful append, and transitively validates the
* sequence numbers and epochs of each new record. Additionally, this class accumulates transaction metadata
* as the incoming records are validated.
*/
@Slf4j
public class ProducerAppendInfo {
private final String topicPartition;
// The id of the producer appending to the log
final long producerId;
// The current entry associated with the producer id which contains metadata for a fixed number of
// the most recent appends made by the producer. Validation of the first incoming append will
// be made against the latest append in the current entry. New appends will replace older appends
// in the current entry so that the space overhead is constant.
private final ProducerStateEntry currentEntry;
// Indicates the origin of to append which implies the extent of validation.
// For example, offset commits, which originate from the group coordinator,
// do not have sequence numbers and therefore only producer epoch validation is done.
// Appends which come through replication are not validated (we assume the validation has already been done)
// and appends from clients require full validation.
private final PartitionLog.AppendOrigin origin;
private final List transactions = Lists.newArrayList();
@VisibleForTesting
final ProducerStateEntry updatedEntry;
public ProducerAppendInfo(String topicPartition,
Long producerId,
ProducerStateEntry currentEntry,
PartitionLog.AppendOrigin origin) {
this.topicPartition = topicPartition;
this.producerId = producerId;
this.currentEntry = currentEntry;
this.origin = origin;
this.updatedEntry = new ProducerStateEntry(producerId, currentEntry.producerEpoch,
currentEntry.coordinatorEpoch, currentEntry.lastTimestamp, currentEntry.currentTxnFirstOffset);
}
void checkProducerEpoch(short producerEpoch) throws InvalidProducerEpochException {
if (updatedEntry.producerEpoch != null
&& producerEpoch < updatedEntry.producerEpoch) {
String message = String.format("Producer %s's epoch in %s is %s, which is smaller than the last seen "
+ "epoch %s", producerId, topicPartition, producerEpoch, currentEntry.producerEpoch);
throw new InvalidProducerEpochException(message);
}
}
public Optional append(RecordBatch batch, long firstOffset, long lastOffset,
@Nullable Position firstPosition) {
if (batch.isControlBatch()) {
final var recordIterator = batch.iterator();
if (recordIterator.hasNext()) {
final var record = recordIterator.next();
final var endTxnMarker = EndTransactionMarker.deserialize(record);
return appendEndTxnMarker(endTxnMarker, batch.producerEpoch(), lastOffset, record.timestamp());
} else {
return Optional.empty();
}
} else {
appendDataBatch(batch.producerEpoch(), batch.maxTimestamp(), firstOffset, lastOffset, firstPosition,
batch.isTransactional());
return Optional.empty();
}
}
@VisibleForTesting
void appendDataBatch(short epoch, long lastTimestamp, long firstOffset, long lastOffset, Position firstPosition,
boolean isTransactional) {
// Here it does not validate sequence like Kafka because it leverages Pulsar's deduplication mechanism
checkProducerEpoch(epoch);
updatedEntry.addBatch(epoch, lastTimestamp);
final var currentTxnFirstOffset = updatedEntry.currentTxnFirstOffset;
if (currentTxnFirstOffset.isPresent() && !isTransactional) {
// Received a non-transactional message while a transaction is active
throw new InvalidTxnStateException("Expected transactional write from producer " + producerId + " at "
+ "offset " + firstOffset + " in partition " + topicPartition);
} else if (currentTxnFirstOffset.isEmpty() && isTransactional) {
// Began a new transaction
updatedEntry.currentTxnFirstOffset = Optional.of(firstOffset);
final var txnMetadata = new TxnMetadata(producerId, firstOffset, lastOffset, firstPosition);
transactions.add(txnMetadata);
}
}
public Optional appendEndTxnMarker(
EndTransactionMarker endTxnMarker,
short producerEpoch,
long offset,
long timestamp) {
checkProducerEpoch(producerEpoch);
// Only emit the `CompletedTxn` for non-empty transactions. A transaction marker
// without any associated data will not have any impact on the last stable offset
// and would not need to be reflected in the transaction index.
Optional completedTxn =
updatedEntry.currentTxnFirstOffset.map(firstOffset ->
new CompletedTxn(producerId, firstOffset, offset,
endTxnMarker.controlType() == ControlRecordType.ABORT));
updatedEntry.maybeUpdateProducerEpoch(producerEpoch);
updatedEntry.currentTxnFirstOffset = Optional.empty();
updatedEntry.lastTimestamp = timestamp;
return completedTxn;
}
public ProducerStateEntry toEntry() {
return updatedEntry;
}
public List startedTransactions() {
return transactions;
}
}