io.streamnative.pulsar.handlers.kop.KafkaRequestHandler Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of pulsar-protocol-handler-kafka Show documentation
Show all versions of pulsar-protocol-handler-kafka Show documentation
Kafka on Pulsar implemented using Pulsar Protocol Handler
/**
* Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
*/
/**
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.streamnative.pulsar.handlers.kop;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration.TENANT_ALLNAMESPACES_PLACEHOLDER;
import static io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration.TENANT_PLACEHOLDER;
import static io.streamnative.pulsar.handlers.kop.utils.KafkaResponseUtils.buildOffsetFetchResponse;
import static io.streamnative.pulsar.handlers.kop.utils.KafkaResponseUtils.newCoordinator;
import static io.streamnative.pulsar.handlers.kop.utils.MessageMetadataUtils.asyncGetCompactedLedger;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsTopic;
import static org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic;
import static org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.ssl.SslHandshakeCompletionEvent;
import io.streamnative.pulsar.handlers.kop.coordinator.group.GroupCoordinator;
import io.streamnative.pulsar.handlers.kop.coordinator.group.GroupMetadata.GroupOverview;
import io.streamnative.pulsar.handlers.kop.coordinator.group.GroupUtils;
import io.streamnative.pulsar.handlers.kop.coordinator.transaction.TransactionCoordinator;
import io.streamnative.pulsar.handlers.kop.exceptions.MetadataCorruptedException;
import io.streamnative.pulsar.handlers.kop.lookup.LookupService;
import io.streamnative.pulsar.handlers.kop.offset.OffsetAndMetadata;
import io.streamnative.pulsar.handlers.kop.offset.OffsetMetadata;
import io.streamnative.pulsar.handlers.kop.scala.Either;
import io.streamnative.pulsar.handlers.kop.security.Authenticator;
import io.streamnative.pulsar.handlers.kop.security.SaslAuthenticator;
import io.streamnative.pulsar.handlers.kop.security.Session;
import io.streamnative.pulsar.handlers.kop.security.SslAuthenticator;
import io.streamnative.pulsar.handlers.kop.security.auth.Authorizer;
import io.streamnative.pulsar.handlers.kop.security.auth.Resource;
import io.streamnative.pulsar.handlers.kop.security.auth.ResourceType;
import io.streamnative.pulsar.handlers.kop.security.auth.SimpleAclAuthorizer;
import io.streamnative.pulsar.handlers.kop.storage.AppendRecordsContext;
import io.streamnative.pulsar.handlers.kop.storage.PartitionLog;
import io.streamnative.pulsar.handlers.kop.storage.ReplicaManager;
import io.streamnative.pulsar.handlers.kop.utils.CoreUtils;
import io.streamnative.pulsar.handlers.kop.utils.GroupIdUtils;
import io.streamnative.pulsar.handlers.kop.utils.KafkaRequestUtils;
import io.streamnative.pulsar.handlers.kop.utils.KafkaResponseUtils;
import io.streamnative.pulsar.handlers.kop.utils.KopTopic;
import io.streamnative.pulsar.handlers.kop.utils.MessageMetadataUtils;
import io.streamnative.pulsar.handlers.kop.utils.MetadataUtils;
import io.streamnative.pulsar.handlers.kop.utils.OffsetFinder;
import io.streamnative.pulsar.handlers.kop.utils.TopicNameUtils;
import io.streamnative.pulsar.handlers.kop.utils.delayed.DelayedOperation;
import io.streamnative.pulsar.handlers.kop.utils.delayed.DelayedOperationKey;
import io.streamnative.pulsar.handlers.kop.utils.delayed.DelayedOperationPurgatory;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.common.util.OrderedScheduler;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
import org.apache.bookkeeper.mledger.Entry;
import org.apache.bookkeeper.mledger.ManagedLedgerException;
import org.apache.bookkeeper.mledger.Position;
import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
import org.apache.bookkeeper.mledger.impl.PositionImpl;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.lang3.tuple.Triple;
import org.apache.kafka.clients.admin.AlterConfigOp;
import org.apache.kafka.common.InvalidRecordException;
import org.apache.kafka.common.IsolationLevel;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.acl.AclOperation;
import org.apache.kafka.common.config.ConfigResource;
import org.apache.kafka.common.errors.ApiException;
import org.apache.kafka.common.errors.AuthenticationException;
import org.apache.kafka.common.errors.InvalidTopicException;
import org.apache.kafka.common.errors.LeaderNotAvailableException;
import org.apache.kafka.common.internals.Topic;
import org.apache.kafka.common.message.AddOffsetsToTxnRequestData;
import org.apache.kafka.common.message.AddOffsetsToTxnResponseData;
import org.apache.kafka.common.message.AddPartitionsToTxnRequestData;
import org.apache.kafka.common.message.AddPartitionsToTxnResponseData;
import org.apache.kafka.common.message.AlterConfigsRequestData;
import org.apache.kafka.common.message.AlterConfigsResponseData;
import org.apache.kafka.common.message.DeleteGroupsRequestData;
import org.apache.kafka.common.message.DeleteRecordsRequestData;
import org.apache.kafka.common.message.DeleteTopicsRequestData;
import org.apache.kafka.common.message.DescribeClusterResponseData;
import org.apache.kafka.common.message.DescribeConfigsRequestData;
import org.apache.kafka.common.message.DescribeConfigsResponseData;
import org.apache.kafka.common.message.DescribeProducersResponseData;
import org.apache.kafka.common.message.DescribeTransactionsResponseData;
import org.apache.kafka.common.message.EndTxnRequestData;
import org.apache.kafka.common.message.EndTxnResponseData;
import org.apache.kafka.common.message.FetchRequestData;
import org.apache.kafka.common.message.FetchResponseData;
import org.apache.kafka.common.message.FindCoordinatorResponseData;
import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData;
import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData;
import org.apache.kafka.common.message.InitProducerIdRequestData;
import org.apache.kafka.common.message.InitProducerIdResponseData;
import org.apache.kafka.common.message.JoinGroupRequestData;
import org.apache.kafka.common.message.LeaveGroupRequestData;
import org.apache.kafka.common.message.ListOffsetsRequestData;
import org.apache.kafka.common.message.ListOffsetsResponseData;
import org.apache.kafka.common.message.ListTransactionsResponseData;
import org.apache.kafka.common.message.OffsetCommitRequestData;
import org.apache.kafka.common.message.OffsetDeleteRequestData;
import org.apache.kafka.common.message.OffsetDeleteResponseData;
import org.apache.kafka.common.message.ProduceRequestData;
import org.apache.kafka.common.message.SaslAuthenticateResponseData;
import org.apache.kafka.common.message.SyncGroupRequestData;
import org.apache.kafka.common.message.TxnOffsetCommitRequestData;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.record.ControlRecordType;
import org.apache.kafka.common.record.EndTransactionMarker;
import org.apache.kafka.common.record.MemoryRecords;
import org.apache.kafka.common.record.MutableRecordBatch;
import org.apache.kafka.common.record.RecordBatch;
import org.apache.kafka.common.requests.AbstractRequest;
import org.apache.kafka.common.requests.AbstractResponse;
import org.apache.kafka.common.requests.AddOffsetsToTxnRequest;
import org.apache.kafka.common.requests.AddOffsetsToTxnResponse;
import org.apache.kafka.common.requests.AddPartitionsToTxnRequest;
import org.apache.kafka.common.requests.AddPartitionsToTxnResponse;
import org.apache.kafka.common.requests.AlterConfigsRequest;
import org.apache.kafka.common.requests.AlterConfigsResponse;
import org.apache.kafka.common.requests.ApiError;
import org.apache.kafka.common.requests.ApiVersionsResponse;
import org.apache.kafka.common.requests.CreatePartitionsRequest;
import org.apache.kafka.common.requests.CreateTopicsRequest;
import org.apache.kafka.common.requests.DeleteGroupsRequest;
import org.apache.kafka.common.requests.DeleteRecordsRequest;
import org.apache.kafka.common.requests.DeleteTopicsRequest;
import org.apache.kafka.common.requests.DescribeClusterRequest;
import org.apache.kafka.common.requests.DescribeClusterResponse;
import org.apache.kafka.common.requests.DescribeConfigsRequest;
import org.apache.kafka.common.requests.DescribeConfigsResponse;
import org.apache.kafka.common.requests.DescribeGroupsRequest;
import org.apache.kafka.common.requests.DescribeProducersRequest;
import org.apache.kafka.common.requests.DescribeProducersResponse;
import org.apache.kafka.common.requests.DescribeTransactionsRequest;
import org.apache.kafka.common.requests.DescribeTransactionsResponse;
import org.apache.kafka.common.requests.EndTxnRequest;
import org.apache.kafka.common.requests.EndTxnResponse;
import org.apache.kafka.common.requests.FetchRequest;
import org.apache.kafka.common.requests.FetchResponse;
import org.apache.kafka.common.requests.FindCoordinatorRequest;
import org.apache.kafka.common.requests.HeartbeatRequest;
import org.apache.kafka.common.requests.HeartbeatResponse;
import org.apache.kafka.common.requests.IncrementalAlterConfigsRequest;
import org.apache.kafka.common.requests.IncrementalAlterConfigsResponse;
import org.apache.kafka.common.requests.InitProducerIdRequest;
import org.apache.kafka.common.requests.InitProducerIdResponse;
import org.apache.kafka.common.requests.JoinGroupRequest;
import org.apache.kafka.common.requests.JoinGroupResponse;
import org.apache.kafka.common.requests.LeaveGroupRequest;
import org.apache.kafka.common.requests.ListGroupsRequest;
import org.apache.kafka.common.requests.ListOffsetRequestV0;
import org.apache.kafka.common.requests.ListOffsetsRequest;
import org.apache.kafka.common.requests.ListOffsetsResponse;
import org.apache.kafka.common.requests.ListTransactionsRequest;
import org.apache.kafka.common.requests.ListTransactionsResponse;
import org.apache.kafka.common.requests.MetadataRequest;
import org.apache.kafka.common.requests.MetadataResponse.PartitionMetadata;
import org.apache.kafka.common.requests.MetadataResponse.TopicMetadata;
import org.apache.kafka.common.requests.OffsetCommitRequest;
import org.apache.kafka.common.requests.OffsetCommitResponse;
import org.apache.kafka.common.requests.OffsetDeleteRequest;
import org.apache.kafka.common.requests.OffsetDeleteResponse;
import org.apache.kafka.common.requests.OffsetFetchRequest;
import org.apache.kafka.common.requests.OffsetFetchResponse;
import org.apache.kafka.common.requests.ProduceRequest;
import org.apache.kafka.common.requests.ProduceResponse;
import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse;
import org.apache.kafka.common.requests.ResponseCallbackWrapper;
import org.apache.kafka.common.requests.SaslAuthenticateResponse;
import org.apache.kafka.common.requests.SyncGroupRequest;
import org.apache.kafka.common.requests.SyncGroupResponse;
import org.apache.kafka.common.requests.TransactionResult;
import org.apache.kafka.common.requests.TxnOffsetCommitRequest;
import org.apache.kafka.common.requests.TxnOffsetCommitResponse;
import org.apache.kafka.common.requests.WriteTxnMarkersRequest;
import org.apache.kafka.common.requests.WriteTxnMarkersResponse;
import org.apache.kafka.common.security.auth.SecurityProtocol;
import org.apache.kafka.common.utils.SystemTime;
import org.apache.kafka.common.utils.Time;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.authentication.AuthenticationProvider;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.schema.KeyValue;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.common.util.Murmur3_32Hash;
import org.apache.pulsar.compaction.TopicCompactionService;
import org.apache.pulsar.metadata.api.MetadataStoreException;
import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
/**
* This class contains all the request handling methods.
*/
@Slf4j
@Getter
public class KafkaRequestHandler extends KafkaCommandDecoder {
private static final int THROTTLE_TIME_MS = 10;
private static final String POLICY_ROOT = "/admin/policies/";
private final PulsarService pulsarService;
private final KafkaTopicManager topicManager;
private final TenantContextManager tenantContextManager;
private final ReplicaManager replicaManager;
private final LookupService lookupService;
@Getter
private final KafkaTopicManagerSharedState kafkaTopicManagerSharedState;
private final String clusterName;
private final ScheduledExecutorService executor;
private final PulsarAdmin admin;
private final MetadataStoreExtended metadataStore;
private Authenticator authenticator;
private Authorizer authorizer;
private final AdminManager adminManager;
private final Boolean tlsEnabled;
private final EndPoint advertisedEndPoint;
private final boolean skipMessagesWithoutIndex;
private final int defaultNumPartitions;
public final int maxReadEntriesNum;
private final int failedAuthenticationDelayMs;
// store the group name for current connected client.
private final ConcurrentHashMap> currentConnectedGroup;
private final ConcurrentSkipListSet currentConnectedClientId;
private final String groupIdStoredPath;
private final Set groupIds = new HashSet<>();
// key is the topic(partition), value is the future that indicates whether the PersistentTopic instance of the key
// is found.
private final Map pendingTopicFuturesMap = new ConcurrentHashMap<>();
// DelayedOperation for produce and fetch
private final DelayedOperationPurgatory fetchPurgatory;
// Flag to manage throttling-publish-buffer by atomically enable/disable read-channel.
private final long maxPendingBytes;
private final long resumeThresholdPendingBytes;
private final AtomicLong pendingBytes = new AtomicLong(0);
private final String defaultNamespacePrefix;
private final String defaultKafkaNamespacePrefix;
private volatile boolean autoReadDisabledPublishBufferLimiting = false;
private final ExecutorService orderedExecutor;
private String getCurrentTenant() {
return getCurrentTenant(kafkaConfig.getKafkaMetadataTenant());
}
private String getCurrentTenant(String defaultTenant) {
if (kafkaConfig.isKafkaEnableMultiTenantMetadata()
&& authenticator != null
&& authenticator.session() != null
&& authenticator.session().getPrincipal() != null
&& authenticator.session().getPrincipal().getTenantSpec() != null) {
String tenantSpec = authenticator.session().getPrincipal().getTenantSpec();
return extractTenantFromTenantSpec(tenantSpec);
}
// fallback to using system (default) tenant
if (log.isDebugEnabled()) {
log.debug("using {} as tenant", defaultTenant);
}
return defaultTenant;
}
public String currentNamespacePrefix() {
String currentTenant = getCurrentTenant(kafkaConfig.getKafkaTenant());
return MetadataUtils.constructUserTopicsNamespace(currentTenant, kafkaConfig);
}
private static String extractTenantFromTenantSpec(String tenantSpec) {
if (tenantSpec != null && !tenantSpec.isEmpty()) {
String tenant = tenantSpec;
// username can be "tenant" or "tenant/namespace"
if (tenantSpec.contains("/")) {
tenant = tenantSpec.substring(0, tenantSpec.indexOf('/'));
}
if (log.isDebugEnabled()) {
log.debug("using {} as tenant", tenant);
}
return tenant;
} else {
return tenantSpec;
}
}
public GroupCoordinator getGroupCoordinator() {
return tenantContextManager.getGroupCoordinator(getCurrentTenant());
}
public TransactionCoordinator getTransactionCoordinator() {
throwIfTransactionCoordinatorDisabled();
return tenantContextManager.getTransactionCoordinator(getCurrentTenant());
}
public KafkaRequestHandler(PulsarService pulsarService,
KafkaServiceConfiguration kafkaConfig,
TenantContextManager tenantContextManager,
ReplicaManager replicaManager,
LookupService lookupService,
AdminManager adminManager,
DelayedOperationPurgatory fetchPurgatory,
Boolean tlsEnabled,
EndPoint advertisedEndPoint,
boolean skipMessagesWithoutIndex,
RequestStats requestStats,
OrderedScheduler sendResponseScheduler,
KafkaTopicManagerSharedState kafkaTopicManagerSharedState,
KafkaTopicLookupService kafkaTopicLookupService) throws Exception {
super(requestStats, kafkaConfig, sendResponseScheduler);
this.pulsarService = pulsarService;
this.tenantContextManager = tenantContextManager;
this.replicaManager = replicaManager;
this.lookupService = lookupService;
this.clusterName = kafkaConfig.getClusterName();
this.executor = pulsarService.getExecutor();
this.admin = pulsarService.getAdminClient();
this.metadataStore = pulsarService.getLocalMetadataStore();
final boolean authenticationEnabled = pulsarService.getBrokerService().isAuthenticationEnabled();
final SecurityProtocol securityProtocol = advertisedEndPoint.getSecurityProtocol();
switch (securityProtocol) {
case SSL:
if (authenticationEnabled) {
AuthenticationProvider tlsAuthenticationProvider = pulsarService.getBrokerService()
.getAuthenticationService()
.getAuthenticationProvider("tls");
if (tlsAuthenticationProvider != null) {
this.authenticator = new SslAuthenticator(tlsAuthenticationProvider, kafkaConfig);
}
}
break;
case SASL_SSL:
case SASL_PLAINTEXT:
if (authenticationEnabled && !kafkaConfig.getSaslAllowedMechanisms().isEmpty()) {
this.authenticator = new SaslAuthenticator(pulsarService, kafkaConfig.getSaslAllowedMechanisms(),
kafkaConfig);
}
break;
case PLAINTEXT:
break;
default:
throw new IllegalArgumentException("Unexpected securityProtocol " + securityProtocol);
}
final boolean authorizationEnabled = pulsarService.getBrokerService().isAuthorizationEnabled();
this.authorizer = authorizationEnabled && this.authenticator != null
? new SimpleAclAuthorizer(pulsarService, kafkaConfig)
: null;
this.adminManager = adminManager;
this.fetchPurgatory = fetchPurgatory;
this.tlsEnabled = tlsEnabled;
this.advertisedEndPoint = advertisedEndPoint;
this.skipMessagesWithoutIndex = skipMessagesWithoutIndex;
this.topicManager = new KafkaTopicManager(this, kafkaTopicLookupService);
this.defaultNumPartitions = kafkaConfig.getDefaultNumPartitions();
this.maxReadEntriesNum = kafkaConfig.getMaxReadEntriesNum();
this.currentConnectedGroup = new ConcurrentHashMap<>();
this.currentConnectedClientId = new ConcurrentSkipListSet<>();
this.groupIdStoredPath = kafkaConfig.getGroupIdZooKeeperPath();
this.maxPendingBytes = kafkaConfig.getMaxMessagePublishBufferSizeInMB() * 1024L * 1024L;
this.resumeThresholdPendingBytes = this.maxPendingBytes / 2;
this.defaultNamespacePrefix = kafkaConfig.getDefaultNamespacePrefix();
this.defaultKafkaNamespacePrefix = defaultNamespacePrefix.replaceAll("/", ".");
this.failedAuthenticationDelayMs = kafkaConfig.getFailedAuthenticationDelayMs();
this.kafkaTopicManagerSharedState = kafkaTopicManagerSharedState;
this.orderedExecutor = pulsarService
.getBrokerService()
.getTopicOrderedExecutor()
.chooseThread(this);
// update alive channel count stats
RequestStats.ALIVE_CHANNEL_COUNT_INSTANCE.incrementAndGet();
}
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
super.channelActive(ctx);
topicManager.setRemoteAddress(ctx.channel().remoteAddress());
// update active channel count stats
RequestStats.ACTIVE_CHANNEL_COUNT_INSTANCE.incrementAndGet();
log.info("channel active: {}", ctx.channel());
}
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
// Handle ssl handshake completion event
if (evt instanceof SslHandshakeCompletionEvent) {
if (((SslHandshakeCompletionEvent) evt).isSuccess()) {
if (this.authenticator != null && this.authenticator instanceof SslAuthenticator) {
this.authenticator.authenticate(ctx,
null,
null,
null,
this::validateTenantAccessForSession);
}
}
} else {
super.userEventTriggered(ctx, evt);
}
}
@Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
super.channelInactive(ctx);
// update active channel count stats
RequestStats.ACTIVE_CHANNEL_COUNT_INSTANCE.decrementAndGet();
close();
}
@Override
protected void close() {
if (isActive.getAndSet(false)) {
super.close();
topicManager.close();
String clientHost = ctx.channel().remoteAddress().toString();
if (currentConnectedGroup.containsKey(clientHost)){
log.info("currentConnectedGroup remove {}", clientHost);
currentConnectedGroup.remove(clientHost);
}
// Try to remove all stored groupID on the metadata store.
if (log.isDebugEnabled()) {
log.debug("Try to remove all stored groupID on the metadata store. Current connected clientIds: {}",
currentConnectedClientId);
}
if (kafkaConfig.isKopEnableGroupLevelConsumerMetrics()) {
currentConnectedClientId.forEach(clientId -> {
String path = groupIdStoredPath + GroupIdUtils.groupIdPathFormat(clientHost, clientId);
metadataStore.delete(path, Optional.empty())
.whenComplete((__, ex) -> {
if (ex != null) {
if (ex.getCause() instanceof MetadataStoreException.NotFoundException) {
if (log.isDebugEnabled()) {
log.debug("The groupId store path doesn't exist. Path: [{}]", path);
}
return;
}
log.error("Delete groupId failed. Path: [{}]", path, ex);
return;
}
if (log.isDebugEnabled()) {
log.debug("Delete groupId success. Path: [{}]", path);
}
});
});
}
if (authenticator != null) {
authenticator.close();
}
// update alive channel count stat
RequestStats.ALIVE_CHANNEL_COUNT_INSTANCE.decrementAndGet();
}
}
@Override
protected boolean hasAuthenticated() {
return authenticator == null || authenticator.complete();
}
@Override
protected void channelPrepare(ChannelHandlerContext ctx,
ByteBuf requestBuf,
BiConsumer registerRequestParseLatency,
BiConsumer registerRequestLatency)
throws AuthenticationException {
if (authenticator != null) {
authenticator.authenticate(ctx, requestBuf, registerRequestParseLatency, registerRequestLatency,
this::validateTenantAccessForSession);
if (authenticator.complete() && kafkaConfig.isKafkaEnableMultiTenantMetadata()) {
setRequestStats(requestStats.forTenant(getCurrentTenant()));
}
}
}
@Override
protected void maybeDelayCloseOnAuthenticationFailure() {
if (this.failedAuthenticationDelayMs > 0) {
this.ctx.executor().schedule(
this::completeCloseOnAuthenticationFailure,
this.failedAuthenticationDelayMs,
TimeUnit.MILLISECONDS);
} else {
this.completeCloseOnAuthenticationFailure();
}
}
@Override
protected void completeCloseOnAuthenticationFailure() {
if (isActive.get() && authenticator != null) {
authenticator.sendAuthenticationFailureResponse(__ -> this.close());
}
}
@Override
protected void handleApiVersionsRequest(KafkaHeaderAndRequest apiVersionRequest,
CompletableFuture resultFuture) {
if (!ApiKeys.API_VERSIONS.isVersionSupported(apiVersionRequest.getHeader().apiVersion())) {
// Notify Client that API_VERSION is UNSUPPORTED.
AbstractResponse apiResponse = overloadDefaultApiVersionsResponse(true);
resultFuture.complete(apiResponse);
} else {
AbstractResponse apiResponse = overloadDefaultApiVersionsResponse(false);
resultFuture.complete(apiResponse);
}
}
public static ApiVersionsResponse overloadDefaultApiVersionsResponse(boolean unsupportedApiVersion) {
if (unsupportedApiVersion){
return KafkaResponseUtils.newApiVersions(Errors.UNSUPPORTED_VERSION);
} else {
List versionList = new ArrayList<>();
for (ApiKeys apiKey : ApiKeys.values()) {
if (apiKey.minRequiredInterBrokerMagic <= RecordBatch.CURRENT_MAGIC_VALUE) {
switch (apiKey) {
case LIST_OFFSETS:
// V0 is needed for librdkafka
versionList.add(new ApiVersion((short) 2, (short) 0, apiKey.latestVersion()));
break;
default:
versionList.add(new ApiVersion(apiKey));
}
}
}
return KafkaResponseUtils.newApiVersions(versionList);
}
}
@Override
protected void handleInactive(KafkaHeaderAndRequest kafkaHeaderAndRequest,
CompletableFuture resultFuture) {
AbstractRequest request = kafkaHeaderAndRequest.getRequest();
AbstractResponse apiResponse = request.getErrorResponse(new LeaderNotAvailableException("Channel is closing!"));
log.error("Kafka API {} is send to a closing channel", kafkaHeaderAndRequest.getHeader().apiKey());
resultFuture.complete(apiResponse);
}
// Leverage pulsar admin to get partitioned topic metadata
// NOTE: the returned future never completes exceptionally
@VisibleForTesting
protected CompletableFuture getTopicMetadataAsync(String kafkaTopic,
String pulsarTopic,
boolean allowAutoTopicCreation) {
final var future = new CompletableFuture>();
final TopicName topicName = TopicName.get(pulsarTopic);
admin.topics().getPartitionedTopicMetadataAsync(pulsarTopic).whenComplete((metadata, e) -> {
if (e == null) {
if (log.isDebugEnabled()) {
log.debug("Topic {} has {} partitions", pulsarTopic, metadata.partitions);
}
if (metadata.properties != null
&& metadata.properties.get(PartitionLog.KAFKA_TOPIC_UUID_PROPERTY_NAME) != null) {
future.complete(Either.right(metadata.partitions));
} else {
admin.topics().updatePropertiesAsync(pulsarTopic, Map.of(
PartitionLog.KAFKA_TOPIC_UUID_PROPERTY_NAME, UUID.randomUUID().toString()
)).whenComplete((__, updateException) -> {
if (updateException == null) {
future.complete(Either.right(metadata.partitions));
return;
}
if (updateException instanceof PulsarAdminException.ConflictException) {
log.warn("[{}] Failed to update properties for topic {}",
ctx.channel(), topicName, updateException);
future.complete(Either.right(metadata.partitions));
return;
}
log.error("[{}] Failed to update properties for topic {}",
ctx.channel(), topicName, updateException);
future.complete(Either.left(Errors.UNKNOWN_SERVER_ERROR));
});
}
} else if (e instanceof PulsarAdminException.NotFoundException) {
(allowAutoTopicCreation ? checkAllowAutoTopicCreation(topicName.getNamespace())
: CompletableFuture.completedFuture(false)
).whenComplete((allowed, err) -> {
if (err != null) {
log.error("[{}] Cannot get policies for namespace {}",
ctx.channel(), topicName.getNamespace(), err);
future.complete(Either.left(Errors.UNKNOWN_SERVER_ERROR));
return;
}
if (allowed) {
Map properties =
Map.of(PartitionLog.KAFKA_TOPIC_UUID_PROPERTY_NAME, UUID.randomUUID().toString());
admin.topics().createPartitionedTopicAsync(pulsarTopic, defaultNumPartitions, properties)
.whenComplete((__, createException) -> {
if (createException == null) {
future.complete(Either.right(defaultNumPartitions));
} else {
log.warn("[{}] Failed to create partitioned topic {}: {}",
ctx.channel(), topicName, createException.getMessage());
future.complete(Either.left(Errors.UNKNOWN_SERVER_ERROR));
}
});
} else {
try {
Topic.validate(topicName.getLocalName());
future.complete(Either.left(Errors.UNKNOWN_TOPIC_OR_PARTITION));
} catch (InvalidTopicException ignored) {
future.complete(Either.left(Errors.INVALID_TOPIC_EXCEPTION));
}
}
});
} else {
log.error("[{}] Failed to get partitioned topic {}", ctx.channel(), pulsarTopic, e);
future.complete(Either.left(Errors.UNKNOWN_SERVER_ERROR));
}
});
return future.thenApply(either -> either.map(partitions ->
new TopicAndMetadata(kafkaTopic, pulsarTopic, partitions, Errors.NONE))
).thenApply(either -> {
if (either.isLeft()) {
return new TopicAndMetadata(kafkaTopic, pulsarTopic, -1, either.getLeft());
} else {
return either.getRight();
}
});
}
private CompletableFuture checkAllowAutoTopicCreation(String namespace) {
return admin.namespaces().getPoliciesAsync(namespace).thenApply(policies -> {
if (policies != null && policies.autoTopicCreationOverride != null) {
return policies.autoTopicCreationOverride.isAllowAutoTopicCreation();
} else {
return kafkaConfig.isAllowAutoTopicCreation();
}
});
}
private CompletableFuture> getAllowedNamespacesAsync() {
return pulsarService.getPulsarResources().getDynamicConfigResources().getDynamicConfigurationAsync()
.thenApply(__ -> __.orElse(Collections.emptyMap()).get(KafkaServiceConfiguration.KOP_ALLOWED_NAMESPACES))
.thenCompose(configValue -> {
if (configValue == null || configValue.isEmpty()) {
return CompletableFuture.completedFuture(kafkaConfig.getKopAllowedNamespaces());
}
final var tenantToNamespaces = new HashMap>();
for (String namespace : configValue.split(",")) {
NamespaceName namespaceName;
try {
namespaceName = NamespaceName.get(namespace);
} catch (Throwable e) {
log.warn("Skip invalid namespace {} in kopAllowedNamespaces: {}", namespace, e.getMessage());
continue;
}
tenantToNamespaces.computeIfAbsent(namespaceName.getTenant(), __ -> new ArrayList<>())
.add(namespaceName);
}
return filterValidNamespaces(tenantToNamespaces);
})
.thenCompose(allowedNamespaces -> {
final var currentTenant = getCurrentTenant(kafkaConfig.getKafkaTenant());
return expandAllowedNamespaces(allowedNamespaces, currentTenant, pulsarService);
})
.thenApply(namespaces -> namespaces.stream().filter(__ -> !__.contains(".")).collect(Collectors.toSet()));
}
@VisibleForTesting
static CompletableFuture> expandAllowedNamespaces(Set allowedNamespaces,
String currentTenant,
PulsarService pulsarService) {
Set result = new CopyOnWriteArraySet<>();
List> results = new ArrayList<>();
for (String namespaceTemplate : allowedNamespaces) {
String namespace = namespaceTemplate.replace(TENANT_PLACEHOLDER, currentTenant);
if (!namespace.endsWith("/" + TENANT_ALLNAMESPACES_PLACEHOLDER)) {
result.add(namespace);
results.add(CompletableFuture.completedFuture(namespace));
} else {
int slash = namespace.lastIndexOf('/');
String tenant = namespace.substring(0, slash);
results.add(pulsarService.getPulsarResources()
.getNamespaceResources()
.listNamespacesAsync(tenant)
.thenAccept(namespaces -> namespaces.forEach(ns -> result.add(tenant + "/" + ns))));
}
}
return CompletableFuture
.allOf(results.toArray(new CompletableFuture[0]))
.thenApply(f -> result);
}
private List analyzeFullTopicNames(final Stream fullTopicNames) {
// key is the topic name, value is a list of the topic's partition indexes
final Map> topicToPartitionIndexes = new HashMap<>();
fullTopicNames.forEach(fullTopicName -> {
final TopicName topicName = TopicName.get(fullTopicName);
// Skip Pulsar's system topic
if (topicName.getLocalName().startsWith("__change_events")
&& topicName.getPartitionedTopicName().endsWith("__change_events")) {
return;
}
topicToPartitionIndexes.computeIfAbsent(
topicName.getPartitionedTopicName(),
ignored -> new ArrayList<>()
).add(topicName.getPartitionIndex());
});
if (topicToPartitionIndexes.isEmpty()) {
return Collections.emptyList();
}
// Check missed partitions
final List topicAndMetadataList = new ArrayList<>();
topicToPartitionIndexes.forEach((topic, partitionIndexes) -> {
Collections.sort(partitionIndexes);
final int lastIndex = partitionIndexes.get(partitionIndexes.size() - 1);
final var kafkaTopic = TopicNameUtils.pulsarToKafka(topic, defaultKafkaNamespacePrefix).topic();
if (lastIndex < 0) { // non-partitioned topic
topicAndMetadataList.add(new TopicAndMetadata(kafkaTopic, topic, 0, Errors.NONE));
} else if (lastIndex == partitionIndexes.size() - 1) {
topicAndMetadataList.add(new TopicAndMetadata(kafkaTopic, topic, partitionIndexes.size(), Errors.NONE));
} else {
// The partitions should be [0, 1, ..., n-1], `n` is the number of partitions. If the last index is not
// `n-1`, there must be some missed partitions.
log.warn("The partitions of topic {} is wrong ({}), try to create missed partitions",
topic, partitionIndexes.size());
admin.topics().createMissedPartitionsAsync(topic);
}
});
return topicAndMetadataList;
}
private CompletableFuture> filterValidNamespaces(
final Map> tenantToNamespaces) {
final var validTenantFutures = tenantToNamespaces.keySet().stream().map(tenant ->
pulsarService.getPulsarResources().getTenantResources().tenantExistsAsync(tenant).thenApply(exists -> {
if (exists) {
return tenant;
} else {
log.warn("Skip non-existent tenant {} in kopAllowedNamespaces", tenant);
return null;
}
})).toList();
return FutureUtil.waitForAll(validTenantFutures).thenCompose(__ -> {
final var validNamespacesFutures = validTenantFutures.stream().map(CompletableFuture::join)
.filter(Objects::nonNull).map(tenantToNamespaces::get).filter(Objects::nonNull)
.flatMap(namespaceNames -> namespaceNames.stream().map(namespaceName ->
pulsarService.getPulsarResources().getNamespaceResources().namespaceExistsAsync(namespaceName)
.thenApply(exist -> {
if (exist) {
return namespaceName.toString();
} else {
log.warn("Skip non-existent namespace {} in kopAllowedNamespaces", namespaceName);
return null;
}
}))).toList();
return FutureUtil.waitForAll(validNamespacesFutures).thenApply(___ -> validNamespacesFutures.stream()
.map(CompletableFuture::join).filter(Objects::nonNull).collect(Collectors.toSet()));
});
}
private CompletableFuture> authorizeNamespacesAsync(final Collection namespaces,
final AclOperation aclOperation) {
final Map> futureMap = namespaces.stream().collect(
Collectors.toMap(
namespace -> namespace,
namespace -> authorize(aclOperation, Resource.of(ResourceType.NAMESPACE, namespace))
));
return CoreUtils.waitForAll(futureMap.values()).thenApply(__ ->
futureMap.entrySet().stream().filter(e -> {
if (!e.getValue().join()) {
log.warn("Failed to authorize {} for ACL operation {}", e.getKey(), aclOperation);
return false;
}
return true;
}).map(Map.Entry::getKey).collect(Collectors.toList())
);
}
private CompletableFuture> listAllTopicsFromNamespacesAsync(final List namespaces) {
return CoreUtils.waitForAll(namespaces.stream()
.map(namespace -> pulsarService.getNamespaceService()
.getListOfPersistentTopics(NamespaceName.get(namespace))
).collect(Collectors.toList()),
topics -> topics.stream().flatMap(List::stream));
}
private CompletableFuture> getTopicsAsync(MetadataRequest request) {
if (request.isAllTopics()) {
return getAllowedNamespacesAsync()
.thenCompose(namespaces -> authorizeNamespacesAsync(namespaces, AclOperation.DESCRIBE))
.thenCompose(this::listAllTopicsFromNamespacesAsync)
.thenApply(this::analyzeFullTopicNames);
} else {
return CoreUtils.waitForAll(request.topics().stream().map(topic -> {
final String pulsarTopic;
try {
pulsarTopic = kafkaToPulsar(topic);
} catch (InvalidTopicException e) {
log.warn("[{}] Invalid topic name {}: {}", ctx.channel(), topic, e.getMessage());
return CompletableFuture.completedFuture(
new TopicAndMetadata(topic, topic, -1, Errors.INVALID_TOPIC_EXCEPTION));
}
return authorize(AclOperation.DESCRIBE, Resource.of(ResourceType.TOPIC, pulsarTopic))
.thenCompose(authorized -> {
if (authorized) {
return getTopicMetadataAsync(topic, pulsarTopic, request.allowAutoTopicCreation());
} else {
return CompletableFuture.completedFuture(
new TopicAndMetadata(topic, pulsarTopic, -1, Errors.TOPIC_AUTHORIZATION_FAILED));
}
});
}).toList(), __ -> __);
}
}
@Override
protected void handleTopicMetadataRequest(KafkaHeaderAndRequest metadataHar,
CompletableFuture resultFuture) {
// Get all kop brokers in local cache
List allNodes = Collections.synchronizedList(
new ArrayList<>(adminManager.getBrokers(advertisedEndPoint.getListenerName())));
// Each Pulsar broker can manage metadata like controller in Kafka,
// Kafka's AdminClient needs to find a controller node for metadata management.
// So here we return an random broker as a controller for the given listenerName.
final int controllerId = adminManager.getControllerId(advertisedEndPoint.getListenerName());
final MetadataRequest request = (MetadataRequest) metadataHar.getRequest();
final String metadataNamespace = kafkaConfig.getKafkaMetadataNamespace();
getTopicsAsync(request).whenComplete((topicAndMetadataList, e) -> {
if (e != null) {
log.error("[{}] Request {}: Exception fetching metadata", ctx.channel(), metadataHar.getHeader(), e);
resultFuture.completeExceptionally(e);
return;
}
final ListPair listPair =
ListPair.split(topicAndMetadataList.stream(), TopicAndMetadata::hasNoError);
CoreUtils.waitForAll(listPair.getSuccessfulList().stream()
.map(topicAndMetadata -> topicAndMetadata.lookupAsync(this::lookup, metadataNamespace))
.collect(Collectors.toList()), successfulTopicMetadataList -> {
final List topicMetadataList = ListUtils.union(successfulTopicMetadataList,
CoreUtils.listToList(listPair.getFailedList(),
metadata -> metadata.toTopicMetadata(metadataNamespace))
);
resultFuture.complete(
KafkaResponseUtils.newMetadata(allNodes, clusterName,
controllerId, topicMetadataList, request.version()));
return null;
})
.exceptionally(lookupException -> {
log.error("[{}] Unexpected exception during lookup", ctx.channel(), lookupException);
resultFuture.completeExceptionally(lookupException);
return null;
});
});
}
private void disableCnxAutoRead() {
if (ctx != null && ctx.channel().config().isAutoRead()) {
ctx.channel().config().setAutoRead(false);
if (log.isDebugEnabled()) {
log.debug("[{}] disable auto read", ctx.channel());
}
}
}
private void enableCnxAutoRead() {
if (ctx != null && !ctx.channel().config().isAutoRead()
&& !autoReadDisabledPublishBufferLimiting) {
// Resume reading from socket if pending-request is not reached to threshold
ctx.channel().config().setAutoRead(true);
// triggers channel read
ctx.read();
if (log.isDebugEnabled()) {
log.debug("[{}] enable auto read", ctx.channel());
}
}
}
private void startSendOperationForThrottling(long msgSize) {
final long currentPendingBytes = pendingBytes.addAndGet(msgSize);
if (currentPendingBytes >= maxPendingBytes && !autoReadDisabledPublishBufferLimiting && maxPendingBytes > 0) {
if (log.isDebugEnabled()) {
log.debug("[{}] disable auto read because currentPendingBytes({}) > maxPendingBytes({})",
ctx.channel(), currentPendingBytes, maxPendingBytes);
}
disableCnxAutoRead();
autoReadDisabledPublishBufferLimiting = true;
setPausedConnections(pulsarService, 1);
}
}
@VisibleForTesting
public static void setPausedConnections(PulsarService pulsarService, int numConnections) {
pulsarService.getBrokerService().pausedConnections(numConnections);
}
private void completeSendOperationForThrottling(long msgSize) {
final long currentPendingBytes = pendingBytes.addAndGet(-msgSize);
if (currentPendingBytes < resumeThresholdPendingBytes && autoReadDisabledPublishBufferLimiting) {
if (log.isDebugEnabled()) {
log.debug("[{}] enable auto read because currentPendingBytes({}) < resumeThreshold({})",
ctx.channel(), currentPendingBytes, resumeThresholdPendingBytes);
}
autoReadDisabledPublishBufferLimiting = false;
enableCnxAutoRead();
resumePausedConnections(pulsarService, 1);
}
}
@VisibleForTesting
public static void resumePausedConnections(PulsarService pulsarService, int numConnections) {
pulsarService.getBrokerService().resumedConnections(numConnections);
}
@Override
protected void handleProduceRequest(KafkaHeaderAndRequest produceHar,
CompletableFuture resultFuture) {
checkArgument(produceHar.getRequest() instanceof ProduceRequest);
ProduceRequest produceRequest = (ProduceRequest) produceHar.getRequest();
final int numPartitions = produceRequest
.data()
.topicData()
.stream()
.mapToInt(t -> t.partitionData().size())
.sum();
if (numPartitions == 0) {
resultFuture.complete(new ProduceResponse(Collections.emptyMap()));
return;
}
final Map unauthorizedTopicResponsesMap = new ConcurrentHashMap<>();
final Map invalidRequestResponses = new HashMap<>();
final Map authorizedRequestInfo = new ConcurrentHashMap<>();
// TODO: handle the produce timeout
short requiredAcks = produceRequest.acks();
String namespacePrefix = currentNamespacePrefix();
final AtomicInteger unfinishedAuthorizationCount = new AtomicInteger(numPartitions);
Runnable completeOne = () -> {
// When complete one authorization or failed, will do the action first.
if (unfinishedAuthorizationCount.decrementAndGet() == 0) {
if (authorizedRequestInfo.isEmpty()) {
resultFuture.complete(new ProduceResponse(unauthorizedTopicResponsesMap));
return;
}
AppendRecordsContext appendRecordsContext = AppendRecordsContext.get(
topicManager,
this::startSendOperationForThrottling,
this::completeSendOperationForThrottling,
pendingTopicFuturesMap,
ctx);
ReplicaManager replicaManager = getReplicaManager();
replicaManager.appendRecords(
requiredAcks,
false,
authorizedRequestInfo,
PartitionLog.AppendOrigin.Client,
appendRecordsContext
).whenComplete((response, ex) -> {
if (ex != null) {
resultFuture.completeExceptionally(ex.getCause());
return;
}
Map mergedResponse = new HashMap<>();
mergedResponse.putAll(response);
mergedResponse.putAll(unauthorizedTopicResponsesMap);
mergedResponse.putAll(invalidRequestResponses);
resultFuture.complete(new ProduceResponse(mergedResponse));
response.keySet().forEach(tp -> {
replicaManager.tryCompleteDelayedFetch(new DelayedOperationKey.TopicPartitionOperationKey(
new KsnTopicPartition(tp, kafkaToPulsar(tp.topic()))));
});
});
}
};
produceRequest.data().topicData().forEach((ProduceRequestData.TopicProduceData topicProduceData) -> {
final var topic = topicProduceData.name();
final String pulsarTopic;
try {
pulsarTopic = kafkaToPulsar(topic);
} catch (InvalidTopicException __) {
topicProduceData.partitionData().forEach(partitionProduceData -> {
unauthorizedTopicResponsesMap.put(new TopicPartition(topic, partitionProduceData.index()),
new PartitionResponse(Errors.INVALID_TOPIC_EXCEPTION));
completeOne.run();
});
return;
}
final var authorizeFuture = authorize(AclOperation.WRITE,
Resource.of(ResourceType.TOPIC, pulsarTopic));
topicProduceData.partitionData().forEach(partitionProduceData -> {
MemoryRecords records = (MemoryRecords) partitionProduceData.records();
int index = partitionProduceData.index();
TopicPartition topicPartition = new TopicPartition(topic, index);
try {
validateRecords(produceHar.getRequest().version(), records);
} catch (ApiException ex) {
invalidRequestResponses.put(topicPartition,
new PartitionResponse(Errors.forException(ex)));
completeOne.run();
return;
}
final String fullPartitionName = KopTopic.toString(topicPartition, namespacePrefix);
authorizeFuture.whenCompleteAsync((isAuthorized, ex) -> {
if (ex != null) {
log.error("Write topic authorize failed, topic - {}. {}",
fullPartitionName, ex.getMessage());
unauthorizedTopicResponsesMap.put(topicPartition,
new PartitionResponse(Errors.TOPIC_AUTHORIZATION_FAILED));
completeOne.run();
return;
}
if (!isAuthorized) {
unauthorizedTopicResponsesMap.put(topicPartition,
new PartitionResponse(Errors.TOPIC_AUTHORIZATION_FAILED));
completeOne.run();
return;
}
authorizedRequestInfo.put(new KsnTopicPartition(topicPartition, pulsarTopic), records);
completeOne.run();
}, orderedExecutor);
});
});
}
private void validateRecords(short version, MemoryRecords records) {
if (version >= 3) {
Iterator iterator = records.batches().iterator();
if (!iterator.hasNext()) {
throw new InvalidRecordException("Produce requests with version " + version + " must have at least "
+ "one record batch");
}
MutableRecordBatch entry = iterator.next();
if (entry.magic() != RecordBatch.MAGIC_VALUE_V2) {
throw new InvalidRecordException("Produce requests with version " + version + " are only allowed to "
+ "contain record batches with magic version 2");
}
if (iterator.hasNext()) {
throw new InvalidRecordException("Produce requests with version " + version + " are only allowed to "
+ "contain exactly one record batch");
}
}
}
@Override
protected void handleFindCoordinatorRequest(KafkaHeaderAndRequest findCoordinator,
CompletableFuture resultFuture) {
checkArgument(findCoordinator.getRequest() instanceof FindCoordinatorRequest);
FindCoordinatorRequest request = (FindCoordinatorRequest) findCoordinator.getRequest();
List coordinatorKeys = request.version() < FindCoordinatorRequest.MIN_BATCHED_VERSION
? Collections.singletonList(request.data().key()) : request.data().coordinatorKeys();
List> futures =
new ArrayList<>(coordinatorKeys.size());
for (String coordinatorKey : coordinatorKeys) {
CompletableFuture future =
findSingleCoordinator(coordinatorKey, findCoordinator);
futures.add(future);
}
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
.whenComplete((ignore, ex) -> {
if (ex != null) {
resultFuture.completeExceptionally(ex);
return;
}
List coordinators = new ArrayList<>(futures.size());
for (CompletableFuture future : futures) {
coordinators.add(future.join());
}
resultFuture.complete(KafkaResponseUtils.newFindCoordinator(coordinators, request.version()));
});
}
private CompletableFuture findSingleCoordinator(
String coordinatorKey, KafkaHeaderAndRequest findCoordinator) {
FindCoordinatorRequest request = (FindCoordinatorRequest) findCoordinator.getRequest();
CompletableFuture findSingleCoordinatorResult =
new CompletableFuture<>();
if (request.data().keyType() == FindCoordinatorRequest.CoordinatorType.TRANSACTION.id()) {
TransactionCoordinator transactionCoordinator = getTransactionCoordinator();
int partition = transactionCoordinator.partitionFor(coordinatorKey);
String pulsarTopicName = transactionCoordinator.getTopicPartitionName(partition);
findBroker(pulsarTopicName)
.whenComplete((KafkaResponseUtils.BrokerLookupResult result, Throwable throwable) -> {
if (result.error != Errors.NONE || throwable != null) {
log.error("[{}] Request {}: Error while find coordinator.",
ctx.channel(), findCoordinator.getHeader(), throwable);
findSingleCoordinatorResult.complete(
newCoordinator(Errors.LEADER_NOT_AVAILABLE, null, coordinatorKey));
return;
}
if (log.isDebugEnabled()) {
log.debug("[{}] Found node {} as coordinator for key {} partition {}.",
ctx.channel(), result.node, request.data().key(), partition);
}
findSingleCoordinatorResult.complete(
newCoordinator(result.error, result.node, coordinatorKey));
});
} else if (request.data().keyType() == FindCoordinatorRequest.CoordinatorType.GROUP.id()) {
authorize(AclOperation.DESCRIBE, Resource.of(ResourceType.GROUP, coordinatorKey))
.whenComplete((isAuthorized, ex) -> {
if (ex != null) {
log.error("Describe group authorize failed, group - {}. {}",
request.data().key(), ex.getMessage());
findSingleCoordinatorResult.complete(
newCoordinator(Errors.GROUP_AUTHORIZATION_FAILED, null, coordinatorKey));
return;
}
if (!isAuthorized) {
findSingleCoordinatorResult.complete(
newCoordinator(Errors.GROUP_AUTHORIZATION_FAILED, null, coordinatorKey));
return;
}
CompletableFuture storeGroupIdFuture;
if (kafkaConfig.isKopEnableGroupLevelConsumerMetrics()) {
String groupIdPath = GroupIdUtils.groupIdPathFormat(findCoordinator.getClientHost(),
findCoordinator.getHeader().clientId());
currentConnectedClientId.add(findCoordinator.getHeader().clientId());
// Store group name to metadata store for current client, use to collect consumer metrics.
storeGroupIdFuture = storeGroupId(coordinatorKey, groupIdPath);
} else {
storeGroupIdFuture = CompletableFuture.completedFuture(null);
}
// Store group name to metadata store for current client, use to collect consumer metrics.
storeGroupIdFuture.whenComplete((__, e) -> {
if (e != null) {
log.warn("Store groupId failed, the groupId might already stored.", e);
}
final var offsetConfig = getGroupCoordinator().getOffsetConfig();
final var topic = offsetConfig.offsetsTopicName() + TopicName.PARTITIONED_TOPIC_SUFFIX
+ GroupUtils.hash(coordinatorKey, offsetConfig.offsetsTopicNumPartitions());
findBroker(topic)
.whenComplete((KafkaResponseUtils.BrokerLookupResult result,
Throwable throwable) -> {
if (result.error != Errors.NONE || throwable != null) {
log.error("[{}] Request {}: Error while find coordinator.",
ctx.channel(), findCoordinator.getHeader(), throwable);
findSingleCoordinatorResult.complete(
newCoordinator(Errors.LEADER_NOT_AVAILABLE, null, coordinatorKey));
return;
}
if (log.isDebugEnabled()) {
log.debug("[{}] Found node {} as coordinator for key {}",
ctx.channel(), result.node, request.data().key());
}
findSingleCoordinatorResult.complete(
newCoordinator(result.error, result.node, coordinatorKey));
});
});
});
} else {
findSingleCoordinatorResult.completeExceptionally(
new NotImplementedException("FindCoordinatorRequest not support unknown type "
+ request.data().keyType()));
}
return findSingleCoordinatorResult;
}
@VisibleForTesting
protected CompletableFuture storeGroupId(String groupId, String groupIdPath) {
String path = groupIdStoredPath + groupIdPath;
CompletableFuture future = new CompletableFuture<>();
metadataStore.put(path, groupId.getBytes(UTF_8), Optional.empty())
.thenAccept(__ -> future.complete(null))
.exceptionally(ex -> {
future.completeExceptionally(ex);
return null;
});
return future;
}
@Override
protected void handleOffsetFetchRequest(KafkaHeaderAndRequest offsetFetch,
CompletableFuture resultFuture) {
checkArgument(offsetFetch.getRequest() instanceof OffsetFetchRequest);
OffsetFetchRequest request = (OffsetFetchRequest) offsetFetch.getRequest();
checkState(getGroupCoordinator() != null,
"Group Coordinator not started");
List> futures = new ArrayList<>();
if (request.version() >= 8) {
request.data().groups().forEach(group -> {
String groupId = group.groupId();
List partitions = new ArrayList<>();
// null topics means no partitions specified, so we should fetch all partitions
if (group.topics() != null) {
group
.topics()
.forEach(topic -> {
topic.partitionIndexes()
.forEach(partition -> partitions.add(new TopicPartition(topic.name(), partition)));
});
}
futures.add(getOffsetFetchForGroup(groupId, request.requireStable(), partitions));
});
} else {
// old clients
String groupId = request.data().groupId();
if (request.data().topics() == null) {
futures.add(getOffsetFetchForGroup(groupId, request.requireStable(), null));
} else {
List partitions = new ArrayList<>();
request.data().topics().forEach(topic -> topic.partitionIndexes()
.forEach(partition -> partitions.add(new TopicPartition(topic.name(), partition))));
futures.add(getOffsetFetchForGroup(groupId, request.requireStable(), partitions));
}
}
FutureUtil.waitForAll(futures).whenComplete((___, error) -> {
if (error != null) {
resultFuture.complete(request.getErrorResponse(error));
return;
}
List partitionsResponses = new ArrayList<>();
futures.forEach(f -> {
partitionsResponses.add(f.join());
});
resultFuture.complete(buildOffsetFetchResponse(partitionsResponses, request.version()));
});
}
protected CompletableFuture getOffsetFetchForGroup(
String groupId,
boolean requireStable,
List partitions
) {
CompletableFuture resultFuture = new CompletableFuture<>();
CompletableFuture> authorizeFuture = new CompletableFuture<>();
List authorizedPartitions = new ArrayList<>();
Map unauthorizedPartitionData =
Maps.newConcurrentMap();
Map unknownPartitionData =
Maps.newConcurrentMap();
if (partitions == null || partitions.isEmpty()) {
// fetch all partitions
authorizeFuture.complete(null);
} else {
AtomicInteger partitionCount = new AtomicInteger(partitions.size());
Runnable completeOneAuthorization = () -> {
if (partitionCount.decrementAndGet() == 0) {
authorizeFuture.complete(authorizedPartitions);
}
};
partitions.forEach(tp -> {
try {
final var fullName = kafkaToPulsar(tp.topic());
authorize(AclOperation.DESCRIBE, Resource.of(ResourceType.TOPIC, fullName))
.whenComplete((isAuthorized, ex) -> {
if (ex != null) {
log.error("Describe topic authorize failed, topic - {}. {}",
fullName, ex.getMessage());
unauthorizedPartitionData.put(tp, OffsetFetchResponse.UNAUTHORIZED_PARTITION);
completeOneAuthorization.run();
return;
}
if (!isAuthorized) {
unauthorizedPartitionData.put(tp, OffsetFetchResponse.UNAUTHORIZED_PARTITION);
completeOneAuthorization.run();
return;
}
authorizedPartitions.add(new KsnTopicPartition(tp, fullName));
completeOneAuthorization.run();
});
} catch (InvalidTopicException e) {
log.warn("Invalid topic name {}: {}", tp.topic(), e.getMessage());
unknownPartitionData.put(tp, new OffsetFetchResponse.PartitionData(-1L,
Optional.empty(), "", Errors.INVALID_TOPIC_EXCEPTION));
}
});
}
authorizeFuture.whenComplete((partitionList, ex) -> {
KeyValue> keyValue =
getGroupCoordinator().handleFetchOffsets(
groupId,
requireStable,
partitionList
);
if (log.isDebugEnabled()) {
log.debug("OFFSET_FETCH Unknown partitions: {}, Unauthorized partitions: {}.",
unknownPartitionData, unauthorizedPartitionData);
}
// recover to original topic name
Map partitionsResponses = keyValue.getValue();
partitionsResponses.putAll(unauthorizedPartitionData);
partitionsResponses.putAll(unknownPartitionData);
Errors errors = keyValue.getKey();
resultFuture.complete(new KafkaResponseUtils.OffsetFetchResponseGroupData(groupId, errors,
partitionsResponses));
});
return resultFuture;
}
private CompletableFuture> fetchOffset(String originalTopicName,
long timestamp,
boolean readCommitted) {
CompletableFuture> partitionData = new CompletableFuture<>();
final String topicName;
if (originalTopicName.startsWith("persistent://")) {
topicName = originalTopicName;
} else {
topicName = "persistent://" + defaultNamespacePrefix + originalTopicName;
}
// The getTopic method may complete the future in `ZKMetadataStore` thread,
// it is better to use `thenAcceptAsync` to avoid blocking the `ZKMetadataStore` thread
// by `replicaManager.getPartitionLog(topicName);` method.
topicManager.getTopic(topicName).thenAcceptAsync((perTopicOpt) -> {
if (!perTopicOpt.isPresent()) {
partitionData.complete(Pair.of(Errors.UNKNOWN_TOPIC_OR_PARTITION, null));
return;
}
PersistentTopic perTopic = perTopicOpt.get();
ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) perTopic.getManagedLedger();
PositionImpl lac = (PositionImpl) managedLedger.getLastConfirmedEntry();
if (lac == null) {
log.error("[{}] Unexpected LastConfirmedEntry for topic {}, managed ledger: {}",
ctx, perTopic.getName(), managedLedger.getName());
partitionData.complete(Pair.of(Errors.UNKNOWN_SERVER_ERROR, -1L));
return;
}
final long logEndOffset = MessageMetadataUtils.getLogEndOffset(managedLedger);
if (timestamp == ListOffsetsRequest.LATEST_TIMESTAMP) {
final CompletableFuture lsoFuture;
if (readCommitted) {
PartitionLog partitionLog = replicaManager.getPartitionLog(topicName);
lsoFuture = partitionLog.awaitInitialisation().thenApply(__ -> {
final long lso = partitionLog.firstUndecidedOffset().orElse(logEndOffset);
if (log.isDebugEnabled()) {
log.debug("[{}] Get last stable offset for topic {} highWatermark: {}, lso: {}",
ctx, perTopic.getName(), logEndOffset, lso);
}
return lso;
});
} else {
lsoFuture = CompletableFuture.completedFuture(logEndOffset);
}
lsoFuture.whenComplete((lso, throwable) -> {
if (throwable != null) {
log.error("[{}] Failed to get last stable offset for topic {}",
ctx, perTopic.getName(), throwable);
partitionData.complete(Pair.of(Errors.UNKNOWN_SERVER_ERROR, null));
return;
}
partitionData.complete(Pair.of(Errors.NONE, lso));
});
} else if (timestamp == ListOffsetsRequest.EARLIEST_TIMESTAMP) {
fetchEarliestOffset(partitionData, managedLedger, lac, timestamp, logEndOffset, perTopic);
} else {
fetchOffsetByTimestamp(partitionData, managedLedger, lac, timestamp, perTopic);
}
}, orderedExecutor).exceptionally(e -> {
Throwable throwable = FutureUtil.unwrapCompletionException(e);
log.error("Failed while get persistentTopic topic: {} ts: {}. ", topicName, timestamp, throwable);
partitionData.complete(Pair.of(Errors.UNKNOWN_TOPIC_OR_PARTITION, null));
return null;
});
return partitionData;
}
private void fetchEarliestOffset(CompletableFuture> partitionData,
ManagedLedgerImpl managedLedger,
PositionImpl lac,
long timestamp,
long logEndOffset,
PersistentTopic perTopic) {
// We need to read the original position first because ledgers may be trimmed after read compactedOffset
PositionImpl position = OffsetFinder.getFirstValidPosition(managedLedger);
CompletableFuture readCompactedOffsetFuture;
if (perTopic.getSubscriptions().containsKey(COMPACTION_SUBSCRIPTION)) {
readCompactedOffsetFuture = fetchFirstCompactedOffset(perTopic.getTopicCompactionService(),
perTopic.getName(), lac);
} else {
readCompactedOffsetFuture = CompletableFuture.completedFuture(null);
}
readCompactedOffsetFuture.thenAccept(firstCompactedOffset -> {
if (firstCompactedOffset != null) {
partitionData.complete(Pair.of(Errors.NONE, firstCompactedOffset));
return;
}
if (position == null) {
log.error("[{}] Failed to find first valid position for topic {}", ctx,
perTopic.getName());
partitionData.complete(Pair.of(Errors.UNKNOWN_SERVER_ERROR, -1L));
return;
}
if (log.isDebugEnabled()) {
log.debug("[{}] Get earliest position for topic {}: {}, lac: {}",
ctx, perTopic.getName(), position, lac);
}
if (position.compareTo(lac) > 0) {
partitionData.complete(Pair.of(Errors.NONE, logEndOffset));
} else {
MessageMetadataUtils.getOffsetOfPosition(managedLedger, position, false,
timestamp, skipMessagesWithoutIndex)
.whenComplete((offset, throwable) -> {
if (throwable != null) {
log.error("[{}] Failed to get offset for position {}",
perTopic, position, throwable);
partitionData.complete(Pair.of(Errors.UNKNOWN_SERVER_ERROR,
null));
return;
}
if (log.isDebugEnabled()) {
log.debug(
"[{}] Get offset of position for topic {}: {}, lac: "
+ "{}, offset: {}, logEndOffset: {}",
ctx, perTopic.getName(), position, lac, offset, logEndOffset);
}
long earliestOffset = Math.min(logEndOffset, offset);
partitionData.complete(Pair.of(Errors.NONE, earliestOffset));
});
}
}).exceptionally(ex -> {
log.error("[{}] Failed to find first valid offset for topic {}", ctx,
perTopic.getName(), ex);
partitionData.complete(Pair.of(Errors.UNKNOWN_SERVER_ERROR, -1L));
return null;
});
}
private CompletableFuture fetchFirstCompactedOffset(TopicCompactionService compactionService,
String topic,
PositionImpl lac) {
return compactionService.getLastCompactedPosition().thenCompose(lastPosition -> {
if (lastPosition != null) {
return compactionService.readCompactedEntries(PositionImpl.EARLIEST, 1);
} else {
return CompletableFuture.completedFuture(null);
}
}).thenApply(entries -> {
if (CollectionUtils.isEmpty(entries)) {
return null;
}
long firstCompactedOffset;
Entry entry = entries.get(0);
PositionImpl firstCompactedPosition = (PositionImpl) entry.getPosition();
try {
firstCompactedOffset = MessageMetadataUtils.peekBaseOffsetFromEntry(entry);
} catch (MetadataCorruptedException e) {
if (e instanceof MetadataCorruptedException.NoBrokerEntryMetadata
&& skipMessagesWithoutIndex) {
log.warn("The entry {} doesn't have BrokerEntryMetadata, return 0 as the offset",
firstCompactedPosition);
firstCompactedOffset = 0L;
} else {
throw new RuntimeException(e);
}
} finally {
entries.forEach(Entry::release);
}
if (firstCompactedPosition.compareTo(lac) > 0) {
firstCompactedOffset = 0L;
}
if (log.isDebugEnabled()) {
log.debug("[{}] Get earliest position for topic {}: {}, lac: {}, offset: {}",
ctx, topic, firstCompactedPosition, lac, firstCompactedOffset);
}
return firstCompactedOffset;
});
}
private void fetchOffsetByTimestamp(CompletableFuture> partitionData,
ManagedLedgerImpl managedLedger,
PositionImpl lac,
long timestamp,
PersistentTopic perTopic) {
// find with real wanted timestamp
OffsetFinder offsetFinder = new OffsetFinder(managedLedger);
final PositionImpl firstPosition = managedLedger.getNextValidPosition(managedLedger.getFirstPosition());
offsetFinder.findMessages(timestamp, new AsyncCallbacks.FindEntryCallback() {
@Override
public void findEntryComplete(Position position, Object ctx) {
final CompletableFuture findCompactedOffsetFuture;
if (perTopic.getSubscriptions().containsKey(COMPACTION_SUBSCRIPTION)
&& (position == null || position.equals(firstPosition))) {
findCompactedOffsetFuture = asyncFindOffsetByTimestampFromCompactedService(
perTopic.getTopicCompactionService(), timestamp);
} else {
findCompactedOffsetFuture = CompletableFuture.completedFuture(null);
}
findCompactedOffsetFuture.whenComplete((compactedOffset, exception) -> {
if (exception != null) {
log.error("Unable to find position for topic {} time {}. Exception:",
perTopic.getName(), timestamp, exception);
partitionData.complete(Pair.of(Errors.UNKNOWN_SERVER_ERROR, null));
return;
}
if (compactedOffset != null) {
partitionData.complete(Pair.of(Errors.NONE, compactedOffset));
return;
}
PositionImpl finalPosition;
if (position == null) {
finalPosition = OffsetFinder.getFirstValidPosition(managedLedger);
if (finalPosition == null) {
log.warn("Unable to find position for topic {} time {}. get NULL position",
perTopic.getName(), timestamp);
partitionData.complete(Pair.of(Errors.UNKNOWN_SERVER_ERROR, null));
return;
}
} else {
finalPosition = (PositionImpl) position;
}
if (log.isDebugEnabled()) {
log.debug("Find position for topic {} time {}. position: {}",
perTopic.getName(), timestamp, finalPosition);
}
if (finalPosition.compareTo(lac) > 0 || MessageMetadataUtils.getCurrentOffset(managedLedger) < 0) {
long offset = Math.max(0, MessageMetadataUtils.getCurrentOffset(managedLedger));
partitionData.complete(Pair.of(Errors.NONE, offset));
} else {
MessageMetadataUtils.getOffsetOfPosition(managedLedger, finalPosition, true,
timestamp, skipMessagesWithoutIndex)
.whenComplete((offset, throwable) -> {
if (throwable != null) {
log.error("[{}] Failed to get offset for position {}", perTopic.getName(),
finalPosition, throwable);
partitionData.complete(Pair.of(Errors.UNKNOWN_SERVER_ERROR, null));
return;
}
partitionData.complete(Pair.of(Errors.NONE, offset));
});
}
});
}
@Override
public void findEntryFailed(ManagedLedgerException exception,
Optional position, Object ctx) {
if (exception instanceof ManagedLedgerException.NonRecoverableLedgerException) {
final long logEndOffset = MessageMetadataUtils.getLogEndOffset(managedLedger);
if (!perTopic.getSubscriptions().containsKey(COMPACTION_SUBSCRIPTION)) {
// The position doesn't exist, it usually happens when the rollover of managed ledger
// leads to the deletion of all expired ledgers. In this case, there's only one empty
// ledger in the managed ledger. So here we complete it with the latest offset.
partitionData.complete(Pair.of(Errors.NONE, logEndOffset));
return;
}
asyncFindOffsetByTimestampFromCompactedService(perTopic.getTopicCompactionService(), timestamp)
.whenComplete((compactedOffset, ex) -> {
if (ex != null) {
log.error("Unable to find position for topic {} time {}. Exception:",
perTopic.getName(), timestamp, exception);
partitionData.complete(Pair.of(Errors.UNKNOWN_SERVER_ERROR, null));
return;
}
partitionData.complete(Pair.of(Errors.NONE,
compactedOffset != null ? compactedOffset : logEndOffset));
});
} else {
log.warn("Unable to find position for topic {} time {}. Exception:",
perTopic.getName(), timestamp, exception);
partitionData.complete(Pair.of(Errors.UNKNOWN_SERVER_ERROR, null));
}
}
});
}
private CompletableFuture asyncFindOffsetByTimestampFromCompactedService(
TopicCompactionService topicCompactionService,
long timestamp) {
return asyncGetCompactedLedger(topicCompactionService).thenCompose(lh -> {
if (lh == null) {
return CompletableFuture.completedFuture(null);
}
return MessageMetadataUtils.asyncFindOffsetByTimestampFromCompactedLeger(lh, timestamp,
skipMessagesWithoutIndex);
});
}
private void waitResponseDataComplete(CompletableFuture resultFuture,
Map>> responseData,
boolean legacy) {
CompletableFuture
.allOf(responseData.values().toArray(new CompletableFuture[0]))
.whenComplete((ignore, ex) -> {
ListOffsetsResponse response = KafkaResponseUtils.newListOffset(
CoreUtils.mapValue(responseData, CompletableFuture::join), legacy);
resultFuture.complete(response);
});
}
private void handleListOffsetRequestV1AndAbove(KafkaHeaderAndRequest listOffset,
CompletableFuture resultFuture) {
ListOffsetsRequest request = (ListOffsetsRequest) listOffset.getRequest();
boolean readCommitted = IsolationLevel.READ_COMMITTED.equals(request.isolationLevel());
Map>> responseData =
Maps.newConcurrentMap();
ListOffsetsRequestData data = request.data();
if (data.topics().size() == 0) {
resultFuture.complete(new ListOffsetsResponse(new ListOffsetsResponseData()));
return;
}
AtomicInteger partitions = new AtomicInteger(
data.topics().stream().map(ListOffsetsRequestData.ListOffsetsTopic::partitions)
.mapToInt(Collection::size).sum()
);
Runnable completeOne = () -> {
if (partitions.decrementAndGet() == 0) {
waitResponseDataComplete(resultFuture, responseData, false);
}
};
request.data().topics().forEach(listOffsetsTopic -> {
final var topic = listOffsetsTopic.name();
final String pulsarTopic;
try {
pulsarTopic = kafkaToPulsar(listOffsetsTopic.name());
} catch (InvalidTopicException __) {
listOffsetsTopic.partitions().forEach(listOffsetsPartition -> {
final var topicPartition = new TopicPartition(topic, listOffsetsPartition.partitionIndex());
responseData.put(topicPartition, CompletableFuture.completedFuture(
Pair.of(Errors.INVALID_TOPIC_EXCEPTION, null)));
completeOne.run();
});
return;
}
final var authorizeFuture = authorize(AclOperation.DESCRIBE,
Resource.of(ResourceType.TOPIC, pulsarTopic));
listOffsetsTopic.partitions().forEach(listOffsetsPartition -> {
final var partition = listOffsetsPartition.partitionIndex();
final var topicPartition = new TopicPartition(topic, partition);
final var fullPartitionName = pulsarTopic + TopicName.PARTITIONED_TOPIC_SUFFIX + partition;
authorizeFuture.whenComplete((isAuthorized, ex) -> {
if (ex != null) {
log.error("Describe topic authorize failed, topic - {}. {}",
fullPartitionName, ex.getMessage());
responseData.put(topicPartition, CompletableFuture.completedFuture(
Pair.of(Errors.TOPIC_AUTHORIZATION_FAILED, null)
));
completeOne.run();
return;
}
if (!isAuthorized) {
responseData.put(topicPartition, CompletableFuture.completedFuture(
Pair.of(Errors.TOPIC_AUTHORIZATION_FAILED, null)
));
completeOne.run();
return;
}
responseData.put(topicPartition, fetchOffset(
fullPartitionName,
listOffsetsPartition.timestamp(),
readCommitted));
completeOne.run();
});
});
});
}
// Some info can be found here
// https://cfchou.github.io/blog/2015/04/23/a-closer-look-at-kafka-offsetrequest/ through web.archive.org
private void handleListOffsetRequestV0(KafkaHeaderAndRequest listOffset,
CompletableFuture resultFuture) {
ListOffsetRequestV0 request =
byteBufToListOffsetRequestV0(listOffset.getBuffer());
boolean readCommitted = IsolationLevel.READ_COMMITTED.equals(request.isolationLevel());
Map>> responseData =
Maps.newConcurrentMap();
if (request.offsetData().size() == 0) {
resultFuture.complete(new ListOffsetsResponse(new ListOffsetsResponseData()));
return;
}
AtomicInteger partitions = new AtomicInteger(request.offsetData().size());
Runnable completeOne = () -> {
if (partitions.decrementAndGet() == 0) {
waitResponseDataComplete(resultFuture, responseData, true);
}
};
// in v0, the iterator is offsetData,
// in v1, the iterator is partitionTimestamps,
if (log.isDebugEnabled()) {
log.debug("received a v0 listOffset: {}", request.toString(true));
}
KafkaRequestUtils.LegacyUtils.forEachListOffsetRequest(request, topic -> times -> maxNumOffsets -> {
final String fullPartitionName;
try {
fullPartitionName = kafkaToPulsar(topic.topic())
+ TopicName.PARTITIONED_TOPIC_SUFFIX + topic.partition();
} catch (InvalidTopicException __) {
responseData.put(topic, CompletableFuture.completedFuture(
Pair.of(Errors.INVALID_TOPIC_EXCEPTION, null)));
completeOne.run();
return;
}
authorize(AclOperation.DESCRIBE, Resource.of(ResourceType.TOPIC, fullPartitionName))
.whenComplete((isAuthorized, ex) -> {
if (ex != null) {
log.error("Describe topic authorize failed, topic - {}. {}",
fullPartitionName, ex.getMessage());
responseData.put(topic, CompletableFuture.completedFuture(
Pair.of(Errors.TOPIC_AUTHORIZATION_FAILED, null)));
completeOne.run();
return;
}
if (!isAuthorized) {
responseData.put(topic, CompletableFuture.completedFuture(
Pair.of(Errors.TOPIC_AUTHORIZATION_FAILED, null)));
completeOne.run();
return;
}
CompletableFuture> partitionData;
// num_num_offsets > 1 is not handled for now, returning an error
if (maxNumOffsets > 1) {
log.warn("request is asking for multiples offsets for {}, not supported for now",
fullPartitionName);
partitionData = new CompletableFuture<>();
partitionData.complete(Pair.of(Errors.UNKNOWN_SERVER_ERROR, null));
}
partitionData = fetchOffset(fullPartitionName, times, readCommitted);
responseData.put(topic, partitionData);
completeOne.run();
});
});
}
// get offset from underline managedLedger
@Override
protected void handleListOffsetRequest(KafkaHeaderAndRequest listOffset,
CompletableFuture resultFuture) {
checkArgument(listOffset.getRequest() instanceof ListOffsetsRequest);
// the only difference between v0 and v1 is the `max_num_offsets => INT32`
// v0 is required because it is used by librdkafka
if (listOffset.getHeader().apiVersion() == 0) {
handleListOffsetRequestV0(listOffset, resultFuture);
} else {
handleListOffsetRequestV1AndAbove(listOffset, resultFuture);
}
}
private Map nonExistingTopicErrors() {
// TODO: The check for the existence of the topic is missing
return Maps.newHashMap();
}
@VisibleForTesting
Map convertOffsetCommitRequestRetentionMs(
Map convertedOffsetData,
long retentionTime,
short apiVersion,
long currentTimeStamp,
long configOffsetsRetentionMs) {
// V2 adds retention time to the request and V5 removes retention time
long offsetRetention;
if (apiVersion <= 1 || apiVersion >= 5 || retentionTime == OffsetCommitRequest.DEFAULT_RETENTION_TIME) {
offsetRetention = configOffsetsRetentionMs;
} else {
offsetRetention = retentionTime;
}
// commit from kafka
// > commit timestamp is always set to now.
// > "default" expiration timestamp is now + retention (and retention may be overridden if v2)
// > expire timestamp is computed differently for v1 and v2.
// > - If v1 and no explicit commit timestamp is provided we use default expiration timestamp.
// > - If v1 and explicit commit timestamp is provided we calculate retention from
// > that explicit commit timestamp
// > - If v2 we use the default expiration timestamp
// commit from kafka
long defaultExpireTimestamp = offsetRetention + currentTimeStamp;
long finalOffsetRetention = offsetRetention;
return CoreUtils.mapValue(convertedOffsetData, (partitionData) -> {
String metadata;
if (partitionData.committedMetadata() == null) {
metadata = OffsetMetadata.NO_METADATA;
} else {
metadata = partitionData.committedMetadata();
}
long expireTimeStamp;
if (partitionData.commitTimestamp() == OffsetCommitRequest.DEFAULT_TIMESTAMP) {
expireTimeStamp = defaultExpireTimestamp;
} else {
expireTimeStamp = finalOffsetRetention + partitionData.commitTimestamp();
}
return OffsetAndMetadata.apply(
partitionData.committedOffset(),
metadata,
currentTimeStamp,
expireTimeStamp);
});
}
@Override
protected void handleOffsetCommitRequest(KafkaHeaderAndRequest offsetCommit,
CompletableFuture resultFuture) {
checkArgument(offsetCommit.getRequest() instanceof OffsetCommitRequest);
checkState(getGroupCoordinator() != null,
"Group Coordinator not started");
OffsetCommitRequest request = (OffsetCommitRequest) offsetCommit.getRequest();
OffsetCommitRequestData data = request.data();
final var failedTopics = new ConcurrentHashMap();
if (data.topics().isEmpty()) {
resultFuture.complete(KafkaResponseUtils.newOffsetCommit(Maps.newHashMap()));
return;
}
final var validOffsetMetadata = new ConcurrentHashMap();
AtomicInteger unfinishedAuthorizationCount = new AtomicInteger(
data.topics().stream().map(OffsetCommitRequestData.OffsetCommitRequestTopic::partitions)
.mapToInt(Collection::size).sum());
Consumer completeOne = (action) -> {
// When complete one authorization or failed, will do the action first.
action.run();
if (unfinishedAuthorizationCount.decrementAndGet() == 0) {
if (validOffsetMetadata.isEmpty()) {
OffsetCommitResponse response = KafkaResponseUtils.newOffsetCommit(failedTopics);
resultFuture.complete(response);
} else {
final var convertedPartitionData =
convertOffsetCommitRequestRetentionMs(
validOffsetMetadata,
KafkaRequestUtils.LegacyUtils.getRetentionTime(request),
offsetCommit.getHeader().apiVersion(),
Time.SYSTEM.milliseconds(),
getGroupCoordinator().getOffsetConfig().offsetsRetentionMs()
);
getGroupCoordinator().handleCommitOffsets(
data.groupId(),
data.memberId(),
data.generationId(),
convertedPartitionData
).thenAccept(offsetCommitResult -> {
offsetCommitResult.putAll(failedTopics);
OffsetCommitResponse response = KafkaResponseUtils.newOffsetCommit(offsetCommitResult);
resultFuture.complete(response);
}).exceptionally(e -> {
log.error("[{}] Failed to handle commit offsets {}",
ctx.channel(), offsetCommit.getHeader(), e);
resultFuture.completeExceptionally(e);
return null;
});
}
}
};
data.topics().forEach((OffsetCommitRequestData.OffsetCommitRequestTopic topicData) -> {
final var topic = topicData.name();
final String pulsarTopic;
try {
pulsarTopic = kafkaToPulsar(topic);
} catch (InvalidTopicException __) {
topicData.partitions().forEach(partitionData -> {
completeOne.accept(() -> failedTopics.put(new TopicPartition(topic, partitionData.partitionIndex()),
Errors.INVALID_TOPIC_EXCEPTION));
});
return;
}
final var authorizeFuture = authorize(AclOperation.READ, Resource.of(ResourceType.TOPIC, pulsarTopic));
topicData.partitions().forEach((OffsetCommitRequestData.OffsetCommitRequestPartition partitionData) -> {
final var topicPartition = new TopicPartition(topic, partitionData.partitionIndex());
final var ksnTopicPartition = new KsnTopicPartition(topicPartition, pulsarTopic);
authorizeFuture.whenComplete((isAuthorized, ex) -> {
if (isAuthorized != null && isAuthorized) {
completeOne.accept(() -> validOffsetMetadata.put(ksnTopicPartition, partitionData));
return;
}
if (ex != null) {
log.error("OffsetCommit authorize failed, topic - {}. {}",
pulsarTopic, ex.getMessage());
}
completeOne.accept(() -> failedTopics.put(topicPartition, Errors.TOPIC_AUTHORIZATION_FAILED));
});
});
});
}
@Override
protected void handleFetchRequest(KafkaHeaderAndRequest fetch,
CompletableFuture resultFuture) {
checkArgument(fetch.getRequest() instanceof FetchRequest);
FetchRequest request = (FetchRequest) fetch.getRequest();
FetchRequestData data = request.data();
if (log.isDebugEnabled()) {
log.debug("[{}] Request {} Fetch request. Size: {}. Each item: ",
ctx.channel(), fetch.getHeader(), data.topics().size());
data.topics().forEach((topicData) -> {
log.debug("Fetch request topic: data:{}.", topicData.toString());
});
}
int numPartitions = data.topics().stream().mapToInt(topic -> topic.partitions().size()).sum();
if (numPartitions == 0) {
resultFuture.complete(new FetchResponse(new FetchResponseData()
.setErrorCode(Errors.NONE.code())
.setSessionId(request.metadata().sessionId())
.setResponses(new ArrayList<>())));
return;
}
ConcurrentHashMap erroneous =
new ConcurrentHashMap<>();
ConcurrentHashMap interesting =
new ConcurrentHashMap<>();
AtomicInteger unfinishedAuthorizationCount = new AtomicInteger(numPartitions);
Runnable completeOne = () -> {
if (unfinishedAuthorizationCount.decrementAndGet() == 0) {
TransactionCoordinator transactionCoordinator = null;
if (request.isolationLevel().equals(IsolationLevel.READ_COMMITTED)
&& kafkaConfig.isKafkaTransactionCoordinatorEnabled()) {
transactionCoordinator = getTransactionCoordinator();
}
String namespacePrefix = currentNamespacePrefix();
int fetchMaxBytes = request.maxBytes();
int fetchMinBytes = Math.min(request.minBytes(), fetchMaxBytes);
if (interesting.isEmpty()) {
if (log.isDebugEnabled()) {
log.debug("Fetch interesting is empty. Partitions: [{}]", data.topics());
}
resultFuture.complete(new FetchResponse(new FetchResponseData()
.setErrorCode(Errors.NONE.code())
.setSessionId(request.metadata().sessionId())
.setResponses(buildFetchResponses(erroneous))));
} else {
MessageFetchContext context = MessageFetchContext
.get(this, transactionCoordinator, maxReadEntriesNum, namespacePrefix,
getKafkaTopicManagerSharedState(), this.executor, fetch);
this.getReplicaManager().fetchMessage(
request.maxWait(),
fetchMinBytes,
fetchMaxBytes,
interesting,
request.isolationLevel(),
context
).thenAccept(resultMap -> {
Map all = new HashMap<>();
resultMap.forEach((tp, results) -> {
all.put(tp, results.toPartitionData());
});
all.putAll(erroneous);
boolean triggeredCompletion = resultFuture.complete(new ResponseCallbackWrapper(
new FetchResponse(new FetchResponseData()
.setErrorCode(Errors.NONE.code())
.setThrottleTimeMs(0)
.setSessionId(request.metadata().sessionId())
.setResponses(buildFetchResponses(all))),
() -> resultMap.forEach((__, readRecordsResult) -> {
readRecordsResult.recycle();
})
));
if (!triggeredCompletion) {
resultMap.forEach((__, readRecordsResult) -> {
readRecordsResult.recycle();
});
}
context.recycle();
}).exceptionally(e -> {
log.error("Failed to handle the fetch request", e);
resultFuture.completeExceptionally(e);
return null;
});
}
}
};
// Regular Kafka consumers need READ permission on each partition they are fetching.
data.topics().forEach(topicData -> {
final var topic = topicData.topic();
final String pulsarTopic;
try {
pulsarTopic = kafkaToPulsar(topic);
} catch (InvalidTopicException __) {
topicData.partitions().forEach(fetchPartition -> {
final var topicPartition = new TopicPartition(topic, fetchPartition.partition());
erroneous.put(topicPartition, errorResponse(Errors.INVALID_TOPIC_EXCEPTION));
completeOne.run();
});
return;
}
final var authorizeFuture = authorize(AclOperation.READ,
Resource.of(ResourceType.TOPIC, pulsarTopic));
topicData.partitions().forEach((partitionData) -> {
final var partition = partitionData.partition();
final var topicPartition = new TopicPartition(topic, partition);
authorizeFuture.whenCompleteAsync((isAuthorized, ex) -> {
if (ex != null) {
log.error("Read topic authorize failed, topic - {}. {}",
topicPartition, ex.getMessage());
erroneous.put(topicPartition, errorResponse(Errors.TOPIC_AUTHORIZATION_FAILED));
completeOne.run();
return;
}
if (!isAuthorized) {
erroneous.put(topicPartition, errorResponse(Errors.TOPIC_AUTHORIZATION_FAILED));
completeOne.run();
return;
}
interesting.put(new KsnTopicPartition(topicPartition, pulsarTopic), partitionData);
completeOne.run();
}, orderedExecutor);
});
});
}
public static List buildFetchResponses(
Map partitionData) {
List result = new ArrayList<>();
partitionData.keySet()
.stream()
.map(topicPartition -> topicPartition.topic())
.distinct()
.forEach(topic -> {
FetchResponseData.FetchableTopicResponse fetchableTopicResponse =
new FetchResponseData.FetchableTopicResponse()
.setTopic(topic)
.setPartitions(new ArrayList<>());
result.add(fetchableTopicResponse);
partitionData.forEach((tp, data) -> {
if (tp.topic().equals(topic)) {
fetchableTopicResponse.partitions().add(new FetchResponseData.PartitionData()
.setPartitionIndex(tp.partition())
.setErrorCode(data.errorCode())
.setHighWatermark(data.highWatermark())
.setLastStableOffset(data.lastStableOffset())
.setLogStartOffset(data.logStartOffset())
.setAbortedTransactions(data.abortedTransactions())
.setPreferredReadReplica(data.preferredReadReplica())
.setRecords(data.records()));
}
});
});
return result;
}
private static FetchResponseData.PartitionData errorResponse(Errors error) {
return new FetchResponseData.PartitionData()
.setErrorCode(error.code())
.setHighWatermark(FetchResponse.INVALID_HIGH_WATERMARK)
.setLastStableOffset(FetchResponse.INVALID_LAST_STABLE_OFFSET)
.setLogStartOffset(FetchResponse.INVALID_LOG_START_OFFSET)
.setRecords(MemoryRecords.EMPTY);
}
@Override
protected void handleJoinGroupRequest(KafkaHeaderAndRequest joinGroup,
CompletableFuture resultFuture) {
checkArgument(joinGroup.getRequest() instanceof JoinGroupRequest);
checkState(getGroupCoordinator() != null,
"Group Coordinator not started");
JoinGroupRequest request = (JoinGroupRequest) joinGroup.getRequest();
JoinGroupRequestData data = request.data();
Map protocols = new HashMap<>();
data.protocols()
.forEach(protocol -> protocols.put(protocol.name(), protocol.metadata()));
getGroupCoordinator().handleJoinGroup(
data.groupId(),
data.memberId(),
joinGroup.getHeader().clientId(),
joinGroup.getClientHost(),
data.rebalanceTimeoutMs(),
data.sessionTimeoutMs(),
data.protocolType(),
protocols
).thenAccept(joinGroupResult -> {
Map members = new HashMap<>(joinGroupResult.getMembers());
JoinGroupResponse response = KafkaResponseUtils.newJoinGroup(
joinGroupResult.getError(),
joinGroupResult.getGenerationId(),
joinGroupResult.getProtocolName(),
joinGroupResult.getProtocolType(),
joinGroupResult.getMemberId(),
joinGroupResult.getLeaderId(),
members,
request.version()
);
if (log.isTraceEnabled()) {
log.trace("Sending join group response {} for correlation id {} to client {}.",
response, joinGroup.getHeader().correlationId(), joinGroup.getHeader().clientId());
}
resultFuture.complete(response);
});
}
@Override
protected void handleSyncGroupRequest(KafkaHeaderAndRequest syncGroup,
CompletableFuture resultFuture) {
checkArgument(syncGroup.getRequest() instanceof SyncGroupRequest);
SyncGroupRequest request = (SyncGroupRequest) syncGroup.getRequest();
SyncGroupRequestData data = request.data();
groupIds.add(data.groupId());
Map assignments = data.assignments()
.stream()
.collect(Collectors.toMap(
SyncGroupRequestData.SyncGroupRequestAssignment::memberId,
SyncGroupRequestData.SyncGroupRequestAssignment::assignment));
getGroupCoordinator().handleSyncGroup(
data.groupId(),
data.generationId(),
data.memberId(),
assignments
).thenAccept(syncGroupResult -> {
SyncGroupResponse response = KafkaResponseUtils.newSyncGroup(
syncGroupResult.getKey(),
data.protocolType(),
data.protocolName(),
syncGroupResult.getValue()
);
resultFuture.complete(response);
});
}
@Override
protected void handleHeartbeatRequest(KafkaHeaderAndRequest heartbeat,
CompletableFuture resultFuture) {
checkArgument(heartbeat.getRequest() instanceof HeartbeatRequest);
HeartbeatRequest request = (HeartbeatRequest) heartbeat.getRequest();
// let the coordinator to handle heartbeat
getGroupCoordinator().handleHeartbeat(
request.data().groupId(),
request.data().memberId(),
request.data().generationId()
).thenAccept(errors -> {
HeartbeatResponse response = KafkaResponseUtils.newHeartbeat(errors);
if (log.isTraceEnabled()) {
log.trace("Sending heartbeat response {} for correlation id {} to client {}.",
response, heartbeat.getHeader().correlationId(), heartbeat.getHeader().clientId());
}
resultFuture.complete(response);
});
}
@Override
protected void handleLeaveGroupRequest(KafkaHeaderAndRequest leaveGroup,
CompletableFuture resultFuture) {
checkArgument(leaveGroup.getRequest() instanceof LeaveGroupRequest);
LeaveGroupRequest request = (LeaveGroupRequest) leaveGroup.getRequest();
LeaveGroupRequestData data = request.data();
Set members = data.members().stream()
.map(LeaveGroupRequestData.MemberIdentity::memberId)
.collect(Collectors.toSet());
if (!data.memberId().isEmpty()) {
// old clients
members.add(data.memberId());
}
// let the coordinator to handle heartbeat
getGroupCoordinator().handleLeaveGroup(
data.groupId(),
members
).thenAccept(errors -> resultFuture.complete(KafkaResponseUtils.newLeaveGroup(errors)));
}
@Override
protected void handleDescribeGroupRequest(KafkaHeaderAndRequest describeGroup,
CompletableFuture resultFuture) {
checkArgument(describeGroup.getRequest() instanceof DescribeGroupsRequest);
DescribeGroupsRequest request = (DescribeGroupsRequest) describeGroup.getRequest();
// let the coordinator to handle heartbeat
resultFuture.complete(KafkaResponseUtils.newDescribeGroups(request.data().groups().stream()
.map(groupId -> Pair.of(groupId, getGroupCoordinator().handleDescribeGroup(groupId)))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight))
));
}
@Override
protected void handleListGroupsRequest(KafkaHeaderAndRequest listGroups,
CompletableFuture resultFuture) {
checkArgument(listGroups.getRequest() instanceof ListGroupsRequest);
ListGroupsRequest request = (ListGroupsRequest) listGroups.getRequest();
List statesFilter = request.data().statesFilter();
final Set states;
if (statesFilter != null) {
states = new HashSet<>(statesFilter);
} else {
states = Set.of();
}
KeyValue> listResult = getGroupCoordinator().handleListGroups(states);
resultFuture.complete(KafkaResponseUtils.newListGroups(listResult.getKey(), listResult.getValue()));
}
@Override
protected void handleDeleteGroupsRequest(KafkaHeaderAndRequest deleteGroups,
CompletableFuture resultFuture) {
checkArgument(deleteGroups.getRequest() instanceof DeleteGroupsRequest);
DeleteGroupsRequest request = (DeleteGroupsRequest) deleteGroups.getRequest();
DeleteGroupsRequestData data = request.data();
resultFuture.complete(KafkaResponseUtils.newDeleteGroups(
getGroupCoordinator().handleDeleteGroups(data.groupsNames())
));
}
@Override
protected void handleSaslAuthenticate(KafkaHeaderAndRequest saslAuthenticate,
CompletableFuture resultFuture) {
resultFuture.complete(new SaslAuthenticateResponse(
new SaslAuthenticateResponseData()
.setErrorCode(Errors.ILLEGAL_SASL_STATE.code())
.setErrorMessage("SaslAuthenticate request received after successful authentication")));
}
@Override
protected void handleSaslHandshake(KafkaHeaderAndRequest saslHandshake,
CompletableFuture resultFuture) {
resultFuture.complete(KafkaResponseUtils.newSaslHandshake(Errors.ILLEGAL_SASL_STATE));
}
@Override
protected void handleCreateTopics(KafkaHeaderAndRequest createTopics,
CompletableFuture resultFuture) {
checkArgument(createTopics.getRequest() instanceof CreateTopicsRequest);
CreateTopicsRequest request = (CreateTopicsRequest) createTopics.getRequest();
final Map result = Maps.newConcurrentMap();
final Map validTopics = Maps.newHashMap();
final Set duplicateTopics = new HashSet<>();
request.data().topics().forEach((CreatableTopic topic) -> {
if (duplicateTopics.add(topic.name())) {
validTopics.put(topic.name(), topic);
} else {
final String errorMessage = "Create topics request from client `" + createTopics.getHeader().clientId()
+ "` contains multiple entries for the following topics: " + duplicateTopics;
result.put(topic.name(), new ApiError(Errors.INVALID_REQUEST, errorMessage));
}
});
if (validTopics.isEmpty()) {
resultFuture.complete(KafkaResponseUtils.newCreateTopics(result));
return;
}
String namespacePrefix = currentNamespacePrefix();
final AtomicInteger validTopicsCount = new AtomicInteger(validTopics.size());
final var authorizedTopics = new ConcurrentHashMap();
Runnable createTopicsAsync = () -> {
if (authorizedTopics.isEmpty()) {
resultFuture.complete(KafkaResponseUtils.newCreateTopics(result));
return;
}
// TODO: handle request.validateOnly()
adminManager.createTopicsAsync(authorizedTopics, request.data().timeoutMs(), namespacePrefix)
.thenApply(validResult -> {
result.putAll(validResult);
resultFuture.complete(KafkaResponseUtils.newCreateTopics(result));
return null;
});
};
BiConsumer completeOneTopic = (topic, topicDetails) -> {
authorizedTopics.put(topic, topicDetails);
if (validTopicsCount.decrementAndGet() == 0) {
createTopicsAsync.run();
}
};
BiConsumer completeOneErrorTopic = (topic, error) -> {
result.put(topic, error);
if (validTopicsCount.decrementAndGet() == 0) {
createTopicsAsync.run();
}
};
validTopics.forEach((topic, details) -> {
final String fullTopicName;
try {
fullTopicName = kafkaToPulsar(topic);
} catch (InvalidTopicException e) {
completeOneErrorTopic.accept(topic, ApiError.fromThrowable(e));
return;
}
authorize(AclOperation.CREATE, Resource.of(ResourceType.TOPIC, fullTopicName))
.whenComplete((isAuthorized, ex) -> {
if (ex != null) {
log.error("CreateTopics authorize failed, topic - {}. {}",
fullTopicName, ex.getMessage());
completeOneErrorTopic
.accept(topic, new ApiError(Errors.TOPIC_AUTHORIZATION_FAILED, ex.getMessage()));
return;
}
if (!isAuthorized) {
log.error("CreateTopics authorize failed, topic - {}.", fullTopicName);
completeOneErrorTopic
.accept(topic, new ApiError(Errors.TOPIC_AUTHORIZATION_FAILED, null));
return;
}
completeOneTopic.accept(new KsnTopicPartition(new TopicPartition(topic, 0), fullTopicName),
details);
});
});
}
@Override
protected void handleAlterConfigs(KafkaHeaderAndRequest describeConfigs,
CompletableFuture resultFuture) {
checkArgument(describeConfigs.getRequest() instanceof AlterConfigsRequest);
AlterConfigsRequest request = (AlterConfigsRequest) describeConfigs.getRequest();
if (request.configs().isEmpty()) {
resultFuture.complete(new AlterConfigsResponse(new AlterConfigsResponseData()));
return;
}
AlterConfigsResponseData data = new AlterConfigsResponseData();
List> futures = new ArrayList<>();
request.data().resources().forEach((AlterConfigsRequestData.AlterConfigsResource resource) -> {
byte resourceType = resource.resourceType();
String resourceName = resource.resourceName();
IncrementalAlterConfigsRequestData.AlterConfigsResource configsResource =
new IncrementalAlterConfigsRequestData.AlterConfigsResource();
configsResource.setResourceName(resourceName);
configsResource.setResourceType(resourceType);
IncrementalAlterConfigsRequestData.AlterableConfigCollection configs =
new IncrementalAlterConfigsRequestData.AlterableConfigCollection();
for (AlterConfigsRequestData.AlterableConfig config : resource.configs()) {
var alterableConfig = new IncrementalAlterConfigsRequestData.AlterableConfig();
alterableConfig.setName(config.name());
alterableConfig.setValue(config.value());
alterableConfig.setConfigOperation(AlterConfigOp.OpType.SET.id());
configs.add(alterableConfig);
}
configsResource.setConfigs(configs);
var future = adminManager.alterConfigs(configsResource, defaultNamespacePrefix).handle((__, ex) -> {
ApiError apiError = ex != null ? ApiError.fromThrowable(ex) : ApiError.NONE;
return new AlterConfigsResponseData.AlterConfigsResourceResponse()
.setErrorCode(apiError.error().code())
.setErrorMessage(apiError.error().message())
.setResourceName(resourceName)
.setResourceType(resourceType);
});
futures.add(future);
});
FutureUtil.waitForAll(futures).whenComplete((__, ex) -> {
if (ex != null) {
resultFuture.completeExceptionally(ex);
return;
}
for (CompletableFuture future : futures) {
data.responses().add(future.join());
}
resultFuture.complete(new AlterConfigsResponse(data));
});
}
@Override
protected void handleIncrementalAlterConfigs(KafkaHeaderAndRequest describeConfigs,
CompletableFuture resultFuture) {
checkArgument(describeConfigs.getRequest() instanceof IncrementalAlterConfigsRequest);
IncrementalAlterConfigsRequest request = (IncrementalAlterConfigsRequest) describeConfigs.getRequest();
if (request.data().resources().isEmpty()) {
resultFuture.complete(new AlterConfigsResponse(new AlterConfigsResponseData()));
return;
}
IncrementalAlterConfigsResponseData data = new IncrementalAlterConfigsResponseData();
List> futures =
new ArrayList<>();
request.data().resources().forEach((IncrementalAlterConfigsRequestData.AlterConfigsResource resource) -> {
byte resourceType = resource.resourceType();
String resourceName = resource.resourceName();
var future = adminManager.alterConfigs(resource, defaultNamespacePrefix).handle((__, ex) -> {
ApiError apiError = ex != null ? ApiError.fromThrowable(ex) : ApiError.NONE;
return new IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse()
.setErrorCode(apiError.error().code())
.setErrorMessage(apiError.error().message())
.setResourceName(resourceName)
.setResourceType(resourceType);
});
futures.add(future);
});
FutureUtil.waitForAll(futures).whenComplete((__, ex) -> {
if (ex != null) {
resultFuture.completeExceptionally(ex);
return;
}
for (CompletableFuture future : futures) {
data.responses().add(future.join());
}
resultFuture.complete(new IncrementalAlterConfigsResponse(data));
});
}
@Override
protected void handleDescribeConfigs(KafkaHeaderAndRequest describeConfigs,
CompletableFuture resultFuture) {
checkArgument(describeConfigs.getRequest() instanceof DescribeConfigsRequest);
DescribeConfigsRequest request = (DescribeConfigsRequest) describeConfigs.getRequest();
DescribeConfigsRequestData data = request.data();
if (data.resources().isEmpty()) {
resultFuture.complete(new DescribeConfigsResponse(new DescribeConfigsResponseData()));
return;
}
Collection authorizedResources = Collections.synchronizedList(new ArrayList<>());
Map failedConfigResourceMap =
Maps.newConcurrentMap();
AtomicInteger unfinishedAuthorizationCount = new AtomicInteger(data.resources().size());
Consumer completeOne = (action) -> {
// When complete one authorization or failed, will do the action first.
action.run();
if (unfinishedAuthorizationCount.decrementAndGet() == 0) {
adminManager.describeConfigsAsync(authorizedResources.stream()
.collect(Collectors.toMap(
configResource -> configResource,
configResource -> data.resources().stream()
.filter(r -> r.resourceName().equals(configResource.name())
&& r.resourceType() == configResource.type().id())
.findAny()
.map(__ -> new HashSet<>())
)),
defaultNamespacePrefix
).thenApply(configResourceConfigMap -> {
DescribeConfigsResponseData responseData = new DescribeConfigsResponseData();
configResourceConfigMap.putAll(failedConfigResourceMap);
configResourceConfigMap.forEach((ConfigResource resource,
DescribeConfigsResponse.Config result) -> {
responseData.results().add(new DescribeConfigsResponseData.DescribeConfigsResult()
.setResourceName(resource.name())
.setResourceType(resource.type().id())
.setErrorCode(result.error().error().code())
.setErrorMessage(result.error().messageWithFallback())
.setConfigs(result.entries().stream().map(c -> {
return new DescribeConfigsResponseData.DescribeConfigsResourceResult()
.setName(c.name())
.setConfigSource(c.source().id())
.setReadOnly(c.isReadOnly())
.setConfigType(c.type().id())
.setValue(c.value())
.setDocumentation("");
}).collect(Collectors.toList())));
});
resultFuture.complete(new DescribeConfigsResponse(responseData));
return null;
});
}
};
// Do authorization for each of resource
data.resources().forEach((DescribeConfigsRequestData.DescribeConfigsResource configRes) -> {
ConfigResource configResource = new ConfigResource(ConfigResource.Type.forId(configRes.resourceType()),
configRes.resourceName());
switch (configResource.type()) {
case TOPIC:
final String pulsarTopic;
try {
pulsarTopic = kafkaToPulsar(configResource.name());
} catch (InvalidTopicException e) {
completeOne.accept(() -> {
final ApiError error = new ApiError(Errors.INVALID_TOPIC_EXCEPTION, null);
failedConfigResourceMap.put(configResource, new DescribeConfigsResponse.Config(
error, Collections.emptyList()));
});
return;
}
authorize(AclOperation.DESCRIBE_CONFIGS, Resource.of(ResourceType.TOPIC, pulsarTopic))
.whenComplete((isAuthorized, ex) -> {
if (ex != null) {
log.error("DescribeConfigs in topic authorize failed, topic - {}. {}",
pulsarTopic, ex.getMessage());
completeOne.accept(() -> failedConfigResourceMap.put(configResource,
new DescribeConfigsResponse.Config(
new ApiError(Errors.TOPIC_AUTHORIZATION_FAILED, null),
Collections.emptyList())));
return;
}
if (isAuthorized) {
completeOne.accept(() -> authorizedResources.add(configResource));
return;
}
completeOne.accept(() -> failedConfigResourceMap.put(configResource,
new DescribeConfigsResponse.Config(
new ApiError(Errors.TOPIC_AUTHORIZATION_FAILED, null),
Collections.emptyList())));
});
break;
case BROKER:
// Current KoP don't support Broker Resource.
// but we are not exposing anything to the client, so it is fine to serve requests.
completeOne.accept(() -> authorizedResources.add(configResource));
break;
case UNKNOWN:
case BROKER_LOGGER:
default:
completeOne.accept(() -> log.error("KoP doesn't support resource type: " + configResource.type()));
break;
}
});
}
@Override
protected void handleDescribeCluster(KafkaHeaderAndRequest describeConfigs,
CompletableFuture resultFuture) {
checkArgument(describeConfigs.getRequest() instanceof DescribeClusterRequest);
DescribeClusterResponseData data = new DescribeClusterResponseData();
List allNodes = new ArrayList<>(adminManager.getBrokers(advertisedEndPoint.getListenerName()));
// Each Pulsar broker can manage metadata like controller in Kafka,
// Kafka's AdminClient needs to find a controller node for metadata management.
// So here we return an random broker as a controller for the given listenerName.
final int controllerId = adminManager.getControllerId(advertisedEndPoint.getListenerName());
DescribeClusterResponse response = new DescribeClusterResponse(data);
data.setControllerId(controllerId);
data.setClusterId(clusterName);
data.setErrorCode(Errors.NONE.code());
data.setErrorMessage(Errors.NONE.message());
allNodes.forEach(node -> {
data.brokers().add(new DescribeClusterResponseData.DescribeClusterBroker()
.setBrokerId(node.id())
.setHost(node.host())
.setPort(node.port()));
});
resultFuture.complete(response);
}
@Override
protected void handleListTransactions(KafkaHeaderAndRequest kafkaHeaderAndRequest,
CompletableFuture resultFuture) {
checkArgument(kafkaHeaderAndRequest.getRequest() instanceof ListTransactionsRequest);
ListTransactionsRequest listTransactionsRequest = (ListTransactionsRequest) kafkaHeaderAndRequest.getRequest();
Set filteredProducerIds = new HashSet<>(listTransactionsRequest.data().producerIdFilters());
Set filteredStates = new HashSet<>(listTransactionsRequest.data().stateFilters());
TransactionCoordinator transactionCoordinator = getTransactionCoordinator();
ListTransactionsResponseData responseData =
transactionCoordinator.handleListTransactions(filteredProducerIds, filteredStates);
// TODO: The response should contain only transactionalIds that the principal
// has `Describe` permission to access.
resultFuture.complete(new ListTransactionsResponse(responseData));
}
@Override
protected void handleDescribeTransactionsRequest(KafkaHeaderAndRequest listGroups,
CompletableFuture response) {
checkArgument(listGroups.getRequest() instanceof DescribeTransactionsRequest);
DescribeTransactionsRequest request = (DescribeTransactionsRequest) listGroups.getRequest();
DescribeTransactionsResponseData describeResult = getTransactionCoordinator()
.handleDescribeTransactions(request.data().transactionalIds());
response.complete(new DescribeTransactionsResponse(describeResult));
}
@Override
protected void handleInitProducerId(KafkaHeaderAndRequest kafkaHeaderAndRequest,
CompletableFuture response) {
InitProducerIdRequest request = (InitProducerIdRequest) kafkaHeaderAndRequest.getRequest();
InitProducerIdRequestData data = request.data();
TransactionCoordinator transactionCoordinator = getTransactionCoordinator();
transactionCoordinator.handleInitProducerId(
data.transactionalId(), data.transactionTimeoutMs(), Optional.empty(),
(resp) -> {
InitProducerIdResponseData responseData = new InitProducerIdResponseData()
.setErrorCode(resp.getError().code())
.setProducerId(resp.getProducerId())
.setProducerEpoch(resp.getProducerEpoch());
response.complete(new InitProducerIdResponse(responseData));
});
}
@Override
protected void handleAddPartitionsToTxn(KafkaHeaderAndRequest kafkaHeaderAndRequest,
CompletableFuture response) {
AddPartitionsToTxnRequest request = (AddPartitionsToTxnRequest) kafkaHeaderAndRequest.getRequest();
AddPartitionsToTxnRequestData data = request.data();
List partitionsToAdd = request.partitions();
Map unauthorizedTopicErrors = Maps.newConcurrentMap();
Map nonExistingTopicErrors = Maps.newConcurrentMap();
Set authorizedPartitions = Sets.newConcurrentHashSet();
TransactionCoordinator transactionCoordinator = getTransactionCoordinator();
AtomicInteger unfinishedAuthorizationCount = new AtomicInteger(partitionsToAdd.size());
Consumer completeOne = (action) -> {
action.run();
if (unfinishedAuthorizationCount.decrementAndGet() == 0) {
if (!unauthorizedTopicErrors.isEmpty() || !nonExistingTopicErrors.isEmpty()) {
Map partitionErrors = Maps.newHashMap();
partitionErrors.putAll(unauthorizedTopicErrors);
partitionErrors.putAll(nonExistingTopicErrors);
for (TopicPartition topicPartition : authorizedPartitions) {
partitionErrors.put(topicPartition, Errors.OPERATION_NOT_ATTEMPTED);
}
response.complete(new AddPartitionsToTxnResponse(0, partitionErrors));
} else {
transactionCoordinator.handleAddPartitionsToTransaction(data.transactionalId(),
data.producerId(), data.producerEpoch(), authorizedPartitions, (errors) -> {
AddPartitionsToTxnResponseData responseData = new AddPartitionsToTxnResponseData();
// TODO: handle PRODUCER_FENCED errors
Map topicPartitionErrorsMap =
addPartitionError(partitionsToAdd, errors);
topicPartitionErrorsMap.keySet()
.stream()
.map(TopicPartition::topic)
.distinct()
.forEach(topicName -> {
AddPartitionsToTxnResponseData.AddPartitionsToTxnTopicResult topicResult =
new AddPartitionsToTxnResponseData.AddPartitionsToTxnTopicResult()
.setName(topicName);
responseData.results().add(topicResult);
topicPartitionErrorsMap.forEach((TopicPartition tp, Errors error) -> {
if (tp.topic().equals(topicName)) {
topicResult.results()
.add(new AddPartitionsToTxnResponseData
.AddPartitionsToTxnPartitionResult()
.setPartitionIndex(tp.partition())
.setErrorCode(error.code()));
}
});
});
response.complete(
new AddPartitionsToTxnResponse(responseData));
});
}
}
};
// key is the Kafka topic name, value is a triple of error code, Pulsar topic and authorize future
final var topicContextMap = new HashMap>>();
partitionsToAdd.forEach(tp -> {
var context = topicContextMap.get(tp.topic());
if (context == null) {
try {
final var pulsarTopic = kafkaToPulsar(tp.topic());
final var authorizeFuture = authorize(AclOperation.WRITE,
Resource.of(ResourceType.TOPIC, pulsarTopic));
context = Triple.of(Errors.NONE, pulsarTopic, authorizeFuture);
topicContextMap.put(tp.topic(), context);
} catch (InvalidTopicException __) {
completeOne.accept(() -> unauthorizedTopicErrors.put(tp, Errors.INVALID_TOPIC_EXCEPTION));
topicContextMap.put(tp.topic(), Triple.of(Errors.INVALID_TOPIC_EXCEPTION, null, null));
return;
}
}
final var pulsarTopic = context.getMiddle();
context.getRight()
.whenComplete((isAuthorized, ex) -> {
if (ex != null) {
log.error("AddPartitionsToTxn topic authorize failed, topic - {}. {}",
pulsarTopic, ex.getMessage());
completeOne.accept(() ->
unauthorizedTopicErrors.put(tp, Errors.TOPIC_AUTHORIZATION_FAILED));
return;
}
if (!isAuthorized) {
completeOne.accept(() ->
unauthorizedTopicErrors.put(tp, Errors.TOPIC_AUTHORIZATION_FAILED));
return;
}
completeOne.accept(() -> authorizedPartitions.add(tp));
});
});
}
@Override
protected void handleAddOffsetsToTxn(KafkaHeaderAndRequest kafkaHeaderAndRequest,
CompletableFuture response) {
AddOffsetsToTxnRequest request = (AddOffsetsToTxnRequest) kafkaHeaderAndRequest.getRequest();
AddOffsetsToTxnRequestData data = request.data();
final var offsetConfig = getGroupCoordinator().getOffsetConfig();
int partition = GroupUtils.hash(data.groupId(), offsetConfig.offsetsTopicNumPartitions());
String offsetTopicName = offsetConfig.offsetsTopicName();
TransactionCoordinator transactionCoordinator = getTransactionCoordinator();
Set topicPartitions = Collections.singleton(new TopicPartition(offsetTopicName, partition));
transactionCoordinator.handleAddPartitionsToTransaction(
data.transactionalId(),
data.producerId(),
data.producerEpoch(),
topicPartitions,
(errors) -> {
AddOffsetsToTxnResponseData responseData = new AddOffsetsToTxnResponseData()
.setErrorCode(errors.code());
// TODO: handle PRODUCER_FENCED errors
response.complete(
new AddOffsetsToTxnResponse(responseData));
});
}
private Map addPartitionError(Collection partitions, Errors errors) {
Map result = Maps.newHashMap();
for (TopicPartition partition : partitions) {
result.put(partition, errors);
}
return result;
}
@Override
protected void handleTxnOffsetCommit(KafkaHeaderAndRequest kafkaHeaderAndRequest,
CompletableFuture response) {
TxnOffsetCommitRequest request = (TxnOffsetCommitRequest) kafkaHeaderAndRequest.getRequest();
TxnOffsetCommitRequestData data = request.data();
if (data.topics().isEmpty()) {
response.complete(new TxnOffsetCommitResponse(0, Maps.newHashMap()));
return;
}
final var failedTopics = new ConcurrentHashMap();
final var validOffsetMetadata = new ConcurrentHashMap();
AtomicInteger unfinishedAuthorizationCount = new AtomicInteger(request.offsets().size());
Consumer completeOne = (action) -> {
action.run();
if (unfinishedAuthorizationCount.decrementAndGet() == 0) {
getGroupCoordinator().handleTxnCommitOffsets(
data.groupId(),
data.producerId(),
data.producerEpoch(),
convertTxnOffsets(validOffsetMetadata)
).thenAccept(offsetCommitResult -> {
offsetCommitResult.putAll(failedTopics);
response.complete(new TxnOffsetCommitResponse(0, offsetCommitResult));
}).exceptionally(e -> {
log.error("[{}] Failed to handle commit offsets {}",
ctx.channel(), kafkaHeaderAndRequest.getHeader(), e);
response.completeExceptionally(e);
return null;
});
}
};
request.offsets().forEach((tp, commitOffset) -> {
final String fullTopicName;
try {
fullTopicName = kafkaToPulsar(tp.topic());
} catch (InvalidTopicException __) {
completeOne.accept(() -> failedTopics.put(tp, Errors.INVALID_TOPIC_EXCEPTION));
return;
}
authorize(AclOperation.READ, Resource.of(ResourceType.TOPIC, fullTopicName))
.whenComplete((isAuthorized, ex) -> {
if (isAuthorized != null && isAuthorized) {
completeOne.accept(() ->
validOffsetMetadata.put(new KsnTopicPartition(tp, fullTopicName), commitOffset));
return;
}
if (ex != null) {
log.error("TxnOffsetCommit authorize failed, topic - {}. {}",
fullTopicName, ex.getMessage());
}
completeOne.accept(() -> failedTopics.put(tp, Errors.TOPIC_AUTHORIZATION_FAILED));
});
});
}
private Map convertTxnOffsets(
Map offsetsMap) {
long currentTimestamp = SystemTime.SYSTEM.milliseconds();
Map offsetAndMetadataMap = new HashMap<>();
for (var entry : offsetsMap.entrySet()) {
TxnOffsetCommitRequest.CommittedOffset partitionData = entry.getValue();
String metadata = KafkaRequestUtils.getMetadata(partitionData);
long offset = KafkaRequestUtils.getOffset(partitionData);
offsetAndMetadataMap.put(entry.getKey(),
OffsetAndMetadata.apply(offset, metadata, currentTimestamp, -1));
}
return offsetAndMetadataMap;
}
@Override
protected void handleEndTxn(KafkaHeaderAndRequest kafkaHeaderAndRequest,
CompletableFuture response) {
EndTxnRequest request = (EndTxnRequest) kafkaHeaderAndRequest.getRequest();
EndTxnRequestData data = request.data();
TransactionCoordinator transactionCoordinator = getTransactionCoordinator();
transactionCoordinator.handleEndTransaction(
data.transactionalId(),
data.producerId(),
data.producerEpoch(),
data.committed() ? TransactionResult.COMMIT : TransactionResult.ABORT,
errors -> response.complete(new EndTxnResponse(new EndTxnResponseData().setErrorCode(errors.code()))));
}
@Override
protected void handleWriteTxnMarkers(KafkaHeaderAndRequest kafkaHeaderAndRequest,
CompletableFuture response) {
WriteTxnMarkersRequest request = (WriteTxnMarkersRequest) kafkaHeaderAndRequest.getRequest();
Map> errors = new ConcurrentHashMap<>();
List markers = request.markers();
AtomicInteger numAppends = new AtomicInteger(markers.size());
if (numAppends.get() == 0) {
response.complete(new WriteTxnMarkersResponse(errors));
return;
}
BiConsumer> updateErrors = (producerId, currentErrors) -> {
Map previousErrors = errors.putIfAbsent(producerId, currentErrors);
if (previousErrors != null) {
previousErrors.putAll(currentErrors);
}
};
Runnable completeOne = () -> {
if (numAppends.decrementAndGet() == 0) {
response.complete(new WriteTxnMarkersResponse(errors));
}
};
for (WriteTxnMarkersRequest.TxnMarkerEntry marker : markers) {
long producerId = marker.producerId();
TransactionResult transactionResult = marker.transactionResult();
Map controlRecords = generateTxnMarkerRecords(marker);
AppendRecordsContext appendRecordsContext = AppendRecordsContext.get(
topicManager,
this::startSendOperationForThrottling,
this::completeSendOperationForThrottling,
this.pendingTopicFuturesMap,
ctx);
getReplicaManager().appendRecords(
(short) 1,
true,
controlRecords,
PartitionLog.AppendOrigin.Coordinator,
appendRecordsContext
).whenComplete((result, ex) -> {
if (ex != null) {
log.error("[{}] Append txn marker ({}) failed.", ctx.channel(), marker, ex);
Map currentErrors = new HashMap<>();
controlRecords.forEach(((ksnTopicPartition, partitionResponse) -> currentErrors.put(
ksnTopicPartition.topicPartition(), Errors.KAFKA_STORAGE_ERROR)));
updateErrors.accept(producerId, currentErrors);
completeOne.run();
return;
}
Map currentErrors = new HashMap<>();
result.forEach(((topicPartition, partitionResponse) -> {
if (log.isDebugEnabled()) {
log.debug("[{}] Append txn marker to topic : [{}], response: [{}].",
ctx.channel(), topicPartition, partitionResponse);
}
currentErrors.put(topicPartition, partitionResponse.error);
}));
updateErrors.accept(producerId, currentErrors);
final String metadataNamespace = kafkaConfig.getKafkaMetadataNamespace();
Set successfulOffsetsPartitions = result.keySet()
.stream()
.filter(topicPartition ->
KopTopic.isGroupMetadataTopicName(topicPartition.topic(), metadataNamespace))
.collect(Collectors.toSet());
if (!successfulOffsetsPartitions.isEmpty()) {
getGroupCoordinator().scheduleHandleTxnCompletion(
producerId,
successfulOffsetsPartitions
.stream().map(TopicPartition::partition).collect(Collectors.toSet()),
transactionResult
).whenComplete((__, e) -> {
if (e != null) {
log.error("Received an exception while trying to update the offsets cache on "
+ "transaction marker append", e);
ConcurrentHashMap updatedErrors = new ConcurrentHashMap<>();
successfulOffsetsPartitions.forEach(partition ->
updatedErrors.put(partition, Errors.forException(e.getCause())));
updateErrors.accept(producerId, updatedErrors);
}
completeOne.run();
});
return;
}
completeOne.run();
});
}
}
private Map generateTxnMarkerRecords(
WriteTxnMarkersRequest.TxnMarkerEntry marker) {
final var txnMarkerRecordsMap = new HashMap();
ControlRecordType controlRecordType = marker.transactionResult().equals(TransactionResult.COMMIT)
? ControlRecordType.COMMIT : ControlRecordType.ABORT;
EndTransactionMarker endTransactionMarker = new EndTransactionMarker(
controlRecordType, marker.coordinatorEpoch());
for (TopicPartition topicPartition : marker.partitions()) {
MemoryRecords memoryRecords = MemoryRecords.withEndTransactionMarker(
marker.producerId(), marker.producerEpoch(), endTransactionMarker);
txnMarkerRecordsMap.put(new KsnTopicPartition(topicPartition, kafkaToPulsar(topicPartition.topic())),
memoryRecords);
}
return txnMarkerRecordsMap;
}
@Override
protected void handleDeleteTopics(KafkaHeaderAndRequest deleteTopics,
CompletableFuture resultFuture) {
checkArgument(deleteTopics.getRequest() instanceof DeleteTopicsRequest);
DeleteTopicsRequest request = (DeleteTopicsRequest) deleteTopics.getRequest();
DeleteTopicsRequestData data = request.data();
List topicsToDelete = data.topics();
if (CollectionUtils.isEmpty(topicsToDelete) && CollectionUtils.isEmpty(data.topicNames())) {
resultFuture.complete(KafkaResponseUtils.newDeleteTopics(Maps.newHashMap()));
return;
}
// compatibility with older(<6) deleteTopics request
if (CollectionUtils.isNotEmpty(data.topicNames())) {
data.topicNames().forEach(n ->
topicsToDelete.add(new DeleteTopicsRequestData.DeleteTopicState().setName(n)));
}
Map deleteTopicsResponse = Maps.newConcurrentMap();
AtomicInteger topicToDeleteCount = new AtomicInteger(topicsToDelete.size());
BiConsumer completeOne = (topic, errors) -> {
deleteTopicsResponse.put(topic, errors);
if (errors == Errors.NONE) {
// create topic ZNode to trigger the coordinator DeleteTopicsEvent event
metadataStore.put(
KopEventManager.getDeleteTopicsPath()
+ "/" + TopicNameUtils.getTopicNameWithUrlEncoded(topic),
new byte[0],
Optional.empty());
}
if (topicToDeleteCount.decrementAndGet() == 0) {
resultFuture.complete(KafkaResponseUtils.newDeleteTopics(deleteTopicsResponse));
}
};
topicsToDelete.forEach((DeleteTopicsRequestData.DeleteTopicState topicState) -> {
String topic = topicState.name();
final String pulsarTopic;
try {
pulsarTopic = kafkaToPulsar(topic);
} catch (InvalidTopicException __) {
completeOne.accept(topic, Errors.INVALID_TOPIC_EXCEPTION);
return;
}
authorize(AclOperation.DELETE, Resource.of(ResourceType.TOPIC, pulsarTopic))
.whenComplete((isAuthorize, ex) -> {
if (ex != null) {
log.error("DeleteTopics authorize failed, topic - {}. {}",
topic, ex.getMessage());
completeOne.accept(topic, Errors.TOPIC_AUTHORIZATION_FAILED);
return;
}
if (!isAuthorize) {
completeOne.accept(topic, Errors.TOPIC_AUTHORIZATION_FAILED);
return;
}
adminManager.deleteTopic(pulsarTopic,
__ -> completeOne.accept(topic, Errors.NONE),
__ -> completeOne.accept(topic, Errors.UNKNOWN_TOPIC_OR_PARTITION));
});
});
}
@Override
protected void handleDeleteRecords(KafkaHeaderAndRequest deleteTopics,
CompletableFuture resultFuture) {
checkArgument(deleteTopics.getRequest() instanceof DeleteRecordsRequest);
DeleteRecordsRequest request = (DeleteRecordsRequest) deleteTopics.getRequest();
Map partitionOffsets = new HashMap<>();
request.data().topics().forEach((DeleteRecordsRequestData.DeleteRecordsTopic topic) -> {
String name = topic.name();
topic.partitions().forEach(partition -> {
TopicPartition topicPartition = new TopicPartition(name, partition.partitionIndex());
partitionOffsets.put(topicPartition, partition.offset());
});
});
if (partitionOffsets.isEmpty()) {
resultFuture.complete(KafkaResponseUtils.newDeleteRecords(Maps.newHashMap()));
return;
}
Map deleteRecordsResponse =
Maps.newConcurrentMap();
AtomicInteger topicToDeleteCount = new AtomicInteger(partitionOffsets.size());
BiConsumer completeOne = (topic, errors) -> {
deleteRecordsResponse.put(topic, errors);
if (topicToDeleteCount.decrementAndGet() == 0) {
resultFuture.complete(KafkaResponseUtils.newDeleteRecords(deleteRecordsResponse));
}
};
// Key is the Kafka topic, value is the pair of the Pulsar topic name and the authorization future.
// A null Pulsar topic represents that the Kafka topic is invalid.
final var context = new HashMap>>();
partitionOffsets.forEach((topicPartition, offset) -> {
final var kafkaTopic = topicPartition.topic();
final String pulsarTopic;
final CompletableFuture authorizeFuture;
final var pair = context.get(kafkaTopic);
if (pair != null) {
if (pair.getLeft() != null) {
pulsarTopic = pair.getLeft();
authorizeFuture = pair.getRight();
} else {
completeOne.accept(topicPartition, Errors.INVALID_TOPIC_EXCEPTION);
return;
}
} else {
try {
pulsarTopic = kafkaToPulsar(kafkaTopic);
} catch (InvalidTopicException __) {
context.put(kafkaTopic, Pair.of(null, null));
completeOne.accept(topicPartition, Errors.INVALID_TOPIC_EXCEPTION);
return;
}
authorizeFuture = authorize(AclOperation.DELETE, Resource.of(ResourceType.TOPIC, pulsarTopic));
context.put(kafkaTopic, Pair.of(pulsarTopic, authorizeFuture));
}
final var fullTopicName = TopicName.getTopicPartitionNameString(pulsarTopic, topicPartition.partition());
authorizeFuture
.whenComplete((isAuthorize, ex) -> {
if (ex != null) {
log.error("DeleteTopics authorize failed, topic - {}. {}",
fullTopicName, ex.getMessage());
completeOne.accept(topicPartition, Errors.TOPIC_AUTHORIZATION_FAILED);
return;
}
if (!isAuthorize) {
completeOne.accept(topicPartition, Errors.TOPIC_AUTHORIZATION_FAILED);
return;
}
topicManager
.getTopicConsumerManager(fullTopicName)
.thenAccept(topicManager -> topicManager.findPositionForIndex(offset)
.thenAccept(
position -> adminManager.truncateTopic(fullTopicName, offset, position,
__ -> completeOne.accept(topicPartition, Errors.NONE),
__ -> completeOne.accept(topicPartition,
Errors.UNKNOWN_TOPIC_OR_PARTITION))));
});
});
}
@Override
protected void handleOffsetDelete(KafkaHeaderAndRequest offsetDelete,
CompletableFuture response) {
checkArgument(offsetDelete.getRequest() instanceof OffsetDeleteRequest);
OffsetDeleteRequest request = (OffsetDeleteRequest) offsetDelete.getRequest();
String groupId = request.data().groupId();
Map topicPartitionErrors = Maps.newConcurrentMap();
List ksnTopicPartitions = new ArrayList<>();
authorize(AclOperation.DELETE, Resource.of(ResourceType.GROUP, groupId)).thenAccept(authorized -> {
if (!authorized) {
response.complete(request.getErrorResponse(-1, Errors.GROUP_AUTHORIZATION_FAILED));
return;
}
request.data().topics().forEach((OffsetDeleteRequestData.OffsetDeleteRequestTopic topic) -> {
String name = topic.name();
final String pulsarTopic;
try {
pulsarTopic = kafkaToPulsar(name);
} catch (InvalidTopicException __) {
topic.partitions().forEach(partition -> topicPartitionErrors.put(
new TopicPartition(name, partition.partitionIndex()), Errors.INVALID_TOPIC_EXCEPTION));
return;
}
final var authorizeFuture = authorize(AclOperation.READ, Resource.of(ResourceType.TOPIC, pulsarTopic));
topic.partitions().forEach(partition -> {
TopicPartition topicPartition = new TopicPartition(name, partition.partitionIndex());
authorizeFuture
.whenComplete((isAuthorize, ex) -> {
if (ex != null) {
log.error("OffsetDelete authorize failed, topic - {}. {}",
pulsarTopic, ex.getMessage());
topicPartitionErrors.put(topicPartition, Errors.TOPIC_AUTHORIZATION_FAILED);
return;
}
if (!isAuthorize) {
topicPartitionErrors.put(topicPartition, Errors.TOPIC_AUTHORIZATION_FAILED);
return;
}
ksnTopicPartitions.add(new KsnTopicPartition(topicPartition, pulsarTopic));
});
});
});
getGroupCoordinator().handleDeleteOffsets(groupId, ksnTopicPartitions)
.thenAccept(offsetDeleteTopicPartitionResponse -> {
Errors groupError = offsetDeleteTopicPartitionResponse.getLeft();
if (groupError != Errors.NONE) {
response.complete(request.getErrorResponse(0, groupError));
return;
}
topicPartitionErrors.putAll(offsetDeleteTopicPartitionResponse.getRight());
OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection topics =
new OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection();
Map> topicPartitionMap =
topicPartitionErrors.keySet().stream()
.collect(Collectors.groupingBy(TopicPartition::topic));
for (Map.Entry> entry : topicPartitionMap.entrySet()) {
String topic = entry.getKey();
List topicPartitionsList = entry.getValue();
OffsetDeleteResponseData.OffsetDeleteResponsePartitionCollection partitions =
new OffsetDeleteResponseData.OffsetDeleteResponsePartitionCollection();
for (TopicPartition topicPartition : topicPartitionsList) {
Errors error = topicPartitionErrors.get(topicPartition);
partitions.add(new OffsetDeleteResponseData.OffsetDeleteResponsePartition()
.setPartitionIndex(topicPartition.partition())
.setErrorCode(error.code()));
}
topics.add(new OffsetDeleteResponseData.OffsetDeleteResponseTopic()
.setName(topic)
.setPartitions(partitions));
}
response.complete(new OffsetDeleteResponse(new OffsetDeleteResponseData()
.setTopics(topics)));
});
}).exceptionally(ex -> {
log.error("OffsetDelete authorize failed, groupId - {}. {}",
groupId, ex.getMessage());
response.complete(request.getErrorResponse(-1, Errors.GROUP_AUTHORIZATION_FAILED));
return null;
});
}
@Override
protected void handleCreatePartitions(KafkaHeaderAndRequest createPartitions,
CompletableFuture resultFuture) {
checkArgument(createPartitions.getRequest() instanceof CreatePartitionsRequest);
CreatePartitionsRequest request = (CreatePartitionsRequest) createPartitions.getRequest();
final Map result = Maps.newConcurrentMap();
final var validTopics = new HashMap();
final Set duplicateTopics = new HashSet<>();
KafkaRequestUtils.forEachCreatePartitionsRequest(request, (topic, newPartition) -> {
if (duplicateTopics.add(topic)) {
validTopics.put(topic, newPartition);
} else {
final String errorMessage = "Create topics partitions request from client `"
+ createPartitions.getHeader().clientId()
+ "` contains multiple entries for the following topics: " + duplicateTopics;
result.put(topic, new ApiError(Errors.INVALID_REQUEST, errorMessage));
}
});
if (validTopics.isEmpty()) {
resultFuture.complete(KafkaResponseUtils.newCreatePartitions(result));
return;
}
final AtomicInteger validTopicsCount = new AtomicInteger(validTopics.size());
final var authorizedTopics = new ConcurrentHashMap();
Runnable createPartitionsAsync = () -> {
if (authorizedTopics.isEmpty()) {
resultFuture.complete(KafkaResponseUtils.newCreatePartitions(result));
return;
}
adminManager.createPartitionsAsync(authorizedTopics, request.data().timeoutMs())
.thenApply(validResult -> {
result.putAll(validResult);
resultFuture.complete(KafkaResponseUtils.newCreatePartitions(result));
return null;
});
};
BiConsumer completeOneTopic =
(ksnTopicPartition, newPartitions) -> {
authorizedTopics.put(ksnTopicPartition, newPartitions);
if (validTopicsCount.decrementAndGet() == 0) {
createPartitionsAsync.run();
}
};
BiConsumer completeOneErrorTopic = (topic, error) -> {
result.put(topic, error);
if (validTopicsCount.decrementAndGet() == 0) {
createPartitionsAsync.run();
}
};
validTopics.forEach((topic, newPartitions) -> {
final String pulsarTopic;
try {
pulsarTopic = kafkaToPulsar(topic);
} catch (InvalidTopicException __) {
completeOneErrorTopic.accept(topic, new ApiError(Errors.INVALID_TOPIC_EXCEPTION, null));
return;
}
authorize(AclOperation.ALTER, Resource.of(ResourceType.TOPIC, pulsarTopic))
.whenComplete((authorized, ex) -> {
if (ex != null) {
log.error("CreatePartitions authorize failed, topic - {}. {}",
pulsarTopic, ex.getMessage());
completeOneErrorTopic.accept(topic,
new ApiError(Errors.TOPIC_AUTHORIZATION_FAILED, ex.getMessage()));
return;
}
if (!authorized) {
completeOneErrorTopic.accept(topic,
new ApiError(Errors.TOPIC_AUTHORIZATION_FAILED, null));
return;
}
completeOneTopic.accept(new KsnTopicPartition(new TopicPartition(topic, 0), pulsarTopic),
newPartitions);
});
});
}
@Override
protected void handleDescribeProducersRequest(KafkaHeaderAndRequest describeGroup,
CompletableFuture responseFuture) {
checkArgument(describeGroup.getRequest() instanceof DescribeProducersRequest);
DescribeProducersRequest request = (DescribeProducersRequest) describeGroup.getRequest();
Map allResponses = Maps.newConcurrentMap();
Map errors = Maps.newConcurrentMap();
final int numPartitions = request.data().topics().stream()
.mapToInt(t -> t.partitionIndexes().size())
.sum();
Runnable completeOne = () -> {
if (errors.size() + allResponses.size() != numPartitions) {
// not enough responses
return;
}
errors.forEach((topicPartition, tpErrors) -> {
var topicResponse = new DescribeProducersResponseData.PartitionResponse()
.setPartitionIndex(topicPartition.partition())
.setErrorCode(tpErrors.code())
.setErrorMessage(tpErrors.message());
allResponses.put(topicPartition, topicResponse);
});
var response = new DescribeProducersResponseData();
allResponses
.entrySet()
.stream()
.collect(Collectors.groupingBy(
entry -> entry.getKey().topic(),
Collectors.mapping(
Map.Entry::getValue,
Collectors.toList()
)
))
.forEach((topic, partitionResponses) -> {
var topicResponse = new DescribeProducersResponseData.TopicResponse()
.setName(topic)
.setPartitions(partitionResponses);
response.topics().add(topicResponse);
});
responseFuture.complete(new DescribeProducersResponse(response));
};
request.data().topics().forEach ((topicRequest) -> {
final var kafkaTopic = topicRequest.name();
final String pulsarTopic;
try {
pulsarTopic = kafkaToPulsar(kafkaTopic);
} catch (InvalidTopicException __) {
topicRequest.partitionIndexes().forEach(partition ->
errors.put(new TopicPartition(kafkaTopic, partition), Errors.INVALID_TOPIC_EXCEPTION));
return;
}
final var authorizeFuture = authorize(AclOperation.READ, Resource.of(ResourceType.TOPIC, pulsarTopic));
topicRequest.partitionIndexes().forEach(partition -> {
final var tp = new TopicPartition(kafkaTopic, partition);
final var ksnTopicPartition = new KsnTopicPartition(tp, pulsarTopic);
authorizeFuture.whenCompleteAsync((isAuthorized, ex) -> {
if (ex != null) {
log.error("DescribeProducers authorize failed, topic - {}. {}",
ksnTopicPartition.fullPartitionName(), ex.getMessage());
errors.put(tp, Errors.TOPIC_AUTHORIZATION_FAILED);
completeOne.run();
return;
}
if (!isAuthorized) {
errors.put(tp, Errors.TOPIC_AUTHORIZATION_FAILED);
completeOne.run();
return;
}
var topicResponse = replicaManager.activeProducerState(ksnTopicPartition);
topicResponse.whenComplete((response, throwable) -> {
if (throwable != null) {
log.error("DescribeProducersRequest failed, topic - {}. {}",
ksnTopicPartition.fullPartitionName(), throwable.getMessage());
errors.put(tp, Errors.UNKNOWN_TOPIC_OR_PARTITION);
} else {
allResponses.put(tp, response);
}
completeOne.run();
});
}, orderedExecutor);
});
});
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
log.error("Caught error in handler, closing channel", cause);
this.close();
}
public CompletableFuture lookup(String topic) {
return findBroker(topic).thenApply(KafkaResponseUtils.BrokerLookupResult::toPartitionMetadata);
}
// The returned future never completes exceptionally
public CompletableFuture findBroker(String topic) {
final CompletableFuture future = new CompletableFuture<>();
lookupService.lookup(TopicName.get(topic), advertisedEndPoint)
.whenComplete((owner, e) -> {
if (e != null) {
log.warn("[{}] Exception while find Broker metadata: {}", topic, e.getMessage());
future.complete(newFailedPartitionMetadata(topic));
} else {
future.complete(newPartitionMetadata(topic, newNode(owner)));
}
});
return future;
}
static Node newNode(InetSocketAddress address) {
if (log.isDebugEnabled()) {
log.debug("Return Broker Node of {}. {}:{}", address, address.getHostString(), address.getPort());
}
return new Node(
Murmur3_32Hash.getInstance().makeHash((address.getHostString() + address.getPort()).getBytes(UTF_8)),
address.getHostString(),
address.getPort());
}
KafkaResponseUtils.BrokerLookupResult newPartitionMetadata(String topic, Node node) {
final var topicPartition = TopicNameUtils.pulsarToKafka(topic, defaultKafkaNamespacePrefix);
if (log.isDebugEnabled()) {
log.debug("Return PartitionMetadata node: {}, topicName: {} ({})", node, topic, topicPartition);
}
return KafkaResponseUtils.newMetadataPartition(topicPartition, node);
}
KafkaResponseUtils.BrokerLookupResult newFailedPartitionMetadata(String topic) {
final var topicPartition = TopicNameUtils.pulsarToKafka(topic, defaultKafkaNamespacePrefix);
log.warn("Failed find Broker metadata for {} ({}), create PartitionMetadata with NOT_LEADER_FOR_PARTITION",
topic, topicPartition);
// most of this error happens when topic is in loading/unloading status,
return KafkaResponseUtils.newMetadataPartition(
Errors.NOT_LEADER_OR_FOLLOWER, topicPartition);
}
private void throwIfTransactionCoordinatorDisabled() {
if (!kafkaConfig.isKafkaTransactionCoordinatorEnabled()) {
throw new IllegalArgumentException("Broker has disabled transaction coordinator, "
+ "please enable it before using transaction.");
}
}
@VisibleForTesting
protected CompletableFuture authorize(AclOperation operation, Resource resource) {
Session session = authenticator != null ? authenticator.session() : null;
return authorize(operation, resource, session);
}
protected CompletableFuture authorize(AclOperation operation, Resource resource, Session session) {
if (authorizer == null) {
return CompletableFuture.completedFuture(true);
}
if (session == null) {
return CompletableFuture.completedFuture(false);
}
CompletableFuture isAuthorizedFuture = null;
switch (operation) {
case READ:
isAuthorizedFuture = authorizer.canConsumeAsync(session.getPrincipal(), resource);
break;
case IDEMPOTENT_WRITE:
case WRITE:
isAuthorizedFuture = authorizer.canProduceAsync(session.getPrincipal(), resource);
break;
case DESCRIBE:
if (resource.getResourceType() == ResourceType.TOPIC) {
isAuthorizedFuture = authorizer.canLookupAsync(session.getPrincipal(), resource);
} else if (resource.getResourceType() == ResourceType.NAMESPACE) {
isAuthorizedFuture = authorizer.canGetTopicList(session.getPrincipal(), resource);
} else if (resource.getResourceType() == ResourceType.GROUP) {
isAuthorizedFuture = authorizer.canDescribeConsumerGroup(session.getPrincipal(), resource);
}
break;
case CREATE:
isAuthorizedFuture = authorizer.canCreateTopicAsync(session.getPrincipal(), resource);
break;
case DELETE:
if (resource.getResourceType() == ResourceType.GROUP) {
isAuthorizedFuture = authorizer.canManageTenantAsync(session.getPrincipal(), null);
break;
}
isAuthorizedFuture = authorizer.canDeleteTopicAsync(session.getPrincipal(), resource);
break;
case ALTER:
isAuthorizedFuture = authorizer.canAlterTopicAsync(session.getPrincipal(), resource);
break;
case DESCRIBE_CONFIGS:
isAuthorizedFuture = authorizer.canManageTenantAsync(session.getPrincipal(), resource);
break;
case ANY:
if (resource.getResourceType() == ResourceType.TENANT) {
isAuthorizedFuture = authorizer.canAccessTenantAsync(session.getPrincipal(), resource);
}
break;
case ALTER_CONFIGS:
case CLUSTER_ACTION:
case UNKNOWN:
case ALL:
default:
break;
}
if (isAuthorizedFuture == null) {
return FutureUtil.failedFuture(
new IllegalStateException("AclOperation [" + operation.name() + "] is not supported."));
}
return isAuthorizedFuture;
}
/**
* If we are using kafkaEnableMultiTenantMetadata we need to ensure
* that the TenantSpec refer to an existing tenant.
* @param session
* @return whether the tenant is accessible
*/
private boolean validateTenantAccessForSession(Session session)
throws AuthenticationException {
if (!kafkaConfig.isKafkaEnableMultiTenantMetadata()) {
// we are not leveraging kafkaEnableMultiTenantMetadata feature
// the client will access only system tenant
return true;
}
String tenantSpec = session.getPrincipal().getTenantSpec();
if (tenantSpec == null) {
// we are not leveraging kafkaEnableMultiTenantMetadata feature
// the client will access only system tenant
return true;
}
String currentTenant = extractTenantFromTenantSpec(tenantSpec);
try {
Boolean granted = authorize(AclOperation.ANY,
Resource.of(ResourceType.TENANT, currentTenant), session)
.get();
return granted != null && granted;
} catch (ExecutionException | InterruptedException err) {
log.error("Internal error while verifying tenant access", err);
throw new AuthenticationException("Internal error while verifying tenant access:" + err, err);
}
}
/**
* Return the threadpool that executes the conversion of data during Fetches.
* We don't want to decode data inside the critical threads like the ManagedLedger Ordered Executor threads.
* @return a executor.
*/
public Executor getDecodeExecutor() {
return this.executor;
}
private String kafkaToPulsar(final String kafkaTopic) throws InvalidTopicException {
return TopicNameUtils.kafkaToPulsar(kafkaTopic, defaultNamespacePrefix);
}
}