All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.kafka.clients.Metadata Maven / Gradle / Ivy

There is a newer version: 3.3.8
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.kafka.clients;

import org.apache.kafka.common.Cluster;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.errors.InvalidMetadataException;
import org.apache.kafka.common.errors.InvalidTopicException;
import org.apache.kafka.common.errors.TopicAuthorizationException;
import org.apache.kafka.common.internals.ClusterResourceListeners;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.record.RecordBatch;
import org.apache.kafka.common.requests.MetadataRequest;
import org.apache.kafka.common.requests.MetadataResponse;
import org.apache.kafka.common.utils.LogContext;
import org.apache.logging.log4j.Logger;

import java.io.Closeable;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.function.Supplier;

/**
 * A class encapsulating some of the logic around metadata.
 * 

* This class is shared by the client thread (for partitioning) and the background sender thread. * * Metadata is maintained for only a subset of topics, which can be added to over time. When we request metadata for a * topic we don't have any metadata for it will trigger a metadata update. *

* If topic expiry is enabled for the metadata, any topic that has not been used within the expiry interval * is removed from the metadata refresh set after an update. Consumers disable topic expiry since they explicitly * manage topics while producers rely on topic expiry to limit the refresh set. */ public class Metadata implements Closeable { private final Logger log; private final long refreshBackoffMs; private final long metadataExpireMs; private int updateVersion; // bumped on every metadata response private int requestVersion; // bumped on every new topic addition private long lastRefreshMs; private long lastSuccessfulRefreshMs; private KafkaException fatalException; private Set invalidTopics; private Set unauthorizedTopics; private MetadataCache cache = MetadataCache.empty(); private boolean needUpdate; private final ClusterResourceListeners clusterResourceListeners; private boolean isClosed; private final Map lastSeenLeaderEpochs; /** * Create a new Metadata instance * * @param refreshBackoffMs The minimum amount of time that must expire between metadata refreshes to avoid busy * polling * @param metadataExpireMs The maximum amount of time that metadata can be retained without refresh * @param logContext Log context corresponding to the containing client * @param clusterResourceListeners List of ClusterResourceListeners which will receive metadata updates. */ public Metadata(long refreshBackoffMs, long metadataExpireMs, LogContext logContext, ClusterResourceListeners clusterResourceListeners) { this.log = logContext.logger(Metadata.class); this.refreshBackoffMs = refreshBackoffMs; this.metadataExpireMs = metadataExpireMs; this.lastRefreshMs = 0L; this.lastSuccessfulRefreshMs = 0L; this.requestVersion = 0; this.updateVersion = 0; this.needUpdate = false; this.clusterResourceListeners = clusterResourceListeners; this.isClosed = false; this.lastSeenLeaderEpochs = new HashMap<>(); this.invalidTopics = Collections.emptySet(); this.unauthorizedTopics = Collections.emptySet(); } /** * Get the current cluster info without blocking */ public synchronized Cluster fetch() { return cache.cluster(); } /** * Return the next time when the current cluster info can be updated (i.e., backoff time has elapsed). * * @param nowMs current time in ms * @return remaining time in ms till the cluster info can be updated again */ public synchronized long timeToAllowUpdate(long nowMs) { return Math.max(this.lastRefreshMs + this.refreshBackoffMs - nowMs, 0); } /** * The next time to update the cluster info is the maximum of the time the current info will expire and the time the * current info can be updated (i.e. backoff time has elapsed); If an update has been request then the expiry time * is now * * @param nowMs current time in ms * @return remaining time in ms till updating the cluster info */ public synchronized long timeToNextUpdate(long nowMs) { long timeToExpire = needUpdate ? 0 : Math.max(this.lastSuccessfulRefreshMs + this.metadataExpireMs - nowMs, 0); return Math.max(timeToExpire, timeToAllowUpdate(nowMs)); } public long metadataExpireMs() { return this.metadataExpireMs; } /** * Request an update of the current cluster metadata info, return the current updateVersion before the update */ public synchronized int requestUpdate() { this.needUpdate = true; return this.updateVersion; } /** * Request an update for the partition metadata iff the given leader epoch is at newer than the last seen leader epoch */ public synchronized boolean updateLastSeenEpochIfNewer(TopicPartition topicPartition, int leaderEpoch) { Objects.requireNonNull(topicPartition, "TopicPartition cannot be null"); return updateLastSeenEpoch(topicPartition, leaderEpoch, oldEpoch -> leaderEpoch > oldEpoch, true); } public Optional lastSeenLeaderEpoch(TopicPartition topicPartition) { return Optional.ofNullable(lastSeenLeaderEpochs.get(topicPartition)); } /** * Conditionally update the leader epoch for a partition * * @param topicPartition topic+partition to update the epoch for * @param epoch the new epoch * @param epochTest a predicate to determine if the old epoch should be replaced * @param setRequestUpdateFlag sets the "needUpdate" flag to true if the epoch is updated * @return true if the epoch was updated, false otherwise */ private synchronized boolean updateLastSeenEpoch(TopicPartition topicPartition, int epoch, Predicate epochTest, boolean setRequestUpdateFlag) { Integer oldEpoch = lastSeenLeaderEpochs.get(topicPartition); log.trace("Determining if we should replace existing epoch {} with new epoch {}", oldEpoch, epoch); if (oldEpoch == null || epochTest.test(oldEpoch)) { log.debug("Updating last seen epoch from {} to {} for partition {}", oldEpoch, epoch, topicPartition); lastSeenLeaderEpochs.put(topicPartition, epoch); if (setRequestUpdateFlag) { this.needUpdate = true; } return true; } else { log.debug("Not replacing existing epoch {} with new epoch {} for partition {}", oldEpoch, epoch, topicPartition); return false; } } /** * Check whether an update has been explicitly requested. * * @return true if an update was requested, false otherwise */ public synchronized boolean updateRequested() { return this.needUpdate; } /** * Return the cached partition info if it exists and a newer leader epoch isn't known about. */ public synchronized Optional partitionInfoIfCurrent(TopicPartition topicPartition) { Integer epoch = lastSeenLeaderEpochs.get(topicPartition); if (epoch == null) { // old cluster format (no epochs) return cache.getPartitionInfo(topicPartition); } else { return cache.getPartitionInfoHavingEpoch(topicPartition, epoch); } } public synchronized void bootstrap(List addresses, long now) { this.needUpdate = true; this.lastRefreshMs = now; this.lastSuccessfulRefreshMs = now; this.updateVersion += 1; this.cache = MetadataCache.bootstrap(addresses); } /** * Update metadata assuming the current request version. This is mainly for convenience in testing. */ public synchronized void update(MetadataResponse response, long now) { this.update(this.requestVersion, response, now); } /** * Updates the cluster metadata. If topic expiry is enabled, expiry time * is set for topics if required and expired topics are removed from the metadata. * * @param requestVersion The request version corresponding to the update response, as provided by * {@link #newMetadataRequestAndVersion()}. * @param response metadata response received from the broker * @param now current time in milliseconds */ public synchronized void update(int requestVersion, MetadataResponse response, long now) { Objects.requireNonNull(response, "Metadata response cannot be null"); if (isClosed()) throw new IllegalStateException("Update requested after metadata close"); if (requestVersion == this.requestVersion) this.needUpdate = false; else requestUpdate(); this.lastRefreshMs = now; this.lastSuccessfulRefreshMs = now; this.updateVersion += 1; String previousClusterId = cache.cluster().clusterResource().clusterId(); this.cache = handleMetadataResponse(response, topic -> retainTopic(topic.topic(), topic.isInternal(), now)); Cluster cluster = cache.cluster(); maybeSetMetadataError(cluster); this.lastSeenLeaderEpochs.keySet().removeIf(tp -> !retainTopic(tp.topic(), false, now)); String newClusterId = cache.cluster().clusterResource().clusterId(); if (!Objects.equals(previousClusterId, newClusterId)) { log.info("Cluster ID: {}", newClusterId); } clusterResourceListeners.onUpdate(cache.cluster().clusterResource()); log.debug("Updated cluster metadata updateVersion {} to {}", this.updateVersion, this.cache); } private void maybeSetMetadataError(Cluster cluster) { clearRecoverableErrors(); checkInvalidTopics(cluster); checkUnauthorizedTopics(cluster); } private void checkInvalidTopics(Cluster cluster) { if (!cluster.invalidTopics().isEmpty()) { log.error("Metadata response reported invalid topics {}", cluster.invalidTopics()); invalidTopics = new HashSet<>(cluster.invalidTopics()); } } private void checkUnauthorizedTopics(Cluster cluster) { if (!cluster.unauthorizedTopics().isEmpty()) { log.error("Topic authorization failed for topics {}", cluster.unauthorizedTopics()); unauthorizedTopics = new HashSet<>(cluster.unauthorizedTopics()); } } /** * Transform a MetadataResponse into a new MetadataCache instance. */ private MetadataCache handleMetadataResponse(MetadataResponse metadataResponse, Predicate topicsToRetain) { Set internalTopics = new HashSet<>(); List partitions = new ArrayList<>(); Map brokersById = metadataResponse.brokersById(); for (MetadataResponse.TopicMetadata metadata : metadataResponse.topicMetadata()) { if (!topicsToRetain.test(metadata)) continue; if (metadata.error() == Errors.NONE) { if (metadata.isInternal()) internalTopics.add(metadata.topic()); for (MetadataResponse.PartitionMetadata partitionMetadata : metadata.partitionMetadata()) { // Even if the partition's metadata includes an error, we need to handle the update to catch new epochs updatePartitionInfo(metadata.topic(), partitionMetadata, metadataResponse.hasReliableLeaderEpochs(), partitionInfoAndEpoch -> { Node leader = partitionInfoAndEpoch.partitionInfo().leader(); if (leader != null && !leader.equals(brokersById.get(leader.id()))) { // If we are reusing metadata from a previous response (which is possible if it // contained a larger epoch), we may not have leader information available in the // latest response. To keep the state consistent, we override the partition metadata // so that the leader is set consistently with the broker metadata PartitionInfo partitionInfo = partitionInfoAndEpoch.partitionInfo(); PartitionInfo partitionInfoWithoutLeader = new PartitionInfo( partitionInfo.topic(), partitionInfo.partition(), brokersById.get(leader.id()), partitionInfo.replicas(), partitionInfo.inSyncReplicas(), partitionInfo.offlineReplicas()); partitions.add(new MetadataCache.PartitionInfoAndEpoch(partitionInfoWithoutLeader, partitionInfoAndEpoch.epoch())); } else { partitions.add(partitionInfoAndEpoch); } }); if (partitionMetadata.error().exception() instanceof InvalidMetadataException) { log.debug("Requesting metadata update for partition {} due to error {}", new TopicPartition(metadata.topic(), partitionMetadata.partition()), partitionMetadata.error()); requestUpdate(); } } } else if (metadata.error().exception() instanceof InvalidMetadataException) { log.debug("Requesting metadata update for topic {} due to error {}", metadata.topic(), metadata.error()); requestUpdate(); } } return new MetadataCache(metadataResponse.clusterId(), brokersById.values(), partitions, metadataResponse.topicsByError(Errors.TOPIC_AUTHORIZATION_FAILED), metadataResponse.topicsByError(Errors.INVALID_TOPIC_EXCEPTION), internalTopics, metadataResponse.controller()); } /** * Compute the correct PartitionInfo to cache for a topic+partition and pass to the given consumer. */ private void updatePartitionInfo(String topic, MetadataResponse.PartitionMetadata partitionMetadata, boolean hasReliableLeaderEpoch, Consumer partitionInfoConsumer) { TopicPartition tp = new TopicPartition(topic, partitionMetadata.partition()); if (hasReliableLeaderEpoch && partitionMetadata.leaderEpoch().isPresent()) { int newEpoch = partitionMetadata.leaderEpoch().get(); // If the received leader epoch is at least the same as the previous one, update the metadata if (updateLastSeenEpoch(tp, newEpoch, oldEpoch -> newEpoch >= oldEpoch, false)) { PartitionInfo info = MetadataResponse.partitionMetaToInfo(topic, partitionMetadata); partitionInfoConsumer.accept(new MetadataCache.PartitionInfoAndEpoch(info, newEpoch)); } else { // Otherwise ignore the new metadata and use the previously cached info cache.getPartitionInfo(tp).ifPresent(partitionInfoConsumer); } } else { // Handle old cluster formats as well as error responses where leader and epoch are missing lastSeenLeaderEpochs.remove(tp); PartitionInfo info = MetadataResponse.partitionMetaToInfo(topic, partitionMetadata); partitionInfoConsumer.accept(new MetadataCache.PartitionInfoAndEpoch(info, RecordBatch.NO_PARTITION_LEADER_EPOCH)); } } /** * If any non-retriable exceptions were encountered during metadata update, clear and throw the exception. * This is used by the consumer to propagate any fatal exceptions or topic exceptions for any of the topics * in the consumer's Metadata. */ public synchronized void maybeThrowAnyException() { clearErrorsAndMaybeThrowException(this::recoverableException); } /** * If any fatal exceptions were encountered during metadata update, throw the exception. This is used by * the producer to abort waiting for metadata if there were fatal exceptions (e.g. authentication failures) * in the last metadata update. */ public synchronized void maybeThrowFatalException() { KafkaException metadataException = this.fatalException; if (metadataException != null) { fatalException = null; throw metadataException; } } /** * If any non-retriable exceptions were encountered during metadata update, throw exception if the exception * is fatal or related to the specified topic. All exceptions from the last metadata update are cleared. * This is used by the producer to propagate topic metadata errors for send requests. */ public synchronized void maybeThrowExceptionForTopic(String topic) { clearErrorsAndMaybeThrowException(() -> recoverableExceptionForTopic(topic)); } private void clearErrorsAndMaybeThrowException(Supplier recoverableExceptionSupplier) { KafkaException metadataException = Optional.ofNullable(fatalException).orElseGet(recoverableExceptionSupplier); fatalException = null; clearRecoverableErrors(); if (metadataException != null) throw metadataException; } // We may be able to recover from this exception if metadata for this topic is no longer needed private KafkaException recoverableException() { if (!unauthorizedTopics.isEmpty()) return new TopicAuthorizationException(unauthorizedTopics); else if (!invalidTopics.isEmpty()) return new InvalidTopicException(invalidTopics); else return null; } private KafkaException recoverableExceptionForTopic(String topic) { if (unauthorizedTopics.contains(topic)) return new TopicAuthorizationException(Collections.singleton(topic)); else if (invalidTopics.contains(topic)) return new InvalidTopicException(Collections.singleton(topic)); else return null; } private void clearRecoverableErrors() { invalidTopics = Collections.emptySet(); unauthorizedTopics = Collections.emptySet(); } /** * Record an attempt to update the metadata that failed. We need to keep track of this * to avoid retrying immediately. */ public synchronized void failedUpdate(long now, KafkaException fatalException) { this.lastRefreshMs = now; this.fatalException = fatalException; } /** * @return The current metadata updateVersion */ public synchronized int updateVersion() { return this.updateVersion; } /** * The last time metadata was successfully updated. */ public synchronized long lastSuccessfulUpdate() { return this.lastSuccessfulRefreshMs; } /** * Close this metadata instance to indicate that metadata updates are no longer possible. */ @Override public synchronized void close() { this.isClosed = true; } /** * Check if this metadata instance has been closed. See {@link #close()} for more information. * * @return True if this instance has been closed; false otherwise */ public synchronized boolean isClosed() { return this.isClosed; } public synchronized void requestUpdateForNewTopics() { // Override the timestamp of last refresh to let immediate update. this.lastRefreshMs = 0; this.requestVersion++; requestUpdate(); } public synchronized MetadataRequestAndVersion newMetadataRequestAndVersion() { return new MetadataRequestAndVersion(newMetadataRequestBuilder(), requestVersion); } protected MetadataRequest.Builder newMetadataRequestBuilder() { return MetadataRequest.Builder.allTopics(); } protected boolean retainTopic(String topic, boolean isInternal, long nowMs) { return true; } public static class MetadataRequestAndVersion { public final MetadataRequest.Builder requestBuilder; public final int requestVersion; private MetadataRequestAndVersion(MetadataRequest.Builder requestBuilder, int requestVersion) { this.requestBuilder = requestBuilder; this.requestVersion = requestVersion; } } public synchronized LeaderAndEpoch leaderAndEpoch(TopicPartition tp) { return partitionInfoIfCurrent(tp) .map(infoAndEpoch -> { Node leader = infoAndEpoch.partitionInfo().leader(); return new LeaderAndEpoch(leader == null ? Node.noNode() : leader, Optional.of(infoAndEpoch.epoch())); }) .orElse(new LeaderAndEpoch(Node.noNode(), lastSeenLeaderEpoch(tp))); } public static class LeaderAndEpoch { public static final LeaderAndEpoch NO_LEADER_OR_EPOCH = new LeaderAndEpoch(Node.noNode(), Optional.empty()); public final Node leader; public final Optional epoch; public LeaderAndEpoch(Node leader, Optional epoch) { this.leader = Objects.requireNonNull(leader); this.epoch = Objects.requireNonNull(epoch); } public static LeaderAndEpoch noLeaderOrEpoch() { return NO_LEADER_OR_EPOCH; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; LeaderAndEpoch that = (LeaderAndEpoch) o; if (!leader.equals(that.leader)) return false; return epoch.equals(that.epoch); } @Override public int hashCode() { int result = leader.hashCode(); result = 31 * result + epoch.hashCode(); return result; } @Override public String toString() { return "LeaderAndEpoch{" + "leader=" + leader + ", epoch=" + epoch.map(Number::toString).orElse("absent") + '}'; } } }





© 2015 - 2024 Weber Informatics LLC | Privacy Policy