io.telicent.smart.cache.sources.kafka.policies.manual.AbstractManualReadPolicy Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of event-source-kafka Show documentation
There is a newer version: 0.25.1
/**
 * Copyright (C) Telicent Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.telicent.smart.cache.sources.kafka.policies.manual;

import io.telicent.smart.cache.sources.kafka.policies.AbstractReadPolicy;
import io.telicent.smart.cache.sources.kafka.policies.automatic.AbstractAutoReadPolicy;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;

/**
 * Abstract base class for Kafka read policies that use the manual partition assignment strategy.
 * 
 * This means that when starting events for a topic the policy first obtains available partitions via
 * {@link org.apache.kafka.clients.consumer.KafkaConsumer#partitionsFor(String)} and then calls
 * {@link org.apache.kafka.clients.consumer.KafkaConsumer#assign(Collection)} to manually assign the relevant partitions
 * for reading.  This is as opposed to the policies derived from {@link AbstractAutoReadPolicy} where Kafka's Consumer
 * Group subscription model is used so the policy is assigned a fair-share of the available partitions.  Therefore any
 * policy derived from this base class will automatically assign itself all available partitions for a topic.
 * 
 * Note that even with this policy Kafka's consumer group offset management is still implicitly used, unless a
 * policy specifically seeks to a specific offset within the partitions.
 * 
 *
 * @param    Key Type
 * @param  Value Type
 */
public class AbstractManualReadPolicy extends AbstractReadPolicy {
    private static final Logger LOGGER = LoggerFactory.getLogger(AbstractManualReadPolicy.class);

    @Override
    public boolean isSubscriptionBased() {
        return false;
    }

    @Override
    public void startEvents(String topic) {
        Set allAssignments = new LinkedHashSet<>(this.consumer.assignment());
        Set currentAssignments = getRelevantPartitions(allAssignments, topic);

        List available = this.consumer.partitionsFor(topic);
        List newPartitions = new ArrayList<>();
        for (PartitionInfo info : available) {
            TopicPartition partition = new TopicPartition(topic, info.partition());
            if (!currentAssignments.contains(partition)) {
                allAssignments.add(partition);
                newPartitions.add(partition);
            }
        }

        if (newPartitions.isEmpty()) {
            LOGGER.warn("Attempted to subscribe to Kafka topic {} but there are no partitions available for it", topic);
            return;
        }

        this.consumer.assign(new ArrayList<>(allAssignments));
        LOGGER.info("Subscribed to Kafka topic {} using manual partition assignment of {} partitions", topic,
                    newPartitions.size());
        seek(newPartitions);
        logPartitionPositions(newPartitions, LOGGER);
    }

    @Override
    public void logReadPositions(String topic) {
        Set allAssignments = new LinkedHashSet<>(this.consumer.assignment());
        Set currentAssignments = getRelevantPartitions(allAssignments, topic);
        this.logPartitionPositions(currentAssignments, LOGGER);
    }

    @Override
    public Long currentLag(String topic) {
        Set allAssignments = new LinkedHashSet<>(this.consumer.assignment());
        Set currentAssignments = getRelevantPartitions(allAssignments, topic);
        return this.calculateLag(currentAssignments);
    }

    @Override
    public void stopEvents(String topic) {
        Set allAssignments = new LinkedHashSet<>(this.consumer.assignment());
        Set currentAssignments = getRelevantPartitions(allAssignments, topic);

        // Want to log our positions within these topic partitions on stop as this will help inform users how much
        // of the partitions has been successfully read
        this.logPartitionPositions(currentAssignments, LOGGER);

        if (!currentAssignments.isEmpty()) {
            allAssignments.removeAll(currentAssignments);
            this.consumer.assign(new ArrayList<>(allAssignments));
        } else {
            LOGGER.debug("Not assigned any partitions for topic {}, caller may be using the API incorrectly", topic);
        }
    }

    @Override
    public void onPartitionsRevoked(Collection partitions) {
        // Should never be called when using manual partition assignment
    }

    @Override
    public void onPartitionsAssigned(Collection partitions) {
        // Should never be called when using manual partition assignment
    }
}