com.epam.eco.commons.kafka.cache.ProjectingKafkaCache Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of commons-kafka Show documentation
A library of utilities, helpers and higher-level APIs for the Kafka client library
There is a newer version: 3.0.5
/*
 * Copyright 2019 EPAM Systems
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License.  You may obtain a copy
 * of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
package com.epam.eco.commons.kafka.cache;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.commons.lang3.Validate;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.epam.eco.commons.kafka.KafkaUtils;
import com.epam.eco.commons.kafka.config.ConsumerConfigBuilder;
import com.epam.eco.commons.kafka.consumer.bootstrap.OffsetInitializer;
import com.epam.eco.commons.kafka.serde.KeyValueDecoder;

/**
 * @author Andrei_Tytsik
 */
public class ProjectingKafkaCache extends AbstractCache {

    private final static Logger LOGGER = LoggerFactory.getLogger(ProjectingKafkaCache.class);

    private final ListCacheConsumer consumer;
    private final ProjectionBuilder projectionBuilder;

    private ProjectingKafkaCache(
            String bootstrapServers,
            String topicName,
            long bootstrapTimeoutInMs,
            Map consumerConfig,
            OffsetInitializer offsetInitializer,
            KeyValueDecoder keyValueDecoder,
            int consumerParallelism,
            ProjectionBuilder projectionBuilder,
            FireUpdateMode fireUpdateMode,
            CacheListener listener) {
        super(true, fireUpdateMode, listener);

        Validate.notBlank(bootstrapServers, "Bootstrap servers list is blank");
        Validate.notBlank(topicName, "Topic name is blank");
        Validate.notNull(projectionBuilder, "Projection builder is null");

        consumer = new ListCacheConsumer<>(
                topicName,
                bootstrapServers,
                bootstrapTimeoutInMs,
                consumerConfig,
                offsetInitializer,
                keyValueDecoder,
                consumerParallelism,
                update -> applyUpdateFromUnderlying(project(update)));

        this.projectionBuilder = projectionBuilder;

        LOGGER.info("Initialized");
    }

    @Override
    public void start() throws Exception {
        consumer.start();

        LOGGER.info("Started");
    }

    @Override
    public void close() {
        KafkaUtils.closeQuietly(consumer);

        LOGGER.info("Closed");
    }

    private Map project(List> update) {
        if (update.isEmpty()) {
            return Collections.emptyMap();
        }

        Map> logs = new HashMap<>();
        for (ConsumerRecord record : update) {
            K key = record.key();
            V value = record.value();

            List log = logs.get(key);
            if (log == null) {
                log = new ArrayList<>();
                logs.put(key, log);
            }

            log.add(value);
        }

        Map projections = new HashMap<>(logs.size(), 1);
        for (Entry> entry : logs.entrySet()) {
            K key = entry.getKey();
            List log = entry.getValue();
            P previous = get(key);
            P projection = projectionBuilder.build(previous, key, log);
            projections.put(key, projection);
        }

        return projections;
    }

    public static  Builder builder() {
        return new Builder<>();
    }

    public static class Builder {

        private String bootstrapServers;
        private String topicName;
        private long bootstrapTimeoutInMs = 30000;
        private Map consumerConfig;
        private ConsumerConfigBuilder consumerConfigBuilder;
        private OffsetInitializer offsetInitializer;
        private KeyValueDecoder keyValueDecoder;
        private int consumerParallelism = 1;
        private ProjectionBuilder projectionBuilder;
        private FireUpdateMode fireUpdateMode = FireUpdateMode.EFFECTIVE;
        private CacheListener listener;

        public Builder bootstrapServers(String bootstrapServers) {
            this.bootstrapServers = bootstrapServers;
            return this;
        }
        public Builder topicName(String topicName) {
            this.topicName = topicName;
            return this;
        }
        public Builder bootstrapTimeoutInMs(long bootstrapTimeoutInMs) {
            this.bootstrapTimeoutInMs = bootstrapTimeoutInMs;
            return this;
        }
        public Builder consumerConfig(Map consumerConfig) {
            this.consumerConfig = consumerConfig;
            return this;
        }
        public Builder consumerConfigBuilder(ConsumerConfigBuilder consumerConfigBuilder) {
            this.consumerConfigBuilder = consumerConfigBuilder;
            return this;
        }
        public Builder offsetInitializer(OffsetInitializer offsetInitializer) {
            this.offsetInitializer = offsetInitializer;
            return this;
        }
        public Builder keyValueDecoder(KeyValueDecoder keyValueDecoder) {
            this.keyValueDecoder = keyValueDecoder;
            return this;
        }
        public Builder consumerParallelism(int consumerParallelism) {
            this.consumerParallelism = consumerParallelism;
            return this;
        }
        public Builder consumerParallelismAvailableCores() {
            this.consumerParallelism = Runtime.getRuntime().availableProcessors();
            return this;
        }
        public Builder projectionBuilder(ProjectionBuilder projectionBuilder) {
            this.projectionBuilder = projectionBuilder;
            return this;
        }
        public Builder fireUpdateModeEffective() {
            return fireUpdateMode(FireUpdateMode.EFFECTIVE);
        }
        public Builder fireUpdateModeAll() {
            return fireUpdateMode(FireUpdateMode.ALL);
        }
        public Builder fireUpdateMode(FireUpdateMode fireUpdateMode) {
            this.fireUpdateMode = fireUpdateMode;
            return this;
        }
        public Builder listener(CacheListener listener) {
            this.listener = listener;
            return this;
        }

        public ProjectingKafkaCache build() {
            return new ProjectingKafkaCache<>(
                    bootstrapServers,
                    topicName,
                    bootstrapTimeoutInMs,
                    consumerConfigBuilder != null ? consumerConfigBuilder.build() : consumerConfig,
                    offsetInitializer,
                    keyValueDecoder,
                    consumerParallelism,
                    projectionBuilder,
                    fireUpdateMode,
                    listener);
        }

    }

}