All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.jkoolcloud.tnt4j.streams.inputs.KafkaConsumerStream Maven / Gradle / Ivy

There is a newer version: 2.3.0
Show newest version
/*
 * Copyright 2014-2018 JKOOL, LLC.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.jkoolcloud.tnt4j.streams.inputs;

import java.io.IOException;
import java.time.Duration;
import java.util.*;

import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.errors.WakeupException;

import com.jkoolcloud.tnt4j.core.OpLevel;
import com.jkoolcloud.tnt4j.sink.EventSink;
import com.jkoolcloud.tnt4j.streams.configure.KafkaStreamProperties;
import com.jkoolcloud.tnt4j.streams.configure.StreamProperties;
import com.jkoolcloud.tnt4j.streams.utils.*;

/**
 * Implements a Kafka topics transmitted activity stream, where each message body is assumed to represent a single
 * activity or event which should be recorded. Topic to listen is defined using "Topic" property in stream
 * configuration.
 * 

* This activity stream requires parsers that can support {@link ConsumerRecord} data like * {@link com.jkoolcloud.tnt4j.streams.parsers.KafkaConsumerRecordParser}. *

* This activity stream supports the following configuration properties (in addition to those supported by * {@link AbstractBufferedStream}): *

    *
  • Topic - defines set of topic names (delimited using '|' character) to listen. (Required)
  • *
  • Offset - defines list of topic offsets (delimited using '|' character) to start consuming messages. Single value * applies to all topics. Default value - {@code -1 (from latest)}. (Optional)
  • *
  • FileName - Kafka Consumer configuration file ({@code "consumer.properties"}) path. (Optional)
  • *
  • List of Kafka Consumer configuration properties. See * Kafka Consumer configuration reference.
  • *
*

* NOTE: those file defined Kafka consumer properties gets merged with ones defined in stream configuration - user * defined properties. So you can take some basic consumer configuration form file and customize it using stream * configuration defined properties. * * @version $Revision: 1 $ * * @see com.jkoolcloud.tnt4j.streams.parsers.ActivityParser#isDataClassSupported(Object) * @see com.jkoolcloud.tnt4j.streams.parsers.KafkaConsumerRecordParser */ public class KafkaConsumerStream extends AbstractBufferedStream> { private static final EventSink LOGGER = LoggerUtils.getLoggerSink(KafkaConsumerStream.class); /** * Kafka consumer user (over stream configuration) defined configuration scope mapping key. */ protected static final String PROP_SCOPE_USER = "user"; // NON-NLS /** * Kafka consumer properties file defined configuration scope mapping key. */ protected static final String PROP_SCOPE_CONSUMER = "consumer"; // NON-NLS private String topicName; private Set topicNames; private String offset; private List offsets; private String cfgFileName; private Map userKafkaProps = new HashMap<>(3); private KafkaDataReceiver kafkaDataReceiver; /** * Constructs a new KafkaConsumerStream. */ public KafkaConsumerStream() { super(); } @Override protected EventSink logger() { return LOGGER; } @Override public void setProperty(String name, String value) { super.setProperty(name, value); if (StreamProperties.PROP_TOPIC_NAME.equalsIgnoreCase(name)) { topicName = value; if (StringUtils.isNotEmpty(value)) { String[] topics = Utils.splitValue(topicName); topicNames = new LinkedHashSet<>(topics.length); for (String topic : topicNames) { String tTopic = topic.trim(); if (!tTopic.isEmpty()) { topicNames.add(tTopic); } } } } else if (StreamProperties.PROP_FILENAME.equalsIgnoreCase(name)) { cfgFileName = value; } else if (KafkaStreamProperties.PROP_OFFSET.equalsIgnoreCase(name)) { offset = value; if (StringUtils.isNotEmpty(value)) { String[] offsetArray = Utils.splitValue(offset); offsets = new ArrayList<>(offsetArray.length); for (String offst : offsetArray) { String tOffst = offst.trim(); offsets.add(tOffst.isEmpty() ? -1 : Integer.parseInt(tOffst)); } } } else if (!StreamsConstants.isStreamCfgProperty(name, KafkaStreamProperties.class)) { addUserKafkaProperty(name, decPassword(value)); } } @Override public Object getProperty(String name) { if (StreamProperties.PROP_TOPIC_NAME.equalsIgnoreCase(name)) { return topicName; } if (KafkaStreamProperties.PROP_OFFSET.equalsIgnoreCase(name)) { return offset; } if (StreamProperties.PROP_FILENAME.equalsIgnoreCase(name)) { return cfgFileName; } Object prop = super.getProperty(name); if (prop == null) { prop = getUserKafkaProperty(name); } return prop; } /** * Adds Kafka configuration property to user defined (from stream configuration) properties map. * * @param pName * fully qualified property name * @param pValue * property value * @return the previous value of the specified property in user's Kafka configuration property list, or {@code null} * if it did not have one */ protected Object addUserKafkaProperty(String pName, String pValue) { if (StringUtils.isEmpty(pName)) { return null; } String[] pParts = tokenizePropertyName(pName); Properties sProps = userKafkaProps.get(pParts[0]); if (sProps == null) { sProps = new Properties(); userKafkaProps.put(pParts[0], sProps); } return sProps.setProperty(pParts[1], pValue); } /** * Gets user defined (from stream configuration) Kafka consumer configuration property value. * * @param pName * fully qualified property name * @return property value, or {@code null} if property is not set */ protected String getUserKafkaProperty(String pName) { if (StringUtils.isEmpty(pName)) { return null; } // String[] pParts = tokenizePropertyName(pName); Properties sProperties = userKafkaProps.get(PROP_SCOPE_USER); return sProperties == null ? null : sProperties.getProperty(pName); } /** * Splits fully qualified property name to property scope and name. * * @param pName * fully qualified property name * @return string array containing property scope and name */ protected static String[] tokenizePropertyName(String pName) { if (StringUtils.isEmpty(pName)) { return null; } int sIdx = pName.indexOf(':'); String[] pParts = new String[2]; if (sIdx >= 0) { pParts[0] = pName.substring(0, sIdx); pParts[1] = pName.substring(sIdx + 1); } else { pParts[1] = pName; } if (StringUtils.isEmpty(pParts[0])) { pParts[0] = PROP_SCOPE_USER; } return pParts; } /** * Returns scope defined properties set. * * @param scope * properties scope key * @return scope defined properties */ protected Properties getScopeProps(String scope) { Properties allScopeProperties = new Properties(); Properties sProperties = userKafkaProps.get(scope); if (sProperties != null) { allScopeProperties.putAll(sProperties); } if (!PROP_SCOPE_USER.equals(scope)) { sProperties = userKafkaProps.get(PROP_SCOPE_USER); if (sProperties != null) { allScopeProperties.putAll(sProperties); } } return allScopeProperties; } @Override protected void applyProperties() throws Exception { super.applyProperties(); if (CollectionUtils.isEmpty(topicNames)) { throw new IllegalStateException(StreamsResources.getStringFormatted(StreamsResources.RESOURCE_BUNDLE_NAME, "TNTInputStream.property.undefined", StreamProperties.PROP_TOPIC_NAME)); } if (CollectionUtils.isNotEmpty(offsets) && offsets.size() > 1 && offsets.size() != topicNames.size()) { throw new IllegalStateException(StreamsResources.getStringFormatted(StreamsResources.RESOURCE_BUNDLE_NAME, "KafkaConsumerStream.offsets.mismatch", offsets.size(), topicNames.size())); } if (StringUtils.isNotEmpty(cfgFileName)) { logger().log(OpLevel.DEBUG, StreamsResources.getBundle(KafkaStreamConstants.RESOURCE_BUNDLE_NAME), "KafkaConsumerStream.consumer.cfgFile.load", cfgFileName); try { Properties fCfgProps = Utils.loadPropertiesFile(cfgFileName); userKafkaProps.put(PROP_SCOPE_CONSUMER, fCfgProps); } catch (IOException exc) { Utils.logThrowable(logger(), OpLevel.WARNING, StreamsResources.getBundle(KafkaStreamConstants.RESOURCE_BUNDLE_NAME), "KafkaConsumerStream.consumer.cfgFile.load.failed", exc); } } } @Override protected void initialize() throws Exception { super.initialize(); logger().log(OpLevel.DEBUG, StreamsResources.getBundle(KafkaStreamConstants.RESOURCE_BUNDLE_NAME), "KafkaConsumerStream.consumer.starting"); kafkaDataReceiver = new KafkaDataReceiver(); kafkaDataReceiver.initialize(getScopeProps(PROP_SCOPE_CONSUMER), topicNames, offsets); } @Override protected void start() throws Exception { super.start(); kafkaDataReceiver.start(); logger().log(OpLevel.DEBUG, StreamsResources.getBundle(StreamsResources.RESOURCE_BUNDLE_NAME), "TNTInputStream.stream.start", getClass().getSimpleName(), getName()); } @Override protected long getActivityItemByteSize(ConsumerRecord activityItem) { return Math.max(activityItem.serializedKeySize(), 0) + Math.max(activityItem.serializedValueSize(), 0); } @Override public boolean isInputEnded() { return kafkaDataReceiver.isInputEnded(); } @Override protected void cleanup() { if (kafkaDataReceiver != null) { kafkaDataReceiver.shutdown(); } userKafkaProps.clear(); super.cleanup(); } private class KafkaDataReceiver extends InputProcessor { private Consumer consumer; private Set topics; private List offsets; private boolean autoCommit = true; private KafkaDataReceiver() { super("KafkaConsumerStream.KafkaDataReceiver"); // NON-NLS } /** * Input data receiver initialization - Kafka consumer configuration. * * @param params * initialization parameters array * * @throws Exception * if fails to initialize data receiver and configure Kafka consumer */ @Override @SuppressWarnings("unchecked") protected void initialize(Object... params) throws Exception { Properties cProperties = (Properties) params[0]; topics = (Set) params[1]; offsets = (List) params[2]; autoCommit = Utils.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, cProperties, true); consumer = new KafkaConsumer<>(cProperties); } /** * Starts Kafka consumer client to receive incoming data. Shuts down this data receiver if exception occurs. */ @Override public void run() { if (consumer != null) { try { consumer.subscribe(topics); if (CollectionUtils.isNotEmpty(offsets)) { if (offsets.size() == 1) { if (offsets.get(0) >= 0) { for (String topic : topics) { consumer.seek(new TopicPartition(topic, 0), offsets.get(0)); } } } else { int idx = 0; for (String topic : topics) { int offset = offsets.get(idx++); if (offset >= 0) { consumer.seek(new TopicPartition(topic, 0), offset); } } } } while (!isHalted()) { ConsumerRecords records = consumer.poll(Duration.ofMillis(Long.MAX_VALUE)); if (autoCommit) { addRecordsToBuffer(records); } else { for (TopicPartition partition : records.partitions()) { List> partitionRecords = records.records(partition); addRecordsToBuffer(partitionRecords); long lastOffset = partitionRecords.get(partitionRecords.size() - 1).offset(); consumer.commitSync( Collections.singletonMap(partition, new OffsetAndMetadata(lastOffset + 1))); logger().log(OpLevel.DEBUG, StreamsResources.getBundle(KafkaStreamConstants.RESOURCE_BUNDLE_NAME), "KafkaConsumerStream.committing.offset", partition, lastOffset); } } } } catch (WakeupException exc) { } finally { consumer.close(); } } } /** * Adds consumer records from provided {@code records} collection to stream input buffer. * * @param records * records collection to add to stream input buffer * * @see #addInputToBuffer(Object) */ protected void addRecordsToBuffer(Iterable> records) { for (ConsumerRecord record : records) { String msgData = Utils.toString(record.value()); logger().log(OpLevel.DEBUG, StreamsResources.getBundle(KafkaStreamConstants.RESOURCE_BUNDLE_NAME), "KafkaConsumerStream.next.message", msgData); addInputToBuffer(record); } } /** * Closes Kafka consume. */ @Override void closeInternals() { if (consumer != null) { consumer.wakeup(); } } } }





© 2015 - 2024 Weber Informatics LLC | Privacy Policy