com.github.jchanghong.kafka.SpringKafkaHelper.kt Maven / Gradle / Ivy

Go to download
package com.github.jchanghong.kafka

import cn.hutool.core.thread.ThreadUtil
import cn.hutool.core.util.RandomUtil
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.clients.producer.ProducerConfig
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.kafka.common.serialization.StringSerializer
import org.slf4j.LoggerFactory
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory
import org.springframework.kafka.core.DefaultKafkaConsumerFactory
import org.springframework.kafka.core.DefaultKafkaProducerFactory
import org.springframework.kafka.core.KafkaTemplate
import org.springframework.kafka.listener.ContainerProperties
import org.springframework.kafka.listener.KafkaMessageListenerContainer
import org.springframework.kafka.listener.MessageListener
import org.springframework.kafka.support.SendResult
import org.springframework.util.concurrent.ListenableFuture
import java.util.function.Consumer

/** 随机key*/
fun KafkaTemplate?.sendAutoKey(
    topic: String,
    data: String
): ListenableFuture?>? {
    return this?.send(topic, "${RandomUtil.randomString(20)}${System.nanoTime()}", data)
}

class SpringKafkaHelper(val ip: String, val port: Int = 9092) {
    val logger = LoggerFactory.getLogger(SpringKafkaHelper::class.java)
    val containerList = mutableListOf>()
    fun startConsumerAsyn(
        topic: String,
        group: String,
        beanName: String,
        concurrency: Int,
        consumer: Consumer>,
        config: Consumer? = null
    ) {
        val containerProps = ContainerProperties(topic)
        config?.accept(containerProps)
        containerProps.messageListener = object : MessageListener {
            override fun onMessage(message: ConsumerRecord) {
                consumer.accept(message)
            }
        }
        for (i in (1..concurrency)) {
            val container = createContainer(containerProps, group)
            container.setBeanName(beanName + i)
            containerList.add(container)
            container.start()
        }
    }

    fun testAutoCommit() {
        val template = createTemplate()
        for (i in (1..20)) {
            template.sendAutoKey("test1", i.toString())
            ThreadUtil.sleep(500)
        }
    }

    fun kafkaListenerContainerFactory(group: String): ConcurrentKafkaListenerContainerFactory {
        val factory = ConcurrentKafkaListenerContainerFactory()
        factory.setConsumerFactory(DefaultKafkaConsumerFactory(consumerProps(group)))
        return factory
    }

    fun createContainer(
        containerProps: ContainerProperties,
        group: String
    ): KafkaMessageListenerContainer {
        val props = consumerProps(group)
        val cf = DefaultKafkaConsumerFactory(props)
        val listenerContainer = KafkaMessageListenerContainer(cf, containerProps)
        return listenerContainer
    }

    fun createTemplate(): KafkaTemplate {
        val senderProps = senderProps()
        val pf = DefaultKafkaProducerFactory(senderProps)
        return KafkaTemplate(pf)
    }

    fun consumerProps(group: String): Map {
        val props: MutableMap = HashMap()
        props[ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG] = "${ip}:${port}"
        props[ConsumerConfig.GROUP_ID_CONFIG] = group
        props[ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG] = true
        props[ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG] = "100"
        props[ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG] = "15000"
        props[ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG] = StringDeserializer::class.java
        props[ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG] = StringDeserializer::class.java
        return props
    }

    fun senderProps(): Map {
        val props: MutableMap = HashMap()
        props[ProducerConfig.BOOTSTRAP_SERVERS_CONFIG] = "${ip}:${port}"
        props[ProducerConfig.RETRIES_CONFIG] = 1
        props[ProducerConfig.BATCH_SIZE_CONFIG] = 16384
        props[ProducerConfig.LINGER_MS_CONFIG] = 1
        props[ProducerConfig.BUFFER_MEMORY_CONFIG] = 33554432
        props[ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG] = StringSerializer::class.java
        props[ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG] = StringSerializer::class.java
        return props
    }
}

fun main() {
    val helper = SpringKafkaHelper("50.1.43.109")
    helper.startConsumerAsyn("test1", "test", "test", 1, {
        println(it.value() + "============" + Thread.currentThread().name)
    })
    helper.testAutoCommit()
}