com.github.jchanghong.kafka.SpringKafkaHelper.kt Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of common-min Show documentation
kotlin java tools
The newest version!
package com.github.jchanghong.kafka

import cn.hutool.core.thread.ThreadUtil
import cn.hutool.core.util.RandomUtil
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.clients.producer.ProducerConfig
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.kafka.common.serialization.StringSerializer
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory
import org.springframework.kafka.core.DefaultKafkaConsumerFactory
import org.springframework.kafka.core.DefaultKafkaProducerFactory
import org.springframework.kafka.core.KafkaTemplate
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer
import org.springframework.kafka.listener.ContainerProperties
import org.springframework.kafka.listener.KafkaMessageListenerContainer
import org.springframework.kafka.listener.MessageListener
import org.springframework.kafka.support.SendResult
import org.springframework.util.concurrent.ListenableFuture
import java.util.function.Consumer
import kotlin.concurrent.thread

/** 随机key*/
fun KafkaTemplate?.sendAutoKey(
    topic: String,
    data: String?
): ListenableFuture?>? {
    return this?.send(topic, "${RandomUtil.randomString(20)}${System.nanoTime()}", data?:"")
}
class SpringKafkaHelper @JvmOverloads constructor(private val ip: String, private val port: Int = 9092) {
    val logger: Logger = LoggerFactory.getLogger(SpringKafkaHelper::class.java)
    var mMessageListenerContainer: ConcurrentMessageListenerContainer?=null
    fun startConsumerAsyn(
        topics: List,
        group: String,
        concurrency: Int,
        consumer: Consumer>,
        config: Consumer? = null
    ) {
        if (mMessageListenerContainer!=null) {
            mMessageListenerContainer!!.start()
            return
        }
        val containerProps = ContainerProperties(*topics.toTypedArray())
        config?.accept(containerProps)
        containerProps.messageListener = object : MessageListener {
            override fun onMessage(message: ConsumerRecord) {
                consumer.accept(message)
            }
        }
        val container: ConcurrentMessageListenerContainer = createContainer(containerProps, group)
        container.beanName = "kafkaMessageListenerContainerJCH"
        container.concurrency=concurrency
        mMessageListenerContainer=container
        container.start()
    }

    /** 后台现场，1秒插入一条数据*/
    fun testAutoCommit(topic: String) {
        val template = createTemplate()
       thread {
           for (i in (1..2000)) {
               template.sendAutoKey(topic, i.toString())?.get()
               ThreadUtil.sleep(1000)
           }
       }
    }

    fun kafkaListenerContainerFactory(group: String): ConcurrentKafkaListenerContainerFactory {
        val factory = ConcurrentKafkaListenerContainerFactory()
        factory.setConsumerFactory(DefaultKafkaConsumerFactory(consumerProps(group)))
        return factory
    }

    fun createContainer(
        containerProps: ContainerProperties,
        group: String
    ): ConcurrentMessageListenerContainer {
        val props = consumerProps(group)
        val cf = DefaultKafkaConsumerFactory(props)
        val listenerContainer = ConcurrentMessageListenerContainer(cf, containerProps)
        return listenerContainer
    }

    fun createTemplate(): KafkaTemplate {
        val senderProps = senderProps()
        val pf = DefaultKafkaProducerFactory(senderProps)
        return KafkaTemplate(pf)
    }

    fun consumerProps(group: String): Map {
        val props: MutableMap = HashMap()
        props[ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG] = "${ip}:${port}"
        props[ConsumerConfig.GROUP_ID_CONFIG] = group
        props[ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG] = true
        props[ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG] = "5000"
//        props[ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG] = "15000"
        props[ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG] = StringDeserializer::class.java
        props[ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG] = StringDeserializer::class.java
        return props
    }

    fun senderProps(): Map {
        val props: MutableMap = HashMap()
        props[ProducerConfig.BOOTSTRAP_SERVERS_CONFIG] = "${ip}:${port}"
//        props[ProducerConfig.RETRIES_CONFIG] = 1
//        props[ProducerConfig.BATCH_SIZE_CONFIG] = 16384
//        props[ProducerConfig.LINGER_MS_CONFIG] = 1
//        props[ProducerConfig.BUFFER_MEMORY_CONFIG] = 33554432
        props[ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG] = StringSerializer::class.java
        props[ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG] = StringSerializer::class.java
        return props
    }
}

fun main() {
    val helper = SpringKafkaHelper("50.1.43.109")
    helper.startConsumerAsyn(listOf("test","test1"), "test",  9, {
        println(it.value() + "======${it.topic()}======" + Thread.currentThread().name)
    })
    helper.testAutoCommit("test")
    helper.testAutoCommit("test1")

//    -----test2 50.1.172.137:9092
//    val helper = SpringKafkaHelper("50.1.172.137",9092)
//    helper.startConsumerAsyn("ORIGIN_SNAP_IMAGE_INFO_TOPIC", "jchtest",  8, {
//        println(it.value() + "============" + Thread.currentThread().name)
//        ThreadUtil.sleep(2000)
//    })
}