com.softwaremill.react.kafka.ProducerProperties.scala Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of reactive-kafka-core_2.10 Show documentation
reactive-kafka-core
The newest version!
package com.softwaremill.react.kafka

import java.util.Properties

import kafka.message.{NoCompressionCodec, SnappyCompressionCodec}
import org.apache.kafka.common.serialization.{ByteArraySerializer, Serializer}

object ProducerProperties {

  /**
   * Producer Properties
   *
   * brokerList
   * This is for bootstrapping and the producer will only use it for getting metadata (topics, partitions and replicas).
   * The socket connections for sending the actual data will be established based on the broker information returned in
   * the metadata. The format is host1:port1,host2:port2, and the list can be a subset of brokers or a VIP pointing to a
   * subset of brokers.
   *
   * topic
   * The high-level API hides the details of brokers from the consumer and allows consuming off the cluster of machines
   * without concern for the underlying topology. It also maintains the state of what has been consumed. The high-level API
   * also provides the ability to subscribe to topics that match a filter expression (i.e., either a whitelist or a blacklist
   * regular expression).  This topic is a whitelist only but can change with re-factoring below on the filterSpec
   *
   */
  def apply[K, V](
    bootstrapServers: String,
    topic: String,
    keySerializer: Serializer[K],
    valueSerializer: Serializer[V],
    partitionizer: V => Option[Int]
  ): ProducerProperties[K, V] = {
    val props = initialMap(bootstrapServers)
    new ProducerProperties(props, topic, keySerializer, valueSerializer, partitionizer)
  }

  def apply[K, V](bootstrapServers: String, topic: String, keySerializer: Serializer[K],
    valueSerializer: Serializer[V]): ProducerProperties[K, V] = {
    val props = initialMap(bootstrapServers)
    new ProducerProperties(props, topic, keySerializer, valueSerializer, (_: V) => None)
  }

  def apply[V](
    bootstrapServers: String,
    topic: String,
    valueSerializer: Serializer[V],
    partitionizer: V => Option[Int]
  ): ProducerProperties[Array[Byte], V] = {
    val props = initialMap(bootstrapServers)
    new ProducerProperties(props, topic, new ByteArraySerializer(), valueSerializer, partitionizer)
  }

  def apply[V](
    bootstrapServers: String,
    topic: String,
    valueSerializer: Serializer[V]
  ): ProducerProperties[Array[Byte], V] = {
    val props = initialMap(bootstrapServers)
    new ProducerProperties(props, topic, new ByteArraySerializer(), valueSerializer, (_: V) => None)
  }

  private def initialMap[T](brokerList: String) = {
    Map[String, AnyRef](
      "bootstrap.servers" -> brokerList
    )
  }

  def apply[K, V](
    params: Map[String, String],
    topic: String,
    keySerializer: Serializer[K],
    valueSerializer: Serializer[V]
  ) = new ProducerProperties[K, V](
    params,
    topic, keySerializer, valueSerializer, (_: V) => None
  )

}

case class ProducerProperties[K, V](
    private val params: Map[String, AnyRef],
    topic: String,
    keySerializer: Serializer[K],
    valueSerializer: Serializer[V],
    partitionizer: V => Option[Int] = (_: V) => None
) {

  /**
   * No Compression
   * Allows you to turn off the compression codec for all data generated by this producer.
   */
  def noCompression(): ProducerProperties[K, V] = {
    copy(params = params + ("compression.type" -> NoCompressionCodec.name))
  }

  /**
   * Use Snappy Compression instead of the default compression
   */
  def useSnappyCompression(): ProducerProperties[K, V] = {
    copy(params = params + ("compression.type" -> SnappyCompressionCodec.name))
  }

  def clientId(id: String): ProducerProperties[K, V] = {
    copy(params = params + ("client.id" -> id))
  }

  /**
   * messageSendMaxRetries
   * This property will cause the producer to automatically retry a failed send request.
   * This property specifies the number of retries when such failures occur. Note that
   * setting a non-zero value here can lead to duplicates in the case of network errors
   * that cause a message to be sent but the acknowledgment to be lost.
   */
  def messageSendMaxRetries(num: Int): ProducerProperties[K, V] = {
    copy(params = params + ("retries" -> num.toString))
  }

  /**
   * requestRequiredAcks
   *  0) which means that the producer never waits for an acknowledgment from the broker (the same behavior as 0.7).
   *     This option provides the lowest latency but the weakest durability guarantees (some data will be lost when a server fails).
   *  1) which means that the producer gets an acknowledgment after the leader replica has received the data. This option provides
   *     better durability as the client waits until the server acknowledges the request as successful (only messages that were
   *     written to the now-dead leader but not yet replicated will be lost).
   * -1) which means that the producer gets an acknowledgment after all in-sync replicas have received the data. This option
   *     provides the best durability, we guarantee that no messages will be lost as long as at least one in sync replica remains.
   */
  def requestRequiredAcks(value: Int): ProducerProperties[K, V] = {
    copy(params = params + ("acks" -> value.toString))
  }

  /**
   * Set any additional properties as needed
   */
  def setProperty(key: String, value: String): ProducerProperties[K, V] = copy(params = params + (key -> value))
  def setProperties(values: (String, String)*): ProducerProperties[K, V] = copy(params = params ++ values)

  def rawProperties = params.foldLeft(new Properties()) { (props, param) => props.put(param._1, param._2); props }

  /**
   * Dump current props for debugging
   */
  def dump: String = params.map { e => f"${e._1}%-20s : ${e._2.toString}" }.mkString("\n")
}