All Downloads are FREE. Search and download functionalities are using the official Maven repository.

tarlake-spark3_2.12.1.3.0.source-code.reference-kafka.conf Maven / Gradle / Ivy

kafka {
  cometOffsetsMode = "STREAM"
  customDeserializers {
   #"deserialize": "io.confluent.kafka.serializers.KafkaAvroDeserializer"
  }
  serverOptions {
    "bootstrap.servers": "localhost:9092"
  }
  topics {
    logs {
      topicName: logs
      maxRead = 0
      fields = ["key as STRING", "value as STRING", "topic as STRING", "partition as INT", "offset as LONG", "timestamp as TIMESTAMP", "timestampType as INT"]
      writeFormat = "parquet"
      createOptions {
        "cleanup.policy": "compact"
      }
      accessOptions {
        "kafka.bootstrap.servers": "localhost:9092"
        "bootstrap.servers": "localhost:9092"
        "key.deserializer": "org.apache.kafka.common.serialization.StringDeserializer",
        "value.deserializer": "org.apache.kafka.common.serialization.StringDeserializer"
        "subscribe": "logs"
      }
    },
    comet_offsets {
      topicName: comet_offsets
      maxRead = 0
      partitions = 1
      replication-factor = 1
      writeFormat = "parquet"
      createOptions {
        "cleanup.policy": "compact"
      }
      accessOptions {
        "kafka.bootstrap.servers": "localhost:9092"
        "auto.offset.reset": "earliest"
        "auto.commit.enable": "false"
        "consumer.timeout.ms": "10"
        "bootstrap.servers": "localhost:9092"
        "key.deserializer": "org.apache.kafka.common.serialization.StringDeserializer",
        "value.deserializer": "org.apache.kafka.common.serialization.StringDeserializer"
        "subscribe": "comet_offsets"
      }
    }
  }
}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy