nstream.adapter.kafka.KafkaIngestingPatch Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of nstream-adapter-kafka Show documentation

Templates for consuming from and producing to Kafka topics with Swim

There is a newer version: 4.15.23

Show newest version

// Copyright 2015-2024 Nstream, inc.
//
// Licensed under the Redis Source Available License 2.0 (RSALv2) Agreement;
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://redis.com/legal/rsalv2-agreement/
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nstream.adapter.kafka;

import java.util.Collection;
import nstream.adapter.common.AdapterUtils;
import nstream.adapter.common.ingress.ValueAssembler;
import nstream.adapter.common.schedule.DeferrableException;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.common.TopicPartition;
import swim.structure.Value;

public class KafkaIngestingPatch extends KafkaIngestingAgent {

  protected ValueAssembler keyAssembler;
  protected ValueAssembler valueAssembler;

  public KafkaIngestingPatch() {
  }

  @Override
  protected void prepareConsumer() {
    loadSettings("kafkaIngressConf");
    this.keyAssembler = ValueAssembler.create(this.ingressSettings.keyMolder());
    this.valueAssembler = ValueAssembler.create(this.ingressSettings.valueMolder());
    // each of the next two lines is blocking and exception-prone
    this.kafkaConsumer = KafkaAdapterUtils.createConsumer(this.ingressSettings);
    this.kafkaConsumer.subscribe(this.ingressSettings.topics(), new ConsumerRebalanceListener() {

      @Override
      public void onPartitionsRevoked(Collection partitions) {
        info(nodeUri() + ": revoked partitions " + partitions);
      }

      @Override
      public void onPartitionsAssigned(Collection partitions) {
        info(nodeUri() + ": assigned partitions " + partitions);
      }

    });
  }

  @Override
  protected void ingest(ConsumerRecord unstructured)
      throws DeferrableException {
    final Value body = KafkaAdapterUtils.assembleConsumerRecord(unstructured,
        this.ingressSettings, this.keyAssembler, this.valueAssembler);
    ingestConsumerRecordStructure(body);
  }

  protected void ingestConsumerRecordStructure(Value responseStructure)
      throws DeferrableException {
    AdapterUtils.ingressDslRelay(this.ingressSettings.relaySchema(),
        agentContext(), responseStructure);
  }

}