nstream.adapter.kafka.KafkaAdapterUtils Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of nstream-adapter-kafka Show documentation
Templates for consuming from and producing to Kafka topics with Swim
There is a newer version: 4.15.23
Show newest version
// Copyright 2015-2024 Nstream, inc.
//
// Licensed under the Redis Source Available License 2.0 (RSALv2) Agreement;
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://redis.com/legal/rsalv2-agreement/
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nstream.adapter.kafka;

import java.util.Properties;
import nstream.adapter.common.AdapterUtils;
import nstream.adapter.common.ext.KafkaIngressSettings;
import nstream.adapter.common.ingress.ContentAssembler;
import nstream.adapter.common.ingress.ValueAssembler;
import nstream.adapter.common.provision.ProvisionLoader;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import swim.structure.Record;
import swim.structure.Value;

public final class KafkaAdapterUtils {

  private KafkaAdapterUtils() {
  }

  private static final String[] CONSUMER_PROVISION_REQUIRED_PROPS = new String[]{
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
      ConsumerConfig.GROUP_ID_CONFIG, // FIXME: not required when we support statically-assigned partitions
      ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
      ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG
  };

  public static  Consumer createConsumer(Value settingsProp) {
    final KafkaIngressSettings settings = parseConsumerCreatingSettings(settingsProp);
    return createConsumer(settings);
  }

  public static  Consumer createConsumer(KafkaIngressSettings settings) {
    final String provisionName = validateConsumerProvisionBasedSettings(settings);
    final Properties consumerProperties = validateConsumerPropertiesProvision(provisionName);
    return new KafkaConsumer<>(consumerProperties);
  }

  private static KafkaIngressSettings parseConsumerCreatingSettings(Value prop) {
    if (prop == null || !prop.isDistinct()) {
      throw new IllegalArgumentException("prop " + prop + " does not yield KafkaIngressSettings");
    }
    return KafkaIngressSettings.form().cast(prop);
  }

  private static String validateConsumerProvisionBasedSettings(KafkaIngressSettings settings) {
    if (settings.topics() == null || settings.topics().isEmpty()) {
      throw new IllegalArgumentException("Must subscribe to at least one topic: " + settings);
    }
    if (settings.consumerPropertiesProvisionName() == null) {
      throw new IllegalArgumentException("Config must include consumerPropertiesProvisionName");
    }
    return settings.consumerPropertiesProvisionName();
  }

  private static Properties validateConsumerPropertiesProvision(String provisionName) {
    final Properties props = ProvisionLoader.getProvision(provisionName).value();
    if (props == null || props.isEmpty()) {
      throw new IllegalArgumentException("Empty Kafka configuration properties");
    }
    for (String prop : CONSUMER_PROVISION_REQUIRED_PROPS) {
      if (!props.containsKey(prop)) {
        throw new IllegalArgumentException("Properties must contain " + prop);
      }
    }
    return props;
  }

  @SuppressWarnings("unchecked")
  private static  Value assembleComponent(Object component, ValueAssembler assembler) {
    if (assembler == null || assembler instanceof ContentAssembler) {
      return AdapterUtils.assembleContent(component, (ContentAssembler) assembler);
    } else {
      return assembler.assemble((V) component);
    }
  }

  public static  Value assembleConsumerRecord(ConsumerRecord record,
                                                    KafkaIngressSettings settings,
                                                    ValueAssembler keyMolder, ValueAssembler valueMolder) {
    final Value key = keyMolder == null ? AdapterUtils.assembleContent(record.key(), settings.keyContentTypeOverride())
        : assembleComponent(record.key(), keyMolder);
    final Value value = valueMolder == null ? AdapterUtils.assembleContent(record.value(), settings.valueContentTypeOverride())
        : assembleComponent(record.value(), valueMolder);
    return Record.create(2).slot("key", key).slot("value", value);
  }

  public static  Value assembleConsumerRecord(ConsumerRecord record,
                                                    KafkaIngressSettings settings) {
    return assembleConsumerRecord(record, settings,
        ValueAssembler.create(settings.keyMolder()), ValueAssembler.create(settings.valueMolder()));
  }

}