com.ibm.cp4waiops.connectors.sdk.KafkaHelper Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of connectors-sdk Show documentation
A developer SDK for creating connectors for CP4WAIOps.
There is a newer version: 2.2.8
package com.ibm.cp4waiops.connectors.sdk;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Properties;

import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kafka.common.config.SslConfigs;
import io.cloudevents.CloudEvent;

import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;
import io.cloudevents.kafka.CloudEventSerializer;

import java.util.logging.Level;
import java.util.logging.Logger;

public class KafkaHelper {
    private static final Logger logger = Logger.getLogger(KafkaHelper.class.getName());

    public Properties KAFKA_CONFIG = new Properties();
    // Properties for Kafka admin clients
    public Properties kafka_admin_properties = new Properties();
    public Properties kafka_prod_properties = new Properties();

    public final String CARTRIDGE_NAME = "cp4waiops-cartridge"; // aiopsedge is unable to set this currently
    public String KAFKA_BROKER = "localhost:9092";
    public Integer KAFKA_PROD_BATCH_SIZE = 16384; // in bytes
    public Integer KAFKA_PROD_LINGER_MS = 10;
    public Integer KAFKA_PROD_BUFFER_MEMORY = 33554432;
    public String KAFKA_PROD_COMPRESSION_TYPE = "lz4";
    public String KAFKA_PROD_ACKS = null;

    public String KAFKA_API_KEY = null;
    public Boolean KAFKA_USE_SSL_CA_TRUSTSTORE = true;
    public String KAFKA_SSL_TRUSTSTORE_PATH = null;
    public String KAFKA_SSL_CA_PASSWORD = null;

    KafkaProducer producer = null;
    KafkaProducer producerCloudEvent = null;

    // By default this is set to false
    protected boolean isDirectToKafkaSupported = true;
    protected boolean isLocalDeployment = false;

    static KafkaHelper helper = new KafkaHelper();

    public static KafkaHelper getKafkaHelperInstance() {
        return helper;
    }

    protected KafkaHelper() {
        logger.log(Level.INFO, "Init KafkaHelper");

        readEnvVariables();

        logger.log(Level.INFO, "isDirectToKafkaSupported: " + isDirectToKafkaSupported()
                + " isDirectToKafkaConsumerEnabled: " + isDirectToKafkaConsumerEnabled());

        try {
            if (isDirectToKafkaSupported()) {
                producer = new KafkaProducer(KAFKA_CONFIG);

                // Add serializers or errors will be thrown
                KAFKA_CONFIG.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
                KAFKA_CONFIG.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, CloudEventSerializer.class);
                producerCloudEvent = new KafkaProducer(KAFKA_CONFIG);
            }
        } catch (Exception e) {
            logger.log(Level.WARNING, "Failed to create Kafka producer " + e.getMessage());
        }
    }

    /*
     * Allows the user to revert back to the server bridge for sending Kafka messages in case there are unforseen issues
     * with direct to Kafka consumption
     */
    public boolean isDirectToKafkaConsumerEnabled() {
        if (System.getenv("DIRECT_TO_KAFKA_CONSUMER_DISABLED") != null
                && !System.getenv("DIRECT_TO_KAFKA_CONSUMER_DISABLED").isEmpty()) {
            if ("true".equals(System.getenv("DIRECT_TO_KAFKA_CONSUMER_DISABLED"))) {
                return false;
            }
        }
        return true;
    }

    /*
     * Only support direct to Kafka if all the Kafka config is correct (e.g. no missing env vars) and if local
     * deployment. This must be checked before using the emitCloudEvent method
     */
    public boolean isDirectToKafkaSupported() {
        return isDirectToKafkaSupported;
    }

    public void emitCloudEvent(String destination, String partitionKey, CloudEvent event) {
        // Do not emit if it isn't supported
        if (!isDirectToKafkaSupported() && getIsLocalDeployment())
            return;

        try {
            String cloudEvent = Util.convertCloudEventToJSON(event);
            logger.log(Level.FINEST,
                    "destination=" + destination + " partition=" + partitionKey + " event=" + cloudEvent);

            // Constant.CE_EXT_STRUCTURED_CONTENT_MODE Kafka messages have the entire CloudEvent pushed
            // into Kafka
            Object obj = event.getExtension(Constant.CE_EXT_STRUCTURED_CONTENT_MODE);
            if (obj != null && Boolean.parseBoolean(obj.toString()) && producer != null) {
                producer.send(new ProducerRecord(destination, partitionKey, cloudEvent));
                logger.log(Level.FINEST, "Message sent sucessfully in structured mode");

                // Return to avoid pushing via unstructured mode
                return;
            }
            // If structured mode is false or not set, then default to unstructured mode
            if (producerCloudEvent != null) {
                // The default is to use the CloudEvent headers in the Kafka headers
                producerCloudEvent.send(new ProducerRecord<>(destination, partitionKey, event));
                logger.log(Level.FINEST, "Message sent sucessfully using non-structured mode");
            }

        } catch (Exception e) {
            logger.log(Level.WARNING, "Error producing CloudEvent: " + e.getMessage());
            // termination of the process has been requested
            Thread.currentThread().interrupt();
        }
    }

    public void setIsLocalDeployment(boolean isLocalDeployment) {
        logger.log(Level.INFO, "setIsLocalDeployment: " + isLocalDeployment);
        this.isLocalDeployment = isLocalDeployment;
    }

    public boolean getIsLocalDeployment() {
        return isLocalDeployment;
    }

    public void readEnvVariables() {
        try {
            // AIOPs uses these settings
            KAFKA_CONFIG.put(SaslConfigs.SASL_MECHANISM, "SCRAM-SHA-512");
            KAFKA_CONFIG.put(SslConfigs.SSL_PROTOCOL_CONFIG, "TLSv1.2");
            KAFKA_CONFIG.put(SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG, "TLSv1.2");
            KAFKA_CONFIG.put(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, "HTTPS");
            KAFKA_CONFIG.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL");

            // Bootstrap info
            if (System.getenv("KAFKA_BOOTSTRAP_SERVERS") != null
                    && !System.getenv("KAFKA_BOOTSTRAP_SERVERS").isEmpty()) {
                KAFKA_BROKER = System.getenv("KAFKA_BOOTSTRAP_SERVERS");
                KAFKA_CONFIG.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, KAFKA_BROKER);
            } else {
                logger.log(Level.WARNING, "KAFKA_BOOTSTRAP_SERVERS environment variable does not exist");
                isDirectToKafkaSupported = false;
            }

            if (System.getenv("KAFKA_SCRAM_PASSWORD_PATH") != null
                    && !System.getenv("KAFKA_SCRAM_PASSWORD_PATH").isEmpty()) {
                KAFKA_API_KEY = readEnvVariable(System.getenv("KAFKA_SCRAM_PASSWORD_PATH"));
            } else {
                logger.log(Level.WARNING, "KAFKA_SCRAM_PASSWORD_PATH environment variable does not exist");
                isDirectToKafkaSupported = false;
            }

            if (System.getenv("KAFKA_TRUSTSTORE_PASSWORD_PATH") != null
                    && !System.getenv("KAFKA_TRUSTSTORE_PASSWORD_PATH").isEmpty()) {
                KAFKA_SSL_CA_PASSWORD = readEnvVariable(System.getenv("KAFKA_TRUSTSTORE_PASSWORD_PATH"));
                KAFKA_CONFIG.put("ssl.truststore.password", KAFKA_SSL_CA_PASSWORD);
            } else {
                logger.log(Level.WARNING, "KAFKA_TRUSTSTORE_PASSWORD_PATH environment variable does not exist");
                isDirectToKafkaSupported = false;
            }

            if (System.getenv("KAFKA_TRUSTSTORE_PATH") != null && !System.getenv("KAFKA_TRUSTSTORE_PATH").isEmpty()) {
                KAFKA_SSL_TRUSTSTORE_PATH = System.getenv("KAFKA_TRUSTSTORE_PATH");
                KAFKA_CONFIG.put("ssl.truststore.location", KAFKA_SSL_TRUSTSTORE_PATH);
            } else {
                logger.log(Level.WARNING, "KAFKA_TRUSTSTORE_PATH environment variable does not exist");
                isDirectToKafkaSupported = false;
            }

            // Kafka Produce Environment Variables
            // References:
            // - https://www.conduktor.io/kafka/kafka-producer-batching/
            // - https://kafka.apache.org/documentation/#acks
            if (System.getenv("KAFKA_PROD_ACKS") != null && !System.getenv("KAFKA_PROD_ACKS").isEmpty()) {
                KAFKA_PROD_ACKS = System.getenv("KAFKA_PROD_ACKS");
                KAFKA_CONFIG.put(ProducerConfig.ACKS_CONFIG, KAFKA_PROD_ACKS);
            } else {
                logger.log(Level.WARNING, "KAFKA_PROD_ACKS environment variable does not exist");
                isDirectToKafkaSupported = false;
            }

            if (System.getenv("KAFKA_PROD_BATCH_SIZE") != null && !System.getenv("KAFKA_PROD_BATCH_SIZE").isEmpty()) {
                KAFKA_PROD_BATCH_SIZE = Integer.parseInt(System.getenv("KAFKA_PROD_BATCH_SIZE"));
                KAFKA_CONFIG.put("batch.size", KAFKA_PROD_BATCH_SIZE.toString());
            } else {
                logger.log(Level.WARNING, "KAFKA_PROD_BATCH_SIZE environment variable does not exist");
                isDirectToKafkaSupported = false;
            }

            if (System.getenv("KAFKA_PROD_LINGER_MS") != null && !System.getenv("KAFKA_PROD_LINGER_MS").isEmpty()) {
                KAFKA_PROD_LINGER_MS = Integer.parseInt(System.getenv("KAFKA_PROD_LINGER_MS"));
                KAFKA_CONFIG.put("linger.ms", KAFKA_PROD_LINGER_MS.toString());
            } else {
                logger.log(Level.WARNING, "KAFKA_PROD_LINGER_MS environment variable does not exist");
                isDirectToKafkaSupported = false;
            }

            if (System.getenv("KAFKA_PROD_COMPRESSION_TYPE") != null
                    && !System.getenv("KAFKA_PROD_COMPRESSION_TYPE").isEmpty()) {
                KAFKA_PROD_COMPRESSION_TYPE = System.getenv("KAFKA_PROD_COMPRESSION_TYPE");
                KAFKA_CONFIG.put("compression.type", KAFKA_PROD_COMPRESSION_TYPE.toString());
            } else {
                logger.log(Level.WARNING, "KAFKA_PROD_COMPRESSION_TYPE environment variable does not exist");
                isDirectToKafkaSupported = false;
            }

            if (System.getenv("KAFKA_PROD_BUFFER_MEMORY") != null
                    && !System.getenv("KAFKA_PROD_BUFFER_MEMORY").isEmpty()) {
                KAFKA_PROD_LINGER_MS = Integer.parseInt(System.getenv("KAFKA_PROD_BUFFER_MEMORY"));
                KAFKA_CONFIG.put("buffer.memory", KAFKA_PROD_LINGER_MS.toString());
            } else {
                logger.log(Level.WARNING, "KAFKA_PROD_BUFFER_MEMORY environment variable does not exist");
                isDirectToKafkaSupported = false;
            }

            String username = System.getenv("KAFKA_SCRAM_USERNAME");
            KAFKA_CONFIG.put("sasl.jaas.config", "org.apache.kafka.common.security.scram.ScramLoginModule"
                    + " required username=\"" + username + "\" password=\"" + KAFKA_API_KEY + "\";");
            KAFKA_CONFIG.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
            KAFKA_CONFIG.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

        } catch (Exception e) {
            logger.log(Level.WARNING, "Error getting environment variables: " + e.getMessage());
        }
    }

    public String readEnvVariable(String file) {
        try {
            File envFile = new File(file);
            if (!envFile.exists()) {
                logger.log(Level.WARNING, "No Env File Found at " + file);
                return null;
            } else {
                FileInputStream inStream = new FileInputStream(envFile);
                BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
                String stringValue = br.readLine();
                inStream.close();
                br.close();
                logger.log(Level.INFO, "Read Env variable from " + file);
                return stringValue;
            }
        } catch (IOException e) {
            return null;
        }
    }
}