com.amazonaws.services.kinesis.multilang.MultiLangDaemonConfig Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of amazon-kinesis-client Show documentation
The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis.
There is a newer version: 1.15.2
Show newest version
/*
 * Copyright 2019 Amazon.com, Inc. or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.amazonaws.services.kinesis.multilang;

import com.amazonaws.ClientConfiguration;
import com.amazonaws.services.kinesis.clientlibrary.config.KinesisClientLibConfigurator;
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.FileInputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * This class captures the configuration needed to run the MultiLangDaemon.
 */
public class MultiLangDaemonConfig {

    private static final Log LOG = LogFactory.getLog(MultiLangDaemonConfig.class);

    private static final String USER_AGENT = "amazon-kinesis-multi-lang-daemon";
    private static final String VERSION = "1.0.1";

    private static final String PROP_EXECUTABLE_NAME = "executableName";
    private static final String PROP_PROCESSING_LANGUAGE = "processingLanguage";
    private static final String PROP_MAX_ACTIVE_THREADS = "maxActiveThreads";

    public static final String PROXY_HOST_PROP = "http.proxyHost";
    public static final String PROXY_PORT_PROP = "http.proxyPort";
    public static final String HTTP_PROXY_ENV_VAR = "HTTP_PROXY";

    private KinesisClientLibConfiguration kinesisClientLibConfig;

    private ExecutorService executorService;

    private MultiLangRecordProcessorFactory recordProcessorFactory;

    /**
     * Constructor.
     *
     * @param propertiesFile The location of the properties file.
     * @throws IOException              Thrown when the properties file can't be accessed.
     * @throws IllegalArgumentException Thrown when the contents of the properties file are not as expected.
     */
    public MultiLangDaemonConfig(String propertiesFile) throws IOException, IllegalArgumentException {
        this(propertiesFile, Thread.currentThread().getContextClassLoader());
    }

    /**
     * @param propertiesFile The location of the properties file.
     * @param classLoader    A classloader, useful if trying to programmatically configure with the daemon, such as in a
     *                       unit test.
     * @throws IOException              Thrown when the properties file can't be accessed.
     * @throws IllegalArgumentException Thrown when the contents of the properties file are not as expected.
     */
    public MultiLangDaemonConfig(String propertiesFile, ClassLoader classLoader)
            throws IOException, IllegalArgumentException {
        this(propertiesFile, classLoader, new KinesisClientLibConfigurator());
    }

    /**
     * @param propertiesFile The location of the properties file.
     * @param classLoader    A classloader, useful if trying to programmatically configure with the daemon, such as in a
     *                       unit test.
     * @param configurator   A configurator to use.
     * @throws IOException              Thrown when the properties file can't be accessed.
     * @throws IllegalArgumentException Thrown when the contents of the properties file are not as expected.
     */
    public MultiLangDaemonConfig(String propertiesFile, ClassLoader classLoader,
            KinesisClientLibConfigurator configurator) throws IOException, IllegalArgumentException {
        Properties properties = loadProperties(classLoader, propertiesFile);
        if (!validateProperties(properties)) {
            throw new IllegalArgumentException(
                    "Must provide an executable name in the properties file, " + "e.g. executableName = sampleapp.py");
        }

        String executableName = properties.getProperty(PROP_EXECUTABLE_NAME);
        String processingLanguage = properties.getProperty(PROP_PROCESSING_LANGUAGE);
        ClientConfiguration clientConfig = buildClientConfig(properties);

        kinesisClientLibConfig = configurator.getConfiguration(properties).withKinesisClientConfig(clientConfig)
                .withCloudWatchClientConfig(clientConfig).withDynamoDBClientConfig(clientConfig);

        executorService = buildExecutorService(properties);
        recordProcessorFactory = new MultiLangRecordProcessorFactory(executableName, executorService,
                kinesisClientLibConfig);

        LOG.info("Running " + kinesisClientLibConfig.getApplicationName() + " to process stream "
                + kinesisClientLibConfig.getStreamName() + " with executable " + executableName);
        prepare(processingLanguage);
    }

    private ClientConfiguration buildClientConfig(Properties properties) {
        ClientConfiguration clientConfig = new ClientConfiguration();
        String proxyHost = null;
        int proxyPort = 0;

        if (properties.getProperty(PROXY_HOST_PROP) != null) {
            LOG.debug("Getting proxy info from properties file.");

            proxyHost = properties.getProperty(PROXY_HOST_PROP);
            proxyPort = Integer.parseInt(properties.getProperty(PROXY_PORT_PROP));
        } else if (System.getProperty(PROXY_HOST_PROP) != null) {
            LOG.debug("Getting proxy info from java system properties");

            proxyHost = System.getProperty(PROXY_HOST_PROP);
            proxyPort = Integer.parseInt(System.getProperty(PROXY_PORT_PROP));
        } else if (System.getenv(HTTP_PROXY_ENV_VAR) != null) {
            LOG.debug("Getting proxy info environment settings");

            try {
                URI proxyAddr = new URI(System.getenv(HTTP_PROXY_ENV_VAR));

                proxyHost = proxyAddr.getHost();
                proxyPort = proxyAddr.getPort();
            } catch (URISyntaxException e) {
                LOG.error("System proxy not set correctly", e);
            }
        }

        if (StringUtils.isNotEmpty(proxyHost) && proxyPort > 0) {
            clientConfig = clientConfig.withProxyHost(proxyHost).withProxyPort(proxyPort);
        } else {
            LOG.debug("Not configuring proxy as none specified");
        }

        return clientConfig;
    }

    private void prepare(String processingLanguage) {
        // Ensure the JVM will refresh the cached IP values of AWS resources (e.g. service endpoints).
        java.security.Security.setProperty("networkaddress.cache.ttl", "60");

        LOG.info("Using workerId: " + kinesisClientLibConfig.getWorkerIdentifier());
        LOG.info("Using credentials with access key id: " + kinesisClientLibConfig.getKinesisCredentialsProvider()
                .getCredentials().getAWSAccessKeyId());

        StringBuilder userAgent = new StringBuilder(KinesisClientLibConfiguration.KINESIS_CLIENT_LIB_USER_AGENT);
        userAgent.append(" ");
        userAgent.append(USER_AGENT);
        userAgent.append("/");
        userAgent.append(VERSION);

        if (processingLanguage != null) {
            userAgent.append(" ");
            userAgent.append(processingLanguage);
        }

        if (recordProcessorFactory.getCommandArray().length > 0) {
            userAgent.append(" ");
            userAgent.append(recordProcessorFactory.getCommandArray()[0]);
        }

        LOG.info(String.format("MultiLangDaemon is adding the following fields to the User Agent: %s",
                userAgent.toString()));
        kinesisClientLibConfig.withUserAgent(userAgent.toString());
    }

    private static Properties loadProperties(ClassLoader classLoader, String propertiesFileName) throws IOException {
        Properties properties = new Properties();
        InputStream propertyStream = null;
        try {
            propertyStream = classLoader.getResourceAsStream(propertiesFileName);
            if (propertyStream == null) {
                File propertyFile = new File(propertiesFileName);
                if (propertyFile.exists()) {
                    propertyStream = new FileInputStream(propertyFile);
                }
            }

            if (propertyStream == null) {
                throw new FileNotFoundException(
                        "Unable to find property file in classpath, or file system: '" + propertiesFileName + "'");
            }

            properties.load(propertyStream);
            return properties;
        } finally {
            if (propertyStream != null) {
                propertyStream.close();
            }
        }

    }

    private static boolean validateProperties(Properties properties) {
        return properties != null && properties.getProperty(PROP_EXECUTABLE_NAME) != null;
    }

    private static int getMaxActiveThreads(Properties properties) {
        return Integer.parseInt(properties.getProperty(PROP_MAX_ACTIVE_THREADS, "0"));
    }

    private static ExecutorService buildExecutorService(Properties properties) {
        int maxActiveThreads = getMaxActiveThreads(properties);
        ThreadFactoryBuilder builder = new ThreadFactoryBuilder().setNameFormat("multi-lang-daemon-%04d");
        LOG.debug(String.format("Value for %s property is %d", PROP_MAX_ACTIVE_THREADS, maxActiveThreads));
        if (maxActiveThreads <= 0) {
            LOG.info("Using a cached thread pool.");
            return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue(),
                    builder.build());
        } else {
            LOG.info(String.format("Using a fixed thread pool with %d max active threads.", maxActiveThreads));
            return new ThreadPoolExecutor(maxActiveThreads, maxActiveThreads, 0L, TimeUnit.MILLISECONDS,
                    new LinkedBlockingQueue(), builder.build());
        }
    }

    /**
     * @return A KinesisClientLibConfiguration object based on the properties file provided.
     */
    public KinesisClientLibConfiguration getKinesisClientLibConfiguration() {
        return kinesisClientLibConfig;
    }

    /**
     * @return An executor service based on the properties file provided.
     */
    public ExecutorService getExecutorService() {
        return executorService;
    }

    /**
     * @return A MultiLangRecordProcessorFactory based on the properties file provided.
     */
    public MultiLangRecordProcessorFactory getRecordProcessorFactory() {
        return recordProcessorFactory;
    }
}