All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.arpnetworking.metrics.mad.Main Maven / Gradle / Ivy

There is a newer version: 1.22.6
Show newest version
/*
 * Copyright 2014 Brandon Arp
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.arpnetworking.metrics.mad;

import ch.qos.logback.classic.LoggerContext;
import com.arpnetworking.commons.builder.Builder;
import com.arpnetworking.commons.jackson.databind.ImmutableObjectMapper;
import com.arpnetworking.commons.jackson.databind.ObjectMapperFactory;
import com.arpnetworking.configuration.jackson.DynamicConfiguration;
import com.arpnetworking.configuration.jackson.HoconFileSource;
import com.arpnetworking.configuration.jackson.JsonNodeFileSource;
import com.arpnetworking.configuration.jackson.JsonNodeSource;
import com.arpnetworking.configuration.jackson.module.pekko.PekkoLoggingModule;
import com.arpnetworking.configuration.triggers.FileTrigger;
import com.arpnetworking.http.Routes;
import com.arpnetworking.http.SupplementalRoutes;
import com.arpnetworking.metrics.MetricsFactory;
import com.arpnetworking.metrics.impl.ApacheHttpSink;
import com.arpnetworking.metrics.impl.TsdMetricsFactory;
import com.arpnetworking.metrics.incubator.PeriodicMetrics;
import com.arpnetworking.metrics.incubator.impl.TsdPeriodicMetrics;
import com.arpnetworking.metrics.jvm.ExecutorServiceMetricsRunnable;
import com.arpnetworking.metrics.jvm.JvmMetricsRunnable;
import com.arpnetworking.metrics.mad.actors.DeadLetterLogger;
import com.arpnetworking.metrics.mad.actors.Status;
import com.arpnetworking.metrics.mad.configuration.AggregatorConfiguration;
import com.arpnetworking.metrics.mad.configuration.PipelineConfiguration;
import com.arpnetworking.metrics.proxy.actors.Telemetry;
import com.arpnetworking.steno.Logger;
import com.arpnetworking.steno.LoggerFactory;
import com.arpnetworking.utility.Configurator;
import com.arpnetworking.utility.Launchable;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Provides;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.pekko.actor.ActorRef;
import org.apache.pekko.actor.ActorSystem;
import org.apache.pekko.actor.DeadLetter;
import org.apache.pekko.actor.Props;
import org.apache.pekko.actor.Terminated;
import org.apache.pekko.dispatch.Dispatcher;
import org.apache.pekko.http.javadsl.ConnectionContext;
import org.apache.pekko.http.javadsl.Http;
import org.apache.pekko.http.javadsl.HttpsConnectionContext;
import org.apache.pekko.stream.Materializer;
import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.openssl.PEMParser;
import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;
import org.bouncycastle.util.io.pem.PemObject;
import scala.concurrent.Await;
import scala.concurrent.ExecutionContextExecutor;
import scala.concurrent.Future;
import scala.concurrent.duration.Duration;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.PrivateKey;
import java.security.SecureRandom;
import java.security.Security;
import java.security.UnrecoverableKeyException;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;

/**
 * Class containing entry point for Metrics Data Aggregator (MAD).
 *
 * @author Brandon Arp (brandon dot arp at inscopemetrics dot io)
 * @author Ville Koskela (ville dot koskela at inscopemetrics dot io)
 */
public final class Main implements Launchable {

    /**
     * Entry point for Metrics Aggregator Daemon (MAD).
     *
     * @param args the command line arguments
     */
    public static void main(final String[] args) {
        // Global initialization
        Thread.setDefaultUncaughtExceptionHandler(
                (thread, throwable) -> {
                    System.err.println("Unhandled exception! exception: " + throwable.toString());
                    throwable.printStackTrace(System.err);
                });

        Thread.currentThread().setUncaughtExceptionHandler(
                (thread, throwable) -> LOGGER.error()
                        .setMessage("Unhandled exception!")
                        .setThrowable(throwable)
                        .log()
        );

        LOGGER.info().setMessage("Launching mad").log();

        Runtime.getRuntime().addShutdownHook(SHUTDOWN_THREAD);

        System.setProperty("org.vertx.logger-delegate-factory-class-name", "org.vertx.java.core.logging.impl.SLF4JLogDelegateFactory");

        // Run the tsd aggregator
        if (args.length != 1) {
            throw new RuntimeException("No configuration file specified");
        }
        LOGGER.debug()
                .setMessage("Loading configuration")
                .addData("file", args[0])
                .log();

        Optional configuration = Optional.empty();
        Optional> configurator = Optional.empty();
        try {
            final File configurationFile = new File(args[0]);
            configurator = Optional.of(new Configurator<>(Main::new, AggregatorConfiguration.class));
            configuration = Optional.of(new DynamicConfiguration.Builder()
                    .setObjectMapper(OBJECT_MAPPER)
                    .addSourceBuilder(getFileSourceBuilder(configurationFile))
                    .addTrigger(
                            new FileTrigger.Builder()
                                    .setFile(configurationFile)
                                    .build())
                    .addListener(configurator.get())
                    .build());

            configuration.ifPresent(DynamicConfiguration::launch);
            // Wait for application shutdown
            SHUTDOWN_SEMAPHORE.acquire();
        } catch (final InterruptedException e) {
            throw new RuntimeException(e);
        } finally {
            configurator.ifPresent(Configurator::shutdown);
            configuration.ifPresent(DynamicConfiguration::shutdown);
            // Notify the shutdown that we're done
            SHUTDOWN_SEMAPHORE.release();
        }
    }

    /**
     * Public constructor.
     *
     * @param configuration Instance of {@link AggregatorConfiguration}.
     */
    public Main(final AggregatorConfiguration configuration) {
        _configuration = configuration;
    }

    @Override
    public synchronized void launch() {
        _actorSystem = launchPekko();
        final Injector injector = launchGuice(_actorSystem);
        launchActors(injector);
        launchPipelines(injector);
        launchJvmMetricsCollector(injector);
    }

    @Override
    public synchronized void shutdown() {
        shutdownJvmMetricsCollector();
        shutdownPipelines();
        shutdownActors();
        shutdownGuice();
        shutdownPekko();
    }

    private void launchJvmMetricsCollector(final Injector injector) {
        LOGGER.info().setMessage("Launching JVM metrics collector.").log();

        final Runnable jvmMetricsRunnable = new JvmMetricsRunnable.Builder()
                .setMetricsFactory(injector.getInstance(MetricsFactory.class))
                .build();

        final Runnable jvmExecutorServiceMetricsRunnable = new ExecutorServiceMetricsRunnable.Builder()
                .setMetricsFactory(injector.getInstance(MetricsFactory.class))
                .setExecutorServices(createExecutorServiceMap(_actorSystem))
                .build();

        _jvmMetricsCollector = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "JVMMetricsCollector"));

        _jvmMetricsCollector.scheduleAtFixedRate(
                jvmMetricsRunnable,
                INITIAL_DELAY_IN_MILLIS,
                _configuration.getJvmMetricsCollectionInterval().toMillis(),
                TIME_UNIT);

        _jvmMetricsCollector.scheduleAtFixedRate(
                jvmExecutorServiceMetricsRunnable,
                INITIAL_DELAY_IN_MILLIS,
                _configuration.getJvmMetricsCollectionInterval().toMillis(),
                TIME_UNIT);
    }

    private void launchPipelines(final Injector injector) {
        LOGGER.info().setMessage("Launching pipelines").log();
        _pipelinesLaunchable = new PipelinesLaunchable(
                PipelineConfiguration.createObjectMapper(injector),
                _configuration.getPipelinesDirectory());
        _pipelinesLaunchable.launch();
    }

    private void launchActors(final Injector injector) {
        LOGGER.info().setMessage("Launching actors").log();

        // Retrieve the actor system
        final ActorSystem actorSystem = injector.getInstance(ActorSystem.class);

        // Create the dead letter logger
        if (_configuration.getLogDeadLetters()) {
            final ActorRef deadMailMan = actorSystem.actorOf(Props.create(DeadLetterLogger.class), "deadmailman");
            actorSystem.eventStream().subscribe(deadMailMan, DeadLetter.class);
        }

        // Create the status actor
        actorSystem.actorOf(Props.create(Status.class), "status");

        // Create the telemetry connection actor
        actorSystem.actorOf(Props.create(Telemetry.class, injector.getInstance(MetricsFactory.class)), "telemetry");

        // Load supplemental routes
        final ImmutableList.Builder supplementalHttpRoutes = ImmutableList.builder();
        _configuration.getSupplementalHttpRoutesClass().ifPresent(
                clazz -> supplementalHttpRoutes.add(injector.getInstance(clazz)));

        // Create and bind Http server
        final Materializer materializer = Materializer.createMaterializer(actorSystem);
        final Routes routes = new Routes(
                actorSystem,
                injector.getInstance(PeriodicMetrics.class),
                _configuration.getHttpHealthCheckPath(),
                _configuration.getHttpStatusPath(),
                supplementalHttpRoutes.build());
        final Http http = Http.get(actorSystem);

        http.newServerAt(_configuration.getHttpHost(), _configuration.getHttpPort()).withMaterializer(materializer).bind(routes);
        if (_configuration.getEnableHttps()) {
            Security.addProvider(new BouncyCastleProvider());
            final HttpsConnectionContext httpsContext = createHttpsContext();
            http.newServerAt(_configuration.getHttpsHost(), _configuration.getHttpsPort())
                    .withMaterializer(materializer).enableHttps(httpsContext)
                    .bind(routes);
        }
    }

    private HttpsConnectionContext createHttpsContext() {
        final PrivateKey privateKey;
        try (PEMParser keyReader = new PEMParser(
                new InputStreamReader(
                        new FileInputStream(_configuration.getHttpsKeyPath()), StandardCharsets.UTF_8))) {
            final Object keyObject = keyReader.readObject();

            if (keyObject instanceof PrivateKeyInfo) {
                final PrivateKeyInfo privateKeyInfo = (PrivateKeyInfo) keyObject;
                privateKey = new JcaPEMKeyConverter().getPrivateKey(privateKeyInfo);
            } else {
                throw new RuntimeException(
                        String.format("Invalid private key format, expected PEM private key, got %s", keyObject.getClass().getName()));
            }
        } catch (final FileNotFoundException e) {
            throw new RuntimeException(String.format("Could not find https key file: %s", _configuration.getHttpsKeyPath()), e);
        } catch (final IOException e) {
            throw new RuntimeException(String.format("Error reading https key file: %s", _configuration.getHttpsKeyPath()), e);
        }

        final X509Certificate cert;
        try (PEMParser certReader = new PEMParser(
                new InputStreamReader(
                        new FileInputStream(_configuration.getHttpsCertificatePath()), StandardCharsets.UTF_8))) {
            final PemObject certObject = certReader.readPemObject();
            final CertificateFactory cf = CertificateFactory.getInstance("X.509");
            cert = (X509Certificate) cf.generateCertificate(new ByteArrayInputStream(certObject.getContent()));
        } catch (final FileNotFoundException e) {
            throw new RuntimeException(
                    String.format("Could not find https certificate file: %s", _configuration.getHttpsCertificatePath()), e);
        } catch (final IOException e) {
            throw new RuntimeException(
                    String.format("Error reading https certificate file: %s", _configuration.getHttpsCertificatePath()), e);
        } catch (final CertificateException e) {
            throw new RuntimeException("Error building X509 certificate", e);
        }

        try {
            final KeyStore ks = KeyStore.getInstance("JKS");
            ks.load(null, null);
            ks.setCertificateEntry("cert", cert);

            ks.setKeyEntry("key", privateKey, new char[0], new Certificate[]{cert});
            final KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("SunX509");
            keyManagerFactory.init(ks, new char[0]);

            final TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
            tmf.init(ks);

            final SSLContext sslContext = SSLContext.getInstance("TLSv1.3");
            sslContext.init(keyManagerFactory.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom());

            return ConnectionContext.httpsServer(sslContext);
        } catch (final UnrecoverableKeyException | KeyManagementException | NoSuchAlgorithmException | IOException
                | KeyStoreException | CertificateException e) {
            throw new RuntimeException("Could not create SSLContext for https configuration", e);
        }
    }

    @SuppressWarnings("deprecation")
    private Injector launchGuice(final ActorSystem actorSystem) {
        LOGGER.info().setMessage("Launching guice").log();

        // Create directories
        if (_configuration.getLogDirectory().mkdirs()) {
            LOGGER.info()
                    .setMessage("Created log directory")
                    .addData("directory", _configuration.getLogDirectory())
                    .log();
        }
        if (_configuration.getPipelinesDirectory().mkdirs()) {
            LOGGER.info()
                    .setMessage("Created pipelines directory")
                    .addData("directory", _configuration.getPipelinesDirectory())
                    .log();
        }

        // Instantiate the metrics factory
        final ImmutableList.Builder monitoringSinksBuilder =
                new ImmutableList.Builder<>();
        if (_configuration.getMetricsClientHost().isPresent()
                || _configuration.getMetricsClientPort().isPresent()) {
            final String endpoint = String.format(
                    "http://%s:%d/metrics/v3/application",
                    _configuration.getMetricsClientHost().orElse("localhost"),
                    _configuration.getMetricsClientPort().orElse(7090));

            monitoringSinksBuilder.add(
                    new ApacheHttpSink.Builder()
                            .setUri(URI.create(endpoint))
                            .build());

        } else {
            monitoringSinksBuilder.addAll(createSinks(_configuration.getMonitoringSinks()));
        }

        final TsdMetricsFactory.Builder metricsFactoryBuilder = new TsdMetricsFactory.Builder()
                .setClusterName(_configuration.getMonitoringCluster())
                .setServiceName(_configuration.getMonitoringService())
                .setSinks(monitoringSinksBuilder.build());

        if (_configuration.getMonitoringHost().isPresent()) {
            metricsFactoryBuilder.setHostName(_configuration.getMonitoringHost().get());
        }

        final MetricsFactory metricsFactory = metricsFactoryBuilder.build();

        final AppShutdown shutdown = new AppShutdown();
        _guiceAppShutdown = shutdown;

        // Instantiate Guice
        return Guice.createInjector(new MainModule(actorSystem, metricsFactory, shutdown));
    }

    @SuppressFBWarnings("REC_CATCH_EXCEPTION")
    static List createSinks(
            final ImmutableList monitoringSinks) {
        // Until we implement the Commons Builder pattern in the metrics client
        // library we need to resort to a more brute-force deserialization
        // style. The benefit of this approach is that it will be forwards
        // compatible with the Commons Builder approach. The drawbacks are
        // the ugly way the configuration is passed around (as JsonNode) and
        // then two-step deserialized.
        final List sinks = new ArrayList<>();
        for (final JsonNode sinkNode : monitoringSinks) {
            @Nullable final JsonNode classNode = sinkNode.get("class");
            try {
                if (classNode != null) {
                    final Class builderClass = Class.forName(classNode.textValue() + "$Builder");
                    final Object builder = OBJECT_MAPPER.treeToValue(sinkNode, builderClass);
                    @SuppressWarnings("unchecked")
                    final com.arpnetworking.metrics.Sink sink =
                            (com.arpnetworking.metrics.Sink) builderClass.getMethod("build").invoke(builder);
                    sinks.add(sink);
                }
                // CHECKSTYLE.OFF: IllegalCatch - There are so many ways this hack can fail!
            } catch (final Exception e) {
                // CHECKSTYLE.ON: IllegalCatch
                throw new RuntimeException("Unable to create sink from: " + sinkNode.toString(), e);
            }
        }
        return sinks;
    }

    private ActorSystem launchPekko() {
        final Config pekkoConfiguration = ConfigFactory.parseMap(_configuration.getPekkoConfiguration());
        return ActorSystem.create("MAD", ConfigFactory.load(pekkoConfiguration));
    }

    private void shutdownJvmMetricsCollector() {
        LOGGER.info().setMessage("Stopping JVM metrics collection").log();
        if (_jvmMetricsCollector != null) {
            _jvmMetricsCollector.shutdown();
        }
    }

    private void shutdownPipelines() {
        LOGGER.info().setMessage("Stopping pipelines").log();
        _pipelinesLaunchable.shutdown();
    }

    private void shutdownActors() {
        LOGGER.info().setMessage("Stopping actors").log();
    }

    private void shutdownPekko() {
        LOGGER.info().setMessage("Stopping pekko").log();

        if (_actorSystem != null) {
            final Future terminate = _actorSystem.terminate();
            try {
                Await.result(terminate, Duration.create(30, TimeUnit.SECONDS));
                // CHECKSTYLE.OFF: IllegalCatch - Await.result throws Exception
            } catch (final Exception e) {
                // CHECKSTYLE.ON: IllegalCatch
                LOGGER.warn()
                        .setMessage("Exception while shutting down actor system")
                        .setThrowable(e)
                        .log();
            }
            _actorSystem = null;
        }
    }

    private void shutdownGuice() {
        LOGGER.info().setMessage("Stopping guice").log();
        if (_guiceAppShutdown != null) {
            _guiceAppShutdown.shutdown();
        }
    }

    private static Map createExecutorServiceMap(
            final ActorSystem actorSystem) {
        final Map executorServices = Maps.newHashMap();

        // Add the default dispatcher
        addExecutorServiceFromExecutionContextExecutor(
                executorServices,
                "pekko/default_dispatcher",
                actorSystem.dispatcher());

        // TODO(ville): Support monitoring additional dispatchers via configuration.

        return executorServices;
    }

    private static void addExecutorServiceFromExecutionContextExecutor(
            final Map executorServices,
            final String name,
            final ExecutionContextExecutor executionContextExecutor) {
        if (executionContextExecutor instanceof Dispatcher) {
            final Dispatcher dispatcher = (Dispatcher) executionContextExecutor;
            addExecutorService(
                    executorServices,
                    name,
                    dispatcher.executorService().executor());
            // TODO(ville): Support other ExecutionContextExecutor types as appropriate
        } else {
            throw new IllegalArgumentException(
                    "Unsupported ExecutionContextExecutor type: " + executionContextExecutor.getClass().getName());
        }
    }

    private static void addExecutorService(
            final Map executorServices,
            final String name,
            final ExecutorService executorService) {
        if (executorService instanceof java.util.concurrent.ForkJoinPool
                || executorService instanceof java.util.concurrent.ThreadPoolExecutor) {
            executorServices.put(name, executorService);
        } else {
            throw new IllegalArgumentException(
                    "Unsupported ExecutorService type: " + executorService.getClass().getName());
        }
    }

    private static Builder getFileSourceBuilder(final File configurationFile) {
        if (configurationFile.getName().toLowerCase(Locale.getDefault()).endsWith(HOCON_FILE_EXTENSION)) {
            return new HoconFileSource.Builder()
                    .setObjectMapper(OBJECT_MAPPER)
                    .setFile(configurationFile);
        }
        return new JsonNodeFileSource.Builder()
                .setObjectMapper(OBJECT_MAPPER)
                .setFile(configurationFile);
    }

    private static ObjectMapper createMadObjectMapper() {
        final ObjectMapper instance = ObjectMapperFactory.createInstance();
        instance.registerModule(new PekkoLoggingModule());
        return ImmutableObjectMapper.of(instance);
    }

    private final AggregatorConfiguration _configuration;

    private volatile PipelinesLaunchable _pipelinesLaunchable;
    private volatile ScheduledExecutorService _jvmMetricsCollector;

    private volatile ActorSystem _actorSystem;
    private volatile AppShutdown _guiceAppShutdown;

    private static final Long INITIAL_DELAY_IN_MILLIS = 0L;
    private static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
    private static final ObjectMapper OBJECT_MAPPER = createMadObjectMapper();
    private static final Logger LOGGER = LoggerFactory.getLogger(Main.class);
    private static final Duration SHUTDOWN_TIMEOUT = Duration.create(30, TimeUnit.SECONDS);
    private static final Semaphore SHUTDOWN_SEMAPHORE = new Semaphore(0);
    private static final Thread SHUTDOWN_THREAD = new ShutdownThread();
    private static final String HOCON_FILE_EXTENSION = ".conf";

    private static final class PipelinesLaunchable implements Launchable, Runnable {

        private PipelinesLaunchable(
                final ObjectMapper objectMapper,
                final File directory) {
            _objectMapper = objectMapper;
            _directory = directory;
            _fileToPipelineLaunchables = Maps.newConcurrentMap();
        }

        @Override
        public synchronized void launch() {
            _pipelinesExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> new Thread(runnable, "PipelineExecutor"));
            _pipelinesExecutor.scheduleAtFixedRate(
                    this,
                    0,  // initial delay
                    1,  // interval
                    TimeUnit.MINUTES);
        }

        @Override
        public synchronized void shutdown() {
            _pipelinesExecutor.shutdown();
            try {
                _pipelinesExecutor.awaitTermination(10, TimeUnit.SECONDS);
            } catch (final InterruptedException e) {
                LOGGER.warn("Unable to shutdown pipeline executor", e);
            }
            _pipelinesExecutor = null;

            _fileToPipelineLaunchables.keySet()
                    .forEach(this::shutdownPipeline);
            _fileToPipelineLaunchables.clear();
        }

        @Override
        public void run() {
            final boolean exists = _directory.exists() && _directory.isDirectory();
            if (exists) {
                final Set missingFiles = Sets.newHashSet(_fileToPipelineLaunchables.keySet());
                for (final File file : Optional.ofNullable(_directory.listFiles()).orElse(EMPTY_FILE_ARRAY)) {
                    missingFiles.remove(file);
                    if (!_fileToPipelineLaunchables.containsKey(file)) {
                        launchPipeline(file);
                    }
                }
                missingFiles.forEach(this::shutdownPipeline);
            } else {
                _fileToPipelineLaunchables.keySet().forEach(this::shutdownPipeline);
            }
        }

        private void launchPipeline(final File file) {
            LOGGER.debug()
                    .setMessage("Creating pipeline")
                    .addData("configuration", file)
                    .log();

            final Configurator pipelineConfigurator =
                    new Configurator<>(Pipeline::new, PipelineConfiguration.class);
            final DynamicConfiguration pipelineConfiguration = new DynamicConfiguration.Builder()
                    .setObjectMapper(_objectMapper)
                    .addSourceBuilder(getFileSourceBuilder(file))
                    .addTrigger(
                            new FileTrigger.Builder()
                                    .setFile(file)
                                    .build())
                    .addListener(pipelineConfigurator)
                    .build();

            LOGGER.debug()
                    .setMessage("Launching pipeline")
                    .addData("pipeline", pipelineConfiguration)
                    .log();

            pipelineConfiguration.launch();

            _fileToPipelineLaunchables.put(file, ImmutableList.of(pipelineConfigurator, pipelineConfiguration));
        }

        private void shutdownPipeline(final File file) {
            LOGGER.debug()
                    .setMessage("Stopping pipeline")
                    .addData("pipeline", file)
                    .log();

            _fileToPipelineLaunchables.remove(file).forEach(com.arpnetworking.utility.Launchable::shutdown);
        }

        private final ObjectMapper _objectMapper;
        private final File _directory;
        private final Map> _fileToPipelineLaunchables;

        private ScheduledExecutorService _pipelinesExecutor;

        private static final File[] EMPTY_FILE_ARRAY = new File[0];
    }

    private static final class ShutdownThread extends Thread {
        private ShutdownThread() {
            super("MADShutdownHook");
        }

        @Override
        public void run() {
            LOGGER.info()
                    .setMessage("Stopping mad")
                    .log();

            // release the main thread waiting for shutdown signal
            SHUTDOWN_SEMAPHORE.release();

            try {
                // wait for it to signal that it has completed shutdown
                if (!SHUTDOWN_SEMAPHORE.tryAcquire(SHUTDOWN_TIMEOUT.toSeconds(), TimeUnit.SECONDS)) {
                    LOGGER.warn()
                            .setMessage("Shutdown did not complete in a timely manner")
                            .log();
                }
            } catch (final InterruptedException e) {
                throw new RuntimeException(e);
            } finally {
                LOGGER.info()
                        .setMessage("Shutdown complete")
                        .log();
                final LoggerContext context = (LoggerContext) org.slf4j.LoggerFactory.getILoggerFactory();
                context.stop();
            }
        }
    }

    private static final class MainModule extends AbstractModule {

        MainModule(final ActorSystem actorSystem, final MetricsFactory metricsFactory, final AppShutdown shutdown) {
            this._actorSystem = actorSystem;
            this._metricsFactory = metricsFactory;
            this._shutdown = shutdown;
        }

        @Override
        public void configure() {
            bind(ActorSystem.class).toInstance(_actorSystem);
            bind(MetricsFactory.class).toInstance(_metricsFactory);
            bind(LifecycleRegistration.class).toInstance(_shutdown);
        }

        @Provides
        @SuppressFBWarnings("UPM_UNCALLED_PRIVATE_METHOD")
        private PeriodicMetrics providePeriodicMetrics(
                final MetricsFactory metricsFactory,
                final LifecycleRegistration lifecycle) {
            final TsdPeriodicMetrics periodicMetrics = new TsdPeriodicMetrics.Builder()
                    .setMetricsFactory(metricsFactory)
                    .build();
            final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
                    r -> new Thread(r, "PeriodicMetricsCloser"));
            final long offsetMillis = 1250 - (System.currentTimeMillis() % 1000);
            executor.scheduleAtFixedRate(periodicMetrics, offsetMillis, 1000, TimeUnit.MILLISECONDS);

            // Register to shutdown the executor when the Guice stack is shutdown.
            lifecycle.registerShutdown(() -> {
                executor.shutdown();
                return CompletableFuture.completedFuture(null);
            });
            return periodicMetrics;
        }

        private final ActorSystem _actorSystem;
        private final MetricsFactory _metricsFactory;
        private final AppShutdown _shutdown;
    }
}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy