All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.graylog2.plugin.inputs.transports.NettyTransport Maven / Gradle / Ivy

There is a newer version: 5.2.7
Show newest version
/**
 * This file is part of Graylog.
 *
 * Graylog is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Graylog is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Graylog.  If not, see .
 */
package org.graylog2.plugin.inputs.transports;

import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.MetricSet;
import com.google.common.annotations.VisibleForTesting;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelPipeline;
import org.graylog2.inputs.transports.netty.EventLoopGroupFactory;
import org.graylog2.inputs.transports.netty.ExceptionLoggingChannelHandler;
import org.graylog2.inputs.transports.netty.PromiseFailureHandler;
import org.graylog2.inputs.transports.netty.RawMessageHandler;
import org.graylog2.plugin.LocalMetricRegistry;
import org.graylog2.plugin.MetricSets;
import org.graylog2.plugin.configuration.Configuration;
import org.graylog2.plugin.configuration.ConfigurationRequest;
import org.graylog2.plugin.configuration.fields.ConfigurationField;
import org.graylog2.plugin.configuration.fields.NumberField;
import org.graylog2.plugin.inputs.MessageInput;
import org.graylog2.plugin.inputs.codecs.CodecAggregator;
import org.graylog2.plugin.inputs.util.PacketInformationDumper;
import org.graylog2.plugin.inputs.util.ThroughputCounter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.Callable;

public abstract class NettyTransport implements Transport {
    public static final String CK_BIND_ADDRESS = "bind_address";
    public static final String CK_PORT = "port";
    public static final String CK_RECV_BUFFER_SIZE = "recv_buffer_size";
    public static final String CK_NUMBER_WORKER_THREADS = "number_worker_threads";
    private static final int DEFAULT_NUMBER_WORKER_THREADS = Runtime.getRuntime().availableProcessors();

    private static final Logger log = LoggerFactory.getLogger(NettyTransport.class);

    protected final EventLoopGroupFactory eventLoopGroupFactory;
    protected final MetricRegistry localRegistry;

    protected final InetSocketAddress socketAddress;
    protected final ThroughputCounter throughputCounter;
    protected final int workerThreads;
    private final int recvBufferSize;

    @Nullable
    private CodecAggregator aggregator;

    public NettyTransport(Configuration configuration,
                          EventLoopGroupFactory eventLoopGroupFactory,
                          ThroughputCounter throughputCounter,
                          LocalMetricRegistry localRegistry) {
        this.throughputCounter = throughputCounter;

        final String hostname = configuration.getString(CK_BIND_ADDRESS);
        if (hostname != null && configuration.intIsSet(CK_PORT)) {
            this.socketAddress = new InetSocketAddress(hostname, configuration.getInt(CK_PORT));
        } else {
            this.socketAddress = null;
        }
        this.recvBufferSize = configuration.intIsSet(CK_RECV_BUFFER_SIZE)
                ? configuration.getInt(CK_RECV_BUFFER_SIZE)
                : MessageInput.getDefaultRecvBufferSize();

        this.eventLoopGroupFactory = eventLoopGroupFactory;

        this.workerThreads = configuration.getInt(CK_NUMBER_WORKER_THREADS, DEFAULT_NUMBER_WORKER_THREADS);

        this.localRegistry = localRegistry;
        localRegistry.registerAll(MetricSets.of(throughputCounter.gauges()));
    }

    protected ChannelInitializer getChannelInitializer(final LinkedHashMap> handlerList) {
        return new ChannelInitializer() {
            @Override
            protected void initChannel(Channel ch) throws Exception {
                final ChannelPipeline p = ch.pipeline();
                Map.Entry> postentry = null;
                for (final Map.Entry> entry : handlerList.entrySet()) {
                    // Handle exceptions at the top of the (bottom-up evaluated) pipeline
                    if (entry.getKey().equals("exception-logger")) {
                        postentry = entry;
                    } else {
                        p.addLast(entry.getKey(), entry.getValue().call());
                    }
                }
                if (postentry != null) {
                    p.addLast(postentry.getKey(), postentry.getValue().call());
                }
            }
        };
    }

    /**
     * Get the local socket address this transport is listening on after being launched.
     *
     * @return the listening address of this transport or {@code null} if the transport hasn't been launched yet.
     */
    @VisibleForTesting
    @Nullable
    public abstract SocketAddress getLocalAddress();

    @Override
    public void setMessageAggregator(@Nullable CodecAggregator aggregator) {
        this.aggregator = aggregator;
    }

    @Nullable
    protected CodecAggregator getAggregator() {
        return aggregator;
    }

    /**
     * Subclasses can override this to add additional {@link ChannelHandler channel handlers} to the
     * Netty {@link ChannelPipeline} to support additional features.
     *
     * Some common use cases are to add connection counters or traffic shapers.
     *
     * @param input The {@link MessageInput} for which these channel handlers are being added
     * @return list of initial {@link ChannelHandler channel handlers} to add to the Netty {@link ChannelPipeline channel pipeline}
     */
    protected LinkedHashMap> getChannelHandlers(final MessageInput input) {
        LinkedHashMap> handlerList = new LinkedHashMap<>();

        handlerList.put("exception-logger", () -> new ExceptionLoggingChannelHandler(input, log));
        handlerList.put("packet-meta-dumper", () -> new PacketInformationDumper(input));
        handlerList.put("output-failure-logger", () -> PromiseFailureHandler.INSTANCE);

        return handlerList;
    }

    /**
     * Subclasses can override this to modify the {@link ChannelHandler channel handlers} at the end of the pipeline for child channels.
     *
     * @param input The {@link MessageInput} for which these child channel handlers are being added
     * @return list of custom {@link ChannelHandler channel handlers} to add to the Netty {@link ChannelPipeline channel pipeline} for child channels
     */
    protected LinkedHashMap> getCustomChildChannelHandlers(final MessageInput input) {
        return new LinkedHashMap<>();
    }

    /**
     * Subclasses can override this to modify the {@link ChannelHandler channel handlers} for child channels.
     *
     * The default handlers in this group are all channel handlers returned by {@link #getCustomChildChannelHandlers(MessageInput)},
     * the optional aggregation handler (e.g. for chunked GELF via UDP) and the {@link RawMessageHandler} (in that order).
     *
     * Usually overriding this method should only be necessary if you have a codec that cannot create a
     * {@link org.graylog2.plugin.journal.RawMessage} for incoming messages at the end of the pipeline.
     *
     * A valid use case would be to insert debug handlers in the middle of the list, though.
     *
     * @param input The {@link MessageInput} for which these child channel handlers are being added
     * @return list of custom {@link ChannelHandler channel handlers} to add to the Netty {@link ChannelPipeline channel pipeline} for child channels
     * @see #getCustomChildChannelHandlers(MessageInput)
     */
    protected abstract LinkedHashMap> getChildChannelHandlers(final MessageInput input);

    protected int getRecvBufferSize() {
        return recvBufferSize;
    }

    @Override
    public MetricSet getMetricSet() {
        return localRegistry;
    }

    public static class Config implements Transport.Config {
        @Override
        public ConfigurationRequest getRequestedConfiguration() {
            final ConfigurationRequest r = new ConfigurationRequest();

            r.addField(ConfigurationRequest.Templates.bindAddress(CK_BIND_ADDRESS));
            r.addField(ConfigurationRequest.Templates.portNumber(CK_PORT, 5555));
            r.addField(ConfigurationRequest.Templates.recvBufferSize(CK_RECV_BUFFER_SIZE, 1024 * 1024));
            r.addField(new NumberField(
                    CK_NUMBER_WORKER_THREADS,
                    "No. of worker threads",
                    DEFAULT_NUMBER_WORKER_THREADS,
                    "Number of worker threads processing network connections for this input.",
                    // Should be mandatory, but then all existing inputs are failing to start
                    ConfigurationField.Optional.OPTIONAL,
                    NumberField.Attribute.ONLY_POSITIVE)
            );

            return r;
        }
    }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy