org.graylog.plugins.beats.BeatsCodec Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of graylog2-server Show documentation
Graylog
There is a newer version: 6.1.4
/*
 * Copyright (C) 2020 Graylog, Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the Server Side Public License, version 1,
 * as published by MongoDB, Inc.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * Server Side Public License for more details.
 *
 * You should have received a copy of the Server Side Public License
 * along with this program. If not, see
 * .
 */
package org.graylog.plugins.beats;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.assistedinject.Assisted;
import org.graylog2.jackson.TypeReferences;
import org.graylog2.plugin.Message;
import org.graylog2.plugin.Tools;
import org.graylog2.plugin.configuration.Configuration;
import org.graylog2.plugin.inputs.annotations.Codec;
import org.graylog2.plugin.inputs.annotations.ConfigClass;
import org.graylog2.plugin.inputs.annotations.FactoryClass;
import org.graylog2.plugin.inputs.codecs.AbstractCodec;
import org.graylog2.plugin.journal.RawMessage;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.inject.Inject;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import static java.util.Objects.requireNonNull;

@Codec(name = "beats-deprecated", displayName = "Beats (deprecated)")
public class BeatsCodec extends AbstractCodec {
    private static final Logger LOG = LoggerFactory.getLogger(BeatsCodec.class);
    private static final String MAP_KEY_SEPARATOR = "_";

    private final ObjectMapper objectMapper;

    @Inject
    public BeatsCodec(@Assisted Configuration configuration, ObjectMapper objectMapper) {
        super(configuration);
        this.objectMapper = requireNonNull(objectMapper);
    }

    @Nullable
    @Override
    public Message decode(@Nonnull RawMessage rawMessage) {
        final byte[] payload = rawMessage.getPayload();
        final Map event;
        try {
            event = objectMapper.readValue(payload, TypeReferences.MAP_STRING_OBJECT);
        } catch (IOException e) {
            LOG.error("Couldn't decode raw message {}", rawMessage);
            return null;
        }

        return parseEvent(event);
    }

    @Nullable
    private Message parseEvent(Map event) {
        @SuppressWarnings("unchecked")
        final Map metadata = (HashMap) event.remove("@metadata");
        final String type;
        if (metadata == null) {
            LOG.warn("Couldn't recognize Beats type");
            type = "unknown";
        } else {
            type = metadata.get("beat");
        }
        final Message gelfMessage;
        switch (type) {
            case "filebeat":
                gelfMessage = parseFilebeat(event);
                break;
            case "topbeat":
                gelfMessage = parseTopbeat(event);
                break;
            case "metricbeat":
                gelfMessage = parseMetricbeat(event);
                break;
            case "packetbeat":
                gelfMessage = parsePacketbeat(event);
                break;
            case "winlogbeat":
                gelfMessage = parseWinlogbeat(event);
                break;
            default:
                LOG.debug("Unknown beats type {}. Using generic handler.", type);
                gelfMessage = parseGenericBeat(event);
                break;
        }

        return gelfMessage;
    }

    private Message createMessage(String message, Map event) {
        @SuppressWarnings("unchecked")
        final Map beat = (Map) event.remove("beat");
        final String hostname;
        final String name;
        if (beat == null) {
            hostname = "unknown";
            name = "unknown";
        } else {
            hostname = String.valueOf(beat.get("hostname"));
            name = String.valueOf(beat.get("name"));
        }
        final String timestampField = String.valueOf(event.remove("@timestamp"));
        final DateTime timestamp = Tools.dateTimeFromString(timestampField);
        final String type = String.valueOf(event.get("type"));
        final Object tags = event.get("tags");

        final Message result = new Message(message, hostname, timestamp);
        result.addField("name", name);
        result.addField("type", type);
        result.addField("tags", tags);

        @SuppressWarnings("unchecked")
        final Map fields = (Map) event.get("fields");
        if (fields != null) {
            result.addFields(fields);
        }

        return result;
    }

    /**
     * @see Filebeat Exported Fields
     */
    private Message parseFilebeat(Map event) {
        final String message = String.valueOf(event.get("message"));
        final Message gelfMessage = createMessage(message, event);
        gelfMessage.addField("facility", "filebeat");
        gelfMessage.addField("file", event.get("source"));
        gelfMessage.addField("input_type", event.get("input_type"));
        gelfMessage.addField("count", event.get("count"));
        gelfMessage.addField("offset", event.get("offset"));

        return gelfMessage;
    }

    /**
     * @see Topbeat Exported Fields
     */
    private Message parseTopbeat(Map event) {
        final Message gelfMessage = createMessage("-", event);
        gelfMessage.addField("facility", "topbeat");
        final Map flattened = MapUtils.flatten(event, "topbeat", MAP_KEY_SEPARATOR);

        // Fix field names containing dots, like "cpu.name"
        final Map withoutDots = MapUtils.replaceKeyCharacter(flattened, '.', MAP_KEY_SEPARATOR.charAt(0));
        gelfMessage.addFields(withoutDots);
        return gelfMessage;
    }

    /**
     * @see Metricbeat Exported Fields
     */
    private Message parseMetricbeat(Map event) {
        final Message gelfMessage = createMessage("-", event);
        gelfMessage.addField("facility", "metricbeat");
        final Map flattened = MapUtils.flatten(event, "metricbeat", MAP_KEY_SEPARATOR);

        // Fix field names containing dots, like "cpu.name"
        final Map withoutDots = MapUtils.replaceKeyCharacter(flattened, '.', MAP_KEY_SEPARATOR.charAt(0));
        gelfMessage.addFields(withoutDots);
        return gelfMessage;
    }

    /**
     * @see Packetbeat Exported Fields
     */
    private Message parsePacketbeat(Map event) {
        final Message gelfMessage = createMessage("-", event);
        gelfMessage.addField("facility", "packetbeat");
        final Map flattened = MapUtils.flatten(event, "packetbeat", MAP_KEY_SEPARATOR);

        // Fix field names containing dots, like "icmp.version"
        final Map withoutDots = MapUtils.replaceKeyCharacter(flattened, '.', MAP_KEY_SEPARATOR.charAt(0));
        gelfMessage.addFields(withoutDots);

        return gelfMessage;
    }

    /**
     * @see Winlogbeat Exported Fields
     */
    private Message parseWinlogbeat(Map event) {
        final String message = String.valueOf(event.remove("message"));
        final Message gelfMessage = createMessage(message, event);
        gelfMessage.addField("facility", "winlogbeat");
        final Map flattened = MapUtils.flatten(event, "winlogbeat", MAP_KEY_SEPARATOR);

        // Fix field names containing dots, like "user.name"
        final Map withoutDots = MapUtils.replaceKeyCharacter(flattened, '.', MAP_KEY_SEPARATOR.charAt(0));
        gelfMessage.addFields(withoutDots);
        return gelfMessage;
    }

    private Message parseGenericBeat(Map event) {
        final String message = String.valueOf(event.remove("message"));
        final Message gelfMessage = createMessage(message, event);
        gelfMessage.addField("facility", "genericbeat");
        final Map flattened = MapUtils.flatten(event, "beat", MAP_KEY_SEPARATOR);

        // Fix field names containing dots
        final Map withoutDots = MapUtils.replaceKeyCharacter(flattened, '.', MAP_KEY_SEPARATOR.charAt(0));
        gelfMessage.addFields(withoutDots);
        return gelfMessage;
    }


    @FactoryClass
    public interface Factory extends AbstractCodec.Factory {
        @Override
        BeatsCodec create(Configuration configuration);

        @Override
        Config getConfig();

        @Override
        Descriptor getDescriptor();
    }

    @ConfigClass
    public static class Config extends AbstractCodec.Config {
    }


    public static class Descriptor extends AbstractCodec.Descriptor {
        @Inject
        public Descriptor() {
            super(BeatsCodec.class.getAnnotation(Codec.class).displayName());
        }
    }
}