org.graylog2.indexer.IndexMapping Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of graylog2-server Show documentation
Graylog
There is a newer version: 6.1.4
/*
 * Copyright (C) 2020 Graylog, Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the Server Side Public License, version 1,
 * as published by MongoDB, Inc.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * Server Side Public License for more details.
 *
 * You should have received a copy of the Server Side Public License
 * along with this program. If not, see
 * .
 */
package org.graylog2.indexer;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.graylog2.indexer.indexset.CustomFieldMappings;
import org.graylog2.indexer.indexset.IndexSetConfig;
import org.graylog2.indexer.indices.Template;
import org.graylog2.plugin.Message;

import java.util.List;
import java.util.Map;

/**
 * Representing the message type mapping in Elasticsearch. This is giving ES more
 * information about what the fields look like and how it should analyze them.
 */
public abstract class IndexMapping implements IndexMappingTemplate {
    public static final String TYPE_MESSAGE = "message";

    @Override
    public Template toTemplate(IndexSetConfig indexSetConfig, String indexPattern, Long order) {
        return messageTemplate(indexPattern, indexSetConfig.indexAnalyzer(), order, indexSetConfig.customFieldMappings());
    }

    protected Map analyzerKeyword() {
        return ImmutableMap.of("analyzer_keyword", ImmutableMap.of(
                "tokenizer", "keyword",
                "filter", "lowercase"));
    }

    public Template messageTemplate(final String indexPattern,
                                               final String analyzer,
                                               final Long order,
                                               final CustomFieldMappings customFieldMappings) {
        var settings = new Template.Settings(Map.of(
                "index", Map.of(
                        "analysis", Map.of("analyzer", analyzerKeyword())
        )
        ));
        var mappings = mapping(analyzer, customFieldMappings);

        return createTemplate(indexPattern, order, settings, mappings);
    }

    Template createTemplate(String indexPattern, Long order, Template.Settings settings, Template.Mappings mappings) {
        return Template.create(indexPattern, mappings, order, settings);
    }

    protected Template.Mappings mapping(final String analyzer,
                                          final CustomFieldMappings customFieldMappings) {
        return new Template.Mappings(ImmutableMap.of(TYPE_MESSAGE, messageMapping(analyzer, customFieldMappings)));
    }

    protected Map messageMapping(final String analyzer,
                                                 final CustomFieldMappings customFieldMappings) {
        return ImmutableMap.of(
                "properties", fieldProperties(analyzer, customFieldMappings),
                "dynamic_templates", dynamicTemplate(),
                "_source", enabled());
    }

    private Map> internalFieldsMapping() {
        return ImmutableMap.of("internal_fields",
                ImmutableMap.of(
                        "match", "gl2_*",
                        "match_mapping_type", "string",
                        "mapping", notAnalyzedString())
        );
    }

    protected List>> dynamicTemplate() {
        final Map> templateInternal = internalFieldsMapping();

        final Map> templateAll = ImmutableMap.of("store_generic", dynamicStrings());

        return ImmutableList.of(templateInternal, templateAll);
    }

    abstract Map dynamicStrings();

    protected Map> fieldProperties(final String analyzer,
                                                               final CustomFieldMappings customFieldMappings) {
        final ImmutableMap.Builder> builder = ImmutableMap.>builder()
                .put(Message.FIELD_MESSAGE, analyzedString(analyzer, false))
                .put(Message.FIELD_FULL_MESSAGE, analyzedString(analyzer, false))
                // http://joda-time.sourceforge.net/api-release/org/joda/time/format/DateTimeFormat.html
                // http://www.elasticsearch.org/guide/reference/mapping/date-format.html
                .put(Message.FIELD_TIMESTAMP, typeTimeWithMillis())
                .put(Message.FIELD_GL2_ACCOUNTED_MESSAGE_SIZE, typeLong())
                .put(Message.FIELD_GL2_RECEIVE_TIMESTAMP, typeTimeWithMillis())
                .put(Message.FIELD_GL2_PROCESSING_TIMESTAMP, typeTimeWithMillis())
                .put(Message.FIELD_GL2_MESSAGE_ID, notAnalyzedString())
                // to support wildcard searches in source we need to lowercase the content (wildcard search lowercases search term)
                .put(Message.FIELD_SOURCE, analyzedString("analyzer_keyword", true))
                .put(Message.FIELD_STREAMS, notAnalyzedString());

        if (customFieldMappings != null) {
            customFieldMappings.forEach(customMapping -> builder.put(customMapping.fieldName(), type(customMapping.toPhysicalType())));
        }

        return builder.build();
    }

    Map notAnalyzedString() {
        return ImmutableMap.of("type", "keyword");
    }
    Map analyzedString(String analyzer, boolean fieldData) {
        return ImmutableMap.of(
                "type", "text",
                "analyzer", analyzer,
                "fielddata", fieldData);
    }

    protected Map typeTimeWithMillis() {
        return ImmutableMap.of(
                "type", "date",
                "format", dateFormat());
    }

    protected Map typeLong() {
        return ImmutableMap.of("type", "long");
    }

    protected Map type(final String type) {
        return ImmutableMap.of("type", type);
    }

    private Map enabled() {
        return ImmutableMap.of("enabled", true);
    }

    protected String dateFormat() {
        return Constants.ES_DATE_FORMAT;
    }
}