org.graylog2.indexer.IndexMapping Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of graylog2-server Show documentation
Graylog
There is a newer version: 6.0.1
/*
 * Copyright (C) 2020 Graylog, Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the Server Side Public License, version 1,
 * as published by MongoDB, Inc.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * Server Side Public License for more details.
 *
 * You should have received a copy of the Server Side Public License
 * along with this program. If not, see
 * .
 */
package org.graylog2.indexer;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.graylog2.indexer.indexset.IndexSetConfig;
import org.graylog2.plugin.Message;

import java.util.Collections;
import java.util.List;
import java.util.Map;

/**
 * Representing the message type mapping in Elasticsearch. This is giving ES more
 * information about what the fields look like and how it should analyze them.
 */
public abstract class IndexMapping implements IndexMappingTemplate {
    public static final String TYPE_MESSAGE = "message";

    @Override
    public Map toTemplate(IndexSetConfig indexSetConfig, String indexPattern, int order) {
        return messageTemplate(indexPattern, indexSetConfig.indexAnalyzer(), order);
    }

    protected Map analyzerKeyword() {
        return ImmutableMap.of("analyzer_keyword", ImmutableMap.of(
                "tokenizer", "keyword",
                "filter", "lowercase"));
    }

    public Map messageTemplate(final String template, final String analyzer, final int order) {
        final Map settings = Collections.singletonMap(
                "analysis", Collections.singletonMap("analyzer", analyzerKeyword())
                );
        final Map mappings = mapping(analyzer);

        return createTemplate(template, order, settings, mappings);
    }

    Map createTemplate(String template, int order, Map settings, Map mappings) {
        return ImmutableMap.of(
                "template", template,
                "order", order,
                "settings", settings,
                "mappings", mappings
        );
    }

    protected Map mapping(String analyzer) {
        return ImmutableMap.of(TYPE_MESSAGE, messageMapping(analyzer));
    }

    protected Map messageMapping(final String analyzer) {
        return ImmutableMap.of(
                "properties", fieldProperties(analyzer),
                "dynamic_templates", dynamicTemplate(),
                "_source", enabled());
    }

    private Map> internalFieldsMapping() {
        return ImmutableMap.of("internal_fields",
                ImmutableMap.of(
                        "match", "gl2_*",
                        "match_mapping_type", "string",
                        "mapping", notAnalyzedString())
        );
    }

    protected List>> dynamicTemplate() {
        final Map> templateInternal = internalFieldsMapping();

        final Map> templateAll = ImmutableMap.of("store_generic", dynamicStrings());

        return ImmutableList.of(templateInternal, templateAll);
    }

    abstract Map dynamicStrings();

    protected Map> fieldProperties(String analyzer) {
        return ImmutableMap.>builder()
                .put("message", analyzedString(analyzer, false))
                .put("full_message", analyzedString(analyzer, false))
                // http://joda-time.sourceforge.net/api-release/org/joda/time/format/DateTimeFormat.html
                // http://www.elasticsearch.org/guide/reference/mapping/date-format.html
                .put("timestamp", typeTimeWithMillis())
                .put(Message.FIELD_GL2_ACCOUNTED_MESSAGE_SIZE, typeLong())
                .put(Message.FIELD_GL2_RECEIVE_TIMESTAMP, typeTimeWithMillis())
                .put(Message.FIELD_GL2_PROCESSING_TIMESTAMP, typeTimeWithMillis())
                // to support wildcard searches in source we need to lowercase the content (wildcard search lowercases search term)
                .put("source", analyzedString("analyzer_keyword", true))
                .put("streams", notAnalyzedString())
                .build();
    }

    Map notAnalyzedString() {
        return ImmutableMap.of("type", "keyword");
    }
    Map analyzedString(String analyzer, boolean fieldData) {
        return ImmutableMap.of(
                "type", "text",
                "analyzer", analyzer,
                "fielddata", fieldData);
    }

    protected Map typeTimeWithMillis() {
        return ImmutableMap.of(
                "type", "date",
                "format", dateFormat());
    }

    protected Map typeLong() {
        return ImmutableMap.of("type", "long");
    }

    private Map enabled() {
        return ImmutableMap.of("enabled", true);
    }

    protected String dateFormat() {
        return Constants.ES_DATE_FORMAT;
    }
}