org.elasticsearch.index.mapper.DynamicFieldsBuilder Maven / Gradle / Ivy

Go to download
/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License
 * 2.0 and the Server Side Public License, v 1; you may not use this file except
 * in compliance with, at your election, the Elastic License 2.0 or the Server
 * Side Public License, v 1.
 */

package org.elasticsearch.index.mapper;

import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.CheckedBiConsumer;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.core.CheckedRunnable;
import org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.xcontent.XContentParser;

import java.io.IOException;
import java.time.format.DateTimeParseException;
import java.util.Map;

/**
 * Encapsulates the logic for dynamically creating fields as part of document parsing.
 * Fields can be mapped under properties, as concrete fields that get indexed,
 * or as runtime fields that are evaluated at search-time and have no indexing overhead.
 * Objects get dynamically mapped only under dynamic:true.
 */
final class DynamicFieldsBuilder {
    private static final Concrete CONCRETE = new Concrete(DocumentParser::parseObjectOrField);
    static final DynamicFieldsBuilder DYNAMIC_TRUE = new DynamicFieldsBuilder(CONCRETE);
    static final DynamicFieldsBuilder DYNAMIC_RUNTIME = new DynamicFieldsBuilder(new Runtime());

    private final Strategy strategy;

    private DynamicFieldsBuilder(Strategy strategy) {
        this.strategy = strategy;
    }

    /**
     * Creates a dynamic field based on the value of the current token being parsed from an incoming document.
     * Makes decisions based on the type of the field being found, looks at matching dynamic templates and
     * delegates to the appropriate strategy which depends on the current dynamic mode.
     * The strategy defines if fields are going to be mapped as ordinary or runtime fields.
     */
    void createDynamicFieldFromValue(final DocumentParserContext context, XContentParser.Token token, String name) throws IOException {
        if (token == XContentParser.Token.VALUE_STRING) {
            String text = context.parser().text();

            boolean parseableAsLong = false;
            try {
                Long.parseLong(text);
                parseableAsLong = true;
            } catch (NumberFormatException e) {
                // not a long number
            }

            boolean parseableAsDouble = false;
            try {
                Double.parseDouble(text);
                parseableAsDouble = true;
            } catch (NumberFormatException e) {
                // not a double number
            }

            if (parseableAsLong && context.root().numericDetection()) {
                createDynamicField(
                    context,
                    name,
                    DynamicTemplate.XContentFieldType.LONG,
                    () -> strategy.newDynamicLongField(context, name)
                );
            } else if (parseableAsDouble && context.root().numericDetection()) {
                createDynamicField(
                    context,
                    name,
                    DynamicTemplate.XContentFieldType.DOUBLE,
                    () -> strategy.newDynamicDoubleField(context, name)
                );
            } else if (parseableAsLong == false && parseableAsDouble == false && context.root().dateDetection()) {
                // We refuse to match pure numbers, which are too likely to be
                // false positives with date formats that include eg.
                // `epoch_millis` or `YYYY`
                for (DateFormatter dateTimeFormatter : context.root().dynamicDateTimeFormatters()) {
                    try {
                        dateTimeFormatter.parse(text);
                    } catch (ElasticsearchParseException | DateTimeParseException | IllegalArgumentException e) {
                        // failure to parse this, continue
                        continue;
                    }
                    createDynamicDateField(
                        context,
                        name,
                        dateTimeFormatter,
                        () -> strategy.newDynamicDateField(context, name, dateTimeFormatter)
                    );
                    return;
                }
                createDynamicField(
                    context,
                    name,
                    DynamicTemplate.XContentFieldType.STRING,
                    () -> strategy.newDynamicStringField(context, name)
                );
            } else {
                createDynamicField(
                    context,
                    name,
                    DynamicTemplate.XContentFieldType.STRING,
                    () -> strategy.newDynamicStringField(context, name)
                );
            }
        } else if (token == XContentParser.Token.VALUE_NUMBER) {
            XContentParser.NumberType numberType = context.parser().numberType();
            if (numberType == XContentParser.NumberType.INT
                || numberType == XContentParser.NumberType.LONG
                || numberType == XContentParser.NumberType.BIG_INTEGER) {
                createDynamicField(
                    context,
                    name,
                    DynamicTemplate.XContentFieldType.LONG,
                    () -> strategy.newDynamicLongField(context, name)
                );
            } else if (numberType == XContentParser.NumberType.FLOAT
                || numberType == XContentParser.NumberType.DOUBLE
                || numberType == XContentParser.NumberType.BIG_DECIMAL) {
                    createDynamicField(
                        context,
                        name,
                        DynamicTemplate.XContentFieldType.DOUBLE,
                        () -> strategy.newDynamicDoubleField(context, name)
                    );
                } else {
                    throw new IllegalStateException("Unable to parse number of type [" + numberType + "]");
                }
        } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
            createDynamicField(
                context,
                name,
                DynamicTemplate.XContentFieldType.BOOLEAN,
                () -> strategy.newDynamicBooleanField(context, name)
            );
        } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
            // runtime binary fields are not supported, hence binary objects always get created as concrete fields
            createDynamicField(
                context,
                name,
                DynamicTemplate.XContentFieldType.BINARY,
                () -> CONCRETE.newDynamicBinaryField(context, name)
            );
        } else {
            createDynamicStringFieldFromTemplate(context, name);
        }
    }

    /**
     * Returns a dynamically created object mapper, eventually based on a matching dynamic template.
     */
    Mapper createDynamicObjectMapper(DocumentParserContext context, String name) {
        Mapper mapper = createObjectMapperFromTemplate(context, name);
        return mapper != null ? mapper : new ObjectMapper.Builder(name).enabled(true).build(MapperBuilderContext.forPath(context.path()));
    }

    /**
     * Returns a dynamically created object mapper, based exclusively on a matching dynamic template, null otherwise.
     */
    Mapper createObjectMapperFromTemplate(DocumentParserContext context, String name) {
        Mapper.Builder templateBuilder = findTemplateBuilderForObject(context, name);
        return templateBuilder == null ? null : templateBuilder.build(MapperBuilderContext.forPath(context.path()));
    }

    /**
     * Creates a dynamic string field based on a matching dynamic template.
     * No field is created in case there is no matching dynamic template.
     */
    void createDynamicStringFieldFromTemplate(DocumentParserContext context, String name) throws IOException {
        createDynamicField(context, name, DynamicTemplate.XContentFieldType.STRING, () -> {});
    }

    private static void createDynamicDateField(
        DocumentParserContext context,
        String name,
        DateFormatter dateFormatter,
        CheckedRunnable createDynamicField
    ) throws IOException {
        createDynamicField(context, name, DynamicTemplate.XContentFieldType.DATE, dateFormatter, createDynamicField);
    }

    private static void createDynamicField(
        DocumentParserContext context,
        String name,
        DynamicTemplate.XContentFieldType matchType,
        CheckedRunnable dynamicFieldStrategy
    ) throws IOException {
        assert matchType != DynamicTemplate.XContentFieldType.DATE;
        createDynamicField(context, name, matchType, null, dynamicFieldStrategy);
    }

    private static void createDynamicField(
        DocumentParserContext context,
        String name,
        DynamicTemplate.XContentFieldType matchType,
        DateFormatter dateFormatter,
        CheckedRunnable dynamicFieldStrategy
    ) throws IOException {
        if (applyMatchingTemplate(context, name, matchType, dateFormatter) == false) {
            dynamicFieldStrategy.run();
        }
    }

    /**
     * Find and apply a matching dynamic template. Returns {@code true} if a template could be found, {@code false} otherwise.
     * @param context        the parse context for this document
     * @param name           the current field name
     * @param matchType      the type of the field in the json document or null if unknown
     * @param dateFormatter  a date formatter to use if the type is a date, null if not a date or is using the default format
     * @return true if a template was found and applied, false otherwise
     */
    private static boolean applyMatchingTemplate(
        DocumentParserContext context,
        String name,
        DynamicTemplate.XContentFieldType matchType,
        DateFormatter dateFormatter
    ) throws IOException {
        DynamicTemplate dynamicTemplate = context.findDynamicTemplate(name, matchType);
        if (dynamicTemplate == null) {
            return false;
        }
        String dynamicType = dynamicTemplate.isRuntimeMapping() ? matchType.defaultRuntimeMappingType() : matchType.defaultMappingType();

        String mappingType = dynamicTemplate.mappingType(dynamicType);
        Map mapping = dynamicTemplate.mappingForName(name, dynamicType);
        if (dynamicTemplate.isRuntimeMapping()) {
            MappingParserContext parserContext = context.dynamicTemplateParserContext(dateFormatter);
            RuntimeField.Parser parser = parserContext.runtimeFieldParser(mappingType);
            String fullName = context.path().pathAsText(name);
            if (parser == null) {
                throw new MapperParsingException("failed to find type parsed [" + mappingType + "] for [" + fullName + "]");
            }
            RuntimeField.Builder builder = parser.parse(fullName, mapping, parserContext);
            Runtime.createDynamicField(builder.createRuntimeField(parserContext), context);
        } else {
            Mapper.Builder builder = parseDynamicTemplateMapping(name, mappingType, mapping, dateFormatter, context);
            CONCRETE.createDynamicField(builder, context);
        }
        return true;
    }

    private static Mapper.Builder findTemplateBuilderForObject(DocumentParserContext context, String name) {
        DynamicTemplate.XContentFieldType matchType = DynamicTemplate.XContentFieldType.OBJECT;
        DynamicTemplate dynamicTemplate = context.findDynamicTemplate(name, matchType);
        if (dynamicTemplate == null) {
            return null;
        }
        String dynamicType = matchType.defaultMappingType();
        String mappingType = dynamicTemplate.mappingType(dynamicType);
        Map mapping = dynamicTemplate.mappingForName(name, dynamicType);
        return parseDynamicTemplateMapping(name, mappingType, mapping, null, context);
    }

    private static Mapper.Builder parseDynamicTemplateMapping(
        String name,
        String mappingType,
        Map mapping,
        DateFormatter dateFormatter,
        DocumentParserContext context
    ) {
        MappingParserContext parserContext = context.dynamicTemplateParserContext(dateFormatter);
        Mapper.TypeParser typeParser = parserContext.typeParser(mappingType);
        if (typeParser == null) {
            throw new MapperParsingException("failed to find type parsed [" + mappingType + "] for [" + name + "]");
        }
        return typeParser.parse(name, mapping, parserContext);
    }

    /**
     * Defines how leaf fields of type string, long, double, boolean and date are dynamically mapped
     */
    private interface Strategy {
        void newDynamicStringField(DocumentParserContext context, String name) throws IOException;

        void newDynamicLongField(DocumentParserContext context, String name) throws IOException;

        void newDynamicDoubleField(DocumentParserContext context, String name) throws IOException;

        void newDynamicBooleanField(DocumentParserContext context, String name) throws IOException;

        void newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) throws IOException;
    }

    /**
     * Dynamically creates concrete fields, as part of the properties section.
     * Use for leaf fields, when their parent object is mapped as dynamic:true
     * @see Dynamic
     */
    private static final class Concrete implements Strategy {
        private final CheckedBiConsumer parseField;

        Concrete(CheckedBiConsumer parseField) {
            this.parseField = parseField;
        }

        void createDynamicField(Mapper.Builder builder, DocumentParserContext context) throws IOException {
            Mapper mapper = builder.build(MapperBuilderContext.forPath(context.path()));
            context.addDynamicMapper(mapper);
            parseField.accept(context, mapper);
        }

        @Override
        public void newDynamicStringField(DocumentParserContext context, String name) throws IOException {
            createDynamicField(
                new TextFieldMapper.Builder(name, context.indexAnalyzers()).addMultiField(
                    new KeywordFieldMapper.Builder("keyword").ignoreAbove(256)
                ),
                context
            );
        }

        @Override
        public void newDynamicLongField(DocumentParserContext context, String name) throws IOException {
            createDynamicField(
                new NumberFieldMapper.Builder(
                    name,
                    NumberFieldMapper.NumberType.LONG,
                    ScriptCompiler.NONE,
                    context.indexSettings().getSettings()
                ),
                context
            );
        }

        @Override
        public void newDynamicDoubleField(DocumentParserContext context, String name) throws IOException {
            // no templates are defined, we use float by default instead of double
            // since this is much more space-efficient and should be enough most of
            // the time
            createDynamicField(
                new NumberFieldMapper.Builder(
                    name,
                    NumberFieldMapper.NumberType.FLOAT,
                    ScriptCompiler.NONE,
                    context.indexSettings().getSettings()
                ),
                context
            );
        }

        @Override
        public void newDynamicBooleanField(DocumentParserContext context, String name) throws IOException {
            createDynamicField(new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE), context);
        }

        @Override
        public void newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateTimeFormatter) throws IOException {
            Settings settings = context.indexSettings().getSettings();
            boolean ignoreMalformed = FieldMapper.IGNORE_MALFORMED_SETTING.get(settings);
            createDynamicField(
                new DateFieldMapper.Builder(
                    name,
                    DateFieldMapper.Resolution.MILLISECONDS,
                    dateTimeFormatter,
                    ScriptCompiler.NONE,
                    ignoreMalformed,
                    context.indexSettings().getIndexVersionCreated()
                ),
                context
            );
        }

        void newDynamicBinaryField(DocumentParserContext context, String name) throws IOException {
            createDynamicField(new BinaryFieldMapper.Builder(name), context);
        }
    }

    /**
     * Dynamically creates runtime fields, in the runtime section.
     * Used for sub-fields of objects that are mapped as dynamic:runtime.
     * @see Dynamic
     */
    private static final class Runtime implements Strategy {
        static void createDynamicField(RuntimeField runtimeField, DocumentParserContext context) {
            context.addDynamicRuntimeField(runtimeField);
        }

        @Override
        public void newDynamicStringField(DocumentParserContext context, String name) {
            String fullName = context.path().pathAsText(name);
            createDynamicField(KeywordScriptFieldType.sourceOnly(fullName), context);
        }

        @Override
        public void newDynamicLongField(DocumentParserContext context, String name) {
            String fullName = context.path().pathAsText(name);
            createDynamicField(LongScriptFieldType.sourceOnly(fullName), context);
        }

        @Override
        public void newDynamicDoubleField(DocumentParserContext context, String name) {
            String fullName = context.path().pathAsText(name);
            createDynamicField(DoubleScriptFieldType.sourceOnly(fullName), context);
        }

        @Override
        public void newDynamicBooleanField(DocumentParserContext context, String name) {
            String fullName = context.path().pathAsText(name);
            createDynamicField(BooleanScriptFieldType.sourceOnly(fullName), context);
        }

        @Override
        public void newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) {
            String fullName = context.path().pathAsText(name);
            createDynamicField(DateScriptFieldType.sourceOnly(fullName, dateFormatter), context);
        }
    }
}