org.elasticsearch.index.mapper.RootObjectMapper Maven / Gradle / Ivy
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.index.mapper;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.logging.DeprecationCategory;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.index.mapper.DynamicTemplate.XContentFieldType;
import org.elasticsearch.index.mapper.MapperService.MergeReason;
import org.elasticsearch.xcontent.ToXContent;
import org.elasticsearch.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter;
public class RootObjectMapper extends ObjectMapper {
private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RootObjectMapper.class);
/**
* Parameter used when serializing {@link RootObjectMapper} and request that the runtime section is skipped.
* This is only needed internally when we compare different versions of mappings and assert that they are the same.
* Runtime fields break these assertions as they can be removed: the master node sends the merged mappings without the runtime fields
* that needed to be removed. Then each local node as part of its assertions merges the incoming mapping with the current mapping,
* and the previously removed runtime fields appear again, which is not desirable. The expectation is that those two versions of the
* mappings are the same, besides runtime fields.
*/
static final String TOXCONTENT_SKIP_RUNTIME = "skip_runtime";
public static class Defaults {
public static final DateFormatter[] DYNAMIC_DATE_TIME_FORMATTERS = new DateFormatter[] {
DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
DateFormatter.forPattern("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis") };
public static final boolean DATE_DETECTION = true;
public static final boolean NUMERIC_DETECTION = false;
}
public static class Builder extends ObjectMapper.Builder {
protected Explicit dynamicTemplates = new Explicit<>(new DynamicTemplate[0], false);
protected Explicit dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false);
protected Explicit dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false);
protected Explicit numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false);
protected Map runtimeFields;
public Builder(String name) {
super(name);
}
public Builder dynamicDateTimeFormatter(Collection dateTimeFormatters) {
this.dynamicDateTimeFormatters = new Explicit<>(dateTimeFormatters.toArray(new DateFormatter[0]), true);
return this;
}
public Builder dynamicTemplates(Collection templates) {
this.dynamicTemplates = new Explicit<>(templates.toArray(new DynamicTemplate[0]), true);
return this;
}
@Override
public RootObjectMapper.Builder add(Mapper.Builder builder) {
super.add(builder);
return this;
}
public RootObjectMapper.Builder setRuntime(Map runtimeFields) {
this.runtimeFields = runtimeFields;
return this;
}
@Override
public RootObjectMapper build(MapperBuilderContext context) {
return new RootObjectMapper(
name,
enabled,
dynamic,
buildMappers(true, context),
runtimeFields == null ? Collections.emptyMap() : runtimeFields,
dynamicDateTimeFormatters,
dynamicTemplates,
dateDetection,
numericDetection
);
}
}
/**
* Removes redundant root includes in {@link NestedObjectMapper} trees to avoid duplicate
* fields on the root mapper when {@code isIncludeInRoot} is {@code true} for a node that is
* itself included into a parent node, for which either {@code isIncludeInRoot} is
* {@code true} or which is transitively included in root by a chain of nodes with
* {@code isIncludeInParent} returning {@code true}.
*/
// TODO it would be really nice to make this an implementation detail of NestedObjectMapper
// and run it as part of the builder, but this does not yet work because of the way that
// index templates are merged together. If merge() was run on Builder objects rather than
// on Mappers then we could move this.
public void fixRedundantIncludes() {
fixRedundantIncludes(this, true);
}
private static void fixRedundantIncludes(ObjectMapper objectMapper, boolean parentIncluded) {
for (Mapper mapper : objectMapper) {
if (mapper instanceof NestedObjectMapper) {
NestedObjectMapper child = (NestedObjectMapper) mapper;
boolean isNested = child.isNested();
boolean includeInRootViaParent = parentIncluded && isNested && child.isIncludeInParent();
boolean includedInRoot = isNested && child.isIncludeInRoot();
if (includeInRootViaParent && includedInRoot) {
child.setIncludeInParent(true);
child.setIncludeInRoot(false);
}
fixRedundantIncludes(child, includeInRootViaParent || includedInRoot);
}
}
}
static final class TypeParser extends ObjectMapper.TypeParser {
@Override
public RootObjectMapper.Builder parse(String name, Map node, MappingParserContext parserContext)
throws MapperParsingException {
RootObjectMapper.Builder builder = new Builder(name);
Iterator> iterator = node.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry entry = iterator.next();
String fieldName = entry.getKey();
Object fieldNode = entry.getValue();
if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)
|| processField(builder, fieldName, fieldNode, parserContext)) {
iterator.remove();
}
}
return builder;
}
@SuppressWarnings("unchecked")
private boolean processField(
RootObjectMapper.Builder builder,
String fieldName,
Object fieldNode,
MappingParserContext parserContext
) {
if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) {
if (fieldNode instanceof List) {
List formatters = new ArrayList<>();
for (Object formatter : (List) fieldNode) {
if (formatter.toString().startsWith("epoch_")) {
throw new MapperParsingException("Epoch [" + formatter + "] is not supported as dynamic date format");
}
formatters.add(parseDateTimeFormatter(formatter));
}
builder.dynamicDateTimeFormatter(formatters);
} else if ("none".equals(fieldNode.toString())) {
builder.dynamicDateTimeFormatter(Collections.emptyList());
} else {
builder.dynamicDateTimeFormatter(Collections.singleton(parseDateTimeFormatter(fieldNode)));
}
return true;
} else if (fieldName.equals("dynamic_templates")) {
// "dynamic_templates" : [
// {
// "template_1" : {
// "match" : "*_test",
// "match_mapping_type" : "string",
// "mapping" : { "type" : "keyword", "store" : "yes" }
// }
// }
// ]
if ((fieldNode instanceof List) == false) {
throw new MapperParsingException("Dynamic template syntax error. An array of named objects is expected.");
}
List tmplNodes = (List) fieldNode;
List templates = new ArrayList<>();
for (Object tmplNode : tmplNodes) {
Map tmpl = (Map) tmplNode;
if (tmpl.size() != 1) {
throw new MapperParsingException("A dynamic template must be defined with a name");
}
Map.Entry entry = tmpl.entrySet().iterator().next();
String templateName = entry.getKey();
Map templateParams = (Map) entry.getValue();
DynamicTemplate template = DynamicTemplate.parse(templateName, templateParams, parserContext.indexVersionCreated());
if (template != null) {
validateDynamicTemplate(parserContext, template);
templates.add(template);
}
}
builder.dynamicTemplates(templates);
return true;
} else if (fieldName.equals("date_detection")) {
builder.dateDetection = new Explicit<>(nodeBooleanValue(fieldNode, "date_detection"), true);
return true;
} else if (fieldName.equals("numeric_detection")) {
builder.numericDetection = new Explicit<>(nodeBooleanValue(fieldNode, "numeric_detection"), true);
return true;
} else if (fieldName.equals("runtime")) {
if (fieldNode instanceof Map) {
Map fields = RuntimeField.parseRuntimeFields(
(Map) fieldNode,
parserContext,
true
);
builder.setRuntime(fields);
return true;
} else {
throw new ElasticsearchParseException("runtime must be a map type");
}
}
return false;
}
}
private Explicit dynamicDateTimeFormatters;
private Explicit dateDetection;
private Explicit numericDetection;
private Explicit dynamicTemplates;
private Map runtimeFields;
RootObjectMapper(
String name,
Explicit enabled,
Dynamic dynamic,
Map mappers,
Map runtimeFields,
Explicit dynamicDateTimeFormatters,
Explicit dynamicTemplates,
Explicit dateDetection,
Explicit numericDetection
) {
super(name, name, enabled, dynamic, mappers);
this.runtimeFields = runtimeFields;
this.dynamicTemplates = dynamicTemplates;
this.dynamicDateTimeFormatters = dynamicDateTimeFormatters;
this.dateDetection = dateDetection;
this.numericDetection = numericDetection;
}
@Override
protected ObjectMapper clone() {
ObjectMapper clone = super.clone();
((RootObjectMapper) clone).runtimeFields = new HashMap<>(this.runtimeFields);
return clone;
}
@Override
RootObjectMapper copyAndReset() {
RootObjectMapper copy = (RootObjectMapper) super.copyAndReset();
// for dynamic updates, no need to carry root-specific options, we just
// set everything to their implicit default value so that they are not
// applied at merge time
copy.dynamicTemplates = new Explicit<>(new DynamicTemplate[0], false);
copy.dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false);
copy.dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false);
copy.numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false);
// also no need to carry the already defined runtime fields, only new ones need to be added
copy.runtimeFields.clear();
return copy;
}
/**
* Public API
*/
public boolean dateDetection() {
return this.dateDetection.value();
}
/**
* Public API
*/
public boolean numericDetection() {
return this.numericDetection.value();
}
/**
* Public API
*/
public DateFormatter[] dynamicDateTimeFormatters() {
return dynamicDateTimeFormatters.value();
}
/**
* Public API
*/
public DynamicTemplate[] dynamicTemplates() {
return dynamicTemplates.value();
}
Collection runtimeFields() {
return runtimeFields.values();
}
RuntimeField getRuntimeField(String name) {
return runtimeFields.get(name);
}
@Override
public RootObjectMapper merge(Mapper mergeWith, MergeReason reason) {
return (RootObjectMapper) super.merge(mergeWith, reason);
}
@Override
protected void doMerge(ObjectMapper mergeWith, MergeReason reason) {
super.doMerge(mergeWith, reason);
RootObjectMapper mergeWithObject = (RootObjectMapper) mergeWith;
if (mergeWithObject.numericDetection.explicit()) {
this.numericDetection = mergeWithObject.numericDetection;
}
if (mergeWithObject.dateDetection.explicit()) {
this.dateDetection = mergeWithObject.dateDetection;
}
if (mergeWithObject.dynamicDateTimeFormatters.explicit()) {
this.dynamicDateTimeFormatters = mergeWithObject.dynamicDateTimeFormatters;
}
if (mergeWithObject.dynamicTemplates.explicit()) {
if (reason == MergeReason.INDEX_TEMPLATE) {
Map templatesByKey = new LinkedHashMap<>();
for (DynamicTemplate template : this.dynamicTemplates.value()) {
templatesByKey.put(template.name(), template);
}
for (DynamicTemplate template : mergeWithObject.dynamicTemplates.value()) {
templatesByKey.put(template.name(), template);
}
DynamicTemplate[] mergedTemplates = templatesByKey.values().toArray(new DynamicTemplate[0]);
this.dynamicTemplates = new Explicit<>(mergedTemplates, true);
} else {
this.dynamicTemplates = mergeWithObject.dynamicTemplates;
}
}
assert this.runtimeFields != mergeWithObject.runtimeFields;
for (Map.Entry runtimeField : mergeWithObject.runtimeFields.entrySet()) {
if (runtimeField.getValue() == null) {
this.runtimeFields.remove(runtimeField.getKey());
} else {
this.runtimeFields.put(runtimeField.getKey(), runtimeField.getValue());
}
}
}
void addRuntimeFields(Collection runtimeFields) {
for (RuntimeField runtimeField : runtimeFields) {
this.runtimeFields.put(runtimeField.name(), runtimeField);
}
}
@Override
protected void doXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
final boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
if (dynamicDateTimeFormatters.explicit() || includeDefaults) {
builder.startArray("dynamic_date_formats");
for (DateFormatter dateTimeFormatter : dynamicDateTimeFormatters.value()) {
builder.value(dateTimeFormatter.pattern());
}
builder.endArray();
}
if (dynamicTemplates.explicit() || includeDefaults) {
builder.startArray("dynamic_templates");
for (DynamicTemplate dynamicTemplate : dynamicTemplates.value()) {
builder.startObject();
builder.field(dynamicTemplate.name(), dynamicTemplate);
builder.endObject();
}
builder.endArray();
}
if (dateDetection.explicit() || includeDefaults) {
builder.field("date_detection", dateDetection.value());
}
if (numericDetection.explicit() || includeDefaults) {
builder.field("numeric_detection", numericDetection.value());
}
if (runtimeFields.size() > 0 && params.paramAsBoolean(TOXCONTENT_SKIP_RUNTIME, false) == false) {
builder.startObject("runtime");
List sortedRuntimeFields = runtimeFields.values()
.stream()
.sorted(Comparator.comparing(RuntimeField::name))
.collect(Collectors.toList());
for (RuntimeField fieldType : sortedRuntimeFields) {
fieldType.toXContent(builder, params);
}
builder.endObject();
}
}
private static void validateDynamicTemplate(MappingParserContext parserContext, DynamicTemplate template) {
if (containsSnippet(template.getMapping(), "{name}")) {
// Can't validate template, because field names can't be guessed up front.
return;
}
final XContentFieldType[] types = template.getXContentFieldTypes();
Exception lastError = null;
for (XContentFieldType fieldType : types) {
String dynamicType = template.isRuntimeMapping() ? fieldType.defaultRuntimeMappingType() : fieldType.defaultMappingType();
String mappingType = template.mappingType(dynamicType);
try {
if (template.isRuntimeMapping()) {
RuntimeField.Parser parser = parserContext.runtimeFieldParser(mappingType);
if (parser == null) {
throw new IllegalArgumentException("No runtime field found for type [" + mappingType + "]");
}
validate(template, dynamicType, (name, mapping) -> parser.parse(name, mapping, parserContext));
} else {
Mapper.TypeParser typeParser = parserContext.typeParser(mappingType);
if (typeParser == null) {
throw new IllegalArgumentException("No mapper found for type [" + mappingType + "]");
}
validate(
template,
dynamicType,
(name, mapping) -> typeParser.parse(name, mapping, parserContext).build(MapperBuilderContext.ROOT)
);
}
lastError = null; // ok, the template is valid for at least one type
break;
} catch (Exception e) {
lastError = e;
}
}
final boolean shouldEmitDeprecationWarning = parserContext.indexVersionCreated().onOrAfter(Version.V_7_7_0);
if (lastError != null && shouldEmitDeprecationWarning) {
String format = "dynamic template [%s] has invalid content [%s], "
+ "attempted to validate it with the following match_mapping_type: %s, caused by [%s]";
String message = String.format(
Locale.ROOT,
format,
template.getName(),
Strings.toString(template),
Arrays.toString(types),
lastError.getMessage()
);
DEPRECATION_LOGGER.critical(DeprecationCategory.TEMPLATES, "invalid_dynamic_template", message);
}
}
private static void validate(DynamicTemplate template, String dynamicType, BiConsumer> mappingConsumer) {
String templateName = "__dynamic__" + template.name();
Map fieldTypeConfig = template.mappingForName(templateName, dynamicType);
mappingConsumer.accept(templateName, fieldTypeConfig);
fieldTypeConfig.remove("type");
if (fieldTypeConfig.isEmpty() == false) {
throw new IllegalArgumentException("Unknown mapping attributes [" + fieldTypeConfig + "]");
}
}
private static boolean containsSnippet(Map map, String snippet) {
for (Map.Entry entry : map.entrySet()) {
String key = entry.getKey().toString();
if (key.contains(snippet)) {
return true;
}
Object value = entry.getValue();
if (containsSnippet(value, snippet)) {
return true;
}
}
return false;
}
private static boolean containsSnippet(List list, String snippet) {
for (Object value : list) {
if (containsSnippet(value, snippet)) {
return true;
}
}
return false;
}
private static boolean containsSnippet(Object value, String snippet) {
if (value instanceof Map) {
return containsSnippet((Map) value, snippet);
} else if (value instanceof List) {
return containsSnippet((List) value, snippet);
} else if (value instanceof String) {
return ((String) value).contains(snippet);
}
return false;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy