org.elasticsearch.index.mapper.RootObjectMapper Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Elasticsearch subproject :server
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.index.mapper;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.logging.DeprecationCategory;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.index.mapper.DynamicTemplate.XContentFieldType;
import org.elasticsearch.index.mapper.MapperService.MergeReason;
import org.elasticsearch.xcontent.ToXContent;
import org.elasticsearch.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter;
public class RootObjectMapper extends ObjectMapper {
private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RootObjectMapper.class);
/**
* Parameter used when serializing {@link RootObjectMapper} and request that the runtime section is skipped.
* This is only needed internally when we compare different versions of mappings and assert that they are the same.
* Runtime fields break these assertions as they can be removed: the master node sends the merged mappings without the runtime fields
* that needed to be removed. Then each local node as part of its assertions merges the incoming mapping with the current mapping,
* and the previously removed runtime fields appear again, which is not desirable. The expectation is that those two versions of the
* mappings are the same, besides runtime fields.
*/
static final String TOXCONTENT_SKIP_RUNTIME = "skip_runtime";
public static class Defaults {
public static final DateFormatter[] DYNAMIC_DATE_TIME_FORMATTERS = new DateFormatter[] {
DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
DateFormatter.forPattern("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis") };
public static final boolean DATE_DETECTION = true;
public static final boolean NUMERIC_DETECTION = false;
}
public static class Builder extends ObjectMapper.Builder {
protected Explicit dynamicTemplates = new Explicit<>(new DynamicTemplate[0], false);
protected Explicit dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false);
protected Explicit dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false);
protected Explicit numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false);
protected Map runtimeFields;
public Builder(String name) {
super(name);
}
public Builder dynamicDateTimeFormatter(Collection dateTimeFormatters) {
this.dynamicDateTimeFormatters = new Explicit<>(dateTimeFormatters.toArray(new DateFormatter[0]), true);
return this;
}
public Builder dynamicTemplates(Collection templates) {
this.dynamicTemplates = new Explicit<>(templates.toArray(new DynamicTemplate[0]), true);
return this;
}
@Override
public RootObjectMapper.Builder add(Mapper.Builder builder) {
super.add(builder);
return this;
}
public RootObjectMapper.Builder setRuntime(Map runtimeFields) {
this.runtimeFields = runtimeFields;
return this;
}
@Override
public RootObjectMapper build(MapperBuilderContext context) {
return new RootObjectMapper(
name,
enabled,
dynamic,
buildMappers(true, context),
runtimeFields == null ? Collections.emptyMap() : runtimeFields,
dynamicDateTimeFormatters,
dynamicTemplates,
dateDetection,
numericDetection
);
}
}
/**
* Removes redundant root includes in {@link NestedObjectMapper} trees to avoid duplicate
* fields on the root mapper when {@code isIncludeInRoot} is {@code true} for a node that is
* itself included into a parent node, for which either {@code isIncludeInRoot} is
* {@code true} or which is transitively included in root by a chain of nodes with
* {@code isIncludeInParent} returning {@code true}.
*/
// TODO it would be really nice to make this an implementation detail of NestedObjectMapper
// and run it as part of the builder, but this does not yet work because of the way that
// index templates are merged together. If merge() was run on Builder objects rather than
// on Mappers then we could move this.
public void fixRedundantIncludes() {
fixRedundantIncludes(this, true);
}
private static void fixRedundantIncludes(ObjectMapper objectMapper, boolean parentIncluded) {
for (Mapper mapper : objectMapper) {
if (mapper instanceof NestedObjectMapper) {
NestedObjectMapper child = (NestedObjectMapper) mapper;
boolean isNested = child.isNested();
boolean includeInRootViaParent = parentIncluded && isNested && child.isIncludeInParent();
boolean includedInRoot = isNested && child.isIncludeInRoot();
if (includeInRootViaParent && includedInRoot) {
child.setIncludeInParent(true);
child.setIncludeInRoot(false);
}
fixRedundantIncludes(child, includeInRootViaParent || includedInRoot);
}
}
}
static final class TypeParser extends ObjectMapper.TypeParser {
@Override
public RootObjectMapper.Builder parse(String name, Map node, MappingParserContext parserContext)
throws MapperParsingException {
RootObjectMapper.Builder builder = new Builder(name);
Iterator> iterator = node.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry entry = iterator.next();
String fieldName = entry.getKey();
Object fieldNode = entry.getValue();
if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)
|| processField(builder, fieldName, fieldNode, parserContext)) {
iterator.remove();
}
}
return builder;
}
@SuppressWarnings("unchecked")
private boolean processField(
RootObjectMapper.Builder builder,
String fieldName,
Object fieldNode,
MappingParserContext parserContext
) {
if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) {
if (fieldNode instanceof List) {
List formatters = new ArrayList<>();
for (Object formatter : (List) fieldNode) {
if (formatter.toString().startsWith("epoch_")) {
throw new MapperParsingException("Epoch [" + formatter + "] is not supported as dynamic date format");
}
formatters.add(parseDateTimeFormatter(formatter));
}
builder.dynamicDateTimeFormatter(formatters);
} else if ("none".equals(fieldNode.toString())) {
builder.dynamicDateTimeFormatter(Collections.emptyList());
} else {
builder.dynamicDateTimeFormatter(Collections.singleton(parseDateTimeFormatter(fieldNode)));
}
return true;
} else if (fieldName.equals("dynamic_templates")) {
// "dynamic_templates" : [
// {
// "template_1" : {
// "match" : "*_test",
// "match_mapping_type" : "string",
// "mapping" : { "type" : "keyword", "store" : "yes" }
// }
// }
// ]
if ((fieldNode instanceof List) == false) {
throw new MapperParsingException("Dynamic template syntax error. An array of named objects is expected.");
}
List tmplNodes = (List) fieldNode;
List templates = new ArrayList<>();
for (Object tmplNode : tmplNodes) {
Map tmpl = (Map) tmplNode;
if (tmpl.size() != 1) {
throw new MapperParsingException("A dynamic template must be defined with a name");
}
Map.Entry entry = tmpl.entrySet().iterator().next();
String templateName = entry.getKey();
Map templateParams = (Map) entry.getValue();
DynamicTemplate template = DynamicTemplate.parse(templateName, templateParams, parserContext.indexVersionCreated());
if (template != null) {
validateDynamicTemplate(parserContext, template);
templates.add(template);
}
}
builder.dynamicTemplates(templates);
return true;
} else if (fieldName.equals("date_detection")) {
builder.dateDetection = new Explicit<>(nodeBooleanValue(fieldNode, "date_detection"), true);
return true;
} else if (fieldName.equals("numeric_detection")) {
builder.numericDetection = new Explicit<>(nodeBooleanValue(fieldNode, "numeric_detection"), true);
return true;
} else if (fieldName.equals("runtime")) {
if (fieldNode instanceof Map) {
Map fields = RuntimeField.parseRuntimeFields(
(Map) fieldNode,
parserContext,
true
);
builder.setRuntime(fields);
return true;
} else {
throw new ElasticsearchParseException("runtime must be a map type");
}
}
return false;
}
}
private Explicit dynamicDateTimeFormatters;
private Explicit dateDetection;
private Explicit numericDetection;
private Explicit dynamicTemplates;
private Map runtimeFields;
RootObjectMapper(
String name,
Explicit enabled,
Dynamic dynamic,
Map mappers,
Map runtimeFields,
Explicit dynamicDateTimeFormatters,
Explicit dynamicTemplates,
Explicit dateDetection,
Explicit numericDetection
) {
super(name, name, enabled, dynamic, mappers);
this.runtimeFields = runtimeFields;
this.dynamicTemplates = dynamicTemplates;
this.dynamicDateTimeFormatters = dynamicDateTimeFormatters;
this.dateDetection = dateDetection;
this.numericDetection = numericDetection;
}
@Override
protected ObjectMapper clone() {
ObjectMapper clone = super.clone();
((RootObjectMapper) clone).runtimeFields = new HashMap<>(this.runtimeFields);
return clone;
}
@Override
RootObjectMapper copyAndReset() {
RootObjectMapper copy = (RootObjectMapper) super.copyAndReset();
// for dynamic updates, no need to carry root-specific options, we just
// set everything to their implicit default value so that they are not
// applied at merge time
copy.dynamicTemplates = new Explicit<>(new DynamicTemplate[0], false);
copy.dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false);
copy.dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false);
copy.numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false);
// also no need to carry the already defined runtime fields, only new ones need to be added
copy.runtimeFields.clear();
return copy;
}
/**
* Public API
*/
public boolean dateDetection() {
return this.dateDetection.value();
}
/**
* Public API
*/
public boolean numericDetection() {
return this.numericDetection.value();
}
/**
* Public API
*/
public DateFormatter[] dynamicDateTimeFormatters() {
return dynamicDateTimeFormatters.value();
}
/**
* Public API
*/
public DynamicTemplate[] dynamicTemplates() {
return dynamicTemplates.value();
}
Collection runtimeFields() {
return runtimeFields.values();
}
RuntimeField getRuntimeField(String name) {
return runtimeFields.get(name);
}
@Override
public RootObjectMapper merge(Mapper mergeWith, MergeReason reason) {
return (RootObjectMapper) super.merge(mergeWith, reason);
}
@Override
protected void doMerge(ObjectMapper mergeWith, MergeReason reason) {
super.doMerge(mergeWith, reason);
RootObjectMapper mergeWithObject = (RootObjectMapper) mergeWith;
if (mergeWithObject.numericDetection.explicit()) {
this.numericDetection = mergeWithObject.numericDetection;
}
if (mergeWithObject.dateDetection.explicit()) {
this.dateDetection = mergeWithObject.dateDetection;
}
if (mergeWithObject.dynamicDateTimeFormatters.explicit()) {
this.dynamicDateTimeFormatters = mergeWithObject.dynamicDateTimeFormatters;
}
if (mergeWithObject.dynamicTemplates.explicit()) {
if (reason == MergeReason.INDEX_TEMPLATE) {
Map templatesByKey = new LinkedHashMap<>();
for (DynamicTemplate template : this.dynamicTemplates.value()) {
templatesByKey.put(template.name(), template);
}
for (DynamicTemplate template : mergeWithObject.dynamicTemplates.value()) {
templatesByKey.put(template.name(), template);
}
DynamicTemplate[] mergedTemplates = templatesByKey.values().toArray(new DynamicTemplate[0]);
this.dynamicTemplates = new Explicit<>(mergedTemplates, true);
} else {
this.dynamicTemplates = mergeWithObject.dynamicTemplates;
}
}
assert this.runtimeFields != mergeWithObject.runtimeFields;
for (Map.Entry runtimeField : mergeWithObject.runtimeFields.entrySet()) {
if (runtimeField.getValue() == null) {
this.runtimeFields.remove(runtimeField.getKey());
} else {
this.runtimeFields.put(runtimeField.getKey(), runtimeField.getValue());
}
}
}
void addRuntimeFields(Collection runtimeFields) {
for (RuntimeField runtimeField : runtimeFields) {
this.runtimeFields.put(runtimeField.name(), runtimeField);
}
}
@Override
protected void doXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
final boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
if (dynamicDateTimeFormatters.explicit() || includeDefaults) {
builder.startArray("dynamic_date_formats");
for (DateFormatter dateTimeFormatter : dynamicDateTimeFormatters.value()) {
builder.value(dateTimeFormatter.pattern());
}
builder.endArray();
}
if (dynamicTemplates.explicit() || includeDefaults) {
builder.startArray("dynamic_templates");
for (DynamicTemplate dynamicTemplate : dynamicTemplates.value()) {
builder.startObject();
builder.field(dynamicTemplate.name(), dynamicTemplate);
builder.endObject();
}
builder.endArray();
}
if (dateDetection.explicit() || includeDefaults) {
builder.field("date_detection", dateDetection.value());
}
if (numericDetection.explicit() || includeDefaults) {
builder.field("numeric_detection", numericDetection.value());
}
if (runtimeFields.size() > 0 && params.paramAsBoolean(TOXCONTENT_SKIP_RUNTIME, false) == false) {
builder.startObject("runtime");
List sortedRuntimeFields = runtimeFields.values()
.stream()
.sorted(Comparator.comparing(RuntimeField::name))
.collect(Collectors.toList());
for (RuntimeField fieldType : sortedRuntimeFields) {
fieldType.toXContent(builder, params);
}
builder.endObject();
}
}
private static void validateDynamicTemplate(MappingParserContext parserContext, DynamicTemplate template) {
if (containsSnippet(template.getMapping(), "{name}")) {
// Can't validate template, because field names can't be guessed up front.
return;
}
final XContentFieldType[] types = template.getXContentFieldTypes();
Exception lastError = null;
for (XContentFieldType fieldType : types) {
String dynamicType = template.isRuntimeMapping() ? fieldType.defaultRuntimeMappingType() : fieldType.defaultMappingType();
String mappingType = template.mappingType(dynamicType);
try {
if (template.isRuntimeMapping()) {
RuntimeField.Parser parser = parserContext.runtimeFieldParser(mappingType);
if (parser == null) {
throw new IllegalArgumentException("No runtime field found for type [" + mappingType + "]");
}
validate(template, dynamicType, (name, mapping) -> parser.parse(name, mapping, parserContext));
} else {
Mapper.TypeParser typeParser = parserContext.typeParser(mappingType);
if (typeParser == null) {
throw new IllegalArgumentException("No mapper found for type [" + mappingType + "]");
}
validate(
template,
dynamicType,
(name, mapping) -> typeParser.parse(name, mapping, parserContext).build(MapperBuilderContext.ROOT)
);
}
lastError = null; // ok, the template is valid for at least one type
break;
} catch (Exception e) {
lastError = e;
}
}
final boolean shouldEmitDeprecationWarning = parserContext.indexVersionCreated().onOrAfter(Version.V_7_7_0);
if (lastError != null && shouldEmitDeprecationWarning) {
String format = "dynamic template [%s] has invalid content [%s], "
+ "attempted to validate it with the following match_mapping_type: %s, caused by [%s]";
String message = String.format(
Locale.ROOT,
format,
template.getName(),
Strings.toString(template),
Arrays.toString(types),
lastError.getMessage()
);
DEPRECATION_LOGGER.critical(DeprecationCategory.TEMPLATES, "invalid_dynamic_template", message);
}
}
private static void validate(DynamicTemplate template, String dynamicType, BiConsumer> mappingConsumer) {
String templateName = "__dynamic__" + template.name();
Map fieldTypeConfig = template.mappingForName(templateName, dynamicType);
mappingConsumer.accept(templateName, fieldTypeConfig);
fieldTypeConfig.remove("type");
if (fieldTypeConfig.isEmpty() == false) {
throw new IllegalArgumentException("Unknown mapping attributes [" + fieldTypeConfig + "]");
}
}
private static boolean containsSnippet(Map map, String snippet) {
for (Map.Entry entry : map.entrySet()) {
String key = entry.getKey().toString();
if (key.contains(snippet)) {
return true;
}
Object value = entry.getValue();
if (containsSnippet(value, snippet)) {
return true;
}
}
return false;
}
private static boolean containsSnippet(List list, String snippet) {
for (Object value : list) {
if (containsSnippet(value, snippet)) {
return true;
}
}
return false;
}
private static boolean containsSnippet(Object value, String snippet) {
if (value instanceof Map) {
return containsSnippet((Map) value, snippet);
} else if (value instanceof List) {
return containsSnippet((List) value, snippet);
} else if (value instanceof String) {
return ((String) value).contains(snippet);
}
return false;
}
}