org.elasticsearch.ingest.ConfigurationUtils Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Elasticsearch subproject :server
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.ingest;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.script.TemplateScript;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.XContentType;
import org.elasticsearch.xcontent.json.JsonXContent;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.script.Script.DEFAULT_TEMPLATE_LANG;
public final class ConfigurationUtils {
public static final String TAG_KEY = "tag";
public static final String DESCRIPTION_KEY = "description";
public static final String[] VALID_MEDIA_TYPES = { "application/json", "text/plain", "application/x-www-form-urlencoded" };
private ConfigurationUtils() {}
/**
* Returns and removes the specified optional property from the specified configuration map.
*
* If the property value isn't of type string a {@link ElasticsearchParseException} is thrown.
*/
public static String readOptionalStringProperty(
String processorType,
String processorTag,
Map configuration,
String propertyName
) {
Object value = configuration.remove(propertyName);
return readString(processorType, processorTag, propertyName, value);
}
/**
* Returns and removes the specified property from the specified configuration map.
*
* If the property value isn't of type string an {@link ElasticsearchParseException} is thrown.
* If the property is missing an {@link ElasticsearchParseException} is thrown
*/
public static String readStringProperty(
String processorType,
String processorTag,
Map configuration,
String propertyName
) {
return readStringProperty(processorType, processorTag, configuration, propertyName, null);
}
/**
* Returns and removes the specified property from the specified configuration map.
*
* If the property value isn't of type string a {@link ElasticsearchParseException} is thrown.
* If the property is missing and no default value has been specified a {@link ElasticsearchParseException} is thrown
*/
public static String readStringProperty(
String processorType,
String processorTag,
Map configuration,
String propertyName,
String defaultValue
) {
Object value = configuration.remove(propertyName);
if (value == null && defaultValue != null) {
return defaultValue;
} else if (value == null) {
throw newConfigurationException(processorType, processorTag, propertyName, "required property is missing");
}
return readString(processorType, processorTag, propertyName, value);
}
private static String readString(String processorType, String processorTag, String propertyName, Object value) {
if (value == null) {
return null;
}
if (value instanceof String string) {
return string;
}
throw newConfigurationException(
processorType,
processorTag,
propertyName,
"property isn't a string, but of type [" + value.getClass().getName() + "]"
);
}
/**
* Returns and removes the specified property from the specified configuration map.
*
* If the property value isn't of type string or int a {@link ElasticsearchParseException} is thrown.
* If the property is missing and no default value has been specified a {@link ElasticsearchParseException} is thrown
*/
public static String readStringOrIntProperty(
String processorType,
String processorTag,
Map configuration,
String propertyName,
String defaultValue
) {
Object value = configuration.remove(propertyName);
if (value == null && defaultValue != null) {
return defaultValue;
} else if (value == null) {
throw newConfigurationException(processorType, processorTag, propertyName, "required property is missing");
}
return readStringOrInt(processorType, processorTag, propertyName, value);
}
private static String readStringOrInt(String processorType, String processorTag, String propertyName, Object value) {
if (value == null) {
return null;
}
if (value instanceof String string) {
return string;
} else if (value instanceof Integer integer) {
return String.valueOf(integer);
}
throw newConfigurationException(
processorType,
processorTag,
propertyName,
"property isn't a string or int, but of type [" + value.getClass().getName() + "]"
);
}
/**
* Returns and removes the specified property from the specified configuration map.
*
* If the property value isn't of type string or int a {@link ElasticsearchParseException} is thrown.
*/
public static String readOptionalStringOrIntProperty(
String processorType,
String processorTag,
Map configuration,
String propertyName
) {
Object value = configuration.remove(propertyName);
if (value == null) {
return null;
}
return readStringOrInt(processorType, processorTag, propertyName, value);
}
private static String readStringOrLong(String processorType, String processorTag, String propertyName, Object value) {
if (value == null) {
return null;
}
if (value instanceof String string) {
return string;
} else if (value instanceof Long || value instanceof Integer) {
return String.valueOf(value);
}
throw newConfigurationException(
processorType,
processorTag,
propertyName,
"property isn't a string or long, but of type [" + value.getClass().getName() + "]"
);
}
/**
* Returns and removes the specified property from the specified configuration map.
*
* If the property value isn't of type string or long a {@link ElasticsearchParseException} is thrown.
*/
public static String readOptionalStringOrLongProperty(
String processorType,
String processorTag,
Map configuration,
String propertyName
) {
Object value = configuration.remove(propertyName);
if (value == null) {
return null;
}
return readStringOrLong(processorType, processorTag, propertyName, value);
}
public static Boolean readBooleanProperty(
String processorType,
String processorTag,
Map configuration,
String propertyName,
boolean defaultValue
) {
Object value = configuration.remove(propertyName);
if (value == null) {
return defaultValue;
} else {
return readBoolean(processorType, processorTag, propertyName, value).booleanValue();
}
}
private static Boolean readBoolean(String processorType, String processorTag, String propertyName, Object value) {
if (value == null) {
return null;
}
if (value instanceof Boolean b) {
return b;
}
throw newConfigurationException(
processorType,
processorTag,
propertyName,
"property isn't a boolean, but of type [" + value.getClass().getName() + "]"
);
}
/**
* Returns and removes the specified property from the specified configuration map.
*
* If the property value isn't of type int a {@link ElasticsearchParseException} is thrown.
* If the property is missing an {@link ElasticsearchParseException} is thrown
*/
public static Integer readIntProperty(
String processorType,
String processorTag,
Map configuration,
String propertyName,
Integer defaultValue
) {
Object value = configuration.remove(propertyName);
if (value == null) {
return defaultValue;
}
try {
return Integer.parseInt(value.toString());
} catch (Exception e) {
throw newConfigurationException(
processorType,
processorTag,
propertyName,
"property cannot be converted to an int [" + value.toString() + "]"
);
}
}
/**
* Returns and removes the specified property from the specified configuration map.
*
* If the property value isn't of type int a {@link ElasticsearchParseException} is thrown.
* If the property is missing an {@link ElasticsearchParseException} is thrown
*/
public static Double readDoubleProperty(
String processorType,
String processorTag,
Map configuration,
String propertyName
) {
Object value = configuration.remove(propertyName);
if (value == null) {
throw newConfigurationException(processorType, processorTag, propertyName, "required property is missing");
}
try {
return Double.parseDouble(value.toString());
} catch (Exception e) {
throw newConfigurationException(
processorType,
processorTag,
propertyName,
"property cannot be converted to a double [" + value.toString() + "]"
);
}
}
/**
* Returns and removes the specified property of type list from the specified configuration map.
*
* If the property value isn't of type list an {@link ElasticsearchParseException} is thrown.
*/
public static List readOptionalList(
String processorType,
String processorTag,
Map configuration,
String propertyName
) {
Object value = configuration.remove(propertyName);
if (value == null) {
return null;
}
return readList(processorType, processorTag, propertyName, value);
}
/**
* Returns and removes the specified property of type list from the specified configuration map.
*
* If the property value isn't of type list an {@link ElasticsearchParseException} is thrown.
* If the property is missing an {@link ElasticsearchParseException} is thrown
*/
public static List readList(String processorType, String processorTag, Map configuration, String propertyName) {
Object value = configuration.remove(propertyName);
if (value == null) {
throw newConfigurationException(processorType, processorTag, propertyName, "required property is missing");
}
return readList(processorType, processorTag, propertyName, value);
}
private static List readList(String processorType, String processorTag, String propertyName, Object value) {
if (value instanceof List) {
@SuppressWarnings("unchecked")
List stringList = (List) value;
return stringList;
} else {
throw newConfigurationException(
processorType,
processorTag,
propertyName,
"property isn't a list, but of type [" + value.getClass().getName() + "]"
);
}
}
/**
* Returns and removes the specified property of type map from the specified configuration map.
*
* If the property value isn't of type map an {@link ElasticsearchParseException} is thrown.
* If the property is missing an {@link ElasticsearchParseException} is thrown
*/
public static Map readMap(
String processorType,
String processorTag,
Map configuration,
String propertyName
) {
Object value = configuration.remove(propertyName);
if (value == null) {
throw newConfigurationException(processorType, processorTag, propertyName, "required property is missing");
}
return readMap(processorType, processorTag, propertyName, value);
}
/**
* Returns and removes the specified property of type map from the specified configuration map.
*
* If the property value isn't of type map an {@link ElasticsearchParseException} is thrown.
*/
public static Map readOptionalMap(
String processorType,
String processorTag,
Map configuration,
String propertyName
) {
Object value = configuration.remove(propertyName);
if (value == null) {
return null;
}
return readMap(processorType, processorTag, propertyName, value);
}
private static Map readMap(String processorType, String processorTag, String propertyName, Object value) {
if (value instanceof Map) {
@SuppressWarnings("unchecked")
Map map = (Map) value;
return map;
} else {
throw newConfigurationException(
processorType,
processorTag,
propertyName,
"property isn't a map, but of type [" + value.getClass().getName() + "]"
);
}
}
/**
* Returns and removes the specified property as an {@link Object} from the specified configuration map.
*/
public static Object readObject(String processorType, String processorTag, Map configuration, String propertyName) {
Object value = configuration.remove(propertyName);
if (value == null) {
throw newConfigurationException(processorType, processorTag, propertyName, "required property is missing");
}
return value;
}
public static String readMediaTypeProperty(
String processorType,
String processorTag,
Map configuration,
String propertyName,
String defaultValue
) {
String mediaType = readStringProperty(processorType, processorTag, configuration, propertyName, defaultValue);
if (Arrays.asList(VALID_MEDIA_TYPES).contains(mediaType) == false) {
throw newConfigurationException(
processorType,
processorTag,
propertyName,
"property does not contain a supported media type [" + mediaType + "]"
);
}
return mediaType;
}
public static ElasticsearchException newConfigurationException(
String processorType,
String processorTag,
String propertyName,
String reason
) {
String msg;
if (propertyName == null) {
msg = reason;
} else {
msg = "[" + propertyName + "] " + reason;
}
ElasticsearchParseException exception = new ElasticsearchParseException(msg);
addMetadataToException(exception, processorType, processorTag, propertyName);
return exception;
}
public static ElasticsearchException newConfigurationException(
String processorType,
String processorTag,
String propertyName,
Exception cause
) {
ElasticsearchException exception = ExceptionsHelper.convertToElastic(cause);
addMetadataToException(exception, processorType, processorTag, propertyName);
return exception;
}
public static List readProcessorConfigs(
List> processorConfigs,
ScriptService scriptService,
Map processorFactories
) throws Exception {
Exception exception = null;
List processors = new ArrayList<>();
if (processorConfigs != null) {
for (Map processorConfigWithKey : processorConfigs) {
for (Map.Entry entry : processorConfigWithKey.entrySet()) {
try {
if (entry.getValue() == null) {
throw newConfigurationException(entry.getKey(), null, null, "processor config cannot be [null]");
} else {
processors.add(readProcessor(processorFactories, scriptService, entry.getKey(), entry.getValue()));
}
} catch (Exception e) {
exception = ExceptionsHelper.useOrSuppress(exception, e);
}
}
}
}
if (exception != null) {
throw exception;
}
return processors;
}
public static TemplateScript.Factory readTemplateProperty(
String processorType,
String processorTag,
Map configuration,
String propertyName,
ScriptService scriptService
) {
String value = readStringProperty(processorType, processorTag, configuration, propertyName, null);
return compileTemplate(processorType, processorTag, propertyName, value, scriptService);
}
public static TemplateScript.Factory compileTemplate(
String processorType,
String processorTag,
String propertyName,
String propertyValue,
ScriptService scriptService
) {
try {
// This check is here because the DEFAULT_TEMPLATE_LANG(mustache) is not
// installed for use by REST tests. `propertyValue` will not be
// modified if templating is not available so a script that simply returns an unmodified `propertyValue`
// is returned.
if (scriptService.isLangSupported(DEFAULT_TEMPLATE_LANG) && propertyValue.contains("{{")) {
Script script = new Script(ScriptType.INLINE, DEFAULT_TEMPLATE_LANG, propertyValue, Collections.emptyMap());
return scriptService.compile(script, TemplateScript.CONTEXT);
} else {
return (params) -> new TemplateScript(params) {
@Override
public String execute() {
return propertyValue;
}
};
}
} catch (Exception e) {
throw ConfigurationUtils.newConfigurationException(processorType, processorTag, propertyName, e);
}
}
private static void addMetadataToException(
ElasticsearchException exception,
String processorType,
String processorTag,
String propertyName
) {
if (processorType != null) {
exception.addMetadata("es.processor_type", processorType);
}
if (processorTag != null) {
exception.addMetadata("es.processor_tag", processorTag);
}
if (propertyName != null) {
exception.addMetadata("es.property_name", propertyName);
}
}
@SuppressWarnings("unchecked")
public static Processor readProcessor(
Map processorFactories,
ScriptService scriptService,
String type,
Object config
) throws Exception {
if (config instanceof Map) {
return readProcessor(processorFactories, scriptService, type, (Map) config);
} else if (config instanceof String && "script".equals(type)) {
Map normalizedScript = Maps.newMapWithExpectedSize(1);
normalizedScript.put(ScriptType.INLINE.getParseField().getPreferredName(), config);
return readProcessor(processorFactories, scriptService, type, normalizedScript);
} else {
throw newConfigurationException(type, null, null, "property isn't a map, but of type [" + config.getClass().getName() + "]");
}
}
public static Processor readProcessor(
Map processorFactories,
ScriptService scriptService,
String type,
Map config
) throws Exception {
String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY);
String description = ConfigurationUtils.readOptionalStringProperty(null, tag, config, DESCRIPTION_KEY);
Script conditionalScript = extractConditional(config);
Processor.Factory factory = processorFactories.get(type);
if (factory != null) {
boolean ignoreFailure = ConfigurationUtils.readBooleanProperty(null, null, config, "ignore_failure", false);
List> onFailureProcessorConfigs = ConfigurationUtils.readOptionalList(
null,
null,
config,
Pipeline.ON_FAILURE_KEY
);
List onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, scriptService, processorFactories);
if (onFailureProcessorConfigs != null && onFailureProcessors.isEmpty()) {
throw newConfigurationException(type, tag, Pipeline.ON_FAILURE_KEY, "processors list cannot be empty");
}
try {
Processor processor = factory.create(processorFactories, tag, description, config);
if (config.isEmpty() == false) {
throw new ElasticsearchParseException(
"processor [{}] doesn't support one or more provided configuration parameters {}",
type,
Arrays.toString(config.keySet().toArray())
);
}
if (onFailureProcessors.size() > 0 || ignoreFailure) {
processor = new CompoundProcessor(ignoreFailure, Collections.singletonList(processor), onFailureProcessors);
}
if (conditionalScript != null) {
processor = new ConditionalProcessor(tag, description, conditionalScript, scriptService, processor);
}
return processor;
} catch (Exception e) {
throw newConfigurationException(type, tag, null, e);
}
}
throw newConfigurationException(type, tag, null, "No processor type exists with name [" + type + "]");
}
private static Script extractConditional(Map config) throws IOException {
Object scriptSource = config.remove("if");
if (scriptSource != null) {
try (
XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent).map(normalizeScript(scriptSource));
InputStream stream = BytesReference.bytes(builder).streamInput();
XContentParser parser = XContentType.JSON.xContent()
.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)
) {
return Script.parse(parser);
}
}
return null;
}
@SuppressWarnings("unchecked")
private static Map normalizeScript(Object scriptConfig) {
if (scriptConfig instanceof Map) {
return (Map) scriptConfig;
} else if (scriptConfig instanceof String) {
return Collections.singletonMap("source", scriptConfig);
} else {
throw newConfigurationException(
"conditional",
null,
"script",
"property isn't a map or string, but of type [" + scriptConfig.getClass().getName() + "]"
);
}
}
}