org.elasticsearch.ingest.ConditionalProcessor Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Elasticsearch subproject :server
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.ingest;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.logging.DeprecationCategory;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.script.DynamicMap;
import org.elasticsearch.script.IngestConditionalScript;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptException;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptType;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.function.LongSupplier;
import java.util.stream.Collectors;
import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
public class ConditionalProcessor extends AbstractProcessor implements WrappingProcessor {
private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DynamicMap.class);
private static final Map> FUNCTIONS = Map.of("_type", value -> {
deprecationLogger.warn(
DeprecationCategory.INDICES,
"conditional-processor__type",
"[types removal] Looking up doc types [_type] in scripts is deprecated."
);
return value;
});
private static final Logger logger = LogManager.getLogger(ConditionalProcessor.class);
static final String TYPE = "conditional";
private final Script condition;
private final ScriptService scriptService;
private final Processor processor;
private final IngestMetric metric;
private final LongSupplier relativeTimeProvider;
private final IngestConditionalScript precompiledConditionScript;
ConditionalProcessor(String tag, String description, Script script, ScriptService scriptService, Processor processor) {
this(tag, description, script, scriptService, processor, System::nanoTime);
}
ConditionalProcessor(
String tag,
String description,
Script script,
ScriptService scriptService,
Processor processor,
LongSupplier relativeTimeProvider
) {
super(tag, description);
this.condition = script;
this.scriptService = scriptService;
this.processor = processor;
this.metric = new IngestMetric();
this.relativeTimeProvider = relativeTimeProvider;
try {
final IngestConditionalScript.Factory factory = scriptService.compile(script, IngestConditionalScript.CONTEXT);
if (ScriptType.INLINE.equals(script.getType())) {
precompiledConditionScript = factory.newInstance(script.getParams());
} else {
// stored script, so will have to compile at runtime
precompiledConditionScript = null;
}
} catch (ScriptException e) {
throw newConfigurationException(TYPE, tag, null, e);
}
}
@Override
public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
assert isAsync() == false;
final boolean matches = evaluate(ingestDocument);
if (matches) {
long startTimeInNanos = relativeTimeProvider.getAsLong();
try {
metric.preIngest();
return processor.execute(ingestDocument);
} catch (Exception e) {
metric.ingestFailed();
throw e;
} finally {
long ingestTimeInNanos = relativeTimeProvider.getAsLong() - startTimeInNanos;
metric.postIngest(ingestTimeInNanos);
}
}
return ingestDocument;
}
@Override
public void execute(IngestDocument ingestDocument, BiConsumer handler) {
assert isAsync();
final boolean matches;
try {
matches = evaluate(ingestDocument);
} catch (Exception e) {
handler.accept(null, e);
return;
}
if (matches) {
final long startTimeInNanos = relativeTimeProvider.getAsLong();
/*
* Our assumption is that the listener passed to the processor is only ever called once. However, there is no way to enforce
* that in all processors and all of the code that they call. If the listener is called more than once it causes problems
* such as the metrics being wrong. The listenerHasBeenCalled variable is used to make sure that the code in the listener
* is only executed once.
*/
final AtomicBoolean listenerHasBeenCalled = new AtomicBoolean(false);
metric.preIngest();
processor.execute(ingestDocument, (result, e) -> {
if (listenerHasBeenCalled.getAndSet(true)) {
logger.warn("A listener was unexpectedly called more than once", new RuntimeException(e));
assert false : "A listener was unexpectedly called more than once";
} else {
long ingestTimeInNanos = relativeTimeProvider.getAsLong() - startTimeInNanos;
metric.postIngest(ingestTimeInNanos);
if (e != null) {
metric.ingestFailed();
handler.accept(null, e);
} else {
handler.accept(result, null);
}
}
});
} else {
handler.accept(ingestDocument, null);
}
}
boolean evaluate(IngestDocument ingestDocument) {
IngestConditionalScript script = precompiledConditionScript;
if (script == null) {
IngestConditionalScript.Factory factory = scriptService.compile(condition, IngestConditionalScript.CONTEXT);
script = factory.newInstance(condition.getParams());
}
return script.execute(new UnmodifiableIngestData(new DynamicMap(ingestDocument.getSourceAndMetadata(), FUNCTIONS)));
}
public Processor getInnerProcessor() {
return processor;
}
IngestMetric getMetric() {
return metric;
}
@Override
public String getType() {
return TYPE;
}
public String getCondition() {
return condition.getIdOrCode();
}
@SuppressWarnings("unchecked")
private static Object wrapUnmodifiable(Object raw) {
// Wraps all mutable types that the JSON parser can create by immutable wrappers.
// Any inputs not wrapped are assumed to be immutable
if (raw instanceof Map) {
return new UnmodifiableIngestData((Map) raw);
} else if (raw instanceof List) {
return new UnmodifiableIngestList((List