org.elasticsearch.xpack.core.ml.datafeed.DatafeedUpdate Maven / Gradle / Ivy

/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License
 * 2.0; you may not use this file except in compliance with the Elastic License
 * 2.0.
 */
package org.elasticsearch.xpack.core.ml.datafeed;

import org.elasticsearch.Version;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.logging.DeprecationCategory;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.messages.Messages;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.ml.utils.QueryProvider;
import org.elasticsearch.xpack.core.ml.utils.XContentObjectTransformer;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;

import static org.elasticsearch.xpack.core.ClientHelper.filterSecurityHeaders;


/**
 * A datafeed update contains partial properties to update a {@link DatafeedConfig}.
 * The main difference between this class and {@link DatafeedConfig} is that here all
 * fields are nullable.
 */
public class DatafeedUpdate implements Writeable, ToXContentObject {

    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DatafeedUpdate.class);
    private static final String DEPRECATION_MESSAGE_ON_JOB_ID_UPDATE = "The ability to update a datafeed's job_id is deprecated.";

    public static final ObjectParser PARSER = new ObjectParser<>("datafeed_update", Builder::new);

    static {
        PARSER.declareString(Builder::setId, DatafeedConfig.ID);
        PARSER.declareString(Builder::setJobId, Job.ID);
        PARSER.declareStringArray(Builder::setIndices, DatafeedConfig.INDEXES);
        PARSER.declareStringArray(Builder::setIndices, DatafeedConfig.INDICES);
        PARSER.declareString((builder, val) -> builder.setQueryDelay(
                TimeValue.parseTimeValue(val, DatafeedConfig.QUERY_DELAY.getPreferredName())), DatafeedConfig.QUERY_DELAY);
        PARSER.declareString((builder, val) -> builder.setFrequency(
                TimeValue.parseTimeValue(val, DatafeedConfig.FREQUENCY.getPreferredName())), DatafeedConfig.FREQUENCY);
        PARSER.declareObject(Builder::setQuery, (p, c) -> QueryProvider.fromXContent(p, false, Messages.DATAFEED_CONFIG_QUERY_BAD_FORMAT),
            DatafeedConfig.QUERY);
        PARSER.declareObject(Builder::setAggregationsSafe,
            (p, c) -> AggProvider.fromXContent(p, false),
            DatafeedConfig.AGGREGATIONS);
        PARSER.declareObject(Builder::setAggregationsSafe,
            (p, c) -> AggProvider.fromXContent(p, false),
            DatafeedConfig.AGGS);
        PARSER.declareObject(Builder::setScriptFields, (p, c) -> {
                List parsedScriptFields = new ArrayList<>();
                while (p.nextToken() != XContentParser.Token.END_OBJECT) {
                    parsedScriptFields.add(new SearchSourceBuilder.ScriptField(p));
            }
            parsedScriptFields.sort(Comparator.comparing(SearchSourceBuilder.ScriptField::fieldName));
            return parsedScriptFields;
        }, DatafeedConfig.SCRIPT_FIELDS);
        PARSER.declareInt(Builder::setScrollSize, DatafeedConfig.SCROLL_SIZE);
        PARSER.declareObject(Builder::setChunkingConfig, ChunkingConfig.STRICT_PARSER, DatafeedConfig.CHUNKING_CONFIG);
        PARSER.declareObject(Builder::setDelayedDataCheckConfig,
            DelayedDataCheckConfig.STRICT_PARSER,
            DatafeedConfig.DELAYED_DATA_CHECK_CONFIG);
        PARSER.declareInt(Builder::setMaxEmptySearches, DatafeedConfig.MAX_EMPTY_SEARCHES);
        PARSER.declareObject(Builder::setIndicesOptions,
            (p, c) -> IndicesOptions.fromMap(p.map(), SearchRequest.DEFAULT_INDICES_OPTIONS),
            DatafeedConfig.INDICES_OPTIONS);
    }

    private final String id;
    private final String jobId;
    private final TimeValue queryDelay;
    private final TimeValue frequency;
    private final List indices;
    private final QueryProvider queryProvider;
    private final AggProvider aggProvider;
    private final List scriptFields;
    private final Integer scrollSize;
    private final ChunkingConfig chunkingConfig;
    private final DelayedDataCheckConfig delayedDataCheckConfig;
    private final Integer maxEmptySearches;
    private final IndicesOptions indicesOptions;

    private DatafeedUpdate(String id, String jobId, TimeValue queryDelay, TimeValue frequency, List indices,
                           QueryProvider queryProvider, AggProvider aggProvider,
                           List scriptFields,
                           Integer scrollSize, ChunkingConfig chunkingConfig, DelayedDataCheckConfig delayedDataCheckConfig,
                           Integer maxEmptySearches, IndicesOptions indicesOptions) {
        this.id = id;
        this.jobId = jobId;
        this.queryDelay = queryDelay;
        this.frequency = frequency;
        this.indices = indices;
        this.queryProvider = queryProvider;
        this.aggProvider = aggProvider;
        this.scriptFields = scriptFields;
        this.scrollSize = scrollSize;
        this.chunkingConfig = chunkingConfig;
        this.delayedDataCheckConfig = delayedDataCheckConfig;
        this.maxEmptySearches = maxEmptySearches;
        this.indicesOptions = indicesOptions;
    }

    public DatafeedUpdate(StreamInput in) throws IOException {
        this.id = in.readString();
        this.jobId = in.readOptionalString();
        this.queryDelay = in.readOptionalTimeValue();
        this.frequency = in.readOptionalTimeValue();
        if (in.readBoolean()) {
            this.indices = in.readStringList();
        } else {
            this.indices = null;
        }
        // This consumes the list of types if there was one.
        if (in.getVersion().before(Version.V_7_0_0)) {
            if (in.readBoolean()) {
                in.readStringList();
            }
        }
        if (in.getVersion().before(Version.V_7_0_0)) {
            this.queryProvider = QueryProvider.fromParsedQuery(in.readOptionalNamedWriteable(QueryBuilder.class));
            this.aggProvider = AggProvider.fromParsedAggs(in.readOptionalWriteable(AggregatorFactories.Builder::new));
        } else {
            this.queryProvider = in.readOptionalWriteable(QueryProvider::fromStream);
            this.aggProvider = in.readOptionalWriteable(AggProvider::fromStream);
        }
        if (in.readBoolean()) {
            this.scriptFields = in.readList(SearchSourceBuilder.ScriptField::new);
        } else {
            this.scriptFields = null;
        }
        this.scrollSize = in.readOptionalVInt();
        this.chunkingConfig = in.readOptionalWriteable(ChunkingConfig::new);
        if (in.getVersion().onOrAfter(Version.V_6_6_0)) {
            delayedDataCheckConfig = in.readOptionalWriteable(DelayedDataCheckConfig::new);
        } else {
            delayedDataCheckConfig = null;
        }
        if (in.getVersion().onOrAfter(Version.V_7_5_0)) {
            maxEmptySearches = in.readOptionalInt();
        } else {
            maxEmptySearches = null;
        }
        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
            indicesOptions = in.readBoolean() ? IndicesOptions.readIndicesOptions(in) : null;
        } else {
            indicesOptions = null;
        }
    }

    /**
     * Get the id of the datafeed to update
     */
    public String getId() {
        return id;
    }

    @Override
    public void writeTo(StreamOutput out) throws IOException {
        out.writeString(id);
        out.writeOptionalString(jobId);
        out.writeOptionalTimeValue(queryDelay);
        out.writeOptionalTimeValue(frequency);
        if (indices != null) {
            out.writeBoolean(true);
            out.writeStringCollection(indices);
        } else {
            out.writeBoolean(false);
        }
        // Write the now removed types to prior versions.
        // An empty list is expected
        if (out.getVersion().before(Version.V_7_0_0)) {
            out.writeBoolean(true);
            out.writeStringCollection(Collections.emptyList());
        }
        if (out.getVersion().before(Version.V_7_0_0)) {
            out.writeOptionalNamedWriteable(queryProvider == null ? null : queryProvider.getParsedQuery());
            out.writeOptionalWriteable(aggProvider == null ? null : aggProvider.getParsedAggs());
        } else {
            out.writeOptionalWriteable(queryProvider);
            out.writeOptionalWriteable(aggProvider);
        }
        if (scriptFields != null) {
            out.writeBoolean(true);
            out.writeList(scriptFields);
        } else {
            out.writeBoolean(false);
        }
        out.writeOptionalVInt(scrollSize);
        out.writeOptionalWriteable(chunkingConfig);
        if (out.getVersion().onOrAfter(Version.V_6_6_0)) {
            out.writeOptionalWriteable(delayedDataCheckConfig);
        }
        if (out.getVersion().onOrAfter(Version.V_7_5_0)) {
            out.writeOptionalInt(maxEmptySearches);
        }
        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
            if (indicesOptions != null) {
                out.writeBoolean(true);
                indicesOptions.writeIndicesOptions(out);
            } else {
                out.writeBoolean(false);
            }
        }
    }

    @Override
    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
        builder.startObject();
        builder.field(DatafeedConfig.ID.getPreferredName(), id);
        addOptionalField(builder, Job.ID, jobId);
        if (queryDelay != null) {
            builder.field(DatafeedConfig.QUERY_DELAY.getPreferredName(), queryDelay.getStringRep());
        }
        if (frequency != null) {
            builder.field(DatafeedConfig.FREQUENCY.getPreferredName(), frequency.getStringRep());
        }
        addOptionalField(builder, DatafeedConfig.INDICES, indices);
        if (queryProvider != null) {
            builder.field(DatafeedConfig.QUERY.getPreferredName(), queryProvider.getQuery());
        }
        if (aggProvider != null) {
            builder.field(DatafeedConfig.AGGREGATIONS.getPreferredName(), aggProvider.getAggs());
        }
        if (scriptFields != null) {
            builder.startObject(DatafeedConfig.SCRIPT_FIELDS.getPreferredName());
            for (SearchSourceBuilder.ScriptField scriptField : scriptFields) {
                scriptField.toXContent(builder, params);
            }
            builder.endObject();
        }
        addOptionalField(builder, DatafeedConfig.SCROLL_SIZE, scrollSize);
        addOptionalField(builder, DatafeedConfig.CHUNKING_CONFIG, chunkingConfig);
        addOptionalField(builder, DatafeedConfig.DELAYED_DATA_CHECK_CONFIG, delayedDataCheckConfig);
        addOptionalField(builder, DatafeedConfig.MAX_EMPTY_SEARCHES, maxEmptySearches);
        if (indicesOptions != null) {
            builder.startObject(DatafeedConfig.INDICES_OPTIONS.getPreferredName());
            indicesOptions.toXContent(builder, params);
            builder.endObject();
        }
        builder.endObject();
        return builder;
    }

    private void addOptionalField(XContentBuilder builder, ParseField field, Object value) throws IOException {
        if (value != null) {
            builder.field(field.getPreferredName(), value);
        }
    }

    public String getJobId() {
        return jobId;
    }

    TimeValue getQueryDelay() {
        return queryDelay;
    }

    TimeValue getFrequency() {
        return frequency;
    }

    List getIndices() {
        return indices;
    }

    Integer getScrollSize() {
        return scrollSize;
    }

    Map getQuery() {
        return queryProvider == null ? null : queryProvider.getQuery();
    }

    QueryBuilder getParsedQuery(NamedXContentRegistry namedXContentRegistry) throws IOException {
        return XContentObjectTransformer.queryBuilderTransformer(namedXContentRegistry).fromMap(queryProvider.getQuery(),
                new ArrayList<>());
    }

    Map getAggregations() {
        return aggProvider == null ? null : aggProvider.getAggs();
    }

    AggregatorFactories.Builder getParsedAgg(NamedXContentRegistry namedXContentRegistry) throws IOException {
        return XContentObjectTransformer.aggregatorTransformer(namedXContentRegistry).fromMap(aggProvider.getAggs(),
            new ArrayList<>());
    }

    /**
     * @return {@code true} when there are non-empty aggregations, {@code false}
     *         otherwise
     */
    boolean hasAggregations() {
        return getAggregations() != null && getAggregations().size() > 0;
    }

    List getScriptFields() {
        return scriptFields == null ? Collections.emptyList() : scriptFields;
    }

    ChunkingConfig getChunkingConfig() {
        return chunkingConfig;
    }

    public DelayedDataCheckConfig getDelayedDataCheckConfig() {
        return delayedDataCheckConfig;
    }

    public Integer getMaxEmptySearches() {
        return maxEmptySearches;
    }

    public IndicesOptions getIndicesOptions() {
        return indicesOptions;
    }

    /**
     * Applies the update to the given {@link DatafeedConfig}
     * @return a new {@link DatafeedConfig} that contains the update
     */
    public DatafeedConfig apply(DatafeedConfig datafeedConfig, Map headers) {
        if (id.equals(datafeedConfig.getId()) == false) {
            throw new IllegalArgumentException("Cannot apply update to datafeedConfig with different id");
        }

        DatafeedConfig.Builder builder = new DatafeedConfig.Builder(datafeedConfig);
        if (jobId != null) {
            if (datafeedConfig.getJobId() != null && datafeedConfig.getJobId().equals(jobId) == false) {
                deprecationLogger.deprecate(DeprecationCategory.API, "update_datafeed_job_id", DEPRECATION_MESSAGE_ON_JOB_ID_UPDATE);
            }
            builder.setJobId(jobId);
        }
        if (queryDelay != null) {
            builder.setQueryDelay(queryDelay);
        }
        if (frequency != null) {
            builder.setFrequency(frequency);
        }
        if (indices != null) {
            builder.setIndices(indices);
        }
        if (queryProvider != null) {
            builder.setQueryProvider(queryProvider);
        }
        if (aggProvider != null) {
            DatafeedConfig.validateAggregations(aggProvider.getParsedAggs());
            builder.setAggProvider(aggProvider);
        }
        if (scriptFields != null) {
            builder.setScriptFields(scriptFields);
        }
        if (scrollSize != null) {
            builder.setScrollSize(scrollSize);
        }
        if (chunkingConfig != null) {
            builder.setChunkingConfig(chunkingConfig);
        }
        if (delayedDataCheckConfig != null) {
            builder.setDelayedDataCheckConfig(delayedDataCheckConfig);
        }
        if (maxEmptySearches != null) {
            builder.setMaxEmptySearches(maxEmptySearches);
        }
        if (indicesOptions != null) {
            builder.setIndicesOptions(indicesOptions);
        }
        if (headers.isEmpty() == false) {
            builder.setHeaders(filterSecurityHeaders(headers));
        }
        return builder.build();
    }

    /**
     * The lists of indices and types are compared for equality but they are not
     * sorted first so this test could fail simply because the indices and types
     * lists are in different orders.
     */
    @Override
    public boolean equals(Object other) {
        if (this == other) {
            return true;
        }

        if (other instanceof DatafeedUpdate == false) {
            return false;
        }

        DatafeedUpdate that = (DatafeedUpdate) other;

        return Objects.equals(this.id, that.id)
                && Objects.equals(this.jobId, that.jobId)
                && Objects.equals(this.frequency, that.frequency)
                && Objects.equals(this.queryDelay, that.queryDelay)
                && Objects.equals(this.indices, that.indices)
                && Objects.equals(this.queryProvider, that.queryProvider)
                && Objects.equals(this.scrollSize, that.scrollSize)
                && Objects.equals(this.aggProvider, that.aggProvider)
                && Objects.equals(this.delayedDataCheckConfig, that.delayedDataCheckConfig)
                && Objects.equals(this.scriptFields, that.scriptFields)
                && Objects.equals(this.chunkingConfig, that.chunkingConfig)
                && Objects.equals(this.maxEmptySearches, that.maxEmptySearches)
                && Objects.equals(this.indicesOptions, that.indicesOptions);
    }

    @Override
    public int hashCode() {
        return Objects.hash(id, jobId, frequency, queryDelay, indices, queryProvider, scrollSize, aggProvider, scriptFields, chunkingConfig,
                delayedDataCheckConfig, maxEmptySearches, indicesOptions);
    }

    @Override
    public String toString() {
        return Strings.toString(this);
    }

    boolean isNoop(DatafeedConfig datafeed) {
        return (frequency == null || Objects.equals(frequency, datafeed.getFrequency()))
                && (queryDelay == null || Objects.equals(queryDelay, datafeed.getQueryDelay()))
                && (indices == null || Objects.equals(indices, datafeed.getIndices()))
                && (queryProvider == null || Objects.equals(queryProvider.getQuery(), datafeed.getQuery()))
                && (scrollSize == null || Objects.equals(scrollSize, datafeed.getScrollSize()))
                && (aggProvider == null || Objects.equals(aggProvider.getAggs(), datafeed.getAggregations()))
                && (scriptFields == null || Objects.equals(scriptFields, datafeed.getScriptFields()))
                && (delayedDataCheckConfig == null || Objects.equals(delayedDataCheckConfig, datafeed.getDelayedDataCheckConfig()))
                && (chunkingConfig == null || Objects.equals(chunkingConfig, datafeed.getChunkingConfig()))
                && (maxEmptySearches == null || Objects.equals(maxEmptySearches, datafeed.getMaxEmptySearches())
                        || (maxEmptySearches == -1 && datafeed.getMaxEmptySearches() == null))
                && (indicesOptions == null || Objects.equals(indicesOptions, datafeed.getIndicesOptions()));
    }

    public static class Builder {

        private String id;
        private String jobId;
        private TimeValue queryDelay;
        private TimeValue frequency;
        private List indices;
        private QueryProvider queryProvider;
        private AggProvider aggProvider;
        private List scriptFields;
        private Integer scrollSize;
        private ChunkingConfig chunkingConfig;
        private DelayedDataCheckConfig delayedDataCheckConfig;
        private Integer maxEmptySearches;
        private IndicesOptions indicesOptions;

        public Builder() {
        }

        public Builder(String id) {
            this.id = ExceptionsHelper.requireNonNull(id, DatafeedConfig.ID.getPreferredName());
        }

        public Builder(DatafeedUpdate config) {
            this.id = config.id;
            this.jobId = config.jobId;
            this.queryDelay = config.queryDelay;
            this.frequency = config.frequency;
            this.indices = config.indices;
            this.queryProvider = config.queryProvider;
            this.aggProvider = config.aggProvider;
            this.scriptFields = config.scriptFields;
            this.scrollSize = config.scrollSize;
            this.chunkingConfig = config.chunkingConfig;
            this.delayedDataCheckConfig = config.delayedDataCheckConfig;
            this.maxEmptySearches = config.maxEmptySearches;
            this.indicesOptions = config.indicesOptions;
        }

        public Builder setId(String datafeedId) {
            id = ExceptionsHelper.requireNonNull(datafeedId, DatafeedConfig.ID.getPreferredName());
            return this;
        }

        public Builder setJobId(String jobId) {
            this.jobId = jobId;
            return this;
        }

        public Builder setIndices(List indices) {
            this.indices = indices;
            return this;
        }

        public Builder setQueryDelay(TimeValue queryDelay) {
            this.queryDelay = queryDelay;
            return this;
        }

        public Builder setFrequency(TimeValue frequency) {
            this.frequency = frequency;
            return this;
        }

        public Builder setQuery(QueryProvider queryProvider) {
            this.queryProvider = queryProvider;
            return this;
        }

        private Builder setAggregationsSafe(AggProvider aggProvider) {
            if (this.aggProvider != null) {
                throw ExceptionsHelper.badRequestException("Found two aggregation definitions: [aggs] and [aggregations]");
            }
            setAggregations(aggProvider);
            return this;
        }

        public Builder setAggregations(AggProvider aggProvider) {
            this.aggProvider = aggProvider;
            return this;
        }

        public Builder setScriptFields(List scriptFields) {
            List sorted = new ArrayList<>(scriptFields);
            sorted.sort(Comparator.comparing(SearchSourceBuilder.ScriptField::fieldName));
            this.scriptFields = sorted;
            return this;
        }

        public Builder setDelayedDataCheckConfig(DelayedDataCheckConfig delayedDataCheckConfig) {
            this.delayedDataCheckConfig = delayedDataCheckConfig;
            return this;
        }

        public Builder setScrollSize(int scrollSize) {
            this.scrollSize = scrollSize;
            return this;
        }

        public Builder setChunkingConfig(ChunkingConfig chunkingConfig) {
            this.chunkingConfig = chunkingConfig;
            return this;
        }

        public Builder setMaxEmptySearches(int maxEmptySearches) {
            if (maxEmptySearches < -1 || maxEmptySearches == 0) {
                String msg = Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE,
                    DatafeedConfig.MAX_EMPTY_SEARCHES.getPreferredName(), maxEmptySearches);
                throw ExceptionsHelper.badRequestException(msg);
            }
            this.maxEmptySearches = maxEmptySearches;
            return this;
        }

        public Builder setIndicesOptions(IndicesOptions indicesOptions) {
            this.indicesOptions = indicesOptions;
            return this;
        }

        public DatafeedUpdate build() {
            return new DatafeedUpdate(id, jobId, queryDelay, frequency, indices, queryProvider, aggProvider, scriptFields, scrollSize,
                    chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions);
        }
    }
}