io.camunda.exporter.tasks.incident.ElasticsearchIncidentUpdateRepository Maven / Gradle / Ivy
/*
* Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under
* one or more contributor license agreements. See the NOTICE file distributed
* with this work for additional information regarding copyright ownership.
* Licensed under the Camunda License 1.0. You may not use this file
* except in compliance with the Camunda License 1.0.
*/
package io.camunda.exporter.tasks.incident;
import co.elastic.clients.elasticsearch.ElasticsearchAsyncClient;
import co.elastic.clients.elasticsearch._types.ErrorCause;
import co.elastic.clients.elasticsearch._types.Refresh;
import co.elastic.clients.elasticsearch._types.SortOrder;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
import co.elastic.clients.elasticsearch.core.BulkRequest;
import co.elastic.clients.elasticsearch.core.SearchRequest;
import co.elastic.clients.elasticsearch.core.SearchResponse;
import co.elastic.clients.elasticsearch.core.bulk.BulkOperation;
import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem;
import co.elastic.clients.elasticsearch.core.bulk.UpdateOperation;
import co.elastic.clients.elasticsearch.core.search.SourceFilter;
import co.elastic.clients.json.JsonData;
import io.camunda.exporter.tasks.incident.IncidentUpdateRepository.NoopIncidentUpdateRepository;
import io.camunda.webapps.schema.descriptors.operate.template.IncidentTemplate;
import io.camunda.webapps.schema.descriptors.operate.template.PostImporterQueueTemplate;
import io.camunda.webapps.schema.entities.operate.IncidentEntity;
import io.camunda.webapps.schema.entities.operate.IncidentState;
import io.camunda.webapps.schema.entities.operate.post.PostImporterActionType;
import io.camunda.zeebe.exporter.api.ExporterException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.Executor;
import java.util.stream.Collectors;
public final class ElasticsearchIncidentUpdateRepository extends NoopIncidentUpdateRepository {
private static final int RETRY_COUNT = 3;
private final int partitionId;
private final String pendingUpdateAlias;
private final String incidentAlias;
private final String listViewAlias;
private final String flowNodeAlias;
private final ElasticsearchAsyncClient client;
private final Executor executor;
public ElasticsearchIncidentUpdateRepository(
final int partitionId,
final String pendingUpdateAlias,
final String incidentAlias,
final String listViewAlias,
final String flowNodeAlias,
final ElasticsearchAsyncClient client,
final Executor executor) {
this.partitionId = partitionId;
this.pendingUpdateAlias = pendingUpdateAlias;
this.incidentAlias = incidentAlias;
this.listViewAlias = listViewAlias;
this.flowNodeAlias = flowNodeAlias;
this.client = client;
this.executor = executor;
}
@Override
public CompletionStage getPendingIncidentsBatch(
final long fromPosition, final int size) {
final var query = createPendingIncidentsBatchQuery(fromPosition);
final var request = createPendingIncidentsBatchRequest(size, query);
return client
.search(request, PendingIncidentUpdate.class)
.thenApplyAsync(this::createPendingIncidentBatch, executor);
}
@Override
public CompletionStage> getIncidentDocuments(
final List incidentIds) {
final var request = createIncidentDocumentsRequest(incidentIds);
return client
.search(request, IncidentEntity.class)
.thenApplyAsync(this::createIncidentDocuments, executor);
}
@Override
public CompletionStage bulkUpdate(final IncidentBulkUpdate bulk) {
final var updates = bulk.stream().map(this::createUpdateOperation).toList();
final var request =
new BulkRequest.Builder()
.operations(updates)
.source(s -> s.fetch(false))
.refresh(Refresh.WaitFor)
.build();
return client
.bulk(request)
.thenComposeAsync(
r -> {
if (r.errors()) {
return CompletableFuture.failedFuture(collectBulkErrors(r.items()));
}
return CompletableFuture.completedFuture(r.items().size());
},
executor);
}
private BulkOperation createUpdateOperation(final DocumentUpdate update) {
return new UpdateOperation.Builder<>()
.index(update.index())
.id(update.id())
.retryOnConflict(RETRY_COUNT)
.action(a -> a.doc(update.doc()))
.routing(update.routing())
.build()
._toBulkOperation();
}
private SearchRequest createIncidentDocumentsRequest(final List incidentIds) {
final var idQ = QueryBuilders.ids(i -> i.values(incidentIds));
final var partitionQ =
QueryBuilders.term(t -> t.field(IncidentTemplate.PARTITION_ID).value(partitionId));
return new SearchRequest.Builder()
.index(incidentAlias)
.query(q -> q.bool(b -> b.must(idQ, partitionQ)))
.allowNoIndices(true)
.ignoreUnavailable(true)
.sort(s -> s.field(f -> f.field(IncidentTemplate.KEY)))
.build();
}
private Map createIncidentDocuments(
final SearchResponse response) {
final Map documents = new HashMap<>();
for (final var hit : response.hits().hits()) {
documents.put(hit.id(), new IncidentDocument(hit.id(), hit.index(), hit.source()));
}
return documents;
}
private SearchRequest createPendingIncidentsBatchRequest(final int size, final Query query) {
final var sourceFilter =
new SourceFilter.Builder()
.includes(
PostImporterQueueTemplate.KEY,
PostImporterQueueTemplate.POSITION,
PostImporterQueueTemplate.INTENT)
.build();
return new SearchRequest.Builder()
.index(pendingUpdateAlias)
.query(query)
.ignoreUnavailable(true)
.allowNoIndices(true)
.source(s -> s.filter(sourceFilter))
.sort(s -> s.field(f -> f.field(PostImporterQueueTemplate.POSITION).order(SortOrder.Asc)))
.size(size)
.build();
}
private Query createPendingIncidentsBatchQuery(final long fromPosition) {
final var positionQ =
QueryBuilders.range(
r -> r.field(PostImporterQueueTemplate.POSITION).gt(JsonData.of(fromPosition)));
final var typeQ =
QueryBuilders.term(
t ->
t.field(PostImporterQueueTemplate.ACTION_TYPE)
.value(PostImporterActionType.INCIDENT.name()));
final var partitionQ =
QueryBuilders.term(t -> t.field(PostImporterQueueTemplate.PARTITION_ID).value(partitionId));
return QueryBuilders.bool(b -> b.must(positionQ, typeQ, partitionQ));
}
private PendingIncidentUpdateBatch createPendingIncidentBatch(
final SearchResponse response) {
final var hits = response.hits().hits();
final Map incidents = new HashMap<>();
// if there are multiple updates for a given incident, keep the latest one only; assuming no
// bugs in the engine, this means it can only be CREATED -> RESOLVED. Since our search results
// are sorted by position, we know the latest will necessarily be the actual "latest" (as in,
// wall clock) result
final var highestPosition = hits.isEmpty() ? -1L : hits.getLast().source().position();
for (final var hit : hits) {
final var entity = hit.source();
final var newState = IncidentState.createFrom(entity.intent());
incidents.put(entity.key(), newState);
}
return new PendingIncidentUpdateBatch(highestPosition, incidents);
}
private Throwable collectBulkErrors(final List items) {
final var collectedErrors = new ArrayList();
items.stream()
.flatMap(item -> Optional.ofNullable(item.error()).stream())
.collect(Collectors.groupingBy(ErrorCause::type))
.forEach(
(type, errors) ->
collectedErrors.add(
String.format(
"Failed to update %d item(s) of bulk update [type: %s, reason: %s]",
errors.size(), type, errors.getFirst().reason())));
return new ExporterException("Failed to flush bulk request: " + collectedErrors);
}
private record PendingIncidentUpdate(long key, long position, String intent) {}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy