org.elasticsearch.index.rankeval.TransportRankEvalAction Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of rank-eval-client Show documentation
Show all versions of rank-eval-client Show documentation
The Rank Eval module adds APIs to evaluate ranking quality.
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.search.MultiSearchRequest;
import org.elasticsearch.action.search.MultiSearchResponse;
import org.elasticsearch.action.search.MultiSearchResponse.Item;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.TemplateScript;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.XContentType;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import static org.elasticsearch.common.xcontent.XContentHelper.createParser;
import static org.elasticsearch.index.rankeval.RatedRequest.validateEvaluatedQuery;
/**
* Instances of this class execute a collection of search intents (read: user
* supplied query parameters) against a set of possible search requests (read:
* search specifications, expressed as query/search request templates) and
* compares the result against a set of annotated documents per search intent.
*
* If any documents are returned that haven't been annotated the document id of
* those is returned per search intent.
*
* The resulting search quality is computed in terms of precision at n and
* returned for each search specification for the full set of search intents as
* averaged precision at n.
*/
public class TransportRankEvalAction extends HandledTransportAction {
private final Client client;
private final ScriptService scriptService;
private final NamedXContentRegistry namedXContentRegistry;
@Inject
public TransportRankEvalAction(
ActionFilters actionFilters,
Client client,
TransportService transportService,
ScriptService scriptService,
NamedXContentRegistry namedXContentRegistry
) {
super(RankEvalAction.NAME, transportService, actionFilters, RankEvalRequest::new);
this.scriptService = scriptService;
this.namedXContentRegistry = namedXContentRegistry;
this.client = client;
}
@Override
protected void doExecute(Task task, RankEvalRequest request, ActionListener listener) {
RankEvalSpec evaluationSpecification = request.getRankEvalSpec();
EvaluationMetric metric = evaluationSpecification.getMetric();
List ratedRequests = evaluationSpecification.getRatedRequests();
Map errors = new ConcurrentHashMap<>(ratedRequests.size());
Map scriptsWithoutParams = new HashMap<>();
for (Entry entry : evaluationSpecification.getTemplates().entrySet()) {
scriptsWithoutParams.put(entry.getKey(), scriptService.compile(entry.getValue(), TemplateScript.CONTEXT));
}
MultiSearchRequest msearchRequest = new MultiSearchRequest();
msearchRequest.maxConcurrentSearchRequests(evaluationSpecification.getMaxConcurrentSearches());
List ratedRequestsInSearch = new ArrayList<>();
for (RatedRequest ratedRequest : ratedRequests) {
SearchSourceBuilder evaluationRequest = ratedRequest.getEvaluationRequest();
if (evaluationRequest == null) {
Map params = ratedRequest.getParams();
String templateId = ratedRequest.getTemplateId();
TemplateScript.Factory templateScript = scriptsWithoutParams.get(templateId);
String resolvedRequest = templateScript.newInstance(params).execute();
try (
XContentParser subParser = createParser(
namedXContentRegistry,
LoggingDeprecationHandler.INSTANCE,
new BytesArray(resolvedRequest),
XContentType.JSON
)
) {
evaluationRequest = SearchSourceBuilder.fromXContent(subParser, false);
// check for parts that should not be part of a ranking evaluation request
validateEvaluatedQuery(evaluationRequest);
} catch (IOException e) {
// if we fail parsing, put the exception into the errors map and continue
errors.put(ratedRequest.getId(), e);
continue;
}
}
if (metric.forcedSearchSize().isPresent()) {
evaluationRequest.size(metric.forcedSearchSize().getAsInt());
}
ratedRequestsInSearch.add(ratedRequest);
List summaryFields = ratedRequest.getSummaryFields();
if (summaryFields.isEmpty()) {
evaluationRequest.fetchSource(false);
} else {
evaluationRequest.fetchSource(summaryFields.toArray(new String[summaryFields.size()]), new String[0]);
}
SearchRequest searchRequest = new SearchRequest(request.indices(), evaluationRequest);
searchRequest.indicesOptions(request.indicesOptions());
searchRequest.searchType(request.searchType());
msearchRequest.add(searchRequest);
}
assert ratedRequestsInSearch.size() == msearchRequest.requests().size();
client.multiSearch(
msearchRequest,
new RankEvalActionListener(
listener,
metric,
ratedRequestsInSearch.toArray(new RatedRequest[ratedRequestsInSearch.size()]),
errors
)
);
}
static class RankEvalActionListener extends ActionListener.Delegating {
private final RatedRequest[] specifications;
private final Map errors;
private final EvaluationMetric metric;
RankEvalActionListener(
ActionListener listener,
EvaluationMetric metric,
RatedRequest[] specifications,
Map errors
) {
super(listener);
this.metric = metric;
this.errors = errors;
this.specifications = specifications;
}
@Override
public void onResponse(MultiSearchResponse multiSearchResponse) {
int responsePosition = 0;
Map responseDetails = new HashMap<>(specifications.length);
for (Item response : multiSearchResponse.getResponses()) {
RatedRequest specification = specifications[responsePosition];
if (response.isFailure() == false) {
SearchHit[] hits = response.getResponse().getHits().getHits();
EvalQueryQuality queryQuality = this.metric.evaluate(specification.getId(), hits, specification.getRatedDocs());
responseDetails.put(specification.getId(), queryQuality);
} else {
errors.put(specification.getId(), response.getFailure());
}
responsePosition++;
}
delegate.onResponse(new RankEvalResponse(this.metric.combine(responseDetails.values()), responseDetails, this.errors));
}
}
}