Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
// Generated by delombok at Thu Oct 10 18:56:33 UTC 2024
/* SPDX-License-Identifier: Apache-2.0
Copyright 2022 Atlan Pte. Ltd. */
package com.atlan.model.search;
import co.elastic.clients.elasticsearch._types.SortOptions;
import co.elastic.clients.elasticsearch._types.SortOrder;
import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import com.atlan.AtlanClient;
import com.atlan.exception.AtlanException;
import com.atlan.exception.ErrorCode;
import com.atlan.exception.LogicException;
import com.atlan.model.assets.Asset;
import com.atlan.net.ApiResource;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
/**
* Captures the response from a search against Atlan. Also provides the ability to iteratively
* page through results, without needing to track or re-run the original query using {@link #getNextPage()}.
*/
public class IndexSearchResponse extends ApiResource implements Iterable {
@java.lang.SuppressWarnings("all")
@lombok.Generated
private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(IndexSearchResponse.class);
private static final long serialVersionUID = 2L;
private static final long MASS_EXTRACT_THRESHOLD = 100000L;
private static final int CHARACTERISTICS = Spliterator.NONNULL | Spliterator.IMMUTABLE | Spliterator.ORDERED;
/**
* Connectivity to the Atlan tenant where the search was run.
*/
@JsonIgnore
AtlanClient client;
/**
* Type of query.
*/
String queryType;
/**
* Parameters for the search.
*/
SearchParameters searchParameters;
/**
* List of results from the search.
*/
@JsonProperty("entities")
List assets;
/**
* Approximate number of total results.
*/
Long approximateCount;
/**
* Map of results for the requested aggregations.
*/
Map aggregations;
/**
* Retrieve the next page of results from this response.
*
* @return next page of results from this response
* @throws AtlanException on any API interaction problem
*/
@JsonIgnore
public IndexSearchResponse getNextPage() throws AtlanException {
IndexSearchDSL dsl = getQuery();
int from = dsl.getFrom() == null ? 0 : dsl.getFrom();
int page = dsl.getSize() == null ? IndexSearchDSL.DEFAULT_PAGE_SIZE : dsl.getSize();
dsl = dsl.toBuilder().from(from + page).build();
IndexSearchRequest.IndexSearchRequestBuilder next = IndexSearchRequest.builder(dsl);
if (searchParameters.getAttributes() != null) {
next = next.attributes(searchParameters.getAttributes());
}
if (searchParameters.getRelationAttributes() != null) {
next = next.relationAttributes(searchParameters.getRelationAttributes());
}
return next.build().search(client);
}
/**
* Retrieve the next page of results from this response, using bulk-oriented paging.
*
* @return next page of results from this response
* @throws AtlanException on any API interaction problem
*/
@JsonIgnore
protected IndexSearchResponse getNextBulkPage() throws AtlanException {
if (getAssets() == null) {
// If there are no assets, return this no-asset page (we're at the end of paging).
return this;
}
IndexSearchDSL dsl = getQuery();
// Check for a timestamp condition to determine if this query is being streamed
Query query = dsl.getQuery();
List rewrittenFilters = new ArrayList<>();
boolean streamed = presortedByTimestamp(dsl.getSort());
if (query.isBool()) {
BoolQuery original = query.bool();
List filters = original.filter();
for (Query candidate : filters) {
if (!isPagingTimestampQuery(candidate)) {
rewrittenFilters.add(candidate);
}
}
}
int page = dsl.getSize() == null ? IndexSearchDSL.DEFAULT_PAGE_SIZE : dsl.getSize();
long firstRecord = -2L;
long lastRecord;
if (getAssets().size() > 1) {
firstRecord = getAssets().get(0).getCreateTime();
lastRecord = getAssets().get(getAssets().size() - 1).getCreateTime();
} else {
lastRecord = -2L;
}
if (streamed && firstRecord != lastRecord) {
// If we're streaming and the first and last record have different timestamps,
// page based on a new timestamp (to keep offsets low)
rewrittenFilters.add(getPagingTimestampQuery(lastRecord));
BoolQuery original = query.bool();
BoolQuery rewritten = BoolQuery.of(b -> b.filter(rewrittenFilters).must(original.must()).mustNot(original.mustNot()).minimumShouldMatch(original.minimumShouldMatch()).should(original.should()).boost(original.boost()));
dsl = dsl.toBuilder().from(0).size(page).query(rewritten._toQuery()).build();
} else {
// If the first and last record in the page have the same timestamp,
// or we're not streaming, use "normal" offset-based paging
int from = dsl.getFrom() == null ? 0 : dsl.getFrom();
dsl = dsl.toBuilder().from(from + getAssets().size()).size(page).build();
}
IndexSearchRequest.IndexSearchRequestBuilder next = IndexSearchRequest.builder(dsl);
if (searchParameters.getAttributes() != null) {
next = next.attributes(searchParameters.getAttributes());
}
if (searchParameters.getRelationAttributes() != null) {
next = next.relationAttributes(searchParameters.getRelationAttributes());
}
return next.build().search(client);
}
private boolean isPagingTimestampQuery(Query candidate) {
return candidate.isRange() && candidate.range().untyped().field().equals(Asset.CREATE_TIME.getInternalFieldName()) && candidate.range().untyped().gte() != null && candidate.range().untyped().gte().to(Long.class) > 0 && candidate.range().untyped().lt() == null && candidate.range().untyped().lte() == null;
}
private Query getPagingTimestampQuery(long lastTimestamp) {
return Asset.CREATE_TIME.gte(lastTimestamp);
}
private IndexSearchResponse getFirstPageTimestampOrdered() throws AtlanException {
IndexSearchDSL dsl = getQuery();
List revisedSort = sortByTimestampFirst(dsl.getSort());
int page = dsl.getSize() == null ? IndexSearchDSL.DEFAULT_PAGE_SIZE : dsl.getSize();
dsl = dsl.toBuilder().from(0).size(page).clearSort().sort(revisedSort).build();
IndexSearchRequest.IndexSearchRequestBuilder first = IndexSearchRequest.builder(dsl);
if (searchParameters.getAttributes() != null) {
first = first.attributes(searchParameters.getAttributes());
}
if (searchParameters.getRelationAttributes() != null) {
first = first.relationAttributes(searchParameters.getRelationAttributes());
}
return first.build().search(client);
}
/**
* Retrieve a specific page of results using the same query used to produce this response.
*
* @param offset starting point for the specific page
* @param pageSize maximum number of results beyond the starting point to retrieve
* @return specific page of results from this response
* @throws AtlanException on any API interaction problem
*/
@JsonIgnore
public List getSpecificPage(int offset, int pageSize) throws AtlanException {
IndexSearchDSL dsl = getQuery().toBuilder().from(offset).size(pageSize).build();
IndexSearchRequest.IndexSearchRequestBuilder next = IndexSearchRequest.builder(dsl);
if (searchParameters.getAttributes() != null) {
next = next.attributes(searchParameters.getAttributes());
}
if (searchParameters.getRelationAttributes() != null) {
next = next.relationAttributes(searchParameters.getRelationAttributes());
}
IndexSearchResponse response = next.build().search(client);
if (response != null && response.getAssets() != null) {
return response.getAssets();
} else {
return Collections.emptyList();
}
}
/**
* {@inheritDoc}
*/
@Override
public Iterator iterator() {
return new IndexSearchResponseIterator(this);
}
/**
* Returns an iterator over the elements of the index search, lazily paged,
* but in such a way that they can be iterated through in bulk (100,000's of results or more).
* Note: this will reorder the results and will NOT retain the sort ordering you have specified (if any).
* (Uses offset-limited sequential paging, to avoid large offsets that effectively need to re-retrieve many
* large numbers of previous pages' results.)
* @return iterator through the assets in the search results
*/
public Iterator biterator() {
return new IndexSearchResponseBulkIterator(this);
}
/**
* {@inheritDoc}
*/
@Override
public Spliterator spliterator() {
long pageSize;
try {
pageSize = getQuery().getSize();
} catch (LogicException e) {
log.warn("Unable to parse page size from query, falling back to {}.", IndexSearchDSL.DEFAULT_PAGE_SIZE, e);
pageSize = IndexSearchDSL.DEFAULT_PAGE_SIZE;
}
IndexSearchResponseSpliterator spliterator = new IndexSearchResponseSpliterator(this, 0, this.getApproximateCount(), pageSize);
List assets = getAssets() == null ? Collections.emptyList() : getAssets();
spliterator.firstPage = assets.spliterator();
return spliterator;
}
/**
* Stream the results (lazily) for processing without needing to manually manage paging.
* Note: if the number of results exceeds the predefined threshold (100,000 assets)
* this will be automatically converted into a bulkStream().
*
* @return a lazily-loaded stream of results from the search
*/
public Stream stream() {
if (approximateCount > MASS_EXTRACT_THRESHOLD) {
log.debug("Results size exceeds threshold ({}), rewriting stream as a bulk stream (ignoring original sorting).", MASS_EXTRACT_THRESHOLD);
return bulkStream();
} else {
return StreamSupport.stream(Spliterators.spliterator(iterator(), approximateCount, CHARACTERISTICS), false);
}
}
/**
* Stream a large number of results (lazily) for processing without needing to manually manage paging.
* Note: this will reorder the results in order to iterate through a large number (more than 100,000) results,
* so any sort ordering you have specified may be ignored.
*
* @return a lazily-loaded stream of results from the search
*/
public Stream bulkStream() {
return StreamSupport.stream(Spliterators.spliterator(biterator(), approximateCount, CHARACTERISTICS), false);
}
/**
* Stream the results in parallel across all pages (may do more than limited to in a request).
* Note: if the number of results exceeds the predefined threshold (100,000 assets)
* this will be automatically converted into a bulkStream().
*
* @return a lazily-loaded stream of results from the search
*/
public Stream parallelStream() {
if (approximateCount > MASS_EXTRACT_THRESHOLD) {
log.debug("Results size exceeds threshold ({}), ignoring request for parallelized streaming and falling back to bulk streaming.", MASS_EXTRACT_THRESHOLD);
return bulkStream();
} else {
return StreamSupport.stream(this::spliterator, CHARACTERISTICS, true);
}
}
/**
* Parse the original query from the search parameters.
*
* @return original query as DSL
* @throws LogicException if the query could not be parsed
*/
@JsonIgnore
public IndexSearchDSL getQuery() throws LogicException {
try {
return client.readValue(searchParameters.getQuery(), IndexSearchDSL.class);
} catch (IOException e) {
throw new LogicException(ErrorCode.UNABLE_TO_PARSE_ORIGINAL_QUERY, e);
}
}
/**
* Allow results to be iterated through in parallel without managing paging retrievals.
* With inspiration from: https://stackoverflow.com/questions/38128274/how-can-i-create-a-general-purpose-paging-spliterator
*/
private static class IndexSearchResponseSpliterator implements Spliterator {
private final IndexSearchResponse response;
private long start;
private final long end;
private final long pageSize;
private Spliterator firstPage;
private Spliterator currentPage;
IndexSearchResponseSpliterator(IndexSearchResponse response, long start, long end, long pageSize) {
this.response = response;
this.start = start;
this.end = end;
this.pageSize = pageSize;
}
/**
* {@inheritDoc}
*/
@Override
public boolean tryAdvance(Consumer action) {
while (true) {
if (ensurePage().tryAdvance(action)) {
// If we're able to advance the page we're on, go for it
return true;
} else if (start >= end) {
// Alternatively, if we're at the end, we can stop now
return false;
} else {
// Otherwise, reset the current page to empty and try
// to advance again
currentPage = null;
}
}
}
/**
* {@inheritDoc}
*/
@Override
public void forEachRemaining(Consumer action) {
do {
ensurePage().forEachRemaining(action);
currentPage = null;
} while (start < end);
}
/**
* {@inheritDoc}
*/
@Override
public Spliterator trySplit() {
if (firstPage != null) {
// If we have a first page already fetched, use it
Spliterator fp = firstPage;
firstPage = null; // clear it once we've used it
start = fp.getExactSizeIfKnown();
return fp;
} else if (currentPage != null) {
// Otherwise, if we have a current page already fetched, use that
return currentPage.trySplit();
} else if (end - start > pageSize) {
// Otherwise, if we know there are remaining elements that we have not yet fetched...
// Calculate a new split-point midway between them (shift rather than divide, to avoid overflow)
long mid = (start + end) >>> 1;
// Round the mid-point to a multiple of the pageSize
mid = mid / pageSize * pageSize;
if (mid == start) {
mid += pageSize;
}
// Create a new spliterator with the new mid-point, while resetting this existing
// spliterator's end-point to the midpoint
return new IndexSearchResponseSpliterator(response, start, start = mid, pageSize);
}
return ensurePage().trySplit();
}
/**
* Only fetch data immediately before traversing or sub-page splitting.
*/
private Spliterator ensurePage() {
if (firstPage != null) {
// If we already have a first page fetched, use it
Spliterator fp = firstPage;
firstPage = null; // clear it once we've used it
currentPage = fp;
start = fp.getExactSizeIfKnown();
return fp;
} else {
// Otherwise, try to use the current page (if any)
Spliterator sp = currentPage;
if (sp == null) {
// ... and if there is no current page...
if (start >= end) {
// Return an empty spliterator if we're at the end
return Spliterators.emptySpliterator();
} else {
// Otherwise (only NOW), go fetch the next specific page of results needed
List assets;
try {
assets = response.getSpecificPage((int) start, (int) Math.min(end - start, pageSize));
} catch (AtlanException e) {
log.warn("Unable to fetch the specific page from {} to {}", start, Math.min(end - start, pageSize), e);
assets = Collections.emptyList();
}
// And update this spliterator's starting point accordingly
sp = assets.spliterator();
if (sp.getExactSizeIfKnown() > 0) {
// If there are any results in the page, increment the start by the size
start += sp.getExactSizeIfKnown();
} else {
// Otherwise, increment the start by the page size (so we skip over this "page"
// entirely rather than attempting to re-retrieve it endlessly)
start += pageSize;
}
currentPage = sp;
}
}
return sp;
}
}
/**
* {@inheritDoc}
*/
@Override
public long estimateSize() {
if (currentPage != null) {
return currentPage.estimateSize();
}
return end - start;
}
/**
* {@inheritDoc}
*/
@Override
public int characteristics() {
return CHARACTERISTICS;
}
}
/**
* Indicates whether the sort options prioritize creation-time in ascending order as the first
* sorting key (true) or anything else (false).
*
* @param sort list of sorting options
* @return true if the sorting options have creation time, ascending as the first option
*/
public static boolean presortedByTimestamp(List sort) {
return sort != null && !sort.isEmpty() && sort.get(0).isField() && sort.get(0).field().field().equals(Asset.CREATE_TIME.getInternalFieldName()) && sort.get(0).field().order() == SortOrder.Asc;
}
/**
* Indicates whether the sort options contain any user-requested sorting (true) or not (false).
*
* @param sort list of sorting options
* @return true if the sorting options have any user-requested sorting
*/
public static boolean hasUserRequestedSort(List sort) {
if (presortedByTimestamp(sort)) {
return false;
}
if (sort != null && !sort.isEmpty() && sort.get(0).isField()) {
String fieldName = sort.get(0).field().field();
return !fieldName.equals(Asset.GUID.getInternalFieldName()) || sort.size() != 1;
}
return true;
}
/**
* Rewrites the sorting options to ensure that sorting by creation time, ascending, is the top
* priority. Adds this condition if it does not already exist, or moves it up to the top sorting
* priority if it does already exist in the list.
*
* @param sort list of sorting options
* @return rewritten sorting options, making sorting by creation time in ascending order the top priority
*/
public static List sortByTimestampFirst(List sort) {
if (sort == null || sort.isEmpty()) {
return List.of(Asset.CREATE_TIME.order(SortOrder.Asc));
} else {
List rewritten = new ArrayList<>();
rewritten.add(Asset.CREATE_TIME.order(SortOrder.Asc));
for (SortOptions candidate : sort) {
if (!candidate.isField() || !candidate.field().field().equals(Asset.CREATE_TIME.getInternalFieldName())) {
rewritten.add(candidate);
}
}
return rewritten;
}
}
/**
* Allow results to be iterated through without managing paging retrievals.
*/
private static class IndexSearchResponseIterator implements Iterator {
private IndexSearchResponse response;
private int i;
public IndexSearchResponseIterator(IndexSearchResponse response) {
this.response = response;
this.i = 0;
}
/**
* {@inheritDoc}
*/
@Override
public boolean hasNext() {
if (response.getAssets() != null && response.getAssets().size() > i) {
return true;
} else {
try {
response = response.getNextPage();
i = 0;
return response.getAssets() != null && response.getAssets().size() > i;
} catch (AtlanException e) {
throw new RuntimeException("Unable to iterate through all pages of search results.", e);
}
}
}
/**
* {@inheritDoc}
*/
@Override
public Asset next() {
return response.getAssets().get(i++);
}
}
/**
* Allow results to be iterated through without managing paging retrievals, and
* without overwhelming system resources, even when iterating through 100,000's or more
* results.
*/
private static class IndexSearchResponseBulkIterator implements Iterator {
private IndexSearchResponse response;
// TODO: Consider optimizing memory using UUID.fromString() to store UUIDs rather than Strings
private final Set processedGuids;
private int i;
public IndexSearchResponseBulkIterator(IndexSearchResponse response) {
try {
IndexSearchDSL dsl = response.getQuery();
if (presortedByTimestamp(dsl.getSort())) {
// If the results are already sorted in ascending order by timestamp, proceed
this.response = response;
} else if (hasUserRequestedSort(dsl.getSort())) {
// Alternatively, if they are sorted by any user-requested sort, we need to throw an error
throw new IllegalArgumentException("Bulk searches can only be sorted by timestamp in ascending order - you must remove your own requested sorting to run a bulk search.");
} else {
// Otherwise, re-fetch the first page sorted first by timestamp
this.response = response.getFirstPageTimestampOrdered();
}
} catch (AtlanException e) {
throw new RuntimeException("Unable to rewrite original query in preparation for iteration.", e);
}
this.processedGuids = new HashSet<>();
this.i = 0;
}
/** {@inheritDoc} */
@Override
public boolean hasNext() {
if (response.getAssets() == null) {
// If there are no assets in this page, then there are no more assets
// so exit straightaway
return false;
}
if (response.getAssets().size() > i) {
Asset candidate = response.getAssets().get(i);
if (candidate != null && !processedGuids.contains(candidate.getGuid())) {
return true;
} else {
for (int j = i; j < response.getAssets().size(); j++) {
candidate = response.getAssets().get(j);
if (candidate != null && !processedGuids.contains(candidate.getGuid())) {
i = j;
return true;
}
}
}
}
try {
response = response.getNextBulkPage();
i = 0;
return response.getAssets() != null && response.getAssets().size() > i;
} catch (AtlanException e) {
throw new RuntimeException("Unable to iterate through all pages of search results.", e);
}
}
/** {@inheritDoc} */
@Override
public Asset next() {
Asset candidate = response.getAssets().get(i++);
processedGuids.add(candidate.getGuid());
return candidate;
}
}
/**
* Connectivity to the Atlan tenant where the search was run.
*/
@java.lang.SuppressWarnings("all")
@lombok.Generated
public AtlanClient getClient() {
return this.client;
}
/**
* Type of query.
*/
@java.lang.SuppressWarnings("all")
@lombok.Generated
public String getQueryType() {
return this.queryType;
}
/**
* Parameters for the search.
*/
@java.lang.SuppressWarnings("all")
@lombok.Generated
public SearchParameters getSearchParameters() {
return this.searchParameters;
}
/**
* List of results from the search.
*/
@java.lang.SuppressWarnings("all")
@lombok.Generated
public List getAssets() {
return this.assets;
}
/**
* Approximate number of total results.
*/
@java.lang.SuppressWarnings("all")
@lombok.Generated
public Long getApproximateCount() {
return this.approximateCount;
}
/**
* Map of results for the requested aggregations.
*/
@java.lang.SuppressWarnings("all")
@lombok.Generated
public Map getAggregations() {
return this.aggregations;
}
@java.lang.Override
@java.lang.SuppressWarnings("all")
@lombok.Generated
public boolean equals(final java.lang.Object o) {
if (o == this) return true;
if (!(o instanceof IndexSearchResponse)) return false;
final IndexSearchResponse other = (IndexSearchResponse) o;
if (!other.canEqual((java.lang.Object) this)) return false;
final java.lang.Object this$approximateCount = this.getApproximateCount();
final java.lang.Object other$approximateCount = other.getApproximateCount();
if (this$approximateCount == null ? other$approximateCount != null : !this$approximateCount.equals(other$approximateCount)) return false;
final java.lang.Object this$client = this.getClient();
final java.lang.Object other$client = other.getClient();
if (this$client == null ? other$client != null : !this$client.equals(other$client)) return false;
final java.lang.Object this$queryType = this.getQueryType();
final java.lang.Object other$queryType = other.getQueryType();
if (this$queryType == null ? other$queryType != null : !this$queryType.equals(other$queryType)) return false;
final java.lang.Object this$searchParameters = this.getSearchParameters();
final java.lang.Object other$searchParameters = other.getSearchParameters();
if (this$searchParameters == null ? other$searchParameters != null : !this$searchParameters.equals(other$searchParameters)) return false;
final java.lang.Object this$assets = this.getAssets();
final java.lang.Object other$assets = other.getAssets();
if (this$assets == null ? other$assets != null : !this$assets.equals(other$assets)) return false;
final java.lang.Object this$aggregations = this.getAggregations();
final java.lang.Object other$aggregations = other.getAggregations();
if (this$aggregations == null ? other$aggregations != null : !this$aggregations.equals(other$aggregations)) return false;
return true;
}
@java.lang.SuppressWarnings("all")
@lombok.Generated
protected boolean canEqual(final java.lang.Object other) {
return other instanceof IndexSearchResponse;
}
@java.lang.Override
@java.lang.SuppressWarnings("all")
@lombok.Generated
public int hashCode() {
final int PRIME = 59;
int result = 1;
final java.lang.Object $approximateCount = this.getApproximateCount();
result = result * PRIME + ($approximateCount == null ? 43 : $approximateCount.hashCode());
final java.lang.Object $client = this.getClient();
result = result * PRIME + ($client == null ? 43 : $client.hashCode());
final java.lang.Object $queryType = this.getQueryType();
result = result * PRIME + ($queryType == null ? 43 : $queryType.hashCode());
final java.lang.Object $searchParameters = this.getSearchParameters();
result = result * PRIME + ($searchParameters == null ? 43 : $searchParameters.hashCode());
final java.lang.Object $assets = this.getAssets();
result = result * PRIME + ($assets == null ? 43 : $assets.hashCode());
final java.lang.Object $aggregations = this.getAggregations();
result = result * PRIME + ($aggregations == null ? 43 : $aggregations.hashCode());
return result;
}
@java.lang.Override
@java.lang.SuppressWarnings("all")
@lombok.Generated
public java.lang.String toString() {
return "IndexSearchResponse(super=" + super.toString() + ", client=" + this.getClient() + ", queryType=" + this.getQueryType() + ", searchParameters=" + this.getSearchParameters() + ", assets=" + this.getAssets() + ", approximateCount=" + this.getApproximateCount() + ", aggregations=" + this.getAggregations() + ")";
}
/**
* Connectivity to the Atlan tenant where the search was run.
*/
@JsonIgnore
@java.lang.SuppressWarnings("all")
@lombok.Generated
public void setClient(final AtlanClient client) {
this.client = client;
}
}
