com.yahoo.elide.async.operation.TableExportOperation Maven / Gradle / Ivy
/*
* Copyright 2021, Yahoo Inc.
* Licensed under the Apache License, Version 2.0
* See LICENSE file in project root for terms.
*/
package com.yahoo.elide.async.operation;
import com.yahoo.elide.Elide;
import com.yahoo.elide.async.AsyncSettings;
import com.yahoo.elide.async.export.formatter.TableExportFormatter;
import com.yahoo.elide.async.export.validator.SingleRootProjectionValidator;
import com.yahoo.elide.async.export.validator.Validator;
import com.yahoo.elide.async.models.AsyncApi;
import com.yahoo.elide.async.models.AsyncApiResult;
import com.yahoo.elide.async.models.FileExtensionType;
import com.yahoo.elide.async.models.TableExport;
import com.yahoo.elide.async.models.TableExportResult;
import com.yahoo.elide.async.service.AsyncExecutorService;
import com.yahoo.elide.async.service.storageengine.ResultStorageEngine;
import com.yahoo.elide.core.PersistentResource;
import com.yahoo.elide.core.RequestScope;
import com.yahoo.elide.core.datastore.DataStoreTransaction;
import com.yahoo.elide.core.exceptions.BadRequestException;
import com.yahoo.elide.core.request.EntityProjection;
import com.yahoo.elide.graphql.GraphQLSettings;
import com.yahoo.elide.jsonapi.JsonApiSettings;
import io.reactivex.Observable;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
/**
* TableExport Execute Operation Interface.
*/
@Slf4j
public abstract class TableExportOperation implements Callable {
private TableExportFormatter formatter;
@Getter private AsyncExecutorService service;
private Integer recordNumber = 0;
private TableExport exportObj;
private RequestScope scope;
private ResultStorageEngine engine;
private List validators = new ArrayList<>(Arrays.asList(new SingleRootProjectionValidator()));
public TableExportOperation(TableExportFormatter formatter, AsyncExecutorService service,
AsyncApi exportObj, RequestScope scope, ResultStorageEngine engine, List validators) {
this.formatter = formatter;
this.service = service;
this.exportObj = (TableExport) exportObj;
this.scope = scope;
this.engine = engine;
this.validators.addAll(validators);
}
@Override
public AsyncApiResult call() {
log.debug("TableExport Object from request: {}", exportObj);
Elide elide = service.getElide();
TableExportResult exportResult = new TableExportResult();
UUID requestId = UUID.fromString(exportObj.getRequestId());
try (DataStoreTransaction tx = elide.getDataStore().beginTransaction()) {
// Do Not Cache Export Results
Map> requestHeaders = new HashMap>();
requestHeaders.put("bypasscache", new ArrayList(Arrays.asList("true")));
RequestScope requestScope = getRequestScope(exportObj, scope, tx, requestHeaders);
Collection projections = getProjections(exportObj, requestScope);
validateProjections(projections);
EntityProjection projection = projections.iterator().next();
Observable observableResults = Observable.empty();
elide.getTransactionRegistry().addRunningTransaction(requestId, tx);
//TODO - we need to add the baseUrlEndpoint to the queryObject.
//TODO - Can we have projectionInfo as null?
requestScope.setEntityProjection(projection);
if (projection != null) {
projection.setPagination(null);
observableResults = PersistentResource.loadRecords(projection, Collections.emptyList(), requestScope);
}
Observable results = Observable.empty();
String preResult = formatter.preFormat(projection, exportObj);
results = observableResults.map(resource -> {
this.recordNumber++;
return formatter.format(resource, recordNumber);
});
String postResult = formatter.postFormat(projection, exportObj);
// Stitch together Pre-Formatted, Formatted, Post-Formatted results of Formatter in single observable.
Observable interimResults = concatStringWithObservable(preResult, results, true);
Observable finalResults = concatStringWithObservable(postResult, interimResults, false);
TableExportResult result = storeResults(exportObj, engine, finalResults);
if (result != null && result.getMessage() != null) {
throw new IllegalStateException(result.getMessage());
}
exportResult.setUrl(new URL(generateDownloadURL(exportObj, scope)));
exportResult.setRecordCount(recordNumber);
tx.flush(requestScope);
elide.getAuditLogger().commit();
tx.commit(requestScope);
} catch (BadRequestException e) {
exportResult.setMessage(e.getMessage());
} catch (MalformedURLException e) {
exportResult.setMessage("Download url generation failure.");
} catch (IOException e) {
log.error("IOException during TableExport", e);
exportResult.setMessage(e.getMessage());
} catch (Exception e) {
exportResult.setMessage(e.getMessage());
} finally {
// Follows same flow as GraphQL. The query may result in failure but request was successfully processed.
exportResult.setHttpStatus(200);
exportResult.setCompletedOn(new Date());
elide.getTransactionRegistry().removeRunningTransaction(requestId);
elide.getAuditLogger().clear();
}
return exportResult;
}
private Observable concatStringWithObservable(String toConcat, Observable observable,
boolean stringFirst) {
if (toConcat == null) {
return observable;
}
return stringFirst ? Observable.just(toConcat).concatWith(observable)
: observable.concatWith(Observable.just(toConcat));
}
/**
* Initializes a new RequestScope for the export operation with the submitted query.
* @param exportObj TableExport type object.
* @param scope RequestScope from the original submission.
* @param tx DataStoreTransaction.
* @param additionalRequestHeaders Additional Request Headers.
* @return RequestScope Type Object
*/
public abstract RequestScope getRequestScope(TableExport exportObj, RequestScope scope, DataStoreTransaction tx,
Map> additionalRequestHeaders);
/**
* Generate Download URL.
* @param exportObj TableExport type object.
* @param scope RequestScope.
* @return URL generated.
*/
public String generateDownloadURL(TableExport exportObj, RequestScope scope) {
AsyncSettings asyncSettings = scope.getElideSettings().getSettings(AsyncSettings.class);
JsonApiSettings jsonApiSettings = scope.getElideSettings().getSettings(JsonApiSettings.class);
GraphQLSettings graphqlSettings = scope.getElideSettings().getSettings(GraphQLSettings.class);
String downloadPath = asyncSettings.getExport().getPath();
String baseURL = scope.getRoute().getBaseUrl();
if (jsonApiSettings != null) {
String jsonApiPath = jsonApiSettings.getPath();
if (jsonApiPath != null && baseURL.endsWith(jsonApiPath)) {
baseURL = baseURL.substring(0, baseURL.length() - jsonApiPath.length());
}
}
if (graphqlSettings != null) {
String graphqlApiPath = graphqlSettings.getPath();
if (graphqlApiPath != null && baseURL.endsWith(graphqlApiPath)) {
baseURL = baseURL.substring(0, baseURL.length() - graphqlApiPath.length());
}
}
String extension = this.engine.isExtensionEnabled()
? exportObj.getResultType().getFileExtensionType().getExtension()
: FileExtensionType.NONE.getExtension();
return baseURL + downloadPath + "/" + exportObj.getId() + extension;
}
/**
* Store Export Results using the ResultStorageEngine.
* @param exportObj TableExport type object.
* @param resultStorageEngine ResultStorageEngine instance.
* @param result Observable of String Results to store.
* @return TableExportResult object.
*/
protected TableExportResult storeResults(TableExport exportObj, ResultStorageEngine resultStorageEngine,
Observable result) {
return resultStorageEngine.storeResults(exportObj, result);
}
private void validateProjections(Collection projections) {
validators.forEach(validator -> validator.validateProjection(projections));
}
/**
* Generate Entity Projection from the query.
* @param exportObj TableExport type object.
* @param requestScope requestScope object.
* @return Collection of EntityProjection object.
*/
public abstract Collection getProjections(TableExport exportObj, RequestScope requestScope);
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy