org.bitbucket.bradleysmithllc.etlunit.feature.database.DbAssertExtender Maven / Gradle / Ivy
package org.bitbucket.bradleysmithllc.etlunit.feature.database;
/*
* #%L
* etlunit-database
* %%
* Copyright (C) 2010 - 2014 bradleysmithllc
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
import org.apache.commons.lang3.ObjectUtils;
import org.bitbucket.bradleysmithllc.etlunit.*;
import org.bitbucket.bradleysmithllc.etlunit.context.VariableContext;
import org.bitbucket.bradleysmithllc.etlunit.feature.Feature;
import org.bitbucket.bradleysmithllc.etlunit.feature.RuntimeOption;
import org.bitbucket.bradleysmithllc.etlunit.feature.database.db.Table;
import org.bitbucket.bradleysmithllc.etlunit.feature.database.json.database._assert.AssertHandler;
import org.bitbucket.bradleysmithllc.etlunit.feature.database.json.database._assert.AssertRequest;
import org.bitbucket.bradleysmithllc.etlunit.feature.file.FileRuntimeSupport;
import org.bitbucket.bradleysmithllc.etlunit.io.file.*;
import org.bitbucket.bradleysmithllc.etlunit.feature.extend.Extender;
import org.bitbucket.bradleysmithllc.etlunit.listener.ClassResponder;
import org.bitbucket.bradleysmithllc.etlunit.metadata.MetaDataPackageContext;
import org.bitbucket.bradleysmithllc.etlunit.parser.ETLTestMethod;
import org.bitbucket.bradleysmithllc.etlunit.parser.ETLTestOperation;
import org.bitbucket.bradleysmithllc.etlunit.parser.ETLTestValueObject;
import org.bitbucket.bradleysmithllc.etlunit.parser.ETLTestValueObjectBuilder;
import javax.inject.Inject;
import javax.inject.Named;
import java.io.File;
import java.sql.Types;
import java.util.*;
public class DbAssertExtender implements Extender, AssertHandler
{
private DataFileManager dataFileManager;
private DiffManager diffManager;
private final DatabaseFeatureModule parent;
private final DatabaseConfiguration databaseConfiguration;
private DatabaseRuntimeSupport databaseRuntimeSupport;
private RuntimeSupport runtimeSupport;
private FileRuntimeSupport fileRuntimeSupport;
@Inject
@Named("database.refreshAssertionData")
private RuntimeOption refreshAssertionData = null;
protected Log applicationLog;
private Map columnTypes;
private List columns;
public DbAssertExtender(DatabaseFeatureModule parent)
{
this.parent = parent;
databaseConfiguration = parent.getDatabaseConfiguration();
columns = Arrays.asList("ROW_NUMBER", "COLUMN_NAME", "SOURCE", "TARGET");
columnTypes = new HashMap();
columnTypes.put("ROW_NUMBER", Types.INTEGER);
columnTypes.put("COLUMN_NAME", Types.VARCHAR);
columnTypes.put("SOURCE", Types.VARCHAR);
columnTypes.put("TARGET", Types.VARCHAR);
}
@Inject
public void receiveDataFileManager(DataFileManager manager)
{
dataFileManager = manager;
}
@Inject
public void receiveRuntimeSupport(RuntimeSupport support)
{
runtimeSupport = support;
}
@Inject
public void receiveFileRuntimeSupport(FileRuntimeSupport support)
{
fileRuntimeSupport = support;
}
@Inject
public void setApplicationLog(@Named("applicationLog") Log log)
{
applicationLog = log;
}
@Inject
public void setDatabaseRuntimeSupport(DatabaseRuntimeSupport databaseRuntimeSupport)
{
this.databaseRuntimeSupport = databaseRuntimeSupport;
}
@Inject
public void receiveDiffManager(DiffManager reporter)
{
diffManager = reporter;
}
@Override
public ClassResponder.action_code process(ETLTestMethod mt, ETLTestOperation op, ETLTestValueObject operands, VariableContext context, ExecutionContext executionContext, int executor) throws TestAssertionFailure, TestExecutionError, TestWarning
{
return action_code.defer;
}
public Feature getFeature()
{
return parent;
}
public action_code _assert(final AssertRequest operation, final ETLTestMethod mt, final ETLTestOperation op, final ETLTestValueObject obj, final VariableContext context, ExecutionContext econtext) throws TestAssertionFailure, TestExecutionError, TestWarning
{
final String sourceTable = operation.getSourceTable();
String qualifiedName = operation.getSourceTable();
String schema = operation.getSourceSchema();
if (schema != null)
{
qualifiedName = schema + "." + sourceTable;
}
DatabaseClassListener.ConnectionMode connMode = DatabaseClassListener.getConnection(parent, obj, context, op.getTestMethod()).get(0);
final DatabaseConnection conn = connMode.conn;
final String mode = connMode.getMode();
// now send it off to the implementation
DatabaseImplementation impl = DatabaseClassListener.getDatabaseImplementation(parent, conn);
final String targetFailureId = connMode.connectionId + "." + connMode.getPrettyMode() + "." + schema + "." + operation.getTarget();
final String sourceFailureId = connMode.connectionId + "." + connMode.getPrettyMode() + "." + qualifiedName;
// dispatch this as an operation (even though this feature will ultimately process)
ETLTestValueObject copy = obj.copy();
final File target;
final String tar = operation.getTarget();
// create a target file attribute for our extract
if (tar != null)
{
target = databaseRuntimeSupport.getGeneratedDataSet(conn, connMode.getPrettyMode(), op.getTestMethod().getQualifiedName() + "_tgt_" + tar);
}
else
{
target = databaseRuntimeSupport.getGeneratedDataSet(conn, connMode.getPrettyMode(), sourceTable + "_" + impl.getDatabaseName(conn, mode) + "_" + op.getTestMethod().getName());
}
// remove the assertion-mode attribute since it does not pass validation
// for the extract operation
/**
* Remove the column list mode and list attributes - I.E., do a full extract,
* so that the data set comparator has a complete data set to work with.
*/
ETLTestValueObjectBuilder cbuilder = new ETLTestValueObjectBuilder(copy);
if (copy.query("assertion-mode") != null)
{
cbuilder = cbuilder.removeKey("assertion-mode");
}
if (copy.query("column-list-mode") != null)
{
cbuilder = cbuilder.removeKey("column-list-mode");
}
if (copy.query("column-list") != null)
{
cbuilder = cbuilder.removeKey("column-list");
}
if (copy.query("failure-id") != null)
{
cbuilder = cbuilder.removeKey("failure-id");
}
// if source is supplied, that should be passed on to the extract operation
ETLTestValueObject trftquery = copy.query("source-reference-file-type");
if (trftquery != null && !trftquery.isNull())
{
cbuilder.key("reference-file-type").value(trftquery.getValueAsString());
cbuilder.removeKey("source-reference-file-type");
}
if (copy.query("target-reference-file-type") != null)
{
cbuilder = cbuilder.removeKey("target-reference-file-type");
}
cbuilder = cbuilder.key("target-file")
.value(target.getAbsolutePath());
copy = cbuilder.toObject();
ETLTestOperation extract_op = op.createSibling("extract", copy);
econtext.process(extract_op, context);
final DataFileSchema extractFileSchema;
// grab the schema the extract produced
if (context.hasVariableBeenDeclared(DatabaseClassListener.LAST_EXTRACT_FILE_SCHEMA))
{
extractFileSchema = (DataFileSchema) context.getValue(DatabaseClassListener.LAST_EXTRACT_FILE_SCHEMA).getValueAsPojo();
}
else
{
extractFileSchema = null;
}
final Table table;
// grab the schema the extract produced
if (context.hasVariableBeenDeclared(DatabaseClassListener.LAST_EXTRACT_TABLE))
{
table = (Table) context.getValue(DatabaseClassListener.LAST_EXTRACT_TABLE).getValueAsPojo();
}
else
{
table = null;
}
// defer to the file runtime support implementation
fileRuntimeSupport.processDataSetAssertion(
new FileRuntimeSupport.DataSetAssertionRequest() {
@Override
public assertMode getAssertMode() {
AssertRequest.AssertionMode assertionMode = operation.getAssertionMode();
return assertionMode != null ? FileRuntimeSupport.DataSetAssertionRequest.assertMode.valueOf(assertionMode.name().toLowerCase()) : assertMode.equals;
}
@Override
public String getActualSchemaName() {
return ObjectUtils.firstNonNull(operation.getSourceTable(), operation.getSqlScript());
}
@Override
public String getActualBackupSchemaName() {
return databaseRuntimeSupport.createSchemaKey(table);
}
@Override
public String getExpectedSchemaName() {
return tar;
}
@Override
public String getExpectedBackupSchemaName() {
return getExpectedSchemaName();
}
@Override
public boolean hasColumnList() {
return operation.getColumnListMode() != null;
}
@Override
public columnListMode getColumnListMode() {
AssertRequest.ColumnListMode columnListMode2 = operation.getColumnListMode();
return columnListMode2 != null ? FileRuntimeSupport.DataSetAssertionRequest.columnListMode.valueOf(columnListMode2.name().toLowerCase()) : columnListMode.exclude;
}
@Override
public Set getColumnList() {
return operation.getColumnList();
}
@Override
public String getFailureId() {
String failureId1 = operation.getFailureId();
if (
getAssertMode() == assertMode.empty
||
getAssertMode() == assertMode.not_empty
)
{
return ObjectUtils.firstNonNull(failureId1, sourceFailureId);
}
else if (
getAssertMode() == assertMode.equals
)
{
return ObjectUtils.firstNonNull(failureId1, targetFailureId);
}
return failureId1;
}
@Override
public DataFile getExpectedDataset(DataFileSchema schema) {
return dataFileManager.loadDataFile(databaseRuntimeSupport.getSourceDataSetForCurrentTest(tar, "assertion-target"), schema);
}
@Override
public DataFile getActualDataset(DataFileSchema schema) {
return dataFileManager.loadDataFile(target, schema);
}
@Override
public ETLTestMethod getTestMethod() {
return mt;
}
@Override
public ETLTestOperation getTestOperation() {
return op;
}
@Override
public ETLTestValueObject getOperationParameters() {
return obj;
}
@Override
public VariableContext getVariableContext() {
return context;
}
@Override
public MetaDataPackageContext getMetaDataPackageContext() {
return null;
}
@Override
public boolean refreshAssertionData() {
return refreshAssertionData.isEnabled();
}
@Override
public DataFileSchema getSourceSchema() {
return extractFileSchema;
}
}
);
if (true)
{
return action_code.handled;
}
/**
* This needs to follow the same pattern as file asertions. Load source or base and use as normal,
* then evaluate target thusly:
* if (tgt specified)
* {
* open target with tgt.fml, then process through the view
* }
* else
* {
* open target with the view
* }
*
* Also, store the schemas in the context as file does for testing. And in the case of refresh assertion data, store usin the
* target schema.
*/
/*
if (a_mode == null)
{
comparisonMode = Assertion.assertion_mode.equals;
}
else
{
switch (a_mode)
{
case EQUALS:
comparisonMode = Assertion.assertion_mode.equals;
break;
case EMPTY:
comparisonMode = Assertion.assertion_mode.empty;
break;
case NOT_EMPTY:
comparisonMode = Assertion.assertion_mode.not_empty;
break;
default:
throw new TestExecutionError("Invalid assertion mode: " + a_mode, DatabaseConstants.ERR_INVALID_ASSERTION_MODE);
}
}
// check the runtime option for refresh data. If so, convert this call into an extract operation for the assert equals case
if (refreshAssertionData.isEnabled() && comparisonMode == Assertion.assertion_mode.equals)
{
// compare to the file in the test folder
File testFile = databaseRuntimeSupport.getSourceDataSetForCurrentTest(tar, "assertion-refresh-target");
ETLTestValueObject copy = obj.copy();
ETLTestValueObjectBuilder cbuilder = new ETLTestValueObjectBuilder(copy);
// get rid of assertion mode and failure-id
if (copy.query("assertion-mode") != null)
{
cbuilder.removeKey("assertion-mode");
}
// if target is supplied, that should be passed on to the extract operation
ETLTestValueObject trftquery = copy.query("target-reference-file-type");
if (trftquery != null)
{
cbuilder.key("reference-file-type").value(trftquery.getValueAsString());
cbuilder.removeKey("target-reference-file-type");
}
if (copy.query("failure-id") != null)
{
cbuilder.removeKey("failure-id");
}
copy = cbuilder.toObject();
ETLTestOperation extract_op = op.createSibling("extract", copy);
econtext.process(extract_op, context);
return ClassResponder.action_code.handled;
}
switch (comparisonMode)
{
case equals:
{
String synthFailureId = connMode.connectionId + "." + connMode.getPrettyMode() + "." + schema + "." + operation.getTarget();
if (tar == null)
{
throw new TestExecutionError("Assert equals requires a target argument", DatabaseConstants.ERR_ASSERT_MISSING_TARGET);
}
// compare to the file in the test folder
File testFile = databaseRuntimeSupport.getSourceDataSetForCurrentTest(tar, "assertion-target");
// throw a checked exception if the target does not exist
if (!testFile.exists())
{
throw new TestExecutionError("Data set [" + tar + "] not found in project", DatabaseConstants.ERR_MISSING_DATA_FILE);
}
List diffReport = null;
try
{
// optionally, check for a source reference file type
DataFileSchema sourceFml = null;
try
{
sourceFml = fileRuntimeSupport.locateReferenceFileSchemaForCurrentTest(tar, tar, obj, FileRuntimeSupport.schemaType.target);
}
catch(ResourceNotFoundException rfne)
{
// don't let this get out - it's not an error
}
DataFile fExpected = null;//dataFileManager.loadDataFile(testFile, extractFileSchema);
DataFile lExpected = null;
// additionally, if a reference-file-type is specified create a view of the table using the exclude columns
if (sourceFml == null)
{
sourceFml = extractFileSchema;
}
// load the data file
fExpected = dataFileManager.loadDataFile(testFile, sourceFml);
// create a view of this schema which intersects the extract
DataFileSchema ffsSourceIntersect = sourceFml.intersect(extractFileSchema, sourceFml.getId() + "-synthetic-" + op.getQualifiedName());
// store the generated file schema for reference
fileRuntimeSupport.persistGeneratedDataFileSchemaForCurrentTest(ffsSourceIntersect);
// create a copy of the source data set to match the new schema
File dataSetLocalTarget = databaseRuntimeSupport.getGeneratedDataSet(conn, connMode.getPrettyMode(), op.getTestMethod().getQualifiedName() + "_src_" + IOUtils.removeExtension(testFile.getName()));
applicationLog.info("Creating cached copy of " + testFile.getAbsolutePath() + "] to [" + dataSetLocalTarget.getAbsolutePath() + "]");
lExpected = dataFileManager.loadDataFile(dataSetLocalTarget, ffsSourceIntersect);
try
{
dataFileManager.copyDataFile(fExpected, lExpected);
}
catch(DataFileMismatchException dfme)
{
throw new TestExecutionError(dfme.getMessage(), DatabaseConstants.ERR_DATA_FILE_SCHEMA_MISMATCH, dfme);
}
DataFile fActual = dataFileManager.loadDataFile(target, extractFileSchema);
//This doesn't work when providing the default reference file type, but not wanting it to apply to the source
applicationLog.info("Asserting equality of [" + lExpected.getSource().getAbsolutePath() + "] and [" + fActual.getSource().getAbsolutePath() + "]");
String variableName = "extract.columns." + tar;
applicationLog.info("Reading extract column list for " + variableName);
try
{
diffReport = dataFileManager.diff(lExpected, fActual);
}
catch(DataFileMismatchException exc)
{
throw new TestExecutionError(exc.toString(), DatabaseConstants.ERR_DATA_FILE_SCHEMA_MISMATCH, exc);
}
if (diffReport.size() != 0)
{
dataFileManager.report(
diffManager,
mt,
op,
failureId != null ? failureId : synthFailureId,
diffReport,
operation.getTarget(),
table != null ? table.getQualifiedName() : sourceTable
);
throw new TestAssertionFailure("Table <" + sourceTable + "> does not match <" + operation.getTarget() + ">", failureId != null ? failureId : synthFailureId);
}
}
catch (TestAssertionFailure e)
{
throw e;
}
catch (TestExecutionError e)
{
throw e;
}
catch (Exception e)
{
throw new TestExecutionError("", e);
}
break;
}
case empty:
// for these two, just dump the table to a file and assert file size is or isn't 0
// visit the file, skipping the header.
// if there are any non-blank or non-comment rows then this is a fail
{
String synthFailureId = connMode.connectionId + "." + connMode.getPrettyMode() + "." + qualifiedName;
try
{
DataFile fSource = dataFileManager.loadDataFile(target, extractFileSchema);
int lines = 0;
DataFile.FileData fdata = fSource.reader();
try
{
Iterator it = fdata.iterator();
while (it.hasNext())
{
it.next();
lines++;
}
}
finally
{
fdata.dispose();
}
if (lines != 0)
{
throw new TestAssertionFailure("Object " + synthFailureId + " not empty", failureId != null ? failureId : synthFailureId);
}
}
catch (IOException e)
{
throw new TestExecutionError("", e);
}
break;
}
case not_empty:
{
String synthFailureId = connMode.connectionId + "." + connMode.getPrettyMode() + "." + qualifiedName;
try
{
DataFile fSource = dataFileManager.loadDataFile(target, extractFileSchema);
int lines = 0;
DataFile.FileData fdata = fSource.reader();
try
{
Iterator it = fdata.iterator();
while (it.hasNext())
{
it.next();
lines++;
}
}
finally
{
fdata.dispose();
}
if (lines == 0)
{
throw new TestAssertionFailure("Object " + synthFailureId + " is empty", failureId != null ? failureId : synthFailureId);
}
}
catch (IOException e)
{
throw new TestExecutionError("", e);
}
break;
}
}
*/
return ClassResponder.action_code.handled;
}
} © 2015 - 2025 Weber Informatics LLC | Privacy Policy