org.bitbucket.bradleysmithllc.etlunit.feature.database.DbAssertExtender Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of etlunit-database Show documentation
There is a newer version: 4.6.0-eu
package org.bitbucket.bradleysmithllc.etlunit.feature.database;

import org.bitbucket.bradleysmithllc.etlunit.*;
import org.bitbucket.bradleysmithllc.etlunit.context.VariableContext;
import org.bitbucket.bradleysmithllc.etlunit.feature.Feature;
import org.bitbucket.bradleysmithllc.etlunit.feature.RuntimeOption;
import org.bitbucket.bradleysmithllc.etlunit.feature.assertion.Assertion;
import org.bitbucket.bradleysmithllc.etlunit.feature.database.db.Table;
import org.bitbucket.bradleysmithllc.etlunit.feature.database.json.database._assert.AssertHandler;
import org.bitbucket.bradleysmithllc.etlunit.feature.database.json.database._assert.AssertRequest;
import org.bitbucket.bradleysmithllc.etlunit.feature.file.FileRuntimeSupport;
import org.bitbucket.bradleysmithllc.etlunit.io.FileBuilder;
import org.bitbucket.bradleysmithllc.etlunit.io.file.*;
import org.bitbucket.bradleysmithllc.etlunit.feature.extend.Extender;
import org.bitbucket.bradleysmithllc.etlunit.metadata.MetaDataArtifact;
import org.bitbucket.bradleysmithllc.etlunit.metadata.MetaDataArtifactContent;
import org.bitbucket.bradleysmithllc.etlunit.metadata.MetaDataContext;
import org.bitbucket.bradleysmithllc.etlunit.metadata.MetaDataPackageContext;
import org.bitbucket.bradleysmithllc.etlunit.parser.ETLTestMethod;
import org.bitbucket.bradleysmithllc.etlunit.parser.ETLTestOperation;
import org.bitbucket.bradleysmithllc.etlunit.parser.ETLTestValueObject;
import org.bitbucket.bradleysmithllc.etlunit.parser.ETLTestValueObjectBuilder;
import org.bitbucket.bradleysmithllc.etlunit.util.IOUtils;

import javax.inject.Inject;
import javax.inject.Named;
import java.io.File;
import java.io.IOException;
import java.sql.Types;
import java.util.*;

public class DbAssertExtender implements Extender, AssertHandler
{
	private DataFileManager dataFileManager;

	private DiffManager diffManager;

	private final DatabaseFeatureModule parent;
	private final DatabaseConfiguration databaseConfiguration;
	private DatabaseRuntimeSupport databaseRuntimeSupport;
	private RuntimeSupport runtimeSupport;
	private FileRuntimeSupport fileRuntimeSupport;

	@Inject
	@Named("database.refreshAssertionData")
	private RuntimeOption refreshAssertionData = null;

	protected Log applicationLog;
	private Map columnTypes;
	private List columns;

	public DbAssertExtender(DatabaseFeatureModule parent)
	{
		this.parent = parent;
		databaseConfiguration = parent.getDatabaseConfiguration();

		columns = Arrays.asList("ROW_NUMBER", "COLUMN_NAME", "SOURCE", "TARGET");

		columnTypes = new HashMap();

		columnTypes.put("ROW_NUMBER", Types.INTEGER);
		columnTypes.put("COLUMN_NAME", Types.VARCHAR);
		columnTypes.put("SOURCE", Types.VARCHAR);
		columnTypes.put("TARGET", Types.VARCHAR);
	}

	@Inject
	public void receiveDataFileManager(DataFileManager manager)
	{
		dataFileManager = manager;
	}

	@Inject
	public void receiveRuntimeSupport(RuntimeSupport support)
	{
		runtimeSupport = support;
	}

	@Inject
	public void receiveFileRuntimeSupport(FileRuntimeSupport support)
	{
		fileRuntimeSupport = support;
	}

	@Inject
	public void setApplicationLog(@Named("applicationLog") Log log)
	{
		applicationLog = log;
	}

	@Inject
	public void setDatabaseRuntimeSupport(DatabaseRuntimeSupport databaseRuntimeSupport)
	{
		this.databaseRuntimeSupport = databaseRuntimeSupport;
	}

	@Inject
	public void receiveDiffManager(DiffManager reporter)
	{
		diffManager = reporter;
	}

	public ClassResponder.action_code process(ETLTestMethod mt, ETLTestOperation op, ETLTestValueObject operands, VariableContext context, ExecutionContext executionContext) throws TestAssertionFailure, TestExecutionError, TestWarning
	{
		return action_code.defer;
	}

	public Feature getFeature()
	{
		return parent;
	}

	public action_code _assert(AssertRequest operation, ETLTestMethod mt, ETLTestOperation op, ETLTestValueObject obj, VariableContext context, ExecutionContext econtext) throws TestAssertionFailure, TestExecutionError, TestWarning
	{
		AssertRequest.AssertionMode a_mode = operation.getAssertionMode();
		final Assertion.assertion_mode comparisonMode;

		if (a_mode == null)
		{
			comparisonMode = Assertion.assertion_mode.equals;
		}
		else
		{
			switch (a_mode)
			{
				case EQUALS:
					comparisonMode = Assertion.assertion_mode.equals;
					break;
				case EMPTY:
					comparisonMode = Assertion.assertion_mode.empty;
					break;
				case NOT_EMPTY:
					comparisonMode = Assertion.assertion_mode.not_empty;
					break;
				default:
					throw new TestExecutionError("Invalid assertion mode: " + a_mode, DatabaseConstants.ERR_INVALID_ASSERTION_MODE);
			}
		}

		String tar = operation.getTarget();

		// check the runtime option for refresh data.  If so, convert this call into an extract operation for the assert equals case
		if (refreshAssertionData.isEnabled() && comparisonMode == Assertion.assertion_mode.equals)
		{
			// compare to the file in the test folder
			File testFile = databaseRuntimeSupport.getSourceDataSetForCurrentTest(tar, "assertion-refresh-target");

			ETLTestValueObject copy = obj.copy();

			ETLTestValueObjectBuilder cbuilder = new ETLTestValueObjectBuilder(copy);

			// get rid of assertion mode and failure-id
			if (copy.query("assertion-mode") != null)
			{
				cbuilder.removeKey("assertion-mode");
			}

			if (copy.query("failure-id") != null)
			{
				cbuilder.removeKey("failure-id");
			}

			copy = cbuilder.toObject();

			ETLTestOperation extract_op = op.createSibling("extract", copy);

			econtext.process(extract_op, context);

			return ClassResponder.action_code.handled;
		}

		String sourceTable = operation.getSourceTable();
		String failureId = operation.getFailureId();

		String qualifiedName = sourceTable;

		String schema = operation.getSourceSchema();

		if (schema != null)
		{
			qualifiedName = schema + "." + sourceTable;
		}

		DatabaseClassListener.ConnectionMode connMode = DatabaseClassListener.getConnection(parent, obj, context, op.getTestMethod()).get(0);

		final DatabaseConnection conn = connMode.conn;

		final String mode = connMode.getMode();

		// now send it off to the implementation
		DatabaseImplementation impl = DatabaseClassListener.getDatabaseImplementation(parent, conn);

		// dispatch this as an operation (even though this feature will ultimately process)
		ETLTestValueObject copy = obj.copy();

		File target;

		// create a target file attribute for our extract
		if (tar != null)
		{
			target = databaseRuntimeSupport.getGeneratedDataSet(conn, connMode.getPrettyMode(), op.getTestMethod().getQualifiedName() + "_tgt_" + tar);
		}
		else
		{
			target = databaseRuntimeSupport.getGeneratedDataSet(conn, connMode.getPrettyMode(), sourceTable + "_" + impl.getDatabaseName(conn, mode) + "_" + op.getTestMethod().getName());
		}

		// remove the assertion-mode attribute since it does not pass validation
		// for the extract operation
		ETLTestValueObjectBuilder cbuilder = new ETLTestValueObjectBuilder(copy);
		if (copy.query("assertion-mode") != null)
		{
			cbuilder = cbuilder.removeKey("assertion-mode");
		}

		if (copy.query("failure-id") != null)
		{
			cbuilder = cbuilder.removeKey("failure-id");
		}

		if (copy.query("reference-file-type") != null)
		{
			cbuilder = cbuilder.removeKey("reference-file-type");
		}

		cbuilder = cbuilder.key("target-file")
				.value(target.getAbsolutePath());

		copy = cbuilder.toObject();

		ETLTestOperation extract_op = op.createSibling("extract", copy);

		econtext.process(extract_op, context);

		DataFileSchema ffs = null;

		// grab the schema the extract produced
		if (context.hasVariableBeenDeclared(DatabaseClassListener.LAST_EXTRACT_FILE_SCHEMA))
		{
			ffs = (DataFileSchema) context.getValue(DatabaseClassListener.LAST_EXTRACT_FILE_SCHEMA).getValueAsPojo();
		}

		Table table = null;

		// grab the schema the extract produced
		if (context.hasVariableBeenDeclared(DatabaseClassListener.LAST_EXTRACT_TABLE))
		{
			table = (Table) context.getValue(DatabaseClassListener.LAST_EXTRACT_TABLE).getValueAsPojo();
		}

		DataFileSchema ffsTarget = ffs;

		switch (comparisonMode)
		{
			case equals:
			{
				String synthFailureId = connMode.connectionId + "." + connMode.getPrettyMode() + "." + schema + "." + operation.getTarget();

				if (tar == null)
				{
					throw new TestExecutionError("Assert equals requires a target argument", DatabaseConstants.ERR_ASSERT_MISSING_TARGET);
				}

				// compare to the file in the test folder
				File testFile = databaseRuntimeSupport.getSourceDataSetForCurrentTest(tar, "assertion-target");

				// throw a checked exception if the target does not exist
				if (!testFile.exists())
				{
					throw new TestExecutionError("Data set [" + tar + "] not found in project", DatabaseConstants.ERR_MISSSING_DATA_FILE);
				}

				List diffReport = null;

				try
				{
					DataFile fExpected = dataFileManager.loadDataFile(testFile, ffs);
					DataFile lExpected = null;

					// additionally, if a reference-file-type is specified create a view of the table using the exclude columns
					if (operation.getReferenceFileType() != null)
					{
						// load the schema
						DataFileSchema ffsSource = databaseRuntimeSupport.getRelationalMetaInfo(conn, schema, sourceTable, sourceTable, obj);
						fExpected = dataFileManager.loadDataFile(testFile, ffsSource);

						// create a view of this schema which intersects the extract
						DataFileSchema ffsSourceIntersect = ffsSource.intersect(ffs, ffs.getId() + "-synthetic-" + op.getQualifiedName());

						// store the generated file schema for reference
						{
							fileRuntimeSupport.persistGeneratedDataFileSchemaForCurrentTest(ffsSourceIntersect);
						}

						// create a copy of the source data set to match the new schema
						File dataSetLocalTarget = databaseRuntimeSupport.getGeneratedDataSet(conn, connMode.getPrettyMode(), op.getTestMethod().getQualifiedName() + "_src_" + IOUtils.removeExtension(testFile.getName()));

						applicationLog.info("Creating cached copy of " + testFile.getAbsolutePath() + "] to [" + dataSetLocalTarget.getAbsolutePath() + "]");

						lExpected = dataFileManager.loadDataFile(dataSetLocalTarget, ffsSourceIntersect);
						dataFileManager.copyDataFile(fExpected, lExpected);
					}
					else
					{
						File dataSetLocal = databaseRuntimeSupport.getGeneratedDataSet(conn, connMode.getPrettyMode(), op.getTestMethod().getQualifiedName() + "_src_" + IOUtils.removeExtension(testFile.getName()));

						applicationLog.info("Creating cached copy of " + testFile.getAbsolutePath() + "] to [" + dataSetLocal.getAbsolutePath() + "]");

						lExpected = dataFileManager.loadDataFile(dataSetLocal, ffs);

						// copy to local cache
						try
						{
							dataFileManager.copyDataFile(fExpected, lExpected);
						}
						catch(DataFileMismatchException exc)
						{
							throw new TestExecutionError("Data set [" + testFile.getName() + "] does not match schema", DatabaseConstants.ERR_DATA_FILE_SCHEMA_MISMATCH);
						}
					}

					DataFile fActual = dataFileManager.loadDataFile(target, ffsTarget);

					applicationLog.info("Asserting equality of [" + lExpected.getSource().getAbsolutePath() + "] and [" + fActual.getSource().getAbsolutePath() + "]");

					String variableName = "extract.columns." + tar;
					applicationLog.info("Reading extract column list for " + variableName);

					/*
					ETLTestValueObject value = context.getValue(variableName);
					if (value == null)
					{
						throw new TestExecutionError("", DatabaseConstants.ERR_MISSING_COL_SPECIFICATION);
					}

					List columns = (List) value.getValueAsPojo();
					 */

					diffReport = dataFileManager.diff(lExpected, fActual);

					if (diffReport.size() != 0)
					{
						dataFileManager.report(
							diffManager,
							op,
							failureId != null ? failureId : synthFailureId,
							diffReport,
							operation.getTarget(),
							table != null ? table.getQualifiedName() : sourceTable
						);

						throw new TestAssertionFailure("Table <" + sourceTable + "> does not match <" + operation.getTarget() + ">", failureId != null ? failureId : synthFailureId);
					}
				}
				/*
				catch (TestExecutionError e)
				{
					throw e;
				}
				 */
				catch (TestAssertionFailure e)
				{
					throw e;
				}
				catch (TestExecutionError e)
				{
					throw e;
				}
				catch (Exception e)
				{
					throw new TestExecutionError("", e);
				}

				break;
			}
			case empty:
				// for these two, just dump the table to a file and assert file size is or isn't 0
				// visit the file, skipping the header.
				// if there are any non-blank or non-comment rows then this is a fail

			{
				String synthFailureId = connMode.connectionId + "." + connMode.getPrettyMode() + "." + qualifiedName;

				try
				{
					DataFile fSource = dataFileManager.loadDataFile(target, ffs);

					int lines = 0;

					DataFile.FileData fdata = fSource.getFileData();

					try
					{
						Iterator it = fdata.iterator();

						while (it.hasNext())
						{
							it.next();
							lines++;
						}
					}
					finally
					{
						fdata.dispose();
					}

					if (lines != 0)
					{
						throw new TestAssertionFailure("Object " + synthFailureId + " not empty", failureId != null ? failureId : synthFailureId);
					}
				}
				catch (IOException e)
				{
					throw new TestExecutionError("", e);
				}

				break;
			}
			case not_empty:
			{
				String synthFailureId = connMode.connectionId + "." + connMode.getPrettyMode() + "." + qualifiedName;

				try
				{
					DataFile fSource = dataFileManager.loadDataFile(target, ffs);

					int lines = 0;

					DataFile.FileData fdata = fSource.getFileData();

					try
					{
						Iterator it = fdata.iterator();

						while (it.hasNext())
						{
							it.next();
							lines++;
						}
					}
					finally
					{
						fdata.dispose();
					}

					if (lines == 0)
					{
						throw new TestAssertionFailure("Object " + synthFailureId + " is empty", failureId != null ? failureId : synthFailureId);
					}
				}
				catch (IOException e)
				{
					throw new TestExecutionError("", e);
				}

				break;
			}
		}

		return ClassResponder.action_code.handled;
	}
}