org.bitbucket.bradleysmithllc.etlunit.feature.file.FileAssertHandler Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of etlunit-file Show documentation
The newest version!
package org.bitbucket.bradleysmithllc.etlunit.feature.file;

/*
 * #%L
 * etlunit-file
 * %%
 * Copyright (C) 2010 - 2023 bradleysmithllc
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import com.google.inject.name.Named;
import org.apache.commons.io.FileUtils;
import org.bitbucket.bradleysmithllc.etlunit.*;
import org.bitbucket.bradleysmithllc.etlunit.context.VariableContext;
import org.bitbucket.bradleysmithllc.etlunit.feature.Feature;
import org.bitbucket.bradleysmithllc.etlunit.feature.RuntimeOption;
import org.bitbucket.bradleysmithllc.etlunit.feature.file.json.file._assert.assert_data_set.AssertDataSetHandler;
import org.bitbucket.bradleysmithllc.etlunit.feature.file.json.file._assert.assert_data_set.AssertDataSetRequest;
import org.bitbucket.bradleysmithllc.etlunit.feature.file.json.file._assert.assert_file.AssertFileHandler;
import org.bitbucket.bradleysmithllc.etlunit.feature.file.json.file._assert.assert_file.AssertFileRequest;
import org.bitbucket.bradleysmithllc.etlunit.io.file.CopyOptionsBuilder;
import org.bitbucket.bradleysmithllc.etlunit.io.file.DataFile;
import org.bitbucket.bradleysmithllc.etlunit.io.file.DataFileManager;
import org.bitbucket.bradleysmithllc.etlunit.io.file.DataFileSchema;
import org.bitbucket.bradleysmithllc.etlunit.io.file.reference_file_type.ReferenceFileTypeRef;
import org.bitbucket.bradleysmithllc.etlunit.listener.ClassResponder;
import org.bitbucket.bradleysmithllc.etlunit.listener.NullClassListener;
import org.bitbucket.bradleysmithllc.etlunit.metadata.MetaDataContext;
import org.bitbucket.bradleysmithllc.etlunit.metadata.MetaDataPackageContext;
import org.bitbucket.bradleysmithllc.etlunit.parser.ETLTestMethod;
import org.bitbucket.bradleysmithllc.etlunit.parser.ETLTestOperation;
import org.bitbucket.bradleysmithllc.etlunit.parser.ETLTestValueObject;
import org.bitbucket.bradleysmithllc.etlunit.util.ObjectUtils;

import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.util.Set;

public class FileAssertHandler extends NullClassListener implements AssertFileHandler, AssertDataSetHandler
{
	private interface ExpectedFinder
	{
		File get(String expectedTargetFile, DataFileSchema.format_type formatType);
	}

	private final FileFeatureModule fileFeatureModule;
	private FileRuntimeSupport fileRuntimeSupport;
	private RuntimeSupport runtimeSupport;

	private DiffManager diffManager;
	private Log applicationLog;

	private DataFileManager dataFileManager;

	@Inject
	@Named("file.refreshAssertionData")
	private RuntimeOption refreshAssertionData = null;

	public FileAssertHandler(FileFeatureModule fileFeatureModule)
	{
		this.fileFeatureModule = fileFeatureModule;
	}

	@Inject
	public void receiveDataFileManager(DataFileManager dfm)
	{
		dataFileManager = dfm;
	}

	@Inject
	public void receiveRuntimeSupport(RuntimeSupport rs)
	{
		runtimeSupport = rs;
	}

	@Inject
	public void receiveApplicationLog(@Named("applicationLog") Log log)
	{
		this.applicationLog = log;
	}

	@Inject
	public void receiveFileRuntimeSupport(FileRuntimeSupport support)
	{
		fileRuntimeSupport = support;
	}

	@Inject
	public void receiveDiffManager(DiffManager reporter)
	{
		diffManager = reporter;
	}

	public Feature getFeature()
	{
		return fileFeatureModule;
	}

	public action_code assertFile(final AssertFileRequest operation, final ETLTestMethod mt, final ETLTestOperation op, final ETLTestValueObject obj, final VariableContext context, ExecutionContext econtext) throws TestAssertionFailure, TestExecutionError, TestWarning
	{
		// find the target before calling
		return assertFile1(new ExpectedFinder()
		{
			@Override
			public File get(String expectedTargetFile, DataFileSchema.format_type formatType)
			{
				return fileRuntimeSupport.getAssertionFileForCurrentTest(expectedTargetFile, formatType);
			}
		}, operation, mt, op, obj, context, econtext);
	}

	public action_code assertFile1(final ExpectedFinder sourceFile, final AssertFileRequest operation, final ETLTestMethod mt, final ETLTestOperation op, final ETLTestValueObject obj, final VariableContext context, ExecutionContext econtext) throws TestAssertionFailure, TestExecutionError, TestWarning
	{
		String actualSourceFile = runtimeSupport.processReference(operation.getSourceFile());
		String expectedTargetFileName = operation.getTarget();

		if (expectedTargetFileName == null)
		{
			expectedTargetFileName = actualSourceFile;
		}

		final String expectedTargetFile = expectedTargetFileName;

		FileProducer fp = fileRuntimeSupport.getRegisteredProducer(operation.getProducer());

		File locatedFile = fp.locateFileWithClassifierAndContext(actualSourceFile, operation.getClassifier(), operation.getContext(), context);
		final File target = runtimeSupport.resolveFile(locatedFile);

		MetaDataContext fcontext = fileFeatureModule.getDataMetaContext();

		final MetaDataPackageContext pcontext = fcontext.createPackageContextForCurrentTest(MetaDataPackageContext.path_type.test_source);

		// look for sql extraction
		String operationSql = operation.getSql();

		if (operationSql == null)
		{
			String sqlScript = operation.getSqlScript();

			if (sqlScript != null)
			{
				try {
					File sourceScriptForCurrentTest = fileRuntimeSupport.getSourceScriptForCurrentTest(sqlScript, "sql-assertion");

					if (sourceScriptForCurrentTest == null)
					{
						throw new TestExecutionError("Error resolving sql script reference: " + sqlScript, FileConstants.ERR_MISSING_SQL_SCRIPT);
					}

					operationSql = FileUtils.readFileToString(sourceScriptForCurrentTest);
				} catch (IOException e) {
					throw new TestExecutionError("Error loading sql script reference: " + sqlScript, FileConstants.ERR_LOAD_SQL_SCRIPT);
				}
			}
		}

		final String sql = operationSql;

		if (fileRuntimeSupport.processDataSetAssertion(new FileRuntimeSupport.DataSetAssertionRequest() {
			@Override
			public void processRefreshedAssertionData(DataFile expected) {
				// TODO: Implement data set for file assertion and stage
				// nothing to do here
			}

			@Override
			public FileRuntimeSupportImpl.DataSetAssertionRequest.assertMode getAssertMode() {
				AssertFileRequest.AssertionMode assertionMode = operation.getAssertionMode();
				return assertionMode != null ? FileRuntimeSupport.DataSetAssertionRequest.assertMode.valueOf(assertionMode.name().toLowerCase()) : assertMode.equals;
			}

			/**
			 * Source-file is always the name of the actual-file-to-assert-against
			 * @return
			 */
			@Override
			public String getActualSchemaName() {
				return operation.getSourceFile();
			}

			/**
			 * When the actual fails, use the expected name.
			 * @return
			 */
			@Override
			public String getActualBackupSchemaName() {
				return getExpectedSchemaName();
			}

			/**
			 * If target is specified, it is the name of the expected data set.  If not specified, use the source file name.
			 * @return
			 */
			@Override
			public String getExpectedSchemaName() {
				return ObjectUtils.firstNotNull(operation.getTarget(), getActualSchemaName());
			}

			@Override
			public String getExpectedBackupSchemaName() {
				return getExpectedSchemaName();
			}

			@Override
			public boolean hasColumnList() {
				return operation.getColumnListMode() != null;
			}

			@Override
			public FileRuntimeSupportImpl.DataSetAssertionRequest.columnListMode getColumnListMode() {
				AssertFileRequest.ColumnListMode columnListMode2 = operation.getColumnListMode();
				return columnListMode2 != null ? FileRuntimeSupport.DataSetAssertionRequest.columnListMode.valueOf(columnListMode2.name().toLowerCase()) : null;
			}

			@Override
			public Set getColumnList() {
				return operation.getColumnList();
			}

			@Override
			public String getFailureId() {
				return operation.getFailureId();
			}

			@Override
			public DataFile getExpectedDataset(
					DataFileSchema explicitActualSchema,
					DataFileSchema actualEffectiveSchema,
					DataFileSchema explicitExpectedSchema,
					DataFileSchema expectedEffetiveSchema
			) {
				// expected file
				File assertionFileForCurrentTest = sourceFile.get(expectedTargetFile, explicitExpectedSchema.getFormatType());
				return dataFileManager.loadDataFile(assertionFileForCurrentTest, expectedEffetiveSchema);
			}

			@Override
			public DataFile getActualDataset(
					DataFileSchema explicitActualSchema,
					DataFileSchema actualEffectiveSchema,
					DataFileSchema explicitExpectedSchema,
					DataFileSchema expectedEffetiveSchema
			) throws IOException, TestExecutionError {
				// check for extraction sql before passing to the assertion processor
				if (sql != null)
				{
					// look for a base source ref type
					DataFileSchema dfs = explicitActualSchema;

					try {
						FileRuntimeSupport.DataFileSchemaQueryResult res = fileRuntimeSupport.locateReferenceFileSchemaForCurrentTest(
							obj,
							FileRuntimeSupport.schemaType.base_source,
							ReferenceFileTypeRef.ref_type.local,
							target.getName()
						);

						if (res.getFmlSearchType() != FileRuntimeSupport.fmlSearchType.notFound)
						{
							dfs = res.getDataFileSchema();
						}
						else
						{
							// if inherited from source, narrow if the base source ref type requires it
							ETLTestValueObject query = obj.query("base-source-reference-file-type");
							if (query != null)
							{
								dfs = fileRuntimeSupport.narrowIfRequired(dfs, ReferenceFileTypeRef.refFromRequest(query, ReferenceFileTypeRef.ref_type.remote));
							}
						}
					} catch (RequestedFmlNotFoundException e) {
						throw new IOException(e);
					}

					applicationLog.info("Resolved to base source reference file type: " + dfs.getId());

					// try to avoid an NPE
					if (!target.exists())
					{
						throw new TestExecutionError("", FileConstants.ERR_SOURCE_FILE_MISSING);
					}

					DataFile targetDf = dataFileManager.loadDataFile(target, dfs);

					// create a temporary target
					DataFile tempDf = dataFileManager.loadDataFile(runtimeSupport.createAnonymousTempFile(), explicitActualSchema);

					// copy using the sql
					dataFileManager.copyDataFile(targetDf, tempDf, new CopyOptionsBuilder().overrideExtractionSql(sql).options());

					// this is our new actual dataset
					return tempDf;
				}

				return dataFileManager.loadDataFile(target, explicitActualSchema);
			}

			@Override
			public ETLTestMethod getTestMethod() {
				return mt;
			}

			@Override
			public ETLTestOperation getTestOperation() {
				return op;
			}

			@Override
			public ETLTestValueObject getOperationParameters() {
				return obj;
			}

			@Override
			public VariableContext getVariableContext() {
				return context;
			}

			@Override
			public MetaDataPackageContext getMetaDataPackageContext() {
				return pcontext;
			}

			@Override
			public boolean refreshAssertionData() {
				return refreshAssertionData.isEnabled();
			}

			@Override
			public DataFileSchema getSourceSchema() {
				return null;
			}
		}))
			return action_code.handled;


		return ClassResponder.action_code.handled;
	}

	public action_code process(ETLTestMethod mt, ETLTestOperation op, ETLTestValueObject parameters, VariableContext vcontext, ExecutionContext econtext, int executor) throws TestAssertionFailure, TestExecutionError, TestWarning
	{
		return action_code.defer;
	}

	@Override
	public action_code assertDataSet(AssertDataSetRequest request, ETLTestMethod testMethod, ETLTestOperation testOperation, ETLTestValueObject valueObject, VariableContext variableContext, ExecutionContext executionContext) throws TestAssertionFailure, TestExecutionError, TestWarning
	{
		final File target = new File(request.getTargetFilePath());

		if (!target.exists() && !refreshAssertionData.isEnabled())
		{
			throw new TestExecutionError("Data set not found", FileConstants.ERR_MISSING_DATA_SET);
		}

		AssertFileRequest asr = new AssertFileRequest();
		asr.setClassifier(request.getClassifier());
		asr.setColumnList(request.getColumnList());

		if (request.getColumnListMode() != null)
		{
			asr.setColumnListMode(AssertFileRequest.ColumnListMode.valueOf(request.getColumnListMode().name()));
		}

		asr.setContext(request.getContext());
		asr.setProducer(request.getProducer());
		asr.setTarget(request.getTarget());
		asr.setSqlScript(request.getSqlScript());
		asr.setSql(request.getSql());
		asr.setSourceFile(request.getSourceFile());
		asr.setFailureId(request.getFailureId());

		if (request.getAssertionMode() != null)
		{
			asr.setAssertionMode(AssertFileRequest.AssertionMode.valueOf(request.getAssertionMode().name()));
		}

		return assertFile1(new ExpectedFinder(){
			@Override
			public File get(String expectedTargetFile, DataFileSchema.format_type formatType)
			{
				return target;
			}
		}, asr, testMethod, testOperation, valueObject, variableContext, executionContext);
	}
}