Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.etlunit.feature.file.FileAssertExtender Maven / Gradle / Ivy
package org.etlunit.feature.file;
import com.google.inject.name.Named;
import org.etlunit.*;
import org.etlunit.context.VariableContext;
import org.etlunit.feature.Feature;
import org.etlunit.feature.RuntimeOption;
import org.etlunit.feature.extend.Extender;
import org.etlunit.io.FileBuilder;
import org.etlunit.io.file.*;
import org.etlunit.json.validator.ResourceNotFoundException;
import org.etlunit.parser.ETLTestOperation;
import org.etlunit.parser.ETLTestValueObject;
import org.etlunit.util.IOUtils;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.util.*;
public class FileAssertExtender implements Extender, AssertOperationProcessor
{
private final FileFeatureModule fileFeatureModule;
private FileRuntimeSupport fileRuntimeSupport;
private RuntimeSupport runtimeSupport;
private DiffManager diffManager;
private Log applicationLog;
private DataFileManager dataFileManager;
private boolean refreshAssertionData = false;
public FileAssertExtender(FileFeatureModule fileFeatureModule)
{
this.fileFeatureModule = fileFeatureModule;
}
@Inject
public void receiveDataFileManager(DataFileManager dfm)
{
dataFileManager = dfm;
}
@Inject
public void receiveRuntimeSupport(RuntimeSupport rs)
{
runtimeSupport = rs;
}
@Inject
public void receiveApplicationLog(@Named("applicationLog") Log log)
{
this.applicationLog = log;
}
@Inject
public void setRefreshAssertionData(@Named("file.refreshAssertionData") RuntimeOption option)
{
refreshAssertionData = option.isEnabled();
applicationLog.info(refreshAssertionData ? "File Module refreshing assertion data" : "File Module not refreshing assertion data");
}
@Inject
public void receiveFileRuntimeSupport(FileRuntimeSupport support)
{
fileRuntimeSupport = support;
}
@Inject
public void receiveDiffManager(DiffManager reporter)
{
diffManager = reporter;
}
public boolean canHandleRequest(ETLTestOperation operation, ETLTestValueObject operands, VariableContext context, ExecutionContext executionContext)
{
// this module requires a file-equals operation
return operands.query("file-equals") != null;
}
public Feature getFeature()
{
return fileFeatureModule;
}
public action_code processAssert(AssertOperation operation, ETLTestOperation op, ETLTestValueObject obj, VariableContext context, ExecutionContext econtext) throws TestAssertionFailure, TestExecutionError, TestWarning
{
ETLTestValueObject fcontext = FileFeatureModule.getFileContextContainer(obj, context);
String fileName = operation.getFile();
File assertionSource = fileRuntimeSupport.getAssertionFile(op.getTestMethod().getTestClass().getPackage(), fileName);
String targetFileName = operation.getTargetFileName();
if (targetFileName == null)
{
targetFileName = fileName;
}
FileProducer fp = fileRuntimeSupport.getRegisteredProducer(operation.getProducer());
File target = fp.locateFileWithClassifierAndContext(targetFileName, operation.getClassifier(), operation.getContext());
// check for the operation type:
AssertOperation.assertionMode_enum assertionMode = operation.getAssertionMode();
if (assertionMode == null)
{
assertionMode = AssertOperation.assertionMode_enum.equals;
}
switch (assertionMode)
{
case equals:
// this is the usual case and is handled outside this switch
break;
case exists:
// target file must exist. It's a failure if it does not.
if (!target.exists())
{
String failureId = FileConstants.FAIL_FILE_NOT_EXIST;
if (operation.getFailureId() != null)
{
failureId = operation.getFailureId();
}
throw new TestAssertionFailure(target.getAbsolutePath(), failureId);
}
return ClassResponder.action_code.handled;
case notExists:
// target file must not exist. It's a failure if it does.
if (target.exists())
{
String failureId = FileConstants.FAIL_FILE_EXISTS;
if (operation.getFailureId() != null)
{
failureId = operation.getFailureId();
}
throw new TestAssertionFailure(target.getAbsolutePath(), failureId);
}
return ClassResponder.action_code.handled;
case empty:
// target file must exist and be empty. It's a failure if it does not exist or contains data.
case notEmpty:
// target file must exist and contain data. It's a failure if it does not exist or does not contain data.
simpleCheckContents(target, obj, operation, assertionMode, op, context);
return ClassResponder.action_code.handled;
}
AssertOperation.columnListMode_enum columnListMode = operation.getColumnListMode();
List actualList = new ArrayList();
List list = operation.getColumnList();
if (columnListMode == null)
{
columnListMode = AssertOperation.columnListMode_enum.exclude;
}
else
{
if (list == null)
{
throw new TestExecutionError("Must specify " + AssertOperation.COLUMNLIST_JSON_NAME + " when the " + AssertOperation.COLUMNLISTMODE_JSON_NAME + " attribute is speicified", FileConstants.ERR_STATE_COLUMN_MODE_STATE);
}
}
/** It's okay for the source file not to exist if refresh assertion data is enabled
* since the source file will be refreshed prior to the assertion being tested
*/
if (!assertionSource.exists() && !refreshAssertionData)
{
throw new TestExecutionError("Source file does not exist: " + assertionSource, FileConstants.ERR_SOURCE_FILE_MISSING);
}
if (target == null || !target.exists())
{
throw new TestExecutionError("Target file does not exist: " + target, FileConstants.ERR_TARGET_FILE_MISSING);
}
DataFileSchema ffs = null;
try
{
ffs = fileRuntimeSupport.locateReferenceFileSchema(op.getTestClass().getPackage(), operation.getFile(), null, obj);
applicationLog.info("Resolved data file schema: " + ffs.getId());
context.declareAndSetStringValue(FileConstants.ASSERT_FILE_SCHEMA, ffs.getId());
}
catch (IllegalArgumentException exc)
{
throw new TestExecutionError("Error loading Flat File Schema", FileConstants.ERR_LOAD_FFML, exc);
}
catch (ResourceNotFoundException exc)
{
throw new TestExecutionError("Error resolving Flat File Schema", FileConstants.ERR_LOAD_FFML, exc);
}
actualList.addAll(ffs.getColumnNames());
if (list != null)
{
switch (columnListMode)
{
case include:
// our list only includes mentioned columns
actualList.clear();
actualList.addAll(list);
break;
case exclude:
// start with all columns, and remove the ones that are called out
actualList.removeAll(list);
break;
}
}
// create a delimited local copy of the remote file
String localID = op.getQualifiedName() + "-target-" + ffs.getId() + "." + target.getName() + ".delimited";
File localSource = assertionSource;
File localTarget = fileRuntimeSupport.getGeneratedDataFile(op.getTestClass().getPackage(), localID);
DataFile sourceFF = null;
DataFile localFF = null;
try
{
DataFileSchema ffView = ffs;
ffView = ffs.createSubViewIncludingColumns(actualList, "synthetic-" + ffs.getId() + "-delimited-" + op.getQualifiedName(), DataFileSchema.format_type.delimited);
ffView.setColumnDelimiter(dataFileManager.getDefaultColumnDelimiter());
ffView.setRowDelimiter(dataFileManager.getDefaultRowDelimiter());
ffView.setNullToken(dataFileManager.getDefaultNullToken());
File fileGen = runtimeSupport.getGeneratedSourceDirectory("file");
IOUtils.writeStringToFile(new FileBuilder(fileGen).subdir("synthetic-fml").mkdirs().name(ffView.getId()).file(), ffView.toJsonString());
// copy both files - source and target, into a local delimited copy
localSource = fileRuntimeSupport.getGeneratedDataFile(op.getTestClass().getPackage(), op.getQualifiedName() + "-source-" + ffs.getId() + "." + assertionSource.getName() + ".delimited");
sourceFF = dataFileManager.loadDataFile(target, ffs);
localFF = dataFileManager.loadDataFile(localTarget, ffView);
applicationLog.info("Caching local copy of target " + target.getName() + " to " + localTarget.getAbsolutePath());
dataFileManager.copyDataFile(sourceFF, localFF);
// before processing the source, refresh the data if necessary
if (refreshAssertionData)
{
// copy from the local target, since it has the schema the source will be validated against
applicationLog.info("Refreshing assertion data " + assertionSource.getName() + " from " + localTarget.getAbsolutePath());
IOUtils.copyFiles(localTarget, assertionSource);
}
sourceFF = dataFileManager.loadDataFile(assertionSource, ffView);
localFF = dataFileManager.loadDataFile(localSource, ffView);
applicationLog.info("Caching local copy of source " + assertionSource.getName() + " to " + localSource.getAbsolutePath());
dataFileManager.copyDataFile(sourceFF, localFF);
// set the file pointers to the new files
localFF = dataFileManager.loadDataFile(localTarget, ffView);
sourceFF = dataFileManager.loadDataFile(localSource, ffView);
// compare localSource to localTarget
// we now have a delimited local file to compare the master file to (created for convenience for debugging)
List diffResults = dataFileManager.diff(sourceFF, localFF);
String failureId = FileConstants.FAIL_FILE_DIFF;
if (operation.getFailureId() != null)
{
failureId = operation.getFailureId();
}
if (diffResults.size() != 0)
{
dataFileManager.report(diffManager, op, failureId, diffResults);
throw new TestAssertionFailure("File : " + assertionSource.getAbsolutePath() + " does not match : " + target.getAbsolutePath(), failureId);
}
}
catch (IOException e)
{
throw new TestExecutionError("Error comparing files: " + e.toString(), FileConstants.ERR_FILE_COMPARE, e);
}
return ClassResponder.action_code.handled;
}
private void simpleCheckContents(File target, ETLTestValueObject obj, AssertOperation operation, AssertOperation.assertionMode_enum mode, ETLTestOperation op, VariableContext context) throws TestAssertionFailure, TestExecutionError
{
String failureId = mode == AssertOperation.assertionMode_enum.notEmpty ? FileConstants.FAIL_FILE_EMPTY : FileConstants.FAIL_FILE_NOT_EMPTY;
if (operation.getFailureId() != null)
{
failureId = operation.getFailureId();
}
if (!target.exists())
{
throw new TestExecutionError(target.getAbsolutePath(), FileConstants.ERR_TARGET_FILE_MISSING);
}
try
{
DataFileSchema ffs = fileRuntimeSupport.locateReferenceFileSchema(op.getTestClass().getPackage(), operation.getFile(), null, obj);
applicationLog.info("Resolved data file schema: " + ffs.getId());
context.declareAndSetStringValue(FileConstants.ASSERT_FILE_SCHEMA, ffs.getId());
// open the file and count the lines
DataFile df = dataFileManager.loadDataFile(target, ffs);
DataFile.FileData data = df.getFileData();
int rows = 0;
try
{
Iterator it = data.iterator();
while (it.hasNext())
{
it.next();
rows++;
}
}
finally
{
data.dispose();
}
switch (mode)
{
case empty:
if (rows != 0)
{
throw new TestAssertionFailure(target.getAbsolutePath() + " contains " + rows + " rows of data", failureId);
}
break;
case notEmpty:
if (rows == 0)
{
throw new TestAssertionFailure(target.getAbsolutePath() + " contains no data", failureId);
}
break;
}
}
catch(IOException exc)
{
throw new TestExecutionError("", FileConstants.ERR_FILE_COMPARE, exc);
}
catch (ResourceNotFoundException exc)
{
// fallback - just check the file length
switch (mode)
{
case empty:
if (target.length() != 0)
{
throw new TestAssertionFailure(target.getAbsolutePath(), failureId);
}
break;
case notEmpty:
if (target.length() == 0)
{
throw new TestAssertionFailure(target.getAbsolutePath(), failureId);
}
break;
}
}
}
public action_code process(ETLTestOperation op, ETLTestValueObject parameters, VariableContext vcontext, ExecutionContext econtext) throws TestAssertionFailure, TestExecutionError, TestWarning
{
throw new UnsupportedOperationException();
}
}
