org.etlunit.io.file.FlatFileDiff Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of etlunit-core Show documentation
There is a newer version: 1.6.9
package org.etlunit.io.file;

import org.apache.commons.lang.StringUtils;
import org.etlunit.DiffGrid;
import org.etlunit.DiffGridRow;
import org.etlunit.DiffManager;
import org.etlunit.parser.ETLTestOperation;

import java.io.BufferedReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

class FlatFileDiff
{
	private final DataFile source;
	private final List sourceColumns;

	public FlatFileDiff(DataFile source) throws IOException
	{
		this.source = source;
		this.sourceColumns = source.getDataFileSchema().getColumns();
	}

	public List diffFile(DataFile target) throws IOException
	{
		return diffFile(target, null);
	}

	public List diffFile(DataFile target, List targetColumnsOfInterest) throws IOException
	{
		List columns = target.getDataFileSchema().getColumns();

		if (targetColumnsOfInterest == null)
		{
			targetColumnsOfInterest = source.getDataFileSchema().getColumnNames();
		}

		List diffList = new ArrayList();

		int sourceLineNumber = 0;
		int targetLineNumber = 0;

		FlatFile.FileData sData = source.getFileData(targetColumnsOfInterest);

		try
		{
			FlatFile.FileData tData = target.getFileData(targetColumnsOfInterest);

			try
			{

				Iterator soit = sData.iterator();
				Iterator tait = tData.iterator();

				DataFile.FileRow sourceData = null;
				DataFile.FileRow targetData = null;

				OrderKey keySource = null;
				OrderKey keyTarget = null;

				while (true)
				{
					if (sourceData == null && soit.hasNext())
					{
						sourceData = soit.next();

						// grab the key for each line and handle accordingly
						keySource = sourceData.getOrderKey();
						sourceLineNumber++;
					}

					if (targetData == null && tait.hasNext())
					{
						targetData = tait.next();

						// grab the key for each line and handle accordingly
						keyTarget = targetData.getOrderKey();
						targetLineNumber++;
					}

					if (sourceData == null && targetData == null)
					{
						break;
					}

					int compRes = 0;

					// fake this out a bit.  If the source or target are null (because we have run past the end of one but not
					// the other, create a fake compare result to trigger the correct action
					if (sourceData == null)
					{
						compRes = 1;
					}
					else if (targetData == null)
					{
						compRes = -1;
					}
					else
					{
						compRes = keySource.compareTo(keyTarget);
					}

					// typical 0, -1, 1 results
					if (compRes == 0)
					{
						compareLines(sourceData, targetData, targetColumnsOfInterest, diffList, sourceLineNumber, targetLineNumber);
						sourceData = null;
						targetData = null;
					}
					else if (compRes > 0)
					{
						diffList.add(new FileDiff(
								targetLineNumber,
								keyTarget,
								prettyPrintData(targetData.getData(), targetColumnsOfInterest),
								FileDiff.diff_type.added
						));

						// get rid of the target row so perhaps the next source row will match
						targetData = null;
					}
					else
					{
						// target missing row
						diffList.add(new FileDiff(
								sourceLineNumber,
								keySource,
								prettyPrintData(sourceData.getData(), targetColumnsOfInterest),
								FileDiff.diff_type.removed
						));

						// get rid of the source row so perhaps the next target row will match
						sourceData = null;
					}
				}
			}
			finally
			{
				tData.dispose();
			}
		}
		finally
		{
			sData.dispose();
		}

		return diffList;
	}

	private String prettyPrintData(Map sourceData, List targetColumnsOfInterest)
	{
		StringBuilder stb = new StringBuilder("[");

		for (String col : targetColumnsOfInterest)
		{
			stb.append(col).append(": '");
			stb.append(sourceData.get(col)).append("' ");
		}

		stb.append("]");

		return stb.toString();
	}

	private String readRow(BufferedReader targetReader) throws IOException
	{
		String line = null;

		while ((line = targetReader.readLine()) != null)
		{
			if (!lineIsIgnored(line))
			{
				break;
			}
		}

		return line;
	}

	public static boolean lineIsIgnored(String line)
	{
		String trim = line.trim();
		return trim.equals("") || (trim.startsWith("/*") && trim.endsWith("*/"));
	}

	private void compareLines(DataFile.FileRow soColumns, DataFile.FileRow taColumns, List targetColumnsOfInterest, List diffList, int sourceLine, int targetLineNumber)
	{
		Map soData = soColumns.getData();
		Map taData = taColumns.getData();

		// now go through the source columns and compare
		for (String key : targetColumnsOfInterest)
		{
			// get source and target and compare
			String sourceValue = soData.get(key);
			String targetValue = taData.get(key);

			if (!StringUtils.equals(sourceValue, targetValue))
			{
				diffList.add(new FileDiff(sourceLine,
						targetLineNumber,
						soColumns.getOrderKey(),
						key,
						sourceValue,
						targetValue));
			}
		}
	}

	public static void report(DiffManager dManager, ETLTestOperation operation, String failureId, List diffs)
	{
		DiffGrid diffGrid = dManager.reportDiff(operation, failureId);

		try
		{
			for (FileDiff diff : diffs)
			{
				DiffGrid.line_type lt = DiffGrid.line_type.changed;

				switch (diff.getDiffType())
				{
					case added:
						lt = DiffGrid.line_type.added;
						break;
					case removed:
						lt = DiffGrid.line_type.removed;
						break;
					case changed:
						lt = DiffGrid.line_type.changed;
						break;
				}

				DiffGridRow diffRow = diffGrid.addRow(diff.getSourceRowNumber(), diff.getTargetRowNumber(), lt);

				try
				{
					diffRow.setOrderKey(String.valueOf(diff.getOrderKey().getPrettyString()));
					diffRow.setColumnName(String.valueOf(diff.getColumnName()));
					diffRow.setSourceValue(String.valueOf(diff.getSourceValue()));
					diffRow.setTargetValue(String.valueOf(diff.getOtherValue()));
				}
				finally
				{
					diffRow.done();
				}
			}
		}
		finally
		{
			diffGrid.done();
		}
	}
}