utsupport.create_test_cases Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of etlunit-core Show documentation
There is a newer version: 1.6.9
package utsupport;

import java.io.*;
import java.util.*;

public class create_test_cases
{
	private static final Map suiteMap = new HashMap();
	static final StringBuffer preBuff = new StringBuffer();
	static final StringBuffer assrtBuff = new StringBuffer();

	private static final File root = new File("..");
	private static final File script = new File(root, "script");
	private static final File sql = new File(root, "sql");
	private static final File desc = new File(root, "test_descriptors");
	private static final File tempDir = new File(root, "temp");
	private static final File runAll = new File(tempDir, "run_all_tests.bat");
	private static File assrtDir = new File(tempDir, "assertions");
	private static File preSqlDir = new File(tempDir, "preSql");
	private static File srcDir = new File(tempDir, "srcSql");
	private static File suiteDir = new File(tempDir, "suite");

	public static void main(String[] argv) throws Exception
	{
		suiteDir.mkdirs();
		srcDir.mkdirs();
		preSqlDir.mkdirs();
		assrtDir.mkdirs();

		final StringBuffer runAllBuff = new StringBuffer();
		runAllBuff.append("ECHO Tests > ..\\log\\run_log.txt\r\n");

		preBuff.append(readFile(new File(sql, "src_frag.sql")));
		assrtBuff.append(readFile(new File(sql, "assrt_frag.sql")));

		desc.listFiles(new FilenameFilter()
		{

			public boolean accept(File dir, String name)
			{
				if (name.endsWith("_test.txt"))
				{
					String testName = name.substring(0, name.length() - 9);
					System.out.println("Generating test " + testName);

					Map> sections = readSections(new File(dir, name));

					runAllBuff.append("ECHO Running test " + testName + "\r\n");
					runAllBuff.append("call " + testName + ".bat\r\n\r\n\r\n");
					runAllBuff.append("ECHO ---------------- >> ..\\log\\run_log.txt\r\n");
					runAllBuff.append("ECHO -              - >> ..\\log\\run_log.txt\r\n");
					runAllBuff.append("ECHO -              - >> ..\\log\\run_log.txt\r\n");

					try
					{
						generateTest(testName, sections);
					}
					catch (Exception exc)
					{
						throw new RuntimeException(exc);
					}
				}

				return false;
			}
		});

		writeFile(runAll, runAllBuff);
	}

	static void writeFile(File file, StringBuffer buff)
	{
		writeFile(file, buff, false);
	}

	static void writeFile(File file, StringBuffer buff, boolean append)
	{
		try
		{
			FileWriter writer = new FileWriter(file, append);

			writer.write(buff.toString());

			writer.close();
		}
		catch (Exception exc)
		{
			throw new RuntimeException("Could not write file", exc);
		}
	}

	public static Map> readSections(File file)
	{
		Map> sections = new HashMap>();

		try
		{
			BufferedReader b = new BufferedReader(new FileReader(file));

			String line = null;
			List currentSection = null;

			while ((line = b.readLine()) != null)
			{
				if (line.equals(""))
				{
					continue;
				}
				else if (line.startsWith("#"))
				{
					continue;
				}
				else if (line.matches("^\\[[A-Z,a-z]+\\]$"))
				{
					line = line.substring(1, line.length() - 1);

					if (sections.get(line) != null)
					{
						throw new RuntimeException("Found section " + line + " twice");
					}

					currentSection = new ArrayList();
					sections.put(line, currentSection);
				}
				else
				{
					if (currentSection == null)
					{
						throw new RuntimeException("Missing section " + line);
					}

					currentSection.add(line.split(" "));
				}
			}

			b.close();
			return sections;
		}
		catch (Exception exc)
		{
			throw new RuntimeException("Dead man testing", exc);
		}
	}

	private static void generateTest(String testName, Map> sections) throws Exception
	{
		StringBuffer
				sb =
				new StringBuffer("CALL ..\\conf\\%COMPUTERNAME%.bat\r\n\r\ncall ..\\bin\\test_report.bat init \""
						+ testName
						+ "\" ..\\results\\TEST-"
						+ testName
						+ ".xml > ..\\log\\"
						+ testName
						+ "_log.txt\r\n\r\n");

		sb.append(
				"ECHO Clearing the database\r\nsqlcmd -E -S %ServerName% -d %DatabaseName% -i ..\\sql\\purge.sql >> ..\\log\\"
						+ testName
						+ "_log.txt\r\n");

		sb.append("IF NOT EXIST \\\\etldev01\\F_DRIVE\\unit_test MKDIR \\\\etldev01\\F_DRIVE\\unit_test\r\n");
		sb.append(
				"IF NOT EXIST \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName% MKDIR \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\r\n");
		sb.append(
				"IF NOT EXIST \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\\log MKDIR \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\\log\r\n");
		sb.append("DEL /Q \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\\log\\*\r\n");

		sb.append(
				"IF NOT EXIST \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\\BadFiles MKDIR \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\\BadFiles\r\n");
		sb.append(
				"IF NOT EXIST \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\\SrcFiles MKDIR \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\\SrcFiles\r\n");
		sb.append(
				"IF NOT EXIST \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\\TgtFiles MKDIR \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\\TgtFiles\r\n");
		sb.append(
				"IF NOT EXIST \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\\parmlib MKDIR \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\\parmlib\r\n");

		String bat_description = null;

		List descs = sections.get("Description");

		if (descs != null)
		{
			StringBuilder desc = new StringBuilder();
			StringBuilder bat_desc = new StringBuilder();

			Iterator it = descs.iterator();

			while (it.hasNext())
			{
				String[] stra = it.next();

				if (stra != null)
				{
					for (int i = 0; i < stra.length; i++)
					{
						if (i != 0)
						{
							desc.append(' ');
							bat_desc.append(' ');
						}
						else
						{
							bat_desc.append("REM ");
						}

						desc.append(stra[i]);
						bat_desc.append(stra[i]);
					}
				}

				desc.append("\r\n");
				bat_desc.append("\r\n");
			}

			bat_description = bat_desc.toString();
		}
		else
		{
			throw new IllegalArgumentException("Missing Description");
		}

		List srcs = sections.get("SrcFiles");

		if (srcs != null)
		{
			Iterator sit = srcs.iterator();

			sb.append("ECHO UPDATING source files >> ..\\log\\" + testName + "_log.txt\r\n");

			while (sit.hasNext())
			{
				String[] dat = sit.next();

				sb.append("copy /Y ..\\SrcFiles\\"
						+ dat[0]
						+ " \\\\etldev01\\F_DRIVE\\unit_test\\%ConnectionName%\\SrcFiles\\ >> ..\\log\\"
						+ testName
						+ "_log.txt\r\n\r\n");
			}
		}

		Map loadedDataMap = new HashMap();

		List datas = sections.get("Data");

		if (datas != null)
		{
			Iterator dit = datas.iterator();

			while (dit.hasNext())
			{
				String[] dat = dit.next();

				String sch = "TST_SRC";

				System.out.println(dat[0]);
				System.out.println(dat[1]);

				if (dat.length == 3)
				{
					sch = dat[2];
				}

				String scriptName = testName + "_" + sch + "_" + dat[0] + "_" + dat[1] + "_pre.sql";
				File fscript = new File(srcDir, scriptName).getCanonicalFile();

				if (!loadedDataMap.containsKey(dat[1]))
				{
					loadData(loadedDataMap, dat[1], testName, sb);
				}

				sb.append("ECHO Copying data source "
						+ dat[1]
						+ " into table "
						+ sch
						+ "."
						+ dat[0]
						+ " >> ..\\log\\"
						+ testName
						+ "_log.txt\r\n");
				sb.append("sqlcmd -E -S %ServerName% -d %DatabaseName% -i "
						+ fscript.getAbsolutePath()
						+ " >> ..\\log\\"
						+ testName
						+ "_log.txt\r\n\r\n");

				StringBuffer
						buff =
						new StringBuffer("DECLARE @SCHEMA VARCHAR(4096)\r\nSET @SCHEMA = '"
								+ sch
								+ "'\r\nDECLARE @TABLE VARCHAR(4096)\r\nSET @TABLE = '"
								+ dat[0]
								+ "'\r\nDECLARE @MST_TABLE VARCHAR(4096)\r\nSET @MST_TABLE = '"
								+ dat[1]
								+ "'\r\n\r\n");
				buff.append(preBuff);

				writeFile(fscript, buff);
			}
		}

		List sql = sections.get("PreSql");

		if (sql != null)
		{
			StringBuffer sqlBuff = new StringBuffer();

			Iterator sit = sql.iterator();

			while (sit.hasNext())
			{
				String[] dat = sit.next();

				for (int i = 0; i < dat.length; i++)
				{
					sqlBuff.append(dat[i]);
					sqlBuff.append(' ');
				}

				sqlBuff.append("\r\n");
			}

			File preSqlF = new File(preSqlDir, testName + "_preSql.sql").getCanonicalFile();

			writeFile(preSqlF, sqlBuff);
			sb.append("ECHO Running pre sql >> ..\\log\\" + testName + "_log.txt\r\n");
			sb.append("sqlcmd -E -S %ServerName% -d %DatabaseName% -i "
					+ preSqlF.getAbsolutePath()
					+ "_preSql.sql >> ..\\log\\"
					+ testName
					+ "_log.txt\r\n");
		}

		List wkfs = sections.get("Workflow");

		if (wkfs == null)
		{
			throw new IllegalArgumentException("Missing Workflow");
		}

		Iterator it = wkfs.iterator();

		while (it.hasNext())
		{
			String[] wkf = it.next();

			sb.append("ECHO Running etlunit >> ..\\log\\" + testName + "_log.txt\r\n");
			sb.append("call ..\\script\\test_workflow "
					+ wkf[0]
					+ " "
					+ wkf[1]
					+ " "
					+ (wkf.length == 3 ? wkf[2] : "")
					+ " >> ..\\log\\"
					+ testName
					+ "_log.txt\r\n");
		}

		List asserts = sections.get("Assert");

		if (asserts == null)
		{
			throw new IllegalArgumentException("Missing Assert");
		}

		Iterator ait = asserts.iterator();

		while (ait.hasNext())
		{
			String[] assrt = ait.next();

			if (assrt[0].equals("TST"))
			{
				sb.append("ECHO Running TSQL assertions >> ..\\log\\" + testName + "_log.txt\r\n");

				sb.append("ECHO Updating TSQL assertion " + assrt[1] + " >> ..\\log\\" + testName + "_log.txt\r\n");
				sb.append("sqlcmd -E -S %ServerName% -d %DatabaseName% -Q \"DROP PROCEDURE DBO.SQLTest_STG_"
						+ assrt[1]
						+ "\" >> ..\\log\\"
						+ testName
						+ "_log.txt\r\n");
				sb.append("sqlcmd -E -S %ServerName% -d %DatabaseName% -i ..\\sql_unit\\STG_"
						+ assrt[1]
						+ ".sql >> ..\\log\\"
						+ testName
						+ "_log.txt\r\n");

				sb.append(
						"sqlcmd -E -S %ServerName% -d %DatabaseName% -Q \"EXEC TST.Runner.RunTest '%DatabaseName%', 'SQLTest_STG_"
								+ assrt[1]
								+ "'\" >> ..\\log\\"
								+ testName
								+ "_log.txt\r\n");
			}
			else if (assrt[0].startsWith("TBL"))
			{
				if (!loadedDataMap.containsKey(assrt[2]))
				{
					loadData(loadedDataMap, assrt[2], testName, sb);
				}

				String pk = "";

				if (assrt[0].length() > 3)
				{
					String extra = assrt[0].substring(3);

					if (extra.equals("(EXCLUDE_SK)"))
					{
						pk = assrt[1] + "_SK";
					}
				}

				File f = new File(assrtDir, testName + "_assert_" + assrt[1] + "_" + assrt[2] + ".sql");

				try
				{
					createSqlCompare(assrt[1], assrt[2], assrt.length == 3 ? null : assrt[3], pk, f);
				}
				catch (Exception exc)
				{
					throw new RuntimeException(exc);
				}

				sb.append("sqlcmd -E -S %ServerName% -d %DatabaseName% -i "
						+ f.getAbsolutePath()
						+ " >> ..\\log\\"
						+ testName
						+ "_log.txt\r\n");
			}
			else if (assrt[0].equals("FILE"))
			{
				sb.append("ECHO Running FILE assertions >> ..\\log\\" + testName + "_log.txt\r\n");

				sb.append("..\\bin\\diff -s \\\\ETLDEV01\\F_DRIVE\\unit_test\\%ConnectionName%\\TgtFiles\\"
						+ assrt[1]
						+ " ..\\data\\TgtFiles\\"
						+ assrt[2]
						+ " >> ..\\log\\"
						+ testName
						+ "_log.txt\r\n");
			}
		}

		sb.append("ECHO copying results to run log >> ..\\log\\" + testName + "_log.txt\r\n");
		sb.append("ECHO Results of test " + testName + " >> ..\\log\\run_log.txt\r\n");
		sb.append("..\\bin\\grep Execution ..\\log\\" + testName + "_log.txt >> ..\\log\\run_log.txt\r\n");
		sb.append("..\\bin\\grep --before-context=1 \"copying result\" ..\\log\\"
				+ testName
				+ "_log.txt >> ..\\log\\run_log.txt\r\n");
		sb.append("..\\bin\\grep \"TST Status\" ..\\log\\" + testName + "_log.txt >> ..\\log\\run_log.txt\r\n");

		sb.append("ECHO Results of test " + testName + ".xml >> ..\\log\\run_log.txt\r\n");

		sb.append("call ..\\bin\\test_report.bat complete \""
				+ testName
				+ "\" ..\\results\\TEST-"
				+ testName
				+ ".xml ..\\log\\"
				+ testName
				+ "_log.txt >> ..\\log\\"
				+ testName
				+ "_log.txt\r\n\r\n");

		List suites = sections.get("Suite");

		StringBuffer buffer = new StringBuffer();

		if (suites != null && !suites.isEmpty())
		{
			it = suites.iterator();

			while (it.hasNext())
			{

				String[] suite = it.next();

				if (suite != null && suite.length != 0)
				{
					for (int i = 0; i < suite.length; i++)
					{
						buffer.setLength(0);

						String suiteTest = suite[i] + ".bat";

						// check for inclusion
						if (!suiteMap.containsKey(suiteTest + "." + testName))
						{
							suiteMap.put(suiteTest + "." + testName, Boolean.TRUE);

							// add this test to the named suite
							buffer.append(bat_description);
							buffer.append("pushd ..\r\ncall ").append(testName).append(".bat\n\rpopd\r\n");

							writeFile(new File(suiteDir, suiteTest), buffer, true);
						}
					}
				}
			}
		}

		writeFile(new File(tempDir, testName + ".bat"), sb);
	}

	private static void createSqlCompare(String table, String mst, String cols, String pk, File out) throws Exception
	{
		StringBuffer
				buff =
				new StringBuffer("DECLARE @COLS VARCHAR(max)\r\n"
						+ (cols != null ? ("SET @COLS = '" + cols + "'\r\n") : "")
						+ "DECLARE @TABLE VARCHAR(4096)\r\nSET @TABLE = '"
						+ table
						+ "'\r\nDECLARE @COL_NAMES TABLE(\r\n\tCOL_NAME VARCHAR(4096) NOT NULL\r\n\tPRIMARY KEY(COL_NAME)\r\n)\r\nINSERT INTO @COL_NAMES (COL_NAME) VALUES ('"
						+ pk
						+ "')\r\nDECLARE @MST_TABLE VARCHAR(4096)\r\nSET @MST_TABLE = '"
						+ mst
						+ "'\r\n\r\n");
		buff.append(assrtBuff);

		writeFile(out, buff);
	}

	public static StringBuffer readFile(File f) throws Exception
	{
		StringBuffer preBuff_l = new StringBuffer();

		BufferedReader reader = new BufferedReader(new FileReader(f));

		try
		{
			String line = null;

			while ((line = reader.readLine()) != null)
			{
				preBuff_l.append(line);
				preBuff_l.append("\r\n");
			}

			reader.close();
		}
		finally
		{
			reader.close();
		}

		return preBuff_l;
	}

	private static void loadData(Map loadedDataMap, String dat, String testName, StringBuffer sb)
	{
		loadedDataMap.put(dat, dat);

		String sqlfile = "..\\data\\SrcData\\TST_MST." + dat + ".Table.sql";
		String bcpfile = "..\\data\\SrcData\\TST_MST." + dat + ".Table.bcp";
		String datend = dat + "_end";

		sb.append("ECHO Loading data source " + dat + " >> ..\\log\\" + testName + "_log.txt\r\n");

		sb.append("sqlcmd -E -S %ServerName% -d %DatabaseName% -i "
				+ sqlfile
				+ " >> ..\\log\\"
				+ testName
				+ "_log.txt\r\n\r\n");
		sb.append("IF NOT EXIST " + bcpfile + " GOTO " + datend + "\r\n");
		sb.append("bcp %DatabaseName%.TST_MST."
				+ dat
				+ " in "
				+ bcpfile
				+ " -T -S %ServerName% -f "
				+ bcpfile
				+ ".fmt >> ..\\log\\"
				+ testName
				+ "_log.txt\r\n\r\n");
		sb.append(":" + datend + "\r\n");
	}
}