org.etlunit.ETLTestVM Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of etlunit-core Show documentation
There is a newer version: 1.6.9
package org.etlunit;

import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.inject.Binder;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.binder.LinkedBindingBuilder;
import com.google.inject.name.Names;
import org.etlunit.context.VariableContext;
import org.etlunit.context.VariableContextImpl;
import org.etlunit.feature.*;
import org.etlunit.feature.debug.DebugFeatureModule;
import org.etlunit.feature.logging.LogFileManager;
import org.etlunit.feature.logging.LogFileManagerImpl;
import org.etlunit.feature.logging.LoggingFeatureModule;
import org.etlunit.feature.report.ReportFeatureModule;
import org.etlunit.feature.results.ResultsFeatureModule;
import org.etlunit.feature.test_locator.DirectoryTestLocatorFeatureModule;
import org.etlunit.io.file.DataFileManager;
import org.etlunit.io.file.DataFileManagerImpl;
import org.etlunit.json.validator.ClasspathSchemaResolver;
import org.etlunit.json.validator.JsonSchema;
import org.etlunit.json.validator.JsonSchemaValidationException;
import org.etlunit.json.validator.JsonValidator;
import org.etlunit.parser.ETLTestValueObject;

import java.util.*;

public class ETLTestVM
{
	private final FeatureLocator featureLocator;
	private final Configuration configuration;
	private final RuntimeSupport runtimeSupport;

	private ClassListener applicationLogListener;

	private Log applicationLog;
	private PrintWriterLog userLog = new PrintWriterLog();

	private final Map discoveredFeaturesMap = new HashMap();
	private final Map installedFeatureMap = new HashMap();
	private final Map> installedFeatureTypeMap = new HashMap>();
	private final List features = new ArrayList();
	private final ResultsFeatureModule resultsFeatureModule;
	private final DiffManagerImpl diffReporter = new HtmlDiffManagerImpl();
	private final VariableContext variableContext = new VariableContextImpl();
	private final DataFileManager dataFileManager = new DataFileManagerImpl();
	private final LogFileManager logFileManager = new LogFileManagerImpl();

	private class ETLModule implements Module
	{
		@Override
		public void configure(Binder binder)
		{
			binder.bind(LogFileManager.class).toInstance(logFileManager);
			binder.bind(Configuration.class).toInstance(configuration);
			binder.bind(RuntimeSupport.class).toInstance(runtimeSupport);
			binder.bind(Log.class).annotatedWith(Names.named("applicationLog")).toInstance(applicationLog);
			binder.bind(Log.class).annotatedWith(Names.named("userLog")).toInstance(userLog);
			binder.bind(DiffManager.class).toInstance(diffReporter);
			binder.bind(VariableContext.class).toInstance(variableContext);
			binder.bind(DataFileManager.class).toInstance(dataFileManager);

			binder.bind(List.class).toInstance(Collections.unmodifiableList(features));

			List options = runtimeSupport.getRuntimeOptions();

			for (RuntimeOption option : options)
			{
				binder.bind(RuntimeOption.class).annotatedWith(Names.named(option.getName())).toInstance(option);
			}
		}
	}

	private ETLModule guiceModule;// = new ETLModule();
	private Injector injector;

	public ETLTestVM(String con)
	{
		this(
				new ServiceLocatorFeatureLocator(),
				new Configuration(con)
				);
	}

	public ETLTestVM(Configuration con)
	{
		this(
				new ServiceLocatorFeatureLocator(),
				con
				);
	}

	public ETLTestVM(FeatureLocator locator, Configuration configuration)
	{
		List lfi = new ArrayList();

		this.configuration = configuration;
		runtimeSupport = new BasicRuntimeSupport(configuration, this);


		diffReporter.setOutputDirectory(runtimeSupport.getReportDirectory("diff"));

		LoggingFeatureModule loggingFeatureModule = new LoggingFeatureModule(runtimeSupport, features);

		applicationLog = loggingFeatureModule.getLog();

		applicationLog.info("Project ["
				+ runtimeSupport.getProjectName()
				+ "] version ["
				+ runtimeSupport.getProjectVersion()
				+ "] running for user ["
				+ runtimeSupport.getProjectUser()
				+ "] using project identifier ["
				+ runtimeSupport.getProjectUID()
				+ "]");

		// install a feature which will validate all annotations
		addFeature(new AnnotationValidationFeature());

		addFeature(loggingFeatureModule);
		addFeature(new DebugFeatureModule());
		addFeature(new ReportFeatureModule());
		addFeature(new TestFeatureModule());
		addFeature(new DirectoryTestLocatorFeatureModule());

		// add a feature to handle contexts
		addFeature(new ContextFeatureModule());

		resultsFeatureModule = new ResultsFeatureModule();
		addFeature(resultsFeatureModule);

		this.featureLocator = locator;

		for (Feature feat : featureLocator.getFeatures())
		{
			discoveredFeaturesMap.put(feat.getFeatureName(), feat);
		}

		ETLTestValueObject res = configuration.query("install-features");

		if (res != null)
		{
			if (res.getValueType() != ETLTestValueObject.value_type.list)
			{
				throw new IllegalArgumentException(
						"install-features must be a list of feature class names (not a map or single string)");
			}

			List flist = res.getValueAsList();

			Iterator fit = flist.iterator();

			while (fit.hasNext())
			{
				ETLTestValueObject fclass = fit.next();

				if (fclass.getValueType() != ETLTestValueObject.value_type.quoted_string)
				{
					throw new IllegalArgumentException(
							"install-features must be a list of feature class names (not a map or single string).  List element is not a string");
				}

				String name = fclass.getValueAsString();

				addFeatureById(name);
			}
		}
	}

	public void installFeatures()
	{
		workOutPrerequisites();

		sortByDependency(features);

		guiceModule = new ETLModule();

		injector = Guice.createInjector(guiceModule);

		injector.injectMembers(this);

		injector.injectMembers(runtimeSupport);
		injector.injectMembers(configuration);
		injector.injectMembers(featureLocator);

		// validate runtime options in the configuration
		List options = runtimeSupport.getRuntimeOptions();

		for (RuntimeOption option : options)
		{
			String name = option.getName();

			// grab the feature.optionName parts
			int index = name.indexOf(".");

			String feature = name.substring(0, index);
			String foption = name.substring(index + 1);

			Feature f = installedFeatureMap.get(feature);

			if (f == null)
			{
				throw new IllegalArgumentException("Feature not installed for option: " + name);
			}

			List optList = f.getMetaInfo().getOptions();

			boolean found = false;

			for (RuntimeOptionDescriptor ro : optList)
			{
				if (ro.getName().equals(foption))
				{
					found = true;
				}
			}

			if (!found)
			{
				throw new IllegalArgumentException("Feature does not export option: " + name);
			}
		}

		// inject all runtime options that each feature exposes
		injector = injector.createChildInjector(new Module()
		{
			@Override
			public void configure(Binder binder)
			{
				Map runtimeOptionMap = new HashMap();

				for (RuntimeOption option : runtimeSupport.getRuntimeOptions())
				{
					runtimeOptionMap.put(option.getName(), "");
				}


				for (Feature feature : features)
				{
					FeatureMetaInfo metaInfo = feature.getMetaInfo();
					if (metaInfo != null)
					{
						for (RuntimeOptionDescriptor featureOption : metaInfo.getOptions())
						{
							String name = feature.getFeatureName() + "." + featureOption.getName();
							if (!runtimeOptionMap.containsKey(name))
							{
								RuntimeOption ro = new RuntimeOption();
								ro.setName(name);
								switch (featureOption.getOptionType())
								{
									case bool:
										ro.setEnabled(featureOption.getDefaultBooleanValue());
										break;
									case integer:
										ro.setIntegerValue(featureOption.getDefaultIntegerValue());
										break;
									case string:
										ro.setStringValue(featureOption.getDefaultStringValue());
										break;
								}

								binder.bind(RuntimeOption.class).annotatedWith(Names.named(ro.getName())).toInstance(ro);
							}
						}
					}
				}
			}
		});

		Iterator i = features.iterator();

		while (i.hasNext())
		{
			final Feature f = i.next();

			applicationLog.info("Initializing feature: " + f.getFeatureName());

			injector = injector.createChildInjector(new Module()
			{
				@Override
				public void configure(Binder binder)
				{
					// cast the builder to remove the type requirement.  Otherwise all types have to be
					// bound by name
					// check to see if more than one of the same class has been installed.
					if (installedFeatureTypeMap.get(f.getClass()).size() != 1)
					{
						binder.bind(Feature.class).annotatedWith(Names.named(f.getFeatureName())).toInstance(f);
					}
					else
					{
						LinkedBindingBuilder linker = binder.bind(f.getClass());
						linker.toInstance(f);
					}

					// reflectively look for a configuration class and inject
					String configurationClassname = f.getClass().getName() + "Configuration";

					ETLTestValueObject query = configuration.query("features." + f.getFeatureName());

					if (query != null)
					{
						// optionally validate
						FeatureMetaInfo meta = f.getMetaInfo();

						JsonSchema featureConfigurationValidator = meta.getFeatureConfigurationValidator();

						if (featureConfigurationValidator != null)
						{
							try
							{
								JsonValidator vlad = new JsonValidator(featureConfigurationValidator, new ClasspathSchemaResolver(f));

								vlad.validate(query.getJsonNode());
							}
							catch (JsonSchemaValidationException e)
							{
								throw new IllegalArgumentException("Invalid configuration for feature ["
										+ f.getFeatureName()
										+ "]: "
										+ e.toString(), e);
							}
						}

						try
						{
							Class conClass = Thread.currentThread().getContextClassLoader().loadClass(configurationClassname);

							// give gson a shot at filling it out
							Gson gson = new GsonBuilder()
									.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_DASHES)
									.create();

							String json = query.getJsonNode().toString();
							Object confObj = gson.fromJson(json, conClass);

							// check to see if more than one of the same class has been installed.
							LinkedBindingBuilder linker = binder.bind(conClass);
							linker.toInstance(confObj);

							applicationLog.info("Configuration class " + conClass + " bound to the session");
						}
						catch (ClassNotFoundException e)
						{
							applicationLog.info("Feature " + f.getFeatureName() + " does not expose a configuration class");
						}

						f.setFeatureConfiguration(query);
					}
				}
			});

			injector.injectMembers(f);
			Injector injectorSub = f.preCreate(injector);

			if (injectorSub != null)
			{
				injector = injectorSub;
			}

			f.initialize(injector);
		}

		//check for a process executor implementation
		ETLTestValueObject procex = configuration.query("runtimeSupport.processExecutor");

		if (procex != null)
		{
			applicationLog.info("User specified process executor: " + procex.getValueAsString());

			String name = procex.getValueAsString();

			try
			{
				Class impl = Class.forName(name);

				if (!ProcessExecutor.class.isAssignableFrom(impl))
				{
					throw new IllegalArgumentException("Class named '"
							+ impl
							+ "' does not implement the ProcessExecutor interface");
				}

				runtimeSupport.installProcessExecutor((ProcessExecutor) postCreate(impl.newInstance()));

				applicationLog.info("User specified process executor: '" + procex.getValueAsString() + "' installed");
			}
			catch (ClassNotFoundException e)
			{
				throw new IllegalArgumentException("Class named '" + name + "' not found", e);
			}
			catch (InstantiationException e)
			{
				throw new IllegalArgumentException("Class named '" + name + "' could not be created", e);
			}
			catch (IllegalAccessException e)
			{
				throw new IllegalArgumentException("Class named '" + name + "' could not be created", e);
			}
		}
	}

	protected List getFeatures()
	{
		return features;
	}

	private void workOutPrerequisites()
	{
		// install any missing features
		boolean installed = false;

		for (Feature feat : new ArrayList(features))
		{
			for (String prereq : feat.getPrerequisites())
			{
				// check if it is already installed
				if (!installedFeatureMap.containsKey(prereq))
				{
					// it hasn't been installed yet, determine if it is a known feature
					Feature feature = discoveredFeaturesMap.get(prereq);

					if (feature == null)
					{
						throw new IllegalArgumentException("Feature named '" + prereq + "' not found");
					}

					addFeature(feature);
					installed = true;
				}
			}
		}

		// recurse if any features were installed
		if (installed)
		{
			workOutPrerequisites();
		}
	}

	public void addFeatureById(String featureId)
	{
		applicationLog.info("Installing feature id " + featureId);
		Feature feature = discoveredFeaturesMap.get(featureId);

		if (feature == null)
		{
			throw new IllegalArgumentException("Feature id '" + featureId + "' not discovered");
		}

		addFeature(feature);
	}

	public void addFeature(Feature feat)
	{
		if (guiceModule != null)
		{
			throw new IllegalStateException("Features may not be added after installFeatures is called");
		}

		applicationLog.info("Installing feature " + feat.getFeatureName());

		if (feat.getPrerequisites().contains(feat.getFeatureName()))
		{
			throw new IllegalArgumentException("A feature may not depend on itself: " + feat.getFeatureName());
		}

		features.add(feat);
		installedFeatureMap.put(feat.getFeatureName(), feat);

		Class aClass = feat.getClass();
		if (!installedFeatureTypeMap.containsKey(aClass))
		{
			installedFeatureTypeMap.put(aClass, new ArrayList());
		}

		installedFeatureTypeMap.get(aClass).add(feat);
	}

	public void addFeature(Class cl) throws InstantiationException, IllegalAccessException
	{
		if (!Feature.class.isAssignableFrom(cl))
		{
			applicationLog.severe("Bad feature added.  Does not implement the FeatureModule interface: " + cl);
		}
		else
		{
			addFeature((Feature) cl.newInstance());
		}
	}

	public RuntimeSupport getRuntimeSupport()
	{
		return runtimeSupport;
	}

	public TestResults runTests()
	{
		ClassBroadcasterImpl cbi = new ClassBroadcasterImpl(
				postCreate(new FeatureClassLocatorProxy(features)),
				postCreate(new FeatureDirectorProxy(features)),
				postCreate(new FeatureListenerProxy(features)),
				applicationLog,
				variableContext
		);

		// add the implicit variables to the context
		variableContext.declareAndSetStringValue("projectName", runtimeSupport.getProjectName());
		variableContext.declareAndSetStringValue("projectUser", runtimeSupport.getProjectUser());
		variableContext.declareAndSetStringValue("projectUID", runtimeSupport.getProjectUID());

		ETLTestCoordinator coordinator = new ETLTestCoordinator(
				cbi,
				postCreate(new FeatureStatusReporterProxy(features))
		);

		coordinator.beginTesting();

		for (Feature feature : features)
		{
			try
			{
				applicationLog.info("Disposing feature: " + feature.getFeatureName());
				feature.dispose();
			}
			catch (Exception exc)
			{
				applicationLog.severe("Error disposing feature", exc);
			}
		}

		diffReporter.dispose();

		return resultsFeatureModule.getTestClassResults();
	}

	public static void sortByDependency(List features)
	{
		// create a map of features
		Map fmap = new HashMap();

		Iterator fit = features.iterator();

		while (fit.hasNext())
		{
			Feature f = fit.next();

			if (fmap.containsKey(f.getFeatureName()))
			{
				System.out
						.println("Feature " + f.getFeatureName() + " already installed as " + fmap.get(f.getFeatureName())
								.getClass() + ", ignoring duplicate " + f.getClass());
			}
			else
			{
				fmap.put(f.getFeatureName(), f);
			}
		}

		// iterate through list and check for prerequisites and circular dependencies
		fit = features.iterator();

		while (fit.hasNext())
		{
			Feature f = fit.next();

			List preq = f.getPrerequisites();

			Iterator prit = preq.iterator();

			while (prit.hasNext())
			{
				String pr = prit.next();

				Feature prf = fmap.get(pr);

				if (prf == null)
				{
					throw new IllegalArgumentException("Missing prerequisite feature: " + pr);
				}

				if (prf.getPrerequisites().contains(f.getFeatureName()))
				{
					throw new IllegalStateException("Circular dependencies: " + pr + " <<>> " + f.getFeatureName());
				}
			}
		}

		final Map priorities = new HashMap();

		// on the first pass, assign each feature its' 'suggested' priority
		for (Feature feature : features)
		{
			priorities.put(feature.getFeatureName(), new Long(feature.getPriorityLevel()));
		}

		// go through the list, and for each dependency, assign this feature 1 higher than it's highest priority
		boolean adjusted = true;
		while (adjusted)
		{
			adjusted = false;

			for (Feature feature : features)
			{
				for (String prereq : feature.getPrerequisites())
				{
					Long prePri = priorities.get(prereq);
					Long pri = priorities.get(feature.getFeatureName());

					Long preNew = new Long(Math.max(prePri.longValue() + 1L, pri.longValue()));

					priorities.put(feature.getFeatureName(), preNew);

					adjusted = adjusted | !preNew.equals(pri);
				}
			}
		}


		// use a sorter to put them into correct order
		Collections.sort(features, new Comparator()
		{
			@Override
			public int compare(Feature feature, Feature feature1)
			{
				Long fp1 = priorities.get(feature.getFeatureName());
				Long fp2 = priorities.get(feature1.getFeatureName());

				return fp1.compareTo(fp2);
			}
		});
	}

	public  T postCreate(T object)
	{
		injector.injectMembers(object);

		return object;
	}
}