org.bitbucket.bradleysmithllc.etlunit.feature.database.DatabaseFeatureModule Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of etlunit-database Show documentation
There is a newer version: 4.6.0-eu
package org.bitbucket.bradleysmithllc.etlunit.feature.database;

/*
 * #%L
 * etlunit-database
 * %%
 * Copyright (C) 2010 - 2014 bradleysmithllc
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import com.google.gson.stream.JsonWriter;
import com.google.inject.Binder;
import com.google.inject.Injector;
import com.google.inject.Module;
import org.apache.commons.io.FileUtils;
import org.bitbucket.bradleysmithllc.etlunit.*;
import com.fasterxml.jackson.databind.JsonNode;
import org.bitbucket.bradleysmithllc.etlunit.listener.ClassListener;
import org.bitbucket.bradleysmithllc.etlunit.feature.AbstractFeature;
import org.bitbucket.bradleysmithllc.etlunit.feature.FeatureModule;
import org.bitbucket.bradleysmithllc.etlunit.feature.database.db.Database;
import org.bitbucket.bradleysmithllc.etlunit.feature.database.util.DimensionalHashMap;
import org.bitbucket.bradleysmithllc.etlunit.feature.database.util.DimensionalMap;
import org.bitbucket.bradleysmithllc.etlunit.io.FileBuilder;
import org.bitbucket.bradleysmithllc.etlunit.metadata.MetaDataContext;
import org.bitbucket.bradleysmithllc.etlunit.metadata.MetaDataManager;
import org.bitbucket.bradleysmithllc.etlunit.util.ObjectUtils;
import org.bitbucket.bradleysmithllc.json.validator.JsonUtils;
import org.bitbucket.bradleysmithllc.etlunit.parser.*;
import org.bitbucket.bradleysmithllc.etlunit.util.IOUtils;
import org.bitbucket.bradleysmithllc.etlunit.util.VelocityUtil;

import javax.inject.Inject;
import java.io.*;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.*;

@FeatureModule
public class DatabaseFeatureModule extends AbstractFeature {
	private MetaDataManager metaDataManager;
	private MetaDataContext dataMetaContext;
	private MetaDataContext scriptsMetaContext;

	private RuntimeSupport runtimeSupport;
	private DatabaseRuntimeSupport databaseRuntimeSupport;

	private static final List prerequisites = Arrays.asList("logging", "file");

	private final Map implementationMap = new HashMap();

	private DatabaseConfiguration databaseConfiguration;
	private DatabaseClassListener classListener;
	private JDBCClientImpl jdbcClient;

	public void addDatabaseImplementation(DatabaseImplementation impl) {
		if (implementationMap.containsKey(impl.getImplementationId())) {
			throw new IllegalArgumentException("Database Implementation " + impl.getImplementationId() + " already installed");
		}

		// create and inject a jdbc implementation
		impl.setJdbcClient(postCreate(new JDBCClientImpl()));
		implementationMap.put(impl.getImplementationId(), postCreate(impl));
	}

	@Inject
	public void receiveMetaDataManager(MetaDataManager manager)
	{
		metaDataManager = manager;
		dataMetaContext = metaDataManager.getMetaData().getOrCreateContext("data");
		scriptsMetaContext= metaDataManager.getMetaData().getOrCreateContext("database");
	}

	@Inject
	public void setRuntimeSupport(RuntimeSupport runtimeSupport) {
		this.runtimeSupport = runtimeSupport;
	}

	@Override
	protected Injector preCreateSub(Injector inj) {
		this.databaseRuntimeSupport = postCreate(new DatabaseRuntimeSupportImpl());
		jdbcClient = postCreate(new JDBCClientImpl());

		Injector childInjector = inj.createChildInjector(new Module() {
			public void configure(Binder binder) {
				binder.bind(DatabaseRuntimeSupport.class).toInstance(databaseRuntimeSupport);
				binder.bind(JDBCClient.class).toInstance(jdbcClient);
			}
		});

		return childInjector;
	}

	@Override
	public void initialize(Injector inj) {
		if (featureConfiguration != null) {
			databaseConfiguration = new DatabaseConfiguration(
					featureConfiguration,
					runtimeSupport.getProjectName(),
					runtimeSupport.getProjectVersion(),
					runtimeSupport.getProjectUser(),
					runtimeSupport.getProjectUID()
			);

			DbAssertExtender object1 = new DbAssertExtender(this);
			inj.injectMembers(object1);

			DbOperationsExecutor object = new DbOperationsExecutor(this);
			inj.injectMembers(object);

			classListener = new DatabaseClassListener(this, databaseConfiguration, object1, object);
			inj.injectMembers(classListener);
		}
	}

	@Override
	public List getPrerequisites() {
		return prerequisites;
	}

	public String getFeatureName() {
		return "database";
	}

	@Override
	public ClassListener getListener() {
		return classListener;
	}

	/*
	private SQLAggregator resolveSQLRef(String ddlRef, DatabaseConnection databaseConnection, SQLAggregator.SQLLocator locator) throws IOException, TestExecutionError {
		String text = locator.locate(ddlRef, databaseConnection);

		SQLAggregatorImpl ddlAgg = new SQLAggregatorImpl(ddlRef, locator, databaseConnection);

		return ddlAgg;
	}
	 */

	public SQLAggregator resolveSQLRef(String text) throws IOException, TestExecutionError {
		SQLAggregatorImpl ddlAgg = new SQLAggregatorImpl(text);

		return ddlAgg;
	}

	public SQLAggregator resolveDDLRef(final DDLSourceRef ddlRef, DatabaseConnection databaseConnection) throws IOException, TestExecutionError {
		String rawDDL = SQLAggregatorImpl.makeText(ddlRef, new SQLAggregator.SQLLocator() {
			public String locate(DDLSourceRef ref, DatabaseConnection connection) throws IOException, TestExecutionError {
				return getDDLText(ref).getText();
			}
		}, databaseConnection);

		File output = new FileBuilder(databaseRuntimeSupport.getGeneratedSourceDirectory(databaseConnection, null)).name("GENERATED_" + ddlRef.urlPath().replace('/', '_')).file();
		IOUtils.writeBufferToFile(output, new StringBuffer(rawDDL));

		try {
			Map bean = new HashMap();
			bean.put("databaseConnection", databaseConnection);
			bean.put("databaseRuntimeSupport", databaseRuntimeSupport);
			bean.put("databaseConfiguration", databaseConfiguration);
			bean.put("runtimeSupport", runtimeSupport);

			String newDDL = VelocityUtil.writeTemplate(rawDDL, bean);

			SQLAggregatorImpl impl = new SQLAggregatorImpl(newDDL);

			File outputProcessed = new FileBuilder(databaseRuntimeSupport.getGeneratedSourceDirectory(databaseConnection, null)).name("PROCESSED_" + ddlRef.urlPath().replace('/', '_')).file();
			IOUtils.writeBufferToFile(outputProcessed, new StringBuffer(newDDL));

			File outputClean = new FileBuilder(databaseRuntimeSupport.getGeneratedSourceDirectory(databaseConnection, null)).name("CLEAN_" + ddlRef.urlPath().replace('/', '_')).file();
			IOUtils.writeBufferToFile(outputClean, new StringBuffer(impl.getCleanText()));

			return impl;
		} catch (Exception e) {
			throw new IOException(e);
		}
	}

	/**
	 * Lookup the ddl text using a search path and return.
	 *
	 * @param ddlRef
	 * @return
	 * @throws IOException
	 * @throws TestExecutionError
	 */
	protected DDLSourceRef getDDLText(final DDLSourceRef ddlRef) throws IOException, TestExecutionError {
		// Enhance this to look at the classpath if the file isn't found
		DDLSourceRef actualRef = ddlRef;

		if (!loadDDLRef(actualRef)) {
			applicationLog.info("Unable to find ddl for reference: " + actualRef.urlPath() + ", trying without the connection ID");

			// try one more time, this time without the connection id
			actualRef = new DDLSourceRef(actualRef.getImplementationId(), null, actualRef.getSource());

			if (!loadDDLRef(actualRef)) {
				// try one more time, this time without the implementation id
				actualRef = new DDLSourceRef(null, null, actualRef.getSource());

				if (!loadDDLRef(actualRef)) {
					throw new TestExecutionError("Error resolving ddl ref: " + actualRef.urlPath(), DatabaseConstants.ERR_MISSING_DDL);
				}
			}
		}

		// instrument the ddl text with ref tags and line numbers
		StringWriter stringWriter = new StringWriter();
		PrintWriter str = new PrintWriter(stringWriter);

		BufferedReader br = new BufferedReader(new StringReader(actualRef.getText()));

		String line;

		int lineNo = 1;

		while ((line = br.readLine()) != null) {
			str.print("/**\tDDL Source [" + actualRef.refPath() + ":" + lineNo++ + "]\t**/");
			str.println(line);
		}

		actualRef.setText(stringWriter.toString());

		return actualRef;
	}

	private boolean loadDDLRef(DDLSourceRef ddl) throws IOException {
		String implementationId = ddl.getImplementationId();
		String databaseId = ddl.getDatabaseId();

		if (implementationId != null && !ddl.isAbsolute()) {
			FileBuilder ddlFile = new FileBuilder(runtimeSupport.getFeatureSourceDirectory(implementationId));

			if (databaseId != null) {
				ddlFile.subdir(databaseId);
			}

			File f = ddlFile.mkdirs().name(ddl.getSource()).file();

			if (f.exists()) {
				ddl.setText(FileUtils.readFileToString(f));
				return true;
			}
		}

		// check for a classpath reference
		URL ref = ObjectUtils.firstNotNull(Thread.currentThread().getContextClassLoader(), getClass().getClassLoader()).getResource(ddl.urlPath());

		if (ref != null) {
			ddl.setText(IOUtils.readURLToString(ref));

			return true;
		}

		return false;
	}

	protected void updateDatabaseFlag(String mode, DatabaseConnection databaseConnection) {
		JsonNode configNode = databaseConnection.getRawDatabaseConfiguration();

		File flagFile = getFlagFile(mode, databaseConnection);

		try {
			IOUtils.writeBufferToFile(flagFile, new StringBuffer(configNode.toString()));

			for (String script : databaseConnection.getSchemaScripts()) {
				File ref = getDDLFlagFile(script, mode, databaseConnection);

				StringBuilder bui = new StringBuilder();

				DDLSourceRef ddlSourceRef = new DDLSourceRef(databaseConnection.getImplementationId(), databaseConnection.getId(), script);
				SQLAggregator SQLAggregator = resolveDDLRef(ddlSourceRef, databaseConnection);
				bui.append(SQLAggregator.getText());
				long crc = IOUtils.checkSumStream(new ByteArrayInputStream(bui.toString().getBytes())).getValue();

				IOUtils.writeBufferToFile(ref, new StringBuffer("{\"checksum\": " + crc + "}"));
			}
		} catch (Exception e) {
			throw new IllegalArgumentException("", e);
		}
	}

	private File getDDLFlagFile(String ref, String mode, DatabaseConnection databaseConnection) {
		return runtimeSupport.createGeneratedSourceFile(databaseConnection.getImplementationId(), "ddl." + ref + "." + databaseConnection.getId() + "." + mode + "." + runtimeSupport.getProjectUID() + ".chk");
	}

	private File getFlagFile(String mode, DatabaseConnection databaseConnection) {
		return runtimeSupport.createGeneratedSourceFile(databaseConnection.getImplementationId(), "database." + databaseConnection.getId() + "." + mode + "." + runtimeSupport.getProjectUID() + ".tag");
	}

	protected boolean databaseOutOfDate(String mode, DatabaseConnection databaseConnection) {
		File flagFile = getFlagFile(mode, databaseConnection);

		JsonNode configNode = databaseConnection.getRawDatabaseConfiguration();

		try {
			if (flagFile.exists()) {
				JsonNode flgNode = JsonUtils.loadJson(flagFile);

				if (flgNode.equals(configNode)) {
					for (String script : databaseConnection.getSchemaScripts()) {
						File ref = getDDLFlagFile(script, mode, databaseConnection);

						if (!ref.exists()) {
							return true;
						}

						StringBuilder bui = new StringBuilder();

						DDLSourceRef ddlSourceRef = new DDLSourceRef(databaseConnection.getImplementationId(), databaseConnection.getId(), script);
						SQLAggregator SQLAggregator = resolveDDLRef(ddlSourceRef, databaseConnection);
						bui.append(SQLAggregator.getText());

						long crc = IOUtils.checkSumStream(new ByteArrayInputStream(bui.toString().getBytes())).getValue();

						JsonNode node = JsonUtils.loadJson(ref);

						if (node.get("checksum").asLong() != crc) {
							return true;
						}
					}

					return false;
				}
			}
		} catch (Exception e) {
			throw new IllegalArgumentException("", e);
		}

		return true;
	}

	protected void propogateProperties(ETLTestValueObjectBuilder builder, DatabaseConnection databaseConnection, String mode) {
		// pass a jdbc url along for external integration
		DatabaseImplementation impl = getImplementation(databaseConnection.getId());

		builder.key("jdbc-url").value(impl.getJdbcUrl(databaseConnection, mode));
		builder.key("jdbc-driver").value(impl.getJdbcDriverClass().getName());

		builder.key("database-name").value(impl.getDatabaseName(databaseConnection, mode));
		builder.key("login-name").value(impl.getLoginName(databaseConnection, mode));
		builder.key("password").value(impl.getPassword(databaseConnection, mode));

		// pass on the default server name if none specified
		builder.key("server-name").value(databaseConnection.getServerName());
		try {
			builder.key("server-address").value(databaseConnection.getServerAddress());
		} catch (UnknownHostException e) {
			throw new RuntimeException("Error getting database address", e);
		}
	}

	DatabaseImplementation getImplementation(String connectionId) {
		DatabaseConnection conn = databaseConfiguration.getDatabaseConnection(connectionId);

		return implementationMap.get(conn.getImplementationId());
	}

	public DatabaseConfiguration getDatabaseConfiguration() {
		return databaseConfiguration;
	}

	@Override
	public void dispose() {
		for (Map.Entry entry : implementationMap.entrySet()) {
			entry.getValue().dispose();
		}
	}

	@Override
	protected List getSupportedFolderNamesSub() {
		return Arrays.asList("data", "sql");
	}

	public MetaDataContext getDataMetaContext() {
		return dataMetaContext;
	}

	public MetaDataContext getDatabaseMetaContext() {
		return scriptsMetaContext;
	}
}