com.gs.obevo.db.apps.reveng.AquaRevengMain Maven / Gradle / Ivy
/**
* Copyright 2017 Goldman Sachs.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.gs.obevo.db.apps.reveng;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.gs.obevo.api.appdata.Schema;
import com.gs.obevo.api.platform.ChangeType;
import com.gs.obevo.api.platform.Platform;
import com.gs.obevo.db.api.appdata.DbEnvironment;
import com.gs.obevo.db.api.platform.DbDeployerAppContext;
import com.gs.obevo.db.api.platform.DbPlatform;
import com.gs.obevo.dbmetadata.api.DaCatalog;
import com.gs.obevo.dbmetadata.api.DaSchemaInfoLevel;
import com.gs.obevo.dbmetadata.api.RuleBinding;
import com.gs.obevo.impl.changetypes.UnclassifiedChangeType;
import com.gs.obevo.impl.util.MultiLineStringSplitter;
import com.gs.obevo.util.FileUtilsCobra;
import com.gs.obevo.util.inputreader.Credential;
import com.gs.obevo.util.inputreader.CredentialReader;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.eclipse.collections.api.RichIterable;
import org.eclipse.collections.api.block.function.Function;
import org.eclipse.collections.api.block.predicate.Predicate;
import org.eclipse.collections.api.collection.ImmutableCollection;
import org.eclipse.collections.api.list.MutableList;
import org.eclipse.collections.api.map.ImmutableMap;
import org.eclipse.collections.api.map.MutableMap;
import org.eclipse.collections.api.multimap.Multimap;
import org.eclipse.collections.api.partition.list.PartitionMutableList;
import org.eclipse.collections.api.set.ImmutableSet;
import org.eclipse.collections.api.tuple.Pair;
import org.eclipse.collections.impl.block.factory.StringFunctions;
import org.eclipse.collections.impl.block.factory.StringPredicates;
import org.eclipse.collections.impl.factory.Lists;
import org.eclipse.collections.impl.factory.Maps;
import org.eclipse.collections.impl.factory.Sets;
import org.eclipse.collections.impl.multimap.list.FastListMultimap;
import org.eclipse.collections.impl.tuple.Tuples;
public class AquaRevengMain {
private static final CredentialReader credentialReader = new CredentialReader();
private static final FileFilter VCS_FILE_FILTER = FileFilterUtils.makeCVSAware(
FileFilterUtils.makeSVNAware(
FileFilterUtils.trueFileFilter()));
private boolean tablespaceToken;
private boolean preprocessSchemaTokens;
@Deprecated
private boolean tokenizeDefaultSchema;
private boolean generateBaseline;
private Multimap ruleBindingMap = FastListMultimap.newMultimap();
private File workDir;
public void execute(AquaRevengArgs args) {
RevengMode mode = args.getMode();
this.workDir = FileUtilsCobra.createTempDir("deploy-aquareveng");
switch (mode) {
case DATA:
CsvStaticDataWriter.start(args, this.workDir);
break;
case SCHEMA:
this.tablespaceToken = args.getTablespaceToken();
this.tokenizeDefaultSchema = args.getTokenizeDefaultSchema();
this.preprocessSchemaTokens = args.isPreprocessSchemaTokens();
this.generateBaseline = args.isGenerateBaseline();
this.doExecute(args);
break;
default:
throw new IllegalArgumentException("No other mode supported for reveng: " + mode);
}
}
private void doExecute(final AquaRevengArgs args) {
File input = args.getInputDir();
File outputDir = args.getOutputDir();
System.out.println("Performing reverse engineering on input " + input);
Validate.notNull(args.getDbSchema(), "dbSchema argument must be specified");
Validate.notNull(input, "Directory " + input + " was not found"); // Maven supplies a null File value in this case
if (args.getDbHost() != null) {
DbEnvironment env = new DbEnvironment();
env.setPlatform(args.getDbPlatform());
env.setSystemDbPlatform(args.getDbPlatform());
env.setDbHost(args.getDbHost());
env.setDbPort(args.getDbPort());
env.setDbServer(args.getDbServer());
if (args.getDriverClass() != null) {
env.setDriverClassName(args.getDriverClass());
}
Schema schema = new Schema(args.getDbSchema());
env.setSchemas(Sets.immutable.with(schema));
Credential credential = credentialReader.getCredential(args.getUsername(), args.getPassword(), false, null, null,
null);
final DbDeployerAppContext ctxt = env.getAppContextBuilder().setCredential(credential).setWorkDir(workDir).buildDbContext();
// Aqua Data Studio can't extract ASE rule bindings, so we do it ourselves
DaCatalog database = ctxt.getDbMetadataManager().getDatabase(args.getDbSchema(), new DaSchemaInfoLevel().setRetrieveRuleBindings(true), false, false);
ImmutableCollection ruleBindings = database.getRuleBindings();
this.ruleBindingMap = ruleBindings.groupBy(RuleBinding.TO_OBJECT);
}
// Sorting it so that the reverse engineering output can be predicatable (good for consistency and unit tests)
MutableList files = Lists.mutable.with(input.listFiles(VCS_FILE_FILTER)).sortThisBy(new Function() {
@Override
public Comparable valueOf(File file) {
return file.getName();
}
});
if (preprocessSchemaTokens && !tablespaceToken) {
files = preprocessSchemaTokens(files, args.getDbSchema(), new File(outputDir, "interim-schemaReplaced"), args.getDbPlatform());
}
this.patternMap = initPatternMap(args.getDbPlatform());
MutableList allRevEngDestinations = Lists.mutable.empty();
for (File file : files) {
if (file.isDirectory()) {
System.out.println("Skipping " + file + " as this is a directory; not expecting this in the format");
continue;
} else if (file.getName().equals(CsvStaticDataWriter.STATIC_DATA_TABLES_FILE_NAME)) {
System.out.println("Skipping the static data table file: " + file);
continue;
}
allRevEngDestinations.addAll(this.calculateRevEngDest(args.getDbPlatform(), file, args.getNameCombinePattern()));
}
File outputWriteFolder = tablespaceToken ? outputDir : new File(outputDir, "final"); // check the tablespaceToken value for backwards-compatibility
new RevengWriter().write(args.getDbPlatform(), allRevEngDestinations, outputWriteFolder, this.generateBaseline, null, args.getJdbcUrl(), args.getDbHost(), args.getDbPort(), args.getDbServer(), args.getExcludeObjects());
}
private ImmutableMap initPatternMap(Platform platform) {
MutableMap params = Maps.mutable.with()
.withKeyValue(ChangeType.SP_STR, Pattern.compile("(?i)create\\s+proc(?:edure)?\\s+(\\w+)", Pattern.DOTALL))
.withKeyValue(ChangeType.FUNCTION_STR, Pattern.compile("(?i)create\\s+func(?:tion)?\\s+(\\w+)", Pattern.DOTALL))
.withKeyValue(ChangeType.VIEW_STR, Pattern.compile("(?i)create\\s+view\\s+(\\w+)", Pattern.DOTALL))
.withKeyValue(ChangeType.SEQUENCE_STR, Pattern.compile("(?i)create\\s+seq(?:uence)?\\s+(\\w+)", Pattern.DOTALL))
.withKeyValue(ChangeType.TABLE_STR, Pattern.compile("(?i)create\\s+table\\s+(\\w+)", Pattern.DOTALL))
.withKeyValue(ChangeType.DEFAULT_STR, Pattern.compile("(?i)create\\s+default\\s+(\\w+)", Pattern.DOTALL))
.withKeyValue(ChangeType.RULE_STR, Pattern.compile("(?i)create\\s+rule\\s+(\\w+)", Pattern.DOTALL))
.withKeyValue(ChangeType.USERTYPE_STR, Pattern.compile("(?i)^\\s*sp_addtype\\s+", Pattern.DOTALL))
.withKeyValue(ChangeType.INDEX_STR, Pattern.compile("(?i)create\\s+(?:unique\\s+)?(?:\\w+\\s+)?index\\s+\\w+\\s+on\\s+(\\w+)", Pattern.DOTALL));
MutableMap patternMap = Maps.mutable.with();
for (String changeTypeName : params.keysView()) {
if (platform.hasChangeType(changeTypeName)) {
ChangeType changeType = platform.getChangeType(changeTypeName);
patternMap.put(changeType, params.get(changeTypeName));
}
}
return patternMap.toImmutable();
}
private MutableList preprocessSchemaTokens(MutableList files, String dbSchema, final File interimFolder, DbPlatform dbPlatform) {
// adding DBO to help w/ Sybase ASE; we should make this code more polymorphic
String schemaSeparatorRegex = dbPlatform.isSubschemaSupported() ? "\\.(?:dbo)?\\." : "\\.";
final Pattern dbSchemaPattern = Pattern.compile(String.format("(?i)%1$s%2$s(\\w+)", dbSchema, schemaSeparatorRegex));
return files.collect(new Function() {
@Override
public File valueOf(File file) {
String fileContent = FileUtilsCobra.readFileToString(file);
final Matcher matcher = dbSchemaPattern.matcher(fileContent);
StringBuffer sb = new StringBuffer(fileContent.length());
while (matcher.find()) {
matcher.appendReplacement(sb, matcher.group(1));
}
matcher.appendTail(sb);
File tempFile = new File(interimFolder, file.getName());
FileUtilsCobra.writeStringToFile(tempFile, sb.toString());
return tempFile;
}
});
}
private ImmutableMap patternMap;
private static Pair getStartIndex(String str, Pattern p) {
Matcher m = p.matcher(str);
while (m.find()) {
String objectName = m.groupCount() > 0 ? m.group(1) : null; // by convention, the second group collected has the name
return Tuples.pair(m.start(), objectName);
}
return Tuples.pair(Integer.MAX_VALUE, null);
}
static String extractName(String objectName, String nameCombinePattern) {
if (nameCombinePattern != null) {
String patternStr = nameCombinePattern.replace("{}", "(.*)");
Pattern namePattern = Pattern.compile(patternStr);
Matcher nameMatcher = namePattern.matcher(objectName);
if (nameMatcher.matches()) {
return nameMatcher.group(1);
}
}
return objectName;
}
private MutableList calculateRevEngDest(DbPlatform dbPlatform, File file, String nameCombinePattern) {
String[] fileNameParts = file.getName().split("\\.");
String wholeFileString = FileUtilsCobra.readFileToString(file);
String schema = fileNameParts[0];
if (fileNameParts.length != 3) {
if (!schema.equals("dbo") || fileNameParts.length != 5) {
return Lists.mutable.with(new ChangeEntry(new RevEngDestination(schema, UnclassifiedChangeType.INSTANCE,
file.getName(), false), wholeFileString, "n/a", null, 0));
}
}
ChangeTypeInfo contentObjectInfo = this.determineChangeType(wholeFileString);
ChangeType objType = contentObjectInfo.getChangeType();
String fileObjectName = this.getObjectNameFromFilename(fileNameParts[1], objType);
// This is for cases in Sybase where teams had backed up db objects, and so it was created w/ a different name
// in the object, but the underlying content points to the original name
// We only fire off this check in case the scan of the content does pick up the object name, as to be
// conservative
// Addendum on 2014-10-15 - in case we know an object must be combined w/ another (e.g. index to parent),
// we don't consider the object diff there as a duplicate
boolean duplicate = dbPlatform.isDuplicateCheckRequiredForReverseEngineering() && !objType.getName().equals(ChangeType.INDEX_STR) && contentObjectInfo.getObjectName() != null &&
!contentObjectInfo.getObjectName().equalsIgnoreCase(fileObjectName);
String objectName = contentObjectInfo.getObjectName() != null
&& !duplicate ? contentObjectInfo.getObjectName() : fileObjectName;
String originalObjectName = objectName;
objectName = extractName(objectName, nameCombinePattern);
RevEngDestination dest = new RevEngDestination(schema, contentObjectInfo.getChangeType(),
objectName, duplicate);
// tokenize default schema name in the objects - this is only done if -tokenizeDefaultSchema is specified
// on the command line
if (this.tokenizeDefaultSchema) {
wholeFileString = this.tokenizeDefaultSchema(schema, wholeFileString, objType, objectName);
}
MutableList changes = Lists.mutable.empty();
// handle any special overriding use cases if needed (currently not needed for view or sequence)
if (objType.getName().equals(ChangeType.TABLE_STR)) {
MutableList statements = MultiLineStringSplitter.createSplitterOnSpaceAndLine("GO").valueOf(wholeFileString);
// append in tablespace if needed
if (this.tablespaceToken) {
statements.set(0, statements.get(0) + " IN ${" + schema.toUpperCase() + "_TABLESPACE" + "}");
}
PartitionMutableList splitStatements = statements.partition(StringPredicates
.contains("FOREIGN KEY"));
MutableList fkStatements = splitStatements.getSelected();
final Pattern triggerPattern = Pattern.compile("(?i).*create.*trigger\\s+(\\w+)\r?\n.*", Pattern.DOTALL);
splitStatements = splitStatements.getRejected().partition(new Predicate() {
@Override
public boolean accept(String s) {
return triggerPattern.matcher(s).matches();
}
});
MutableList triggerStatements = splitStatements.getSelected();
final Pattern indexPattern = Pattern.compile("(?i).*create.*index\\s+(\\w+)\r?\n.*", Pattern.DOTALL);
splitStatements = splitStatements.getRejected().partition(new Predicate() {
@Override
public boolean accept(String s) {
Matcher m = indexPattern.matcher(s);
return m.matches();
}
});
MutableList indexStatements = splitStatements.getSelected();
MutableList nonFkStatements = splitStatements.getRejected();
int selfOrder = 0;
String endingString;
if (nonFkStatements.isEmpty() || nonFkStatements.getLast().endsWith("GO") || StringUtils.isBlank(nonFkStatements.getLast())) {
endingString = "\n";
} else {
endingString = "\nGO\n";
}
changes.add(new ChangeEntry(dest, nonFkStatements.makeString("\n", "\nGO\n", endingString), "init", null,
selfOrder++));
if (!fkStatements.isEmpty()) {
changes.add(new ChangeEntry(dest,
fkStatements.collect(StringFunctions.trim()).makeString("", "\nGO\n", "\nGO\n"), "initFk",
"FK", selfOrder++));
}
for (String index : indexStatements) {
Matcher m = indexPattern.matcher(index);
if (m.matches()) {
String indexName = m.group(1);
changes.add(new ChangeEntry(dest, index.trim() + "\nGO", indexName, "INDEX", selfOrder++));
} else {
throw new IllegalStateException("Invalid state - this should have already had the index in it: "
+ index);
}
}
for (String trigger : triggerStatements) {
Matcher m = triggerPattern.matcher(trigger);
if (m.matches()) {
String name = m.group(1);
changes.add(new ChangeEntry(dest, trigger.trim() + "\nGO", name, "TRIGGER", selfOrder++));
} else {
throw new IllegalStateException(
"Invalid state - this should have already had the trigger name in it: " + trigger);
}
}
RichIterable ruleBindings = this.ruleBindingMap.get(objectName);
if (ruleBindings != null) {
int i = 1;
for (RuleBinding binding : ruleBindings) {
changes.add(new ChangeEntry(dest, binding.getSql().trim(), "binding" + i++, "FK", selfOrder++));
}
}
} else if (objType.getName().equals(ChangeType.INDEX_STR)) {
RevEngDestination newDest = new RevEngDestination(dest.getSchema(), dbPlatform.getChangeType(ChangeType.TABLE_STR),
dest.getObjectName(), dest.isDuplicate());
// the fileObjectName.replace line is to replace the file name w/ the object name in case it differs (e.g
// . coalescing the different table files into one)
changes.add(new ChangeEntry(newDest, wholeFileString.trim()
, "index_" + fileObjectName.replace(originalObjectName, objectName)
, "INDEX", 100));
// } else if (objType == ChangeType.TRIGGER) {
//
// RevEngDestination newDest = new RevEngDestination(dest.getSchema(), ChangeType.TABLE,
// dest.getObjectName(), dest.isDuplicate());
// // the fileObjectName.replace line is to replace the file name w/ the object name in case it differs (e.g
// // . coalescing the different table files into one)
// changes.add(new ChangeEntry(newDest, wholeFileString.trim()
// , "trigger_" + fileObjectName.replace(originalObjectName, objectName)
// , "TRIGGER", 200));
} else {
RichIterable ruleBindings = this.ruleBindingMap.get(objectName);
if (ruleBindings != null && !ruleBindings.isEmpty()) {
for (RuleBinding binding : ruleBindings) {
wholeFileString = wholeFileString + "\nGO\n" + binding.getSql().trim() + "\nGO";
}
}
changes.add(new ChangeEntry(dest, wholeFileString));
}
dbPlatform.postProcessChangeForRevEng(changes.getFirst(), wholeFileString);
return changes;
}
private String tokenizeDefaultSchema(String schema, String content, ChangeType objType, String objectName) {
ImmutableSet tokenizableChangeTypes = Sets.immutable.with(ChangeType.FUNCTION_STR, ChangeType.SP_STR, ChangeType.VIEW_STR, UnclassifiedChangeType.INSTANCE.getName());
// only tokenize the schema name for functions, sps, views, and other
if (tokenizableChangeTypes.contains(objType.getName())) {
File tempFile;
try {
// ensure the prefix is at least 3 chars long. Otherwise it will fail
String padName = "";
if (objectName.length() < 3) {
padName = "123";
}
tempFile = File.createTempFile(objectName + padName, ".sql");
} catch (IOException e) {
throw new RuntimeException("failed creating a temp file for " + objectName, e);
}
System.out.println("using temp file " + tempFile);
// tokenize the default schema
String tokenName = "${" + schema.toLowerCase() + ".token}";
content = content.replace(schema.toLowerCase() + ".", tokenName + ".");
content = content.replace(schema.toUpperCase() + ".", tokenName + ".");
FileUtilsCobra.writeStringToFile(tempFile, content);
return content;
}
return content;
}
private String getObjectNameFromFilename(String fname, ChangeType objType) {
if (!objType.getName().equals(ChangeType.SP_STR) && !objType.getName().equals(ChangeType.FUNCTION_STR)) {
return fname;
}
String result = fname;
// only for db2 for specific-names bit
int hyphenIndex = fname.lastIndexOf('-');
if (hyphenIndex != -1) {
// use the name to the right of the hyphen - ensure it doesn't start with SQL
// if it does, use what is on the right side
result = fname.substring(hyphenIndex + 1);
if (result.indexOf("SQL") == 0) {
result = fname.substring(0, hyphenIndex);
}
}
return result;
}
private ChangeTypeInfo determineChangeType(final String wholeFileString) {
RichIterable changeTypeInfos = this.patternMap.keyValuesView().collect(
new Function, ChangeTypeInfo>() {
@Override
public ChangeTypeInfo valueOf(Pair object) {
Pair contentInfo = getStartIndex(wholeFileString, object.getTwo());
return new ChangeTypeInfo(object.getOne()
, contentInfo.getOne()
, contentInfo.getTwo()
);
}
});
ChangeTypeInfo chosenChangeTypeInfo = changeTypeInfos.minBy(ChangeTypeInfo.TO_START_INDEX);
if (chosenChangeTypeInfo.getStartIndex() == Integer.MAX_VALUE) {
return new ChangeTypeInfo(UnclassifiedChangeType.INSTANCE, Integer.MAX_VALUE, null);
} else {
return chosenChangeTypeInfo;
}
}
private static class ChangeTypeInfo {
private final ChangeType changeType;
private final int startIndex;
private final String objectName;
static final Function TO_START_INDEX = new Function() {
@Override
public Integer valueOf(ChangeTypeInfo object) {
return object.getStartIndex();
}
};
private ChangeTypeInfo(ChangeType changeType, int startIndex, String objectName) {
this.changeType = changeType;
this.startIndex = startIndex;
this.objectName = objectName;
}
ChangeType getChangeType() {
return this.changeType;
}
int getStartIndex() {
return this.startIndex;
}
String getObjectName() {
return this.objectName;
}
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy