com.gs.obevo.impl.graph.GraphEnricherImpl Maven / Gradle / Ivy
/**
* Copyright 2017 Goldman Sachs.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.gs.obevo.impl.graph;
import com.gs.obevo.api.appdata.CodeDependency;
import com.gs.obevo.api.appdata.CodeDependencyType;
import com.gs.obevo.api.platform.ChangeType;
import org.eclipse.collections.api.RichIterable;
import org.eclipse.collections.api.block.function.Function;
import org.eclipse.collections.api.block.procedure.primitive.ObjectIntProcedure;
import org.eclipse.collections.api.list.MutableList;
import org.eclipse.collections.api.map.MutableMap;
import org.eclipse.collections.api.multimap.Multimap;
import org.eclipse.collections.api.tuple.Pair;
import org.eclipse.collections.impl.block.factory.Functions;
import org.eclipse.collections.impl.factory.Lists;
import org.eclipse.collections.impl.factory.Maps;
import org.eclipse.collections.impl.map.mutable.UnifiedMap;
import org.eclipse.collections.impl.tuple.Tuples;
import org.jgrapht.DirectedGraph;
import org.jgrapht.graph.DefaultDirectedGraph;
import org.jgrapht.graph.DefaultEdge;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Created a graph out of the input changes, so that the graph can be shared by other components
*/
public class GraphEnricherImpl implements GraphEnricher {
private static final Logger LOG = LoggerFactory.getLogger(GraphEnricherImpl.class);
private final Function convertDbObjectName;
public GraphEnricherImpl(Function convertDbObjectName) {
this.convertDbObjectName = convertDbObjectName;
}
@Override
public DirectedGraph createDependencyGraph(RichIterable inputs, boolean rollback) {
final RichIterable> changeIndexes = Lists.mutable.of(
new ObjectIndex(),
new SchemaObjectIndex(),
new ObjectChangeIndex(),
new SchemaChangeObjectIndex()
);
for (ChangeIndex changeIndex : changeIndexes) {
for (T change : inputs) {
changeIndex.add(change);
}
}
final DefaultDirectedGraph graph = new DefaultDirectedGraph(DefaultEdge.class);
// First - add the core objects to the graph
for (T change : inputs) {
graph.addVertex(change);
}
// Now add the declared dependencies to the graph
for (T changeGroup : inputs) {
for (SortableDependency change : changeGroup.getComponents()) {
if (change.getCodeDependencies() != null) {
for (CodeDependency dependency : change.getCodeDependencies()) {
T dependencyVertex = null;
for (ChangeIndex changeIndex : changeIndexes) {
dependencyVertex = changeIndex.retrieve(change.getObjectKey().getSchema(), dependency.getTarget());
if (dependencyVertex != null) {
if (LOG.isTraceEnabled()) {
LOG.trace("Discovered dependency from {} to {} using index {}",
dependencyVertex,
change.getObjectKey() + "-" + change.getChangeName(),
changeIndex);
}
break;
}
}
if (dependencyVertex == null) {
LOG.trace("Dependency not found; likely due to not enriching the full graph in source. Should be OK to ignore: {} - {}", dependency, change);
} else {
graph.addEdge(dependencyVertex, changeGroup, new DependencyEdge(dependencyVertex, changeGroup, dependency.getCodeDependencyType()));
}
}
}
}
}
// Add in changes within incremental files to ensure proper order
RichIterable> groupToComponentPairs = inputs.flatCollect(new Function>>() {
@Override
public Iterable> valueOf(T group) {
return group.getComponents().collect(Functions.pair(Functions.getFixedValue(group), Functions.getPassThru()));
}
});
final Multimap> incrementalChangeByObjectMap = groupToComponentPairs.groupBy(new Function, String>() {
@Override
public String valueOf(Pair pair) {
SortableDependency tSortMetadata = pair.getTwo();
String changeType = tSortMetadata.getObjectKey().getChangeType().getName();
if (changeType.equals(ChangeType.TRIGGER_INCREMENTAL_OLD_STR) || changeType.equals(ChangeType.FOREIGN_KEY_STR)) {
changeType = ChangeType.TABLE_STR;
}
return changeType + ":" + tSortMetadata.getObjectKey().getSchema() + ":" + convertDbObjectName.valueOf(tSortMetadata.getObjectKey().getObjectName());
}
});
for (RichIterable> sortMetadataPairs : incrementalChangeByObjectMap.multiValuesView()) {
final MutableList> sortedChanges = sortMetadataPairs.toSortedListBy(Functions.chain(Functions.secondOfPair(), new Function() {
@Override
public Integer valueOf(SortableDependency sortableDependency) {
return sortableDependency.getOrderWithinObject();
}
}));
if (sortedChanges.size() > 1) {
if (rollback) {
sortedChanges.forEachWithIndex(0, sortedChanges.size() - 2, new ObjectIntProcedure>() {
@Override
public void value(Pair each, int index) {
// for rollback, we go in reverse-order (each change follows the one after it in the file)
Pair nextChange = sortedChanges.get(index + 1);
graph.addEdge(nextChange.getOne(), each.getOne(), new DependencyEdge(nextChange.getOne(), each.getOne(), CodeDependencyType.IMPLICIT));
}
});
} else {
sortedChanges.forEachWithIndex(1, sortedChanges.size() - 1, new ObjectIntProcedure>() {
@Override
public void value(Pair each, int index) {
// for regular mode, we go in regular-order (each change follows the one before it in the file)
Pair previousChange = sortedChanges.get(index - 1);
graph.addEdge(previousChange.getOne(), each.getOne(), new DependencyEdge(previousChange.getOne(), each.getOne(), CodeDependencyType.IMPLICIT));
}
});
}
}
}
// validate
GraphUtil.validateNoCycles(graph,
new Function() {
@Override
public String valueOf(T t) {
return t.getComponents().collect(new Function() {
@Override
public String valueOf(SortableDependency sortableDependency) {
return "[" + sortableDependency.getObjectKey().getObjectName() + "." + sortableDependency.getChangeName() + "]";
}
}).makeString(", ");
}
},
new Function() {
@Override
public String valueOf(DefaultEdge dependencyEdge) {
return "-" + ((DependencyEdge) dependencyEdge).getEdgeType();
}
});
return graph;
}
private interface ChangeIndex {
void add(T change);
T retrieve(String schema, String dependency);
}
/**
* Looks for the given dependency/object
*/
private class ObjectIndex implements ChangeIndex {
private final MutableMap, T> schemaToObjectMap = Maps.mutable.empty();
@Override
public void add(T changeGroup) {
for (SortableDependency change : changeGroup.getComponents()) {
T existingChange = retrieve(change.getObjectKey().getSchema(), convertDbObjectName.valueOf(change.getObjectKey().getObjectName()));
// TODO getFirst is not ideal here
if (existingChange == null || existingChange.getComponents().getFirst().getOrderWithinObject() < change.getOrderWithinObject()) {
// only keep the latest (why latest vs earliest?)
schemaToObjectMap.put(Tuples.pair(change.getObjectKey().getSchema(), convertDbObjectName.valueOf(change.getObjectKey().getObjectName())), changeGroup);
}
}
}
@Override
public T retrieve(String schema, String dependency) {
return (T) schemaToObjectMap.get(Tuples.pair(schema, convertDbObjectName.valueOf(dependency)));
}
}
private class SchemaObjectIndex implements ChangeIndex {
private final MutableMap objectMap = UnifiedMap.newMap();
@Override
public void add(T changeGroup) {
for (SortableDependency change : changeGroup.getComponents()) {
T existingChange = retrieve(change.getObjectKey().getSchema(), convertDbObjectName.valueOf(change.getObjectKey().getObjectName()));
// TODO getFirst is not ideal here
if (existingChange == null || existingChange.getComponents().getFirst().getOrderWithinObject() < change.getOrderWithinObject()) {
// only keep the latest (why latest vs earliest?)
objectMap.put(convertDbObjectName.valueOf(change.getObjectKey().getSchema() + "." + change.getObjectKey().getObjectName()), changeGroup);
}
}
}
@Override
public T retrieve(String schema, String dependency) {
return objectMap.get(convertDbObjectName.valueOf(dependency));
}
}
private class ObjectChangeIndex implements ChangeIndex {
private final MutableMap, T> schemaToObjectMap = Maps.mutable.empty();
@Override
public void add(T changeGroup) {
for (SortableDependency change : changeGroup.getComponents()) {
schemaToObjectMap.put(Tuples.pair(change.getObjectKey().getSchema(), convertDbObjectName.valueOf(change.getObjectKey().getObjectName() + "." + change.getChangeName())), changeGroup);
}
}
@Override
public T retrieve(String schema, String dependency) {
return (T) schemaToObjectMap.get(Tuples.pair(schema, convertDbObjectName.valueOf(dependency)));
}
}
private class SchemaChangeObjectIndex implements ChangeIndex {
private final MutableMap objectMap = UnifiedMap.newMap();
@Override
public void add(T changeGroup) {
for (SortableDependency change : changeGroup.getComponents()) {
objectMap.put(convertDbObjectName.valueOf(change.getObjectKey().getSchema() + "." + change.getObjectKey().getObjectName() + "." + change.getChangeName()), changeGroup);
}
}
@Override
public T retrieve(String schema, String dependency) {
return objectMap.get(convertDbObjectName.valueOf(dependency));
}
}
/**
* Custom edge type to allow for better error logging for cycles, namely to show the dependency edge type.
*/
private static class DependencyEdge extends DefaultEdge {
private final T source;
private final T target;
private final CodeDependencyType edgeType;
DependencyEdge(T source, T target, CodeDependencyType edgeType) {
this.source = source;
this.target = target;
this.edgeType = edgeType;
}
@Override
public T getSource() {
return source;
}
@Override
public T getTarget() {
return target;
}
CodeDependencyType getEdgeType() {
return edgeType;
}
@Override
public String toString() {
return edgeType.name();
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy