All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.regnosys.rosetta.translate.synonymmap.ModelCombiner Maven / Gradle / Ivy

There is a newer version: 11.25.1
Show newest version
package com.regnosys.rosetta.translate.synonymmap;

import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.codahale.metrics.Timer;
import com.codahale.metrics.Timer.Context;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.regnosys.rosetta.common.util.StreamUtils;
import com.regnosys.rosetta.generator.object.ExpandedAttribute;
import com.regnosys.rosetta.rosetta.RosettaEnumeration;
import com.regnosys.rosetta.translate.IngesterGenerator;
import com.regnosys.rosetta.translate.MappingError;
import com.regnosys.rosetta.translate.MappingError.MappingErrorLevel;
import com.regnosys.rosetta.translate.datamodel.Attribute;
import com.regnosys.rosetta.translate.datamodel.Cardinality;
import com.regnosys.rosetta.translate.datamodel.Entity;
import com.regnosys.rosetta.translate.datamodel.EntityImpl;
import com.regnosys.rosetta.translate.datamodel.NamespaceName;
import com.regnosys.rosetta.translate.datamodel.Schema;

public class ModelCombiner {

	private final static Logger LOGGER = LoggerFactory.getLogger(ModelCombiner.class);
	private static final String BASIC_ERROR_STRING = "Type Mismatch: The rosettaPath %s.%s is of type %s "
			+ "while the schema path %s.%s is a basic type";

	public List combineModels(Schema model, List synonyms, List errors,
			Collection topEntityNames) {
		Timer timer = IngesterGenerator.GENERATOR_METRICS.timer("combining models");
		Context time = timer.time();

		List topEntities = null;
		if (topEntityNames.isEmpty()) {
			EntityImpl topEnt = new EntityImpl(new NamespaceName("top", "top"), false);
			topEnt.getAttributes().addAll(model.getAttributes());
			topEntities = Collections.singletonList(topEnt);
		} else {
			topEntities = model.getAttributes().stream().filter(a -> topEntityNames.contains(a.getName()))
					.map(a -> a.getType()).collect(Collectors.toList());
		}

		Collection globals = model.getGlobalTypes();

		for (Entity topEntity : topEntities) {
			for (SynonymMap sm : synonyms) {
				combine(sm, topEntity, true, globals, errors);
				if (sm.getSuperMap() != null) {
					combine(sm.getSuperMap(), topEntity, true, globals, errors);
				}
			}
		}
		for (SynonymMap sm : synonyms) {
			combineGlobals(sm, globals, errors);
		}
		time.stop();
		return synonyms;
	}

	public List prune(List synonyms) {
		HashMultimap removed = HashMultimap.create();

		List prune = prune(synonyms, removed);
		for (Entry rem : removed.entries()) {
			String location = rem.getKey().getEnclosingType() + "." + rem.getKey().getName();
			LOGGER.trace("Removed broken synonyms on " + location + "[" + rem.getValue().toPathsString() + "]");
		}
		return prune;
	}

	public List prune(List synonyms, Multimap removed) {
		Timer timer = IngesterGenerator.GENERATOR_METRICS.timer("pruning model");
		Context time = timer.time();
		Map replacedMaps = new HashMap<>();
		List pruned = synonyms.stream().map(s -> removeUnconnectedEntities(s, replacedMaps, removed))
				.collect(Collectors.toList());
		time.stop();
		return pruned;
	}

	/**
	 * Combines a synonymMap (representing a rosetta class) with an entity any
	 * synonym of the class will be matched against the attributes of the entity at
	 * the end of a synonym chain we can deduce a possible mapping for the next
	 * synonym map
	 *
	 * @param map
	 * @param entity
	 * @param actualMatch - if the RosettaClass->entity match is a proper match or
	 *                    just a possible one based on the existence of xsi:type
	 * @param errors      Accumulator for errors found while combining
	 */
	private void combine(SynonymMap map, Entity entity, boolean actualMatch, Collection globals,
			List errors) {
		if (map.getLinkedEntities().contains(entity)) {
			return;
		}

		if (actualMatch) {
			map.getLinkedEntities().add(entity);
		}

		for (AttributeGroupMapping mappedGroup : map.getMappedGroups()) {
			for (SynonymGroup sg : mappedGroup.getGroup().getSynonymGroups()) {
				for (SynonymValue sv : sg.getSynonymValues()) {
					Entity endEntity = combinePath(entity, sv, mappedGroup, actualMatch, globals, map, errors);
					if (endEntity != null) {
						// The class mapped to by this pair maps to the current entity
						combine(mappedGroup.getMappings(), endEntity, true, globals, errors);
						SynonymMap superMap = mappedGroup.getMappings().getSuperMap();
						while (superMap != null) {
							combine(superMap, endEntity, true, globals, errors);
							superMap = superMap.getSuperMap();
						}
					}
				}

				// now match all the synonyms used in conditions
				for (SynonymCondition condition : sg.getConditions()) {
					for (SynonymTest test : condition.getCondition()) {
						for (SynonymValue sv : test.getPaths()) {
							combinePath(entity, sv, mappedGroup, actualMatch, globals, map, errors);
						}
					}
				}
			}
		}
	}

	// The get all attributes function is relatively expensive and called a lot with
	// the same arguments a cache of the results is a worthwhile saving (eg 2
	// seconds - .09 seconds for a typical run)
	private final Map> entityAttributeCache = new HashMap<>();

	private Set getAllAttributes(Entity parentEntity, Collection globals) {
		return entityAttributeCache.computeIfAbsent(parentEntity, e -> {
			Set result = new HashSet<>();
			Queue entitiesToCheck = new ArrayDeque<>();
			Set checked = new HashSet<>();
			entitiesToCheck.add(parentEntity);

			while (!entitiesToCheck.isEmpty()) {
				Entity entity = entitiesToCheck.remove();
				if (checked.contains(entity))
					continue;
				checked.add(entity);
				
				List attributes = entity.getAttributes();
				result.addAll(attributes);

				Set substitutedAttributes = getSubstitutedAttributes(parentEntity, entity, globals);
				result.addAll(substitutedAttributes);
				
				entitiesToCheck.addAll(entity.getKnownExtendingEntities());
			}
			return result;
		});
	}

	private Set getSubstitutedAttributes(Entity parentEntity, Entity currentEntity, Collection globals) {
		Set substitutedAttributes = new HashSet<>();

		Entity globalParentEntity = globals.stream().filter(globalEntity -> parentEntity.equals(globalEntity))
				.map(Entity::getExtendedEntity).filter(Objects::nonNull).findFirst().orElse(null);

		if (globalParentEntity == null) {
			return substitutedAttributes;
		}

		for (Attribute attribute : currentEntity.getAttributes()) {
			addSubstitutedAttributes(globalParentEntity, attribute.getType(), globals, substitutedAttributes);
		}
		return substitutedAttributes;
	}

	private void addSubstitutedAttributes(Entity parentEntity, Entity attrEntity, Collection globals,
			Set result) {
		for (Entity knownExtendingEntity : attrEntity.getKnownExtendingEntities()) {
			Set substitutedAttributes = parentEntity.getAttributes().stream()
					.filter(a -> knownExtendingEntity.equals(a.getType())).collect(Collectors.toSet());

			if (!substitutedAttributes.isEmpty()) {
				substitutedAttributes.forEach(substitutedAttribute -> {
					if (!result.contains(substitutedAttribute)) {
						LOGGER.trace("Adding substituted attribute {}{} from {} to parentEntity {}",
								substitutedAttribute.getName(),
								substitutedAttribute.getCardinality().toShortString(),
								knownExtendingEntity.getName().getName(),
								parentEntity.getName().getName());
						result.add(substitutedAttribute);
					}
				});
				
			}

			// recursively check extending entities
			addSubstitutedAttributes(parentEntity, knownExtendingEntity, globals, result);
		}
	}

	private Entity combinePath(Entity entity, SynonymValue sv, AttributeGroupMapping mappedGroup, boolean actualMatch,
			Collection globals, SynonymMap startPoint, List errors) {
		Entity currentEntity = entity;
		Entity result = null;

		for (Element pathElement : sv.getSynonymPath()) {
			String pathElementName = pathElement.getName();
			// if we are trying to match to global types anything can match ANYWHERE
			boolean matched = false || !actualMatch;

			Set atts = getAllAttributes(currentEntity, globals);

			for (Attribute att : atts) {
				String attName = att.getName();
				Entity attEntity = att.getType();

				if (attName.equals(pathElementName)) {
					matched = true;
					// we have discovered that this element corresponds to this attribute
					if (pathElement.getEntity() != null) {
						// something already matched to this - this is relatively normal
						// pick the more complicated cardinality case
						Cardinality card = Cardinality.max(pathElement.getCardinality(), att.getCardinality());
						pathElement.setEntity(pathElement.getEntity(), card);
					} else {
						pathElement.setEntity(attEntity, att.getCardinality());
					}

					if (attEntity.hasData() && !mappedGroup.getMappings().isBasic() && actualMatch) {
						// the synonym is a leaf but we haven't finished the path -
						// we ignore coincidence matches caused by the whole global entity matching
						// thing
						errors.add(new MappingError(MappingErrorLevel.ERROR,
								String.format(BASIC_ERROR_STRING, startPoint.getRosetta().getName(),
										mappedGroup.getGroup().attsToString(),
										mappedGroup.getMappings().getRosetta().getName(), entity.getName().getName(),
										sv.toPathString())));
					}

					currentEntity = attEntity;
					result = attEntity;
				}
			}
			if (!matched)
				return null;

		}
		return result;
	}

	/**
	 * The xml xsi:type attribute means that any xml element can be replaced with
	 * any global type therefore any element can effectively have any of the
	 * attributes of the global types we should attempt to combine every mapping
	 * with every global type
	 *
	 * @param mapping
	 * @param globals
	 */
	private void combineGlobals(SynonymMap mapping, Collection globals, List errors) {
		List maps = StreamUtils.flattenTreeC(mapping, sm -> sm.childMappings())
				.collect(Collectors.toList());
		for (SynonymMap map : maps) {
			for (Entity e : globals) {
				combine(map, e, false, globals, errors);
				while (map.getSuperMap() != null) {
					combine(map.getSuperMap(), e, false, globals, errors);
					map = map.getSuperMap();
				}
			}
		}
	}

	/**
	 * Go through all the synonym entities we have we created and remove the ones
	 * that we cannot map to.
	 *
	 * @param replacedMaps
	 */

	private SynonymMap removeUnconnectedEntities(SynonymMap synonym, Map replacedMaps,
			Multimap removed) {
		if (synonym == null)
			return null;
		if (replacedMaps.containsKey(synonym)) {
			SynonymMap synonymMap = replacedMaps.get(synonym);
			return synonymMap;
		}

		if (synonym.getRosetta() instanceof RosettaEnumeration) {
			return synonym;// Many enum value synonyms don't come from xsd directly
		}

		SynonymMap parentMap = removeUnconnectedEntities(synonym.getSuperMap(), replacedMaps, removed);
		SynonymMap result = new SynonymMap(synonym.getRosetta(), parentMap);
		replacedMaps.put(synonym, result);

		for (AttributeGroupMapping mappedGroups : synonym.getMappedGroups()) {
			AttributeGroup group = removeUnconnected(mappedGroups.getGroup(), removed);
			if (group != null) {
				SynonymMap newMap = removeUnconnectedEntities(mappedGroups.getMappings(), replacedMaps, removed);
				result.addMapping(group, newMap);
				// copy all merge synonyms
				result.getMergeSynonyms().putAll(synonym.getMergeSynonyms());
			}
		}

		for (Entry s : synonym.getConditionalCaptures().entries()) {
			if (isConnected(s.getKey())) {
				result.getConditionalCaptures().put(s.getKey(), s.getValue());
			}
		}
		return result;
	}

	private AttributeGroup removeUnconnected(AttributeGroup pair, Multimap removed) {
		List synonymPaths = pair.getSynonymGroups().stream().sequential()
				.map(sg -> removeUnconnected(sg, removed, pair)).filter(Objects::nonNull).collect(Collectors.toList());
		if (synonymPaths.isEmpty())
			return null;
		return new AttributeGroup(synonymPaths, pair.getAttributePath());
	}

	private SynonymGroup removeUnconnected(SynonymGroup sg, Multimap removed,
			AttributeGroup pair) {
		boolean prunedCondition = sg.getConditions().stream().anyMatch(c -> isUnconnected(c));
		if (prunedCondition) {
			ExpandedAttribute expandedAttribute = pair.getAttributePath().get(pair.getAttributePath().size() - 1);
			removed.put(expandedAttribute, sg);
			return null;
		}
		// special conditional groups are empty to start with - we need to keep them
		if (sg.getSynonymValues().isEmpty()) {
			return sg;
		}

		List paths = new ArrayList<>();
		for (SynonymValue val : sg.getSynonymValues()) {
			if (isConnected(val)) {
				paths.add(val);
			}
		}
		if (paths.isEmpty()) {
			ExpandedAttribute expandedAttribute = pair.getAttributePath().get(pair.getAttributePath().size() - 1);
			removed.put(expandedAttribute, sg);
			return null;
		}

		return new SynonymGroup(paths, sg.getConditions(), sg.getMapperName(), sg.getFormatString(),
				sg.getPatternMatcher(), sg.getPatternReplace(), sg.isRemoveHtml());
	}

	private boolean isUnconnected(SynonymCondition c) {
		return c.getCondition().stream().anyMatch(t -> isUnconnected(t));
	}

	private boolean isUnconnected(SynonymTest t) {
		return t.getPaths().stream().anyMatch(v -> !isConnected(v));
	}

	private boolean isConnected(SynonymValue sv) {
		return sv.getSynonymPath().stream().anyMatch(e -> e.getEntity() != null);
	}
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy