com.yahoo.config.application.OverrideProcessor Maven / Gradle / Ivy
The newest version!
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.config.application;
import com.yahoo.config.provision.CloudName;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.Tags;
import com.yahoo.text.XML;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import javax.xml.transform.TransformerException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
/**
* Handles overrides in a XML document according to the rules defined for multi-environment application packages.
*
* Rules:
*
* 1. A directive specifying both environment and region will override a more generic directive specifying only one of them
* 2. Directives are inherited in child elements
* 3. When multiple XML elements with the same name is specified (i.e. when specifying search or docproc chains),
* the id attribute of the element is used together with the element name when applying directives
*
* @author Ulf Lilleengen
*/
class OverrideProcessor implements PreProcessor {
private static final Logger log = Logger.getLogger(OverrideProcessor.class.getName());
private final InstanceName instance;
private final Environment environment;
private final RegionName region;
private final CloudName cloud;
private final Tags tags;
private static final String ID_ATTRIBUTE = "id";
private static final String IDREF_ATTRIBUTE = "idref";
private static final String INSTANCE_ATTRIBUTE = "instance";
private static final String ENVIRONMENT_ATTRIBUTE = "environment";
private static final String REGION_ATTRIBUTE = "region";
private static final String CLOUD_ATTRIBUTE = "cloud";
private static final String TAGS_ATTRIBUTE = "tags";
public OverrideProcessor(InstanceName instance, Environment environment, RegionName region, CloudName cloud, Tags tags) {
this.instance = instance;
this.environment = environment;
this.region = region;
this.cloud = cloud;
this.tags = tags;
}
public Document process(Document input) throws TransformerException {
log.log(Level.FINE, () -> "Preprocessing overrides with " + environment + "." + region);
Document ret = Xml.copyDocument(input);
Element root = ret.getDocumentElement();
applyOverrides(root, Context.empty());
return ret;
}
private void applyOverrides(Element parent, Context context) {
context = getParentContext(parent, context);
Map> elementsByTagName = elementsByTagNameAndId(XML.getChildren(parent));
retainOverriddenElements(elementsByTagName);
// For each tag name, prune overrides
for (Map.Entry> entry : elementsByTagName.entrySet()) {
pruneOverrides(parent, entry.getValue(), context);
}
// Repeat for remaining children;
for (Element child : XML.getChildren(parent)) {
applyOverrides(child, context);
// Remove attributes
child.removeAttributeNS(XmlPreProcessor.deployNamespaceUri, INSTANCE_ATTRIBUTE);
child.removeAttributeNS(XmlPreProcessor.deployNamespaceUri, ENVIRONMENT_ATTRIBUTE);
child.removeAttributeNS(XmlPreProcessor.deployNamespaceUri, REGION_ATTRIBUTE);
child.removeAttributeNS(XmlPreProcessor.deployNamespaceUri, CLOUD_ATTRIBUTE);
child.removeAttributeNS(XmlPreProcessor.deployNamespaceUri, TAGS_ATTRIBUTE);
}
}
private Context getParentContext(Element parent, Context context) {
Set instances = context.instances;
Set environments = context.environments;
Set regions = context.regions;
Set clouds = context.clouds;
Tags tags = context.tags;
if (instances.isEmpty())
instances = getInstances(parent);
if (environments.isEmpty())
environments = getEnvironments(parent);
if (regions.isEmpty())
regions = getRegions(parent);
if(clouds.isEmpty())
clouds = getClouds(parent);
if (tags.isEmpty())
tags = getTags(parent);
return Context.create(instances, environments, regions, clouds, tags);
}
/**
* Prune overrides from parent according to deploy override rules.
*
* @param parent parent {@link Element} above children.
* @param children children where one {@link Element} will remain as the overriding element
* @param context current context with instance, environment, region and cloud
*/
private void pruneOverrides(Element parent, List children, Context context) {
checkConsistentInheritance(children, context);
pruneNonMatching(parent, children);
retainMostSpecific(parent, children, context);
}
private void checkConsistentInheritance(List children, Context context) {
for (Element child : children) {
Set instances = getInstances(child);
if ( ! instances.isEmpty() && ! context.instances.isEmpty() && ! context.instances.containsAll(instances)) {
throw new IllegalArgumentException("Instances in child (" + instances +
") are not a subset of those of the parent (" + context.instances + ") at " + child);
}
Set environments = getEnvironments(child);
if ( ! environments.isEmpty() && ! context.environments.isEmpty() && ! context.environments.containsAll(environments)) {
throw new IllegalArgumentException("Environments in child (" + environments +
") are not a subset of those of the parent (" + context.environments + ") at " + child);
}
Set regions = getRegions(child);
if ( ! regions.isEmpty() && ! context.regions.isEmpty() && ! context.regions.containsAll(regions)) {
throw new IllegalArgumentException("Regions in child (" + regions +
") are not a subset of those of the parent (" + context.regions + ") at " + child);
}
Set clouds = getClouds(child);
if ( ! clouds.isEmpty() && ! context.clouds.isEmpty() && ! context.clouds.containsAll(clouds)) {
throw new IllegalArgumentException("Clouds in child (" + regions +
") are not a subset of those of the parent (" + context.clouds + ") at " + child);
}
Tags tags = getTags(child);
if ( ! tags.isEmpty() && ! context.tags.isEmpty() && ! context.tags.containsAll(tags)) {
throw new IllegalArgumentException("Tags in child (" + environments +
") are not a subset of those of the parent (" + context.tags + ") at " + child);
}
}
}
/** Prune elements that are not matching our environment and region. */
private void pruneNonMatching(Element parent, List children) {
Iterator elemIt = children.iterator();
while (elemIt.hasNext()) {
Element child = elemIt.next();
if ( ! matches(getInstances(child), getEnvironments(child), getRegions(child), getClouds(child), getTags(child))) {
parent.removeChild(child);
elemIt.remove();
}
}
}
private boolean matches(Set elementInstances,
Set elementEnvironments,
Set elementRegions,
Set elementClouds,
Tags elementTags) {
if ( ! elementInstances.isEmpty()) { // match instance
if ( ! elementInstances.contains(instance)) return false;
}
if ( ! elementEnvironments.isEmpty()) { // match environment
if ( ! elementEnvironments.contains(environment)) return false;
}
if ( ! elementRegions.isEmpty()) { // match region
if ( ! elementRegions.contains(region)) return false;
}
if ( ! elementClouds.isEmpty()) { // match cloud
if ( ! elementClouds.contains(cloud)) return false;
}
if ( ! elementTags.isEmpty()) { // match tags
if ( ! elementTags.intersects(tags)) return false;
// Tags are set on instances. Having a tag match for a deployment to a non-prod environment
// disables the usual downscaling of the cluster, which is surprising. We therefore either
// require the tags match to either also match an environment directive, or the implicit prod.
if (elementEnvironments.isEmpty() && environment != Environment.prod) return false;
}
return true;
}
/** Find the most specific element and remove all others. */
private void retainMostSpecific(Element parent, List children, Context context) {
// Keep track of elements with the highest number of matches (might be more than one element with same tag, need a list)
List bestMatches = new ArrayList<>();
int bestMatch = 0;
for (Element child : children) {
bestMatch = updateBestMatches(bestMatches, child, bestMatch, context);
}
if (bestMatch > 0) { // there was a region/environment specific override
doElementSpecificProcessingOnOverride(bestMatches);
for (Element child : children) {
if ( ! bestMatches.contains(child)) {
parent.removeChild(child);
}
}
}
}
private int updateBestMatches(List bestMatches, Element child, int bestMatch, Context context) {
int overrideCount = getNumberOfOverrides(child, context);
if (overrideCount >= bestMatch) {
if (overrideCount > bestMatch)
bestMatches.clear();
bestMatches.add(child);
return overrideCount;
} else {
return bestMatch;
}
}
private int getNumberOfOverrides(Element child, Context context) {
int currentMatch = 0;
Set elementInstances = hasInstance(child) ? getInstances(child) : context.instances;
Set elementEnvironments = hasEnvironment(child) ? getEnvironments(child) : context.environments;
Set elementRegions = hasRegion(child) ? getRegions(child) : context.regions;
Set elementClouds = hasCloud(child) ? getClouds(child) : context.clouds;
Tags elementTags = hasTag(child) ? getTags(child) : context.tags;
if ( ! elementInstances.isEmpty() && elementInstances.contains(instance))
currentMatch++;
if ( ! elementEnvironments.isEmpty() && elementEnvironments.contains(environment))
currentMatch++;
if ( ! elementRegions.isEmpty() && elementRegions.contains(region))
currentMatch++;
if ( ! elementClouds.isEmpty() && elementClouds.contains(cloud))
currentMatch++;
if ( elementTags.intersects(tags))
currentMatch++;
return currentMatch;
}
/** Called on each element which is selected by matching some override condition */
private void doElementSpecificProcessingOnOverride(List elements) {
// if node capacity is specified explicitly for some combination we should require that capacity
elements.forEach(element -> {
if (element.getTagName().equals("nodes"))
if (!hasChildWithTagName(element, "node")) // specifies capacity, not a list of nodes
element.setAttribute("required", "true");
});
}
private static boolean hasChildWithTagName(Element element, String childName) {
for (var child : XML.getChildren(element)) {
if (child.getTagName().equals(childName))
return true;
}
return false;
}
/** Retains all elements where at least one element is overridden. Removes non-overridden elements from map. */
private void retainOverriddenElements(Map> elementsByTagName) {
Iterator>> it = elementsByTagName.entrySet().iterator();
while (it.hasNext()) {
List elements = it.next().getValue();
boolean hasOverrides = false;
for (Element element : elements) {
if (hasInstance(element) || hasEnvironment(element) || hasRegion(element) || hasCloud(element) || hasTag(element)) {
hasOverrides = true;
}
}
if (!hasOverrides) {
it.remove();
}
}
}
private boolean hasInstance(Element element) {
return element.hasAttributeNS(XmlPreProcessor.deployNamespaceUri, INSTANCE_ATTRIBUTE);
}
private boolean hasRegion(Element element) {
return element.hasAttributeNS(XmlPreProcessor.deployNamespaceUri, REGION_ATTRIBUTE);
}
private boolean hasCloud(Element element) {
return element.hasAttributeNS(XmlPreProcessor.deployNamespaceUri, CLOUD_ATTRIBUTE);
}
private boolean hasEnvironment(Element element) {
return element.hasAttributeNS(XmlPreProcessor.deployNamespaceUri, ENVIRONMENT_ATTRIBUTE);
}
private boolean hasTag(Element element) {
return element.hasAttributeNS(XmlPreProcessor.deployNamespaceUri, TAGS_ATTRIBUTE);
}
private Set getInstances(Element element) {
String instance = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, INSTANCE_ATTRIBUTE);
if (instance.isEmpty()) return Set.of();
return Arrays.stream(instance.split(" ")).map(InstanceName::from).collect(Collectors.toSet());
}
private Set getEnvironments(Element element) {
String env = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, ENVIRONMENT_ATTRIBUTE);
if (env.isEmpty()) return Set.of();
return Arrays.stream(env.split(" ")).map(Environment::from).collect(Collectors.toSet());
}
private Set getRegions(Element element) {
String reg = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, REGION_ATTRIBUTE);
if (reg.isEmpty()) return Set.of();
return Arrays.stream(reg.split(" ")).map(RegionName::from).collect(Collectors.toSet());
}
private Set getClouds(Element element) {
String reg = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, CLOUD_ATTRIBUTE);
if (reg.isEmpty()) return Set.of();
return Arrays.stream(reg.split(" ")).map(CloudName::from).collect(Collectors.toSet());
}
private Tags getTags(Element element) {
String env = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, TAGS_ATTRIBUTE);
if (env.isEmpty()) return Tags.empty();
return Tags.fromString(env);
}
private Map> elementsByTagNameAndId(List children) {
Map> elementsByTagName = new LinkedHashMap<>();
// Index by tag name and optionally add "id" or "idref" to key if they are set
for (Element child : children) {
String key = child.getTagName();
if (child.hasAttribute(ID_ATTRIBUTE))
key += child.getAttribute(ID_ATTRIBUTE);
if (child.hasAttribute(IDREF_ATTRIBUTE))
key += child.getAttribute(IDREF_ATTRIBUTE);
if ( ! elementsByTagName.containsKey(key)) {
elementsByTagName.put(key, new ArrayList<>());
}
elementsByTagName.get(key).add(child);
}
return elementsByTagName;
}
// For debugging
private static String getPrintableElement(Element element) {
StringBuilder sb = new StringBuilder(element.getTagName());
final NamedNodeMap attributes = element.getAttributes();
for (int i = 0; i < attributes.getLength(); i++) {
sb.append(" ").append(attributes.item(i).getNodeName());
}
return sb.toString();
}
// For debugging
private static String getPrintableElementRecursive(Element element) {
StringBuilder sb = new StringBuilder();
sb.append(element.getTagName());
final NamedNodeMap attributes = element.getAttributes();
for (int i = 0; i < attributes.getLength(); i++) {
sb.append(" ")
.append(attributes.item(i).getNodeName())
.append("=")
.append(attributes.item(i).getNodeValue());
}
final List children = XML.getChildren(element);
if (children.size() > 0) {
sb.append("\n");
for (Element e : children)
sb.append("\t").append(getPrintableElementRecursive(e));
}
return sb.toString();
}
/**
* Represents environments, regions, instances, clouds and tags in a given context.
*/
private static final class Context {
final Set instances;
final Set environments;
final Set regions;
final Set clouds;
final Tags tags;
private Context(Set instances,
Set environments,
Set regions,
Set clouds,
Tags tags) {
this.instances = Set.copyOf(instances);
this.environments = Set.copyOf(environments);
this.regions = Set.copyOf(regions);
this.clouds = Set.copyOf(clouds);
this.tags = tags;
}
static Context empty() {
return new Context(Set.of(), Set.of(), Set.of(), Set.of(), Tags.empty());
}
public static Context create(Set instances,
Set environments,
Set regions,
Set clouds,
Tags tags) {
return new Context(instances, environments, regions, clouds, tags);
}
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy