org.opensearch.indices.SystemIndices Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of opensearch Show documentation
Show all versions of opensearch Show documentation
OpenSearch subproject :server
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/
package org.opensearch.indices;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.automaton.Automata;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.MinimizationOperations;
import org.apache.lucene.util.automaton.Operations;
import org.opensearch.common.Nullable;
import org.opensearch.common.collect.Tuple;
import org.opensearch.common.regex.Regex;
import org.opensearch.core.index.Index;
import org.opensearch.tasks.TaskResultsService;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static java.util.Collections.unmodifiableList;
import static java.util.Collections.unmodifiableMap;
import static org.opensearch.tasks.TaskResultsService.TASK_INDEX;
/**
* This class holds the {@link SystemIndexDescriptor} objects that represent system indices the
* node knows about. Methods for determining if an index should be a system index are also provided
* to reduce the locations within the code that need to deal with {@link SystemIndexDescriptor}s.
*
* @opensearch.internal
*/
public class SystemIndices {
private static final Logger logger = LogManager.getLogger(SystemIndices.class);
private static final Map> SERVER_SYSTEM_INDEX_DESCRIPTORS = singletonMap(
TaskResultsService.class.getName(),
singletonList(new SystemIndexDescriptor(TASK_INDEX + "*", "Task Result Index"))
);
private final CharacterRunAutomaton runAutomaton;
private final Collection systemIndexDescriptors;
public SystemIndices(Map> pluginAndModulesDescriptors) {
final Map> descriptorsMap = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors);
checkForOverlappingPatterns(descriptorsMap);
this.systemIndexDescriptors = unmodifiableList(
descriptorsMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList())
);
this.runAutomaton = buildCharacterRunAutomaton(systemIndexDescriptors);
}
/**
* Determines whether a given index is a system index by comparing its name to the collection of loaded {@link SystemIndexDescriptor}s
* @param index the {@link Index} object to check against loaded {@link SystemIndexDescriptor}s
* @return true if the {@link Index}'s name matches a pattern from a {@link SystemIndexDescriptor}
*/
public boolean isSystemIndex(Index index) {
return isSystemIndex(index.getName());
}
/**
* Determines whether a given index is a system index by comparing its name to the collection of loaded {@link SystemIndexDescriptor}s
* @param indexName the index name to check against loaded {@link SystemIndexDescriptor}s
* @return true if the index name matches a pattern from a {@link SystemIndexDescriptor}
*/
public boolean isSystemIndex(String indexName) {
return runAutomaton.run(indexName);
}
/**
* Finds a single matching {@link SystemIndexDescriptor}, if any, for the given index name.
* @param name the name of the index
* @return The matching {@link SystemIndexDescriptor} or {@code null} if no descriptor is found
* @throws IllegalStateException if multiple descriptors match the name
*/
public @Nullable SystemIndexDescriptor findMatchingDescriptor(String name) {
final List matchingDescriptors = systemIndexDescriptors.stream()
.filter(descriptor -> descriptor.matchesIndexPattern(name))
.collect(Collectors.toList());
if (matchingDescriptors.isEmpty()) {
return null;
} else if (matchingDescriptors.size() == 1) {
return matchingDescriptors.get(0);
} else {
// This should be prevented by failing on overlapping patterns at startup time, but is here just in case.
StringBuilder errorMessage = new StringBuilder().append("index name [")
.append(name)
.append("] is claimed as a system index by multiple system index patterns: [")
.append(
matchingDescriptors.stream()
.map(
descriptor -> "pattern: ["
+ descriptor.getIndexPattern()
+ "], description: ["
+ descriptor.getDescription()
+ "]"
)
.collect(Collectors.joining("; "))
);
// Throw AssertionError if assertions are enabled, or a regular exception otherwise:
assert false : errorMessage.toString();
throw new IllegalStateException(errorMessage.toString());
}
}
/**
* Validates (if this index has a dot-prefixed name) and it is system index.
* @param index The name of the index in question
*/
public boolean validateSystemIndex(String index) {
if (index.charAt(0) == '.') {
SystemIndexDescriptor matchingDescriptor = findMatchingDescriptor(index);
if (matchingDescriptor != null) {
logger.trace(
"index [{}] is a system index because it matches index pattern [{}] with description [{}]",
index,
matchingDescriptor.getIndexPattern(),
matchingDescriptor.getDescription()
);
return true;
}
}
return false;
}
private static CharacterRunAutomaton buildCharacterRunAutomaton(Collection descriptors) {
Optional automaton = descriptors.stream()
.map(descriptor -> Regex.simpleMatchToAutomaton(descriptor.getIndexPattern()))
.reduce(Operations::union);
return new CharacterRunAutomaton(MinimizationOperations.minimize(automaton.orElse(Automata.makeEmpty()), Integer.MAX_VALUE));
}
/**
* Given a collection of {@link SystemIndexDescriptor}s and their sources, checks to see if the index patterns of the listed
* descriptors overlap with any of the other patterns. If any do, throws an exception.
*
* @param sourceToDescriptors A map of source (plugin) names to the SystemIndexDescriptors they provide.
* @throws IllegalStateException Thrown if any of the index patterns overlaps with another.
*/
static void checkForOverlappingPatterns(Map> sourceToDescriptors) {
List> sourceDescriptorPair = sourceToDescriptors.entrySet()
.stream()
.flatMap(entry -> entry.getValue().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor)))
.sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getIndexPattern())) // Consistent ordering -> consistent error message
.collect(Collectors.toList());
// This is O(n^2) with the number of system index descriptors, and each check is quadratic with the number of states in the
// automaton, but the absolute number of system index descriptors should be quite small (~10s at most), and the number of states
// per pattern should be low as well. If these assumptions change, this might need to be reworked.
sourceDescriptorPair.forEach(descriptorToCheck -> {
List> descriptorsMatchingThisPattern = sourceDescriptorPair.stream()
.filter(d -> descriptorToCheck.v2() != d.v2()) // Exclude the pattern currently being checked
.filter(d -> overlaps(descriptorToCheck.v2(), d.v2()))
.collect(Collectors.toList());
if (descriptorsMatchingThisPattern.isEmpty() == false) {
throw new IllegalStateException(
"a system index descriptor ["
+ descriptorToCheck.v2()
+ "] from ["
+ descriptorToCheck.v1()
+ "] overlaps with other system index descriptors: ["
+ descriptorsMatchingThisPattern.stream()
.map(descriptor -> descriptor.v2() + " from [" + descriptor.v1() + "]")
.collect(Collectors.joining(", "))
);
}
});
}
private static boolean overlaps(SystemIndexDescriptor a1, SystemIndexDescriptor a2) {
Automaton a1Automaton = Regex.simpleMatchToAutomaton(a1.getIndexPattern());
Automaton a2Automaton = Regex.simpleMatchToAutomaton(a2.getIndexPattern());
return Operations.isEmpty(Operations.intersection(a1Automaton, a2Automaton)) == false;
}
private static Map> buildSystemIndexDescriptorMap(
Map> pluginAndModulesMap
) {
final Map> map = new HashMap<>(
pluginAndModulesMap.size() + SERVER_SYSTEM_INDEX_DESCRIPTORS.size()
);
map.putAll(pluginAndModulesMap);
// put the server items last since we expect less of them
SERVER_SYSTEM_INDEX_DESCRIPTORS.forEach((source, descriptors) -> {
if (map.putIfAbsent(source, descriptors) != null) {
throw new IllegalArgumentException(
"plugin or module attempted to define the same source [" + source + "] as a built-in system index"
);
}
});
return unmodifiableMap(map);
}
}