Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under
* one or more contributor license agreements. See the NOTICE file distributed
* with this work for additional information regarding copyright ownership.
* Licensed under the Camunda License 1.0. You may not use this file
* except in compliance with the Camunda License 1.0.
*/
package io.camunda.operate.schema;
import static io.camunda.operate.util.CollectionUtil.map;
import com.google.common.collect.Maps;
import io.camunda.operate.exceptions.OperateRuntimeException;
import io.camunda.operate.property.OperateProperties;
import io.camunda.operate.schema.IndexMapping.IndexMappingProperty;
import io.camunda.operate.schema.indices.IndexDescriptor;
import io.camunda.operate.schema.migration.SemanticVersion;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class IndexSchemaValidator {
private static final Logger LOGGER = LoggerFactory.getLogger(IndexSchemaValidator.class);
private static final Pattern VERSION_PATTERN = Pattern.compile(".*-(\\d+\\.\\d+\\.\\d+.*)_.*");
@Autowired Set indexDescriptors;
@Autowired SchemaManager schemaManager;
@Autowired private OperateProperties operateProperties;
private Set getAllIndexNamesForIndex(final String index) {
final String indexPattern = String.format("%s-%s*", getIndexPrefix(), index);
LOGGER.debug("Getting all indices for {}", indexPattern);
final Set indexNames = schemaManager.getIndexNames(indexPattern);
// since we have indices with similar names, we need to additionally filter index names
// e.g. task and task-variable
final String patternWithVersion = String.format("%s-%s-\\d.*", getIndexPrefix(), index);
return indexNames.stream()
.filter(n -> n.matches(patternWithVersion))
.collect(Collectors.toSet());
}
private String getIndexPrefix() {
return schemaManager.getIndexPrefix();
}
public Set newerVersionsForIndex(final IndexDescriptor indexDescriptor) {
final SemanticVersion currentVersion =
SemanticVersion.fromVersion(indexDescriptor.getVersion());
final Set versions = versionsForIndex(indexDescriptor);
return versions.stream()
.filter(version -> SemanticVersion.fromVersion(version).isNewerThan(currentVersion))
.collect(Collectors.toSet());
}
public Set olderVersionsForIndex(final IndexDescriptor indexDescriptor) {
final SemanticVersion currentVersion =
SemanticVersion.fromVersion(indexDescriptor.getVersion());
final Set versions = versionsForIndex(indexDescriptor);
return versions.stream()
.filter(version -> currentVersion.isNewerThan(SemanticVersion.fromVersion(version)))
.collect(Collectors.toSet());
}
private Set versionsForIndex(final IndexDescriptor indexDescriptor) {
final Set allIndexNames = getAllIndexNamesForIndex(indexDescriptor.getIndexName());
return allIndexNames.stream()
.map(this::getVersionFromIndexName)
.filter(Optional::isPresent)
.map(Optional::get)
.collect(Collectors.toSet());
}
private Optional getVersionFromIndexName(final String indexName) {
final Matcher matcher = VERSION_PATTERN.matcher(indexName);
if (matcher.matches() && matcher.groupCount() > 0) {
return Optional.of(matcher.group(1));
}
return Optional.empty();
}
/**
* Validates whether there are no more than one older versions, which would mean not finished
* migration. Or whether there are newer index versions, meaning that older Operate version is
* run.
*/
public void validateIndexVersions() {
if (!hasAnyOperateIndices()) {
return;
}
final Set errors = new HashSet<>();
indexDescriptors.forEach(
indexDescriptor -> {
final Set oldVersions = olderVersionsForIndex(indexDescriptor);
final Set newerVersions = newerVersionsForIndex(indexDescriptor);
if (oldVersions.size() > 1) {
errors.add(
String.format(
"More than one older version for %s (%s) found: %s",
indexDescriptor.getIndexName(), indexDescriptor.getVersion(), oldVersions));
}
if (!newerVersions.isEmpty()) {
errors.add(
String.format(
"Newer version(s) for %s (%s) already exists: %s",
indexDescriptor.getIndexName(), indexDescriptor.getVersion(), newerVersions));
}
});
if (!errors.isEmpty()) {
throw new OperateRuntimeException("Error(s) in index schema: " + String.join(";", errors));
}
}
public boolean hasAnyOperateIndices() {
final Set indices = schemaManager.getIndexNames(schemaManager.getIndexPrefix() + "*");
return !indices.isEmpty();
}
public boolean schemaExists() {
try {
final Set indices = schemaManager.getIndexNames(schemaManager.getIndexPrefix() + "*");
final List allIndexNames =
map(indexDescriptors, IndexDescriptor::getFullQualifiedName);
final Set aliases =
schemaManager.getAliasesNames(schemaManager.getIndexPrefix() + "*");
final List allAliasesNames = map(indexDescriptors, IndexDescriptor::getAlias);
return indices.containsAll(allIndexNames) && aliases.containsAll(allAliasesNames);
} catch (final Exception e) {
LOGGER.error("Check for existing schema failed", e);
return false;
}
}
/**
* Validates existing indices mappings against schema files defined in codebase.
*
* @return newFields map with the new field definitions per index
* @throws OperateRuntimeException in case some fields would need to be deleted or have different
* settings
*/
public Map> validateIndexMappings() {
final Map> newFields = new HashMap<>();
final Map indexMappings =
schemaManager.getIndexMappings(schemaManager.getIndexPrefix() + "*");
for (final IndexDescriptor indexDescriptor : indexDescriptors) {
final Map indexMappingsGroup =
filterIndexMappings(indexMappings, indexDescriptor);
// we don't check indices that were not yet created
if (!indexMappingsGroup.isEmpty()) {
final IndexMappingDifference difference =
getDifference(indexDescriptor, indexMappingsGroup);
validateDifferenceAndCollectNewFields(indexDescriptor, difference, newFields);
}
}
return newFields;
}
private IndexMappingDifference getDifference(
final IndexDescriptor indexDescriptor, final Map indexMappingsGroup) {
return getIndexMappingDifference(indexDescriptor, indexMappingsGroup);
}
private void validateDifferenceAndCollectNewFields(
final IndexDescriptor indexDescriptor,
final IndexMappingDifference difference,
final Map> newFields) {
if (difference != null && !difference.isEqual()) {
LOGGER.debug(
String.format(
"Index fields differ from expected. Index name: %s. Difference: %s.",
indexDescriptor.getIndexName(), difference));
if (!difference.getEntriesDiffering().isEmpty()) {
// This call will throw an exception unless the index is dynamic, in which case
// field differences will be ignored. In the case of a dynamic index, we still want
// to collect any new fields, so we should continue to the next checks instead of making
// this part of the if/else block
validateFieldsDifferBetweenIndices(difference, indexDescriptor);
}
if (!difference.getEntriesOnlyOnRight().isEmpty()) {
final String message =
String.format(
"Index name: %s. Field deletion is requested, will be ignored. Fields: %s",
indexDescriptor.getIndexName(), difference.getEntriesOnlyOnRight());
LOGGER.info(message);
} else if (!difference.getEntriesOnlyOnLeft().isEmpty()) {
// Collect the new fields
newFields.put(indexDescriptor, difference.getEntriesOnlyOnLeft());
}
} else {
LOGGER.debug(
String.format(
"Index fields are up to date. Index name: %s.", indexDescriptor.getIndexName()));
}
}
private IndexMappingDifference getIndexMappingDifference(
final IndexDescriptor indexDescriptor, final Map indexMappingsGroup) {
final IndexMapping indexMappingMustBe = schemaManager.getExpectedIndexFields(indexDescriptor);
IndexMappingDifference difference = null;
// compare every index in group
for (final Map.Entry singleIndexMapping : indexMappingsGroup.entrySet()) {
final IndexMappingDifference currentDifference =
new IndexMappingDifference.IndexMappingDifferenceBuilder()
.setLeft(indexMappingMustBe)
.setRight(singleIndexMapping.getValue())
.build();
if (!currentDifference.isEqual()) {
if (difference == null) {
difference = currentDifference;
// If there is a difference between the template and the existing runtime/data indices,
// all those indices should have the same difference. Compare based only on the
// differences (exclude the IndexMapping fields in the comparison)
} else if (!difference.checkEqualityForDifferences(currentDifference)) {
throw new OperateRuntimeException(
"Ambiguous schema update. First bring runtime and date indices to one schema. Difference 1: "
+ difference
+ ". Difference 2: "
+ currentDifference);
}
}
}
return difference;
}
/**
* Leave only runtime and dated indices that correspond to the given IndexDescriptor.
*
* @param indexMappings
* @param indexDescriptor
* @return
*/
private Map filterIndexMappings(
final Map indexMappings, final IndexDescriptor indexDescriptor) {
return Maps.filterEntries(
indexMappings,
e -> e.getKey().matches(indexDescriptor.getAllVersionsIndexNameRegexPattern()));
}
private boolean indexIsDynamic(final IndexMapping mapping) {
if (mapping == null) {
return false;
}
if (mapping.getDynamic() == null) {
return true;
}
return Boolean.parseBoolean(mapping.getDynamic());
}
private void validateFieldsDifferBetweenIndices(
final IndexMappingDifference difference, final IndexDescriptor indexDescriptor) {
if (indexIsDynamic(difference.getLeftIndexMapping())) {
LOGGER.debug(
String.format(
"Left index name: %s is dynamic, ignoring changes found: %s",
indexDescriptor.getIndexName(), difference.getEntriesDiffering()));
} else if (indexIsDynamic(difference.getRightIndexMapping())) {
LOGGER.debug(
String.format(
"Right index name: %s is dynamic, ignoring changes found: %s",
indexDescriptor.getIndexName(), difference.getEntriesDiffering()));
} else {
final String errorMsg =
String.format(
"Index name: %s. Not supported index changes are introduced. Data migration is required. Changes found: %s",
indexDescriptor.getIndexName(), difference.getEntriesDiffering());
LOGGER.error(errorMsg);
throw new OperateRuntimeException(errorMsg);
}
}
}
