org.apache.jackrabbit.oak.indexversion.IndexVersionOperation Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of oak-run-commons Show documentation
The newest version!
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.jackrabbit.oak.indexversion;

import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;

import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.index.IndexName;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_DISABLED;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;

/*
    Main operation of this class is to mark IndexName's with operations
 */
public abstract class IndexVersionOperation {
    private static final Logger LOG = LoggerFactory.getLogger(IndexVersionOperation.class);

    private IndexName indexName;
    private Operation operation;

    public IndexVersionOperation(IndexName indexName) {
        this.indexName = indexName;
        this.operation = Operation.NOOP;
    }

    public void setOperation(Operation operation) {
        this.operation = operation;
    }

    public Operation getOperation() {
        return this.operation;
    }

    public IndexName getIndexName() {
        return this.indexName;
    }

    @Override
    public String toString() {
        return this.getIndexName() + " operation:" + this.getOperation();
    }

    /**
     * Generate list of index version operation over a list of indexes have same index base. This will purge base index.
     *
     * @param rootNode             NodeState of root
     * @param parentPath           parent path of baseIndex
     * @param indexNameObjectList  This is a list of IndexName Objects with same baseIndexName on which operations will be applied.
     * @param purgeThresholdMillis after which a fully functional index is eligible for purge operations
     *
     * @return This method returns an IndexVersionOperation list i.e indexNameObjectList marked with operations
     */
    public List generateIndexVersionOperationList(NodeState rootNode, String parentPath,
                                                                                List indexNameObjectList, long purgeThresholdMillis) {
        return generateIndexVersionOperationList(rootNode, parentPath, indexNameObjectList, purgeThresholdMillis, true);
    }

    /**
     * Generate list of index version operation over a list of indexes have same index base.
     *
     * @param rootNode             NodeState of root
     * @param parentPath           parent path of baseIndex
     * @param indexNameObjectList  This is a list of IndexName Objects with same baseIndexName on which operations will be applied.
     * @param purgeThresholdMillis after which a fully functional index is eligible for purge operations
     * @param shouldPurgeBaseIndex If set to true, will apply purge operations on active base index i.e. DELETE or DELETE_HIDDEN_AND_DISABLE
     *
     * @return This method returns an IndexVersionOperation list i.e indexNameObjectList marked with operations
     */
    public List generateIndexVersionOperationList(NodeState rootNode, String parentPath,
            List indexNameObjectList, long purgeThresholdMillis, boolean shouldPurgeBaseIndex) {
        NodeState indexDefParentNode = NodeStateUtils.getNode(rootNode, parentPath);
        List reverseSortedIndexNameList = getReverseSortedIndexNameList(indexNameObjectList);
        LOG.info("Reverse Sorted list {} ", reverseSortedIndexNameList);
        List indexVersionOperationList = new LinkedList<>();

        List disableIndexNameObjectList = removeDisabledCustomIndexesFromList(indexDefParentNode, reverseSortedIndexNameList);
        // Remove the disabled indexes from the reverseSortedIndexNameList since these would be handled differently
        reverseSortedIndexNameList.removeAll(disableIndexNameObjectList);
        LOG.info("Disabled index list {}, new reverse sorted list after removing disabled indexes{}", disableIndexNameObjectList, reverseSortedIndexNameList);
        // for disabled indexes, do full deletion if checkIfDisabledIndexCanBeMarkedForDeletion = true, otherwise no action needed
        // checkIfDisabledIndexCanBeMarkedForDeletion - this has different implementations for lucene and elastic.
        // Check individual LuceneIndexVersionOperation#checkIfDisabledIndexCanBeMarkedForDeletion and ElasticIndexVersionOperation#checkIfDisabledIndexCanBeMarkedForDeletion for more details.
        for (IndexName indexNameObject : disableIndexNameObjectList) {
            String indexName = indexNameObject.getNodeName();
            NodeState indexNode = indexDefParentNode.getChildNode(PathUtils.getName(indexName));
            IndexVersionOperation indexVersionOperation = getIndexVersionOperationInstance(indexNameObject);
            if (checkIfDisabledIndexCanBeMarkedForDeletion(indexNode)) {
                indexVersionOperation.setOperation(Operation.DELETE);
            }
            indexVersionOperationList.add(indexVersionOperation);
        }

        if (!reverseSortedIndexNameList.isEmpty()) {
            IndexName activeIndexNameObject = getActiveIndex(reverseSortedIndexNameList, parentPath, rootNode);
            if (activeIndexNameObject == null) {
                LOG.warn("Cannot find any active index from the list: {}", reverseSortedIndexNameList);
            } else {
                if (!isSameProductVersionBaseIndexPresent(reverseSortedIndexNameList, activeIndexNameObject)) {
                    LOG.warn("Repository don't have base index:{} with product version same as active index:{}",
                            activeIndexNameObject.getBaseName() + "-" + activeIndexNameObject.getProductVersion(), activeIndexNameObject.toString());
                }
                NodeState activeIndexNode = indexDefParentNode.getChildNode(PathUtils.getName(activeIndexNameObject.getNodeName()));
                boolean isActiveIndexOldEnough = isActiveIndexOldEnough(activeIndexNameObject, activeIndexNode, purgeThresholdMillis);
                int activeProductVersion = activeIndexNameObject.getProductVersion();
                indexVersionOperationList.add(getIndexVersionOperationInstance(activeIndexNameObject));

                // the reverseSortedIndexNameList will now contain the remaining indexes,
                // the active index and disabled index was removed from that list already
                for (IndexName indexNameObject : reverseSortedIndexNameList) {
                    String indexName = indexNameObject.getNodeName();
                    NodeState indexNode = indexDefParentNode.getChildNode(PathUtils.getName(indexName));
                    IndexVersionOperation indexVersionOperation = getIndexVersionOperationInstance(indexNameObject);
                    // if active index not long enough, NOOP for all indexes
                    if (isActiveIndexOldEnough) {
                        if (indexNameObject.getProductVersion() == activeProductVersion && indexNameObject.getCustomerVersion() == 0) {
                            if (shouldPurgeBaseIndex) {
                                indexVersionOperation.setOperation(Operation.DELETE_HIDDEN_AND_DISABLE);
                            } else {
                                indexVersionOperation.setOperation(Operation.NOOP);
                            }
                        } else if (indexNameObject.getProductVersion() <= activeProductVersion ) {
                            // the check hidden oak mount logic only works when passing through the proper composite store
                            if (isHiddenOakMountExists(indexNode)) {
                                LOG.info("Found hidden oak mount node for: '{}', disable it but no index definition deletion", indexName);
                                indexVersionOperation.setOperation(Operation.DELETE_HIDDEN_AND_DISABLE);
                            } else {
                                indexVersionOperation.setOperation(Operation.DELETE);
                            }
                        } else {
                            // if the index product version is larger than active index, leave it as is
                            // for instance: if there is active damAssetLucene-7, the inactive damAssetLucene-8 will be leave there as is
                            LOG.info("The index '{}' leave as is since the version is larger than current active index", indexName);
                            indexVersionOperation.setOperation(Operation.NOOP);
                        }
                    }
                    LOG.info("The operation for index '{}' will be: '{}'", indexName, indexVersionOperation.getOperation());
                    indexVersionOperationList.add(indexVersionOperation);
                }
            }
        }
        if (indexVersionOperationList.isEmpty()) {
            LOG.info("Not valid version operation list: '{}', skip all", indexNameObjectList);
            indexVersionOperationList = Collections.emptyList();
        }
        return indexVersionOperationList;
    }

    private static boolean isSameProductVersionBaseIndexPresent(List reverseSortedIndexNameList, IndexName activeIndexNameObject) {
        return reverseSortedIndexNameList.stream().filter(n -> (n.getBaseName().equals(activeIndexNameObject.getBaseName())
                && n.getProductVersion() == activeIndexNameObject.getProductVersion()
                && n.getCustomerVersion() == 0)).findFirst().isPresent();
    }

    // do index purge ready based on the active index's last reindexing time is longer enough, we do this for prevent rollback
    private static boolean isActiveIndexOldEnough(IndexName activeIndexName, NodeState activeIndexNode, long purgeThresholdMillis) {
        // the 1st index must be active
        String indexName = activeIndexName.getNodeName();
        if (activeIndexNode.hasChildNode(IndexDefinition.STATUS_NODE)) {
            if (activeIndexNode.getChildNode(IndexDefinition.STATUS_NODE)
                    .getProperty(IndexDefinition.REINDEX_COMPLETION_TIMESTAMP) != null) {
                String reindexCompletionTime = activeIndexNode.getChildNode(IndexDefinition.STATUS_NODE)
                        .getProperty(IndexDefinition.REINDEX_COMPLETION_TIMESTAMP)
                        .getValue(Type.DATE);
                long reindexCompletionTimeInMillis = PurgeOldVersionUtils.getMillisFromString(reindexCompletionTime);
                long currentTimeInMillis = System.currentTimeMillis();
                if (currentTimeInMillis - reindexCompletionTimeInMillis > purgeThresholdMillis) {
                    LOG.info("Found active index {} is old enough", indexName);
                    return true;
                } else {
                    LOG.info("The last index time '{}' isn't old enough for: {}", reindexCompletionTime, indexName);
                }
            } else {
                LOG.warn("{} property is not set for index {}", IndexDefinition.REINDEX_COMPLETION_TIMESTAMP, indexName);
            }
        }
        LOG.info("The active index '{}' indexing time isn't old enough", indexName);
        return false;
    }

    protected static boolean isHiddenOakMountExists(NodeState indexNode) {
        for (String nodeName : indexNode.getChildNodeNames()) {
            if (nodeName.startsWith(IndexDefinition.HIDDEN_OAK_MOUNT_PREFIX)) {
                return true;
            }
        }
        return false;
    }

    /*
        returns IndexNameObjects in descending order of version i.e from newest version to oldest
    */
    private static List getReverseSortedIndexNameList(List indexNameObjectList) {
        List reverseSortedIndexNameObjectList = new ArrayList<>(indexNameObjectList);
        Collections.sort(reverseSortedIndexNameObjectList, Collections.reverseOrder());
        return reverseSortedIndexNameObjectList;
    }


    private static List removeDisabledCustomIndexesFromList(NodeState indexDefParentNode,
                                                            List indexNameObjectList) {
        List disableIndexNameObjectList = new ArrayList<>();
        for (int i = 0; i < indexNameObjectList.size(); i++) {
            NodeState indexNode = indexDefParentNode
                    .getChildNode(PathUtils.getName(indexNameObjectList.get(i).getNodeName()));
            if (indexNode.getProperty(TYPE_PROPERTY_NAME) != null && TYPE_DISABLED.equals(indexNode.getProperty(TYPE_PROPERTY_NAME).getValue(Type.STRING))) {
                disableIndexNameObjectList.add(indexNameObjectList.get(i));
            }
        }
        return disableIndexNameObjectList;
    }

    /*
        NOOP means : No operation to be performed index Node
        DELETE_HIDDEN_AND_DISABLE: This operation means that we should disable this indexNode and delete all hidden nodes under it
        DELETE: Delete this index altogether
     */
    enum Operation {
        NOOP,
        DELETE_HIDDEN_AND_DISABLE,
        DELETE
    }

    protected abstract IndexVersionOperation getIndexVersionOperationInstance(IndexName indexName);

    /**
     *
     * @param indexNode - NodeState of a disabled index
     * @return true if the disabled index with NodeState indexNode can be marked for deletion or not.
     */
    protected abstract boolean checkIfDisabledIndexCanBeMarkedForDeletion(NodeState indexNode);

    /**
     *
     * @param reverseSortedIndexNameList
     * @param parentPath
     * @param rootNode
     * @return Highest versioned active index's IndexName
     */
    @Nullable
    protected abstract IndexName getActiveIndex(List reverseSortedIndexNameList, String parentPath, NodeState rootNode);
}