org.apache.solr.handler.IncrementalShardBackup Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.solr.handler;

import org.apache.commons.math3.util.Precision;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.backup.BackupFilePaths;
import org.apache.solr.core.backup.Checksum;
import org.apache.solr.core.backup.ShardBackupId;
import org.apache.solr.core.backup.ShardBackupMetadata;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.time.Instant;
import java.util.Collection;
import java.util.Optional;
import java.util.UUID;

/**
 * Responsible for orchestrating the actual incremental backup process.
 *
 * If this is the first backup for a collection, all files are uploaded.  But if previous backups exist, uses the most recent
 * {@link ShardBackupMetadata} file to determine which files already exist in the repository and can be skipped.
 */
public class IncrementalShardBackup {
    private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
    private SolrCore solrCore;

    private BackupFilePaths incBackupFiles;
    private BackupRepository backupRepo;

    private ShardBackupId prevShardBackupId;
    private ShardBackupId shardBackupId;
    private Optional commitNameOption;

    /**
     *
     * @param prevShardBackupId previous ShardBackupMetadata file which will be used for skipping
     *                             uploading index files already present in this file.
     * @param shardBackupId file where all meta data of this backup will be stored to.
     */
    public IncrementalShardBackup(BackupRepository backupRepo, SolrCore solrCore, BackupFilePaths incBackupFiles,
                                  ShardBackupId prevShardBackupId, ShardBackupId shardBackupId,
                                  Optional commitNameOption) {
        this.backupRepo = backupRepo;
        this.solrCore = solrCore;
        this.incBackupFiles = incBackupFiles;
        this.prevShardBackupId = prevShardBackupId;
        this.shardBackupId = shardBackupId;
        this.commitNameOption = commitNameOption;
    }

    @SuppressWarnings({"rawtypes"})
    public NamedList backup() throws Exception {
        final IndexCommit indexCommit = getAndSaveIndexCommit();
        try {
            return backup(indexCommit);
        } finally {
            solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration());
        }
    }

    /**
     * Returns {@link IndexDeletionPolicyWrapper#getAndSaveLatestCommit} unless a particular commitName was requested.
     * 
     * Note:
     * 

     *  This method does error handling when the commit can't be found and wraps them in {@link SolrException}
     *  
     *  If this method returns, the result will be non null, and the caller MUST
     *      call {@link IndexDeletionPolicyWrapper#releaseCommitPoint} when finished
     *  
     * 
     */
    private IndexCommit getAndSaveIndexCommit() throws IOException {
        if (commitNameOption.isPresent()) {
            return SnapShooter.getAndSaveNamedIndexCommit(solrCore, commitNameOption.get());
        }

        final IndexDeletionPolicyWrapper delPolicy = solrCore.getDeletionPolicy();
        final IndexCommit commit = delPolicy.getAndSaveLatestCommit();
        if (null == commit) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Index does not yet have any commits for core " +
                    solrCore.getName());
        }
        if (log.isDebugEnabled())   {
            log.debug("Using latest commit: generation={}", commit.getGeneration());
        }
        return commit;
    }

    private IndexCommit getAndSaveLatestIndexCommit(IndexDeletionPolicyWrapper delPolicy) {
        final IndexCommit commit = delPolicy.getAndSaveLatestCommit();
        if (null == commit) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Index does not yet have any commits for core " +
                    solrCore.getName());
        }
        if (log.isDebugEnabled())   {
            log.debug("Using latest commit: generation={}", commit.getGeneration());
        }
        return commit;
    }

    // note: remember to reserve the indexCommit first so it won't get deleted concurrently
    @SuppressWarnings({"rawtypes"})
    protected NamedList backup(final IndexCommit indexCommit) throws Exception {
        assert indexCommit != null;
        URI backupLocation = incBackupFiles.getBackupLocation();
        log.info("Creating backup snapshot at {} shardBackupMetadataFile:{}", backupLocation, shardBackupId);
        NamedList