org.apache.solr.handler.SnapShooter Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Consumer;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.backup.repository.BackupRepository.PathType;
import org.apache.solr.core.backup.repository.LocalFileSystemRepository;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Provides functionality equivalent to the snapshooter script
* This is no longer used in standard replication.
*
*
* @since solr 1.4
*/
public class SnapShooter {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private SolrCore solrCore;
private String snapshotName = null;
private String directoryName = null;
private URI baseSnapDirPath = null;
private URI snapshotDirPath = null;
private BackupRepository backupRepo = null;
private String commitName; // can be null
@Deprecated
public SnapShooter(SolrCore core, String location, String snapshotName) {
String snapDirStr = null;
// Note - This logic is only applicable to the usecase where a shared file-system is exposed via
// local file-system interface (primarily for backwards compatibility). For other use-cases, users
// will be required to specify "location" where the backup should be stored.
if (location == null) {
snapDirStr = core.getDataDir();
} else {
snapDirStr = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString();
}
initialize(new LocalFileSystemRepository(), core, Paths.get(snapDirStr).toUri(), snapshotName, null);
}
public SnapShooter(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) {
initialize(backupRepo, core, location, snapshotName, commitName);
}
private void initialize(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) {
this.solrCore = Objects.requireNonNull(core);
this.backupRepo = Objects.requireNonNull(backupRepo);
this.baseSnapDirPath = location;
this.snapshotName = snapshotName;
if (snapshotName != null) {
directoryName = "snapshot." + snapshotName;
} else {
SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT, Locale.ROOT);
directoryName = "snapshot." + fmt.format(new Date());
}
this.snapshotDirPath = backupRepo.resolve(location, directoryName);
this.commitName = commitName;
}
public BackupRepository getBackupRepository() {
return backupRepo;
}
/**
* Gets the parent directory of the snapshots. This is the {@code location}
* given in the constructor.
*/
public URI getLocation() {
return this.baseSnapDirPath;
}
public void validateDeleteSnapshot() {
Objects.requireNonNull(this.snapshotName);
boolean dirFound = false;
String[] paths;
try {
paths = backupRepo.listAll(baseSnapDirPath);
for (String path : paths) {
if (path.equals(this.directoryName)
&& backupRepo.getPathType(baseSnapDirPath.resolve(path)) == PathType.DIRECTORY) {
dirFound = true;
break;
}
}
if(dirFound == false) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Snapshot " + snapshotName + " cannot be found in directory: " + baseSnapDirPath);
}
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to find snapshot " + snapshotName + " in directory: " + baseSnapDirPath, e);
}
}
protected void deleteSnapAsync(final ReplicationHandler replicationHandler) {
new Thread(() -> deleteNamedSnapshot(replicationHandler)).start();
}
public void validateCreateSnapshot() throws IOException {
// Note - Removed the current behavior of creating the directory hierarchy.
// Do we really need to provide this support?
if (!backupRepo.exists(baseSnapDirPath)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
" Directory does not exist: " + snapshotDirPath);
}
if (backupRepo.exists(snapshotDirPath)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Snapshot directory already exists: " + snapshotDirPath);
}
}
public NamedList createSnapshot() throws Exception {
final IndexCommit indexCommit = getAndSaveIndexCommit();
try {
return createSnapshot(indexCommit);
} finally {
solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration());
}
}
/**
* If {@link #commitName} is non-null, then fetches the generation from the
* {@link SolrSnapshotMetaDataManager} and then returns
* {@link IndexDeletionPolicyWrapper#getAndSaveCommitPoint}, otherwise it returns
* {@link IndexDeletionPolicyWrapper#getAndSaveLatestCommit}.
*
* Either way:
*
* - This method does error handling for all cases where the commit can't be found
* and wraps them in {@link SolrException}
*
* - If this method returns, the result will be non null, and the caller MUST
* call {@link IndexDeletionPolicyWrapper#releaseCommitPoint} when finished
*
*
*/
private IndexCommit getAndSaveIndexCommit() throws IOException {
final IndexDeletionPolicyWrapper delPolicy = solrCore.getDeletionPolicy();
if (null != commitName) {
final SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager();
// We're going to tell the delPolicy to "save" this commit -- even though it's a named snapshot
// that will already be protected -- just in case another thread deletes the name.
// Because of this, we want to sync on the delPolicy to ensure there is no window of time after
// snapshotMgr confirms commitName exists, but before we have a chance to 'save' it, when
// the commitName might be deleted *and* the IndexWriter might call onCommit()
synchronized (delPolicy) {
final Optional namedCommit = snapshotMgr.getIndexCommitByName(commitName);
if (namedCommit.isPresent()) {
final IndexCommit commit = namedCommit.get();
log.debug("Using named commit: name={}, generation={}", commitName, commit.getGeneration());
delPolicy.saveCommitPoint(commit.getGeneration());
return commit;
}
} // else...
throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to find an index commit with name " +
commitName + " for core " + solrCore.getName());
}
// else: not a named commit...
final IndexCommit commit = delPolicy.getAndSaveLatestCommit();
if (null == commit) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Index does not yet have any commits for core " +
solrCore.getName());
}
log.debug("Using latest commit: generation={}", commit.getGeneration());
return commit;
}
public void createSnapAsync(final int numberToKeep, Consumer result) throws IOException {
//TODO should use Solr's ExecutorUtil
new Thread(() -> {
NamedList snapShootDetails;
try {
snapShootDetails = createSnapshot();
} catch (Exception e) {
log.error("Exception while creating snapshot", e);
snapShootDetails = new NamedList<>();
snapShootDetails.add("exception", e.getMessage());
}
if (snapshotName == null) {
try {
deleteOldBackups(numberToKeep);
} catch (IOException e) {
log.warn("Unable to delete old snapshots ", e);
}
}
if (null != snapShootDetails) result.accept(snapShootDetails);
}).start();
}
/**
* Handles the logic of creating a snapshot
*
* NOTE: The caller MUST ensure that the {@link IndexCommit} is saved prior to
* calling this method, and released after calling this method, or there is no no garuntee that the
* method will function correctly.
*
*
* @see IndexDeletionPolicyWrapper#saveCommitPoint
* @see IndexDeletionPolicyWrapper#releaseCommitPoint
*/
protected NamedList createSnapshot(final IndexCommit indexCommit) throws Exception {
assert indexCommit != null;
log.info("Creating backup snapshot " + (snapshotName == null ? "" : snapshotName) + " at " + baseSnapDirPath);
boolean success = false;
try {
NamedList