All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.solr.handler.SnapShooter Maven / Gradle / Ivy

Go to download 

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.handler;

import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Consumer;

import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.backup.repository.BackupRepository.PathType;
import org.apache.solr.core.backup.repository.LocalFileSystemRepository;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * 
 Provides functionality equivalent to the snapshooter script 

 * This is no longer used in standard replication.
 *
 *
 * @since solr 1.4
 */
public class SnapShooter {
  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
  private SolrCore solrCore;
  private String snapshotName = null;
  private String directoryName = null;
  private URI baseSnapDirPath = null;
  private URI snapshotDirPath = null;
  private BackupRepository backupRepo = null;
  private String commitName; // can be null

  @Deprecated
  public SnapShooter(SolrCore core, String location, String snapshotName) {
    String snapDirStr = null;
    // Note - This logic is only applicable to the usecase where a shared file-system is exposed via
    // local file-system interface (primarily for backwards compatibility). For other use-cases, users
    // will be required to specify "location" where the backup should be stored.
    if (location == null) {
      snapDirStr = core.getDataDir();
    } else {
      snapDirStr = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString();
    }
    initialize(new LocalFileSystemRepository(), core, Paths.get(snapDirStr).toUri(), snapshotName, null);
  }

  public SnapShooter(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) {
    initialize(backupRepo, core, location, snapshotName, commitName);
  }

  private void initialize(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) {
    this.solrCore = Objects.requireNonNull(core);
    this.backupRepo = Objects.requireNonNull(backupRepo);
    this.baseSnapDirPath = location;
    this.snapshotName = snapshotName;
    if (snapshotName != null) {
      directoryName = "snapshot." + snapshotName;
    } else {
      SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT, Locale.ROOT);
      directoryName = "snapshot." + fmt.format(new Date());
    }
    this.snapshotDirPath = backupRepo.resolve(location, directoryName);
    this.commitName = commitName;
  }

  public BackupRepository getBackupRepository() {
    return backupRepo;
  }

  /**
   * Gets the parent directory of the snapshots. This is the {@code location}
   * given in the constructor.
   */
  public URI getLocation() {
    return this.baseSnapDirPath;
  }

  public void validateDeleteSnapshot() {
    Objects.requireNonNull(this.snapshotName);

    boolean dirFound = false;
    String[] paths;
    try {
      paths = backupRepo.listAll(baseSnapDirPath);
      for (String path : paths) {
        if (path.equals(this.directoryName)
            && backupRepo.getPathType(baseSnapDirPath.resolve(path)) == PathType.DIRECTORY) {
          dirFound = true;
          break;
        }
      }
      if(dirFound == false) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Snapshot " + snapshotName + " cannot be found in directory: " + baseSnapDirPath);
      }
    } catch (IOException e) {
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to find snapshot " + snapshotName + " in directory: " + baseSnapDirPath, e);
    }
  }

  protected void deleteSnapAsync(final ReplicationHandler replicationHandler) {
    new Thread(() -> deleteNamedSnapshot(replicationHandler)).start();
  }

  public void validateCreateSnapshot() throws IOException {
    // Note - Removed the current behavior of creating the directory hierarchy.
    // Do we really need to provide this support?
    if (!backupRepo.exists(baseSnapDirPath)) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
          " Directory does not exist: " + snapshotDirPath);
    }

    if (backupRepo.exists(snapshotDirPath)) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
          "Snapshot directory already exists: " + snapshotDirPath);
    }
  }

  public NamedList createSnapshot() throws Exception {
    final IndexCommit indexCommit = getAndSaveIndexCommit();
    try {
      return createSnapshot(indexCommit);
    } finally {
      solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration());
    }
  }

  /**
   * If {@link #commitName} is non-null, then fetches the generation from the 
   * {@link SolrSnapshotMetaDataManager} and then returns 
   * {@link IndexDeletionPolicyWrapper#getAndSaveCommitPoint}, otherwise it returns 
   * {@link IndexDeletionPolicyWrapper#getAndSaveLatestCommit}.
   * 

   * Either way:
   * 
    
   *  
  • This method does error handling for all cases where the commit can't be found 
   *       and wraps them in {@link SolrException}
   *  
    • 
   *  
  • If this method returns, the result will be non null, and the caller MUST 
   *      call {@link IndexDeletionPolicyWrapper#releaseCommitPoint} when finished
   *  
    • 
   * 

   */
  private IndexCommit getAndSaveIndexCommit() throws IOException {
    final IndexDeletionPolicyWrapper delPolicy = solrCore.getDeletionPolicy();
    if (null != commitName) {
      final SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager();
      // We're going to tell the delPolicy to "save" this commit -- even though it's a named snapshot
      // that will already be protected -- just in case another thread deletes the name.
      // Because of this, we want to sync on the delPolicy to ensure there is no window of time after
      // snapshotMgr confirms commitName exists, but before we have a chance to 'save' it, when
      // the commitName might be deleted *and* the IndexWriter might call onCommit()
      synchronized (delPolicy) { 
        final Optional namedCommit = snapshotMgr.getIndexCommitByName(commitName);
        if (namedCommit.isPresent()) {
          final IndexCommit commit = namedCommit.get();
          log.debug("Using named commit: name={}, generation={}", commitName, commit.getGeneration());
          delPolicy.saveCommitPoint(commit.getGeneration());
          return commit;
        }
      } // else...
      throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to find an index commit with name " +
                                commitName + " for core " + solrCore.getName());
    }
    // else: not a named commit...
    final IndexCommit commit = delPolicy.getAndSaveLatestCommit();
    if (null == commit) {
      throw new SolrException(ErrorCode.BAD_REQUEST, "Index does not yet have any commits for core " +
                              solrCore.getName());
    }
    log.debug("Using latest commit: generation={}", commit.getGeneration());
    return commit;
  }

  public void createSnapAsync(final int numberToKeep, Consumer result) throws IOException {
    //TODO should use Solr's ExecutorUtil
    new Thread(() -> {
      NamedList snapShootDetails;
      try {
        snapShootDetails = createSnapshot();
      } catch (Exception e) {
        log.error("Exception while creating snapshot", e);
        snapShootDetails = new NamedList<>();
        snapShootDetails.add("exception", e.getMessage());
      }
      if (snapshotName == null) {
        try {
          deleteOldBackups(numberToKeep);
        } catch (IOException e) {
          log.warn("Unable to delete old snapshots ", e);
        }
      }
      if (null != snapShootDetails) result.accept(snapShootDetails);
    }).start();

  }

  /**
   * Handles the logic of creating a snapshot
   * 

   * NOTE: The caller MUST ensure that the {@link IndexCommit} is saved prior to 
   * calling this method, and released after calling this method, or there is no no garuntee that the 
   * method will function correctly.
   * 

   *
   * @see IndexDeletionPolicyWrapper#saveCommitPoint
   * @see IndexDeletionPolicyWrapper#releaseCommitPoint
   */
  protected NamedList createSnapshot(final IndexCommit indexCommit) throws Exception {
    assert indexCommit != null;
    log.info("Creating backup snapshot " + (snapshotName == null ? "" : snapshotName) + " at " + baseSnapDirPath);
    boolean success = false;
    try {
      NamedList details = new NamedList<>();
      details.add("startTime", new Date().toString());//bad; should be Instant.now().toString()

      Collection files = indexCommit.getFileNames();
      Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
      try {
        for(String fileName : files) {
          log.debug("Copying fileName={} from dir={} to snapshot={}", fileName, dir, snapshotDirPath);
          backupRepo.copyFileFrom(dir, fileName, snapshotDirPath);
        }
      } finally {
        solrCore.getDirectoryFactory().release(dir);
      }

      details.add("fileCount", files.size());
      details.add("status", "success");
      details.add("snapshotCompletedAt", new Date().toString());//bad; should be Instant.now().toString()
      details.add("snapshotName", snapshotName);
      details.add("directoryName", directoryName);
      log.info("Done creating backup snapshot: {} into {}", (snapshotName == null ? "" : snapshotName), snapshotDirPath);
      success = true;
      return details;
    } finally {
      if (!success) {
        try {
          backupRepo.deleteDirectory(snapshotDirPath);
        } catch (Exception excDuringDelete) {
          log.warn("Failed to delete "+snapshotDirPath+" after snapshot creation failed due to: "+excDuringDelete);
        }
      }
    }
  }

  private void deleteOldBackups(int numberToKeep) throws IOException {
    String[] paths = backupRepo.listAll(baseSnapDirPath);
    List dirs = new ArrayList<>();
    for (String f : paths) {
      if (backupRepo.getPathType(baseSnapDirPath.resolve(f)) == PathType.DIRECTORY) {
        OldBackupDirectory obd = new OldBackupDirectory(baseSnapDirPath, f);
        if (obd.getTimestamp().isPresent()) {
          dirs.add(obd);
        }
      }
    }
    if (numberToKeep > dirs.size() -1) {
      return;
    }
    Collections.sort(dirs);
    int i=1;
    for (OldBackupDirectory dir : dirs) {
      if (i++ > numberToKeep) {
        backupRepo.deleteDirectory(dir.getPath());
      }
    }
  }

  protected void deleteNamedSnapshot(ReplicationHandler replicationHandler) {
    log.info("Deleting snapshot: " + snapshotName);

    NamedList details = new NamedList<>();
    details.add("snapshotName", snapshotName);
      
    try {
      URI path = baseSnapDirPath.resolve("snapshot." + snapshotName);
      backupRepo.deleteDirectory(path);

      details.add("status", "success");
      details.add("snapshotDeletedAt", new Date().toString());

    } catch (IOException e) {
      details.add("status", "Unable to delete snapshot: " + snapshotName);
      log.warn("Unable to delete snapshot: " + snapshotName, e);
    }

    replicationHandler.snapShootDetails = details;
  }

  public static final String DATE_FMT = "yyyyMMddHHmmssSSS";

}
 















© 2015 - 2024 Weber Informatics LLC | Privacy Policy