All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.solr.update.SolrIndexSplitter Maven / Gradle / Ivy

Go to download

Apache Solr (module: core)

There is a newer version: 9.7.0
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.update;

import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.FilterCodecReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SlowCodecReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.HardlinkCopyDirectoryWrapper;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.Lock;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.CompositeIdRouter;
import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.HashBasedRouter;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.IndexFetcher;
import org.apache.solr.handler.SnapShooter;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.BitsFilteredPostingsEnum;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.RTimerTree;
import org.apache.solr.util.RefCounted;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class SolrIndexSplitter {
  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

  private static final String INDEX_PREFIX = "index.";

  public enum SplitMethod {
    REWRITE,
    LINK;

    public static SplitMethod get(String p) {
      if (p != null) {
        try {
          return SplitMethod.valueOf(p.toUpperCase(Locale.ROOT));
        } catch (Exception ex) {
          return null;
        }
      }
      return null;
    }

    public String toLower() {
      return toString().toLowerCase(Locale.ROOT);
    }
  }

  SolrIndexSearcher searcher;
  SchemaField field;
  List ranges;
  DocRouter.Range[] rangesArr; // same as ranges list, but an array for extra speed in inner loops
  List paths;
  List cores;
  HashBasedRouter hashRouter;
  int numPieces;
  String routeFieldName;
  String splitKey;
  SplitMethod splitMethod;
  RTimerTree timings = new RTimerTree();

  public SolrIndexSplitter(SplitIndexCommand cmd) {
    searcher = cmd.getReq().getSearcher();
    ranges = cmd.ranges;
    paths = cmd.paths;
    cores = cmd.cores;
    hashRouter = cmd.router instanceof HashBasedRouter ? (HashBasedRouter)cmd.router : null;

    if (ranges == null) {
      numPieces =  paths != null ? paths.size() : cores.size();
    } else  {
      numPieces = ranges.size();
      rangesArr = ranges.toArray(new DocRouter.Range[ranges.size()]);
    }
    routeFieldName = cmd.routeFieldName;
    if (routeFieldName == null) {
      field = searcher.getSchema().getUniqueKeyField();
    } else  {
      field = searcher.getSchema().getField(routeFieldName);
    }
    if (cmd.splitKey != null) {
      splitKey = getRouteKey(cmd.splitKey);
    }
    if (cores == null) {
      this.splitMethod = SplitMethod.REWRITE;
    } else {
      this.splitMethod = cmd.splitMethod;
    }
  }

  public void split(NamedList results) throws IOException {
    SolrCore parentCore = searcher.getCore();
    Directory parentDirectory = searcher.getRawReader().directory();
    Lock parentDirectoryLock = null;
    UpdateLog ulog = parentCore.getUpdateHandler().getUpdateLog();
    if (ulog == null && splitMethod == SplitMethod.LINK) {
      log.warn("No updateLog in parent core, switching to use potentially slower 'splitMethod=rewrite'");
      splitMethod = SplitMethod.REWRITE;
    }
    if (splitMethod == SplitMethod.LINK) {
      RTimerTree t = timings.sub("closeParentIW");
      try {
        // start buffering updates
        ulog.bufferUpdates();
        parentCore.getSolrCoreState().closeIndexWriter(parentCore, false);
        // make sure we can lock the directory for our exclusive use
        parentDirectoryLock = parentDirectory.obtainLock(IndexWriter.WRITE_LOCK_NAME);
        log.info("Splitting in 'link' mode: closed parent IndexWriter...");
        t.stop();
      } catch (Exception e) {
        if (parentDirectoryLock != null) {
          IOUtils.closeWhileHandlingException(parentDirectoryLock);
        }
        try {
          parentCore.getSolrCoreState().openIndexWriter(parentCore);
          ulog.applyBufferedUpdates();
        } catch (Exception e1) {
          log.error("Error reopening IndexWriter after failed close", e1);
          log.error("Original error closing IndexWriter:", e);
          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reopening IndexWriter after failed close", e1);
        }
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error closing current IndexWriter, aborting 'link' split...", e);
      }
    }
    boolean success = false;
    try {
      RTimerTree t = timings.sub("doSplit");
      doSplit();
      t.stop();
      success = true;
    } catch (Exception e) {
      results.add("failed", e.toString());
      throw e;
    } finally {
      if (splitMethod == SplitMethod.LINK) {
        IOUtils.closeWhileHandlingException(parentDirectoryLock);
        RTimerTree t = timings.sub("reopenParentIW");
        parentCore.getSolrCoreState().openIndexWriter(parentCore);
        t.stop();
        t = timings.sub("parentApplyBufferedUpdates");
        ulog.applyBufferedUpdates();
        t.stop();
        log.info("Splitting in 'link' mode " + (success? "finished" : "FAILED") +
            ": re-opened parent IndexWriter.");
      }
    }
    results.add(CommonParams.TIMING, timings.asNamedList());
  }

  public void doSplit() throws IOException {

    List leaves = searcher.getRawReader().leaves();
    Directory parentDirectory = searcher.getRawReader().directory();
    List segmentDocSets = new ArrayList<>(leaves.size());
    SolrIndexConfig parentConfig = searcher.getCore().getSolrConfig().indexConfig;
    String timestamp = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date());

    log.info("SolrIndexSplitter: partitions=" + numPieces + " segments=" + leaves.size());
    RTimerTree t;

    // this tracks round-robin assignment of docs to partitions
    AtomicInteger currentPartition = new AtomicInteger();

    if (splitMethod != SplitMethod.LINK) {
      t = timings.sub("findDocSetsPerLeaf");
      for (LeafReaderContext readerContext : leaves) {
        assert readerContext.ordInParent == segmentDocSets.size();  // make sure we're going in order
        FixedBitSet[] docSets = split(readerContext, numPieces, field, rangesArr, splitKey, hashRouter, currentPartition, false);
        segmentDocSets.add(docSets);
      }
      t.stop();
    }


    Map docsToDeleteCache = new ConcurrentHashMap<>();

    // would it be more efficient to write segment-at-a-time to each new index?
    // - need to worry about number of open descriptors
    // - need to worry about if IW.addIndexes does a sync or not...
    // - would be more efficient on the read side, but prob less efficient merging
    for (int partitionNumber=0; partitionNumber iwRef = null;
      IndexWriter iw;
      if (cores != null && splitMethod != SplitMethod.LINK) {
        SolrCore subCore = cores.get(partitionNumber);
        iwRef = subCore.getUpdateHandler().getSolrCoreState().getIndexWriter(subCore);
        iw = iwRef.get();
      } else {
        if (splitMethod == SplitMethod.LINK) {
          SolrCore subCore = cores.get(partitionNumber);
          String path = subCore.getDataDir() + INDEX_PREFIX + timestamp;
          t = timings.sub("hardLinkCopy");
          t.resume();
          // copy by hard-linking
          Directory splitDir = subCore.getDirectoryFactory().get(path, DirectoryFactory.DirContext.DEFAULT, subCore.getSolrConfig().indexConfig.lockType);
          // the wrapper doesn't hold any resources itself so it doesn't need closing
          HardlinkCopyDirectoryWrapper hardLinkedDir = new HardlinkCopyDirectoryWrapper(splitDir);
          boolean copiedOk = false;
          try {
            for (String file : parentDirectory.listAll()) {
              // we've closed the IndexWriter, so ignore write.lock
              // its file may be present even when IndexWriter is closed but
              // we've already checked that the lock is not held by anyone else
              if (file.equals(IndexWriter.WRITE_LOCK_NAME)) {
                continue;
              }
              hardLinkedDir.copyFrom(parentDirectory, file, file, IOContext.DEFAULT);
            }
            copiedOk = true;
          } finally {
            if (!copiedOk) {
              subCore.getDirectoryFactory().doneWithDirectory(splitDir);
              subCore.getDirectoryFactory().remove(splitDir);
            }
          }
          t.pause();
          IndexWriterConfig iwConfig = parentConfig.toIndexWriterConfig(subCore);
          // don't run merges at this time
          iwConfig.setMergePolicy(NoMergePolicy.INSTANCE);
          t = timings.sub("createSubIW");
          t.resume();
          iw = new SolrIndexWriter(partitionName, splitDir, iwConfig);
          t.pause();
        } else {
          SolrCore core = searcher.getCore();
          String path = paths.get(partitionNumber);
          t = timings.sub("createSubIW");
          t.resume();
          iw = SolrIndexWriter.create(core, partitionName, path,
              core.getDirectoryFactory(), true, core.getLatestSchema(),
              core.getSolrConfig().indexConfig, core.getDeletionPolicy(), core.getCodec());
          t.pause();
        }
      }

      try {
        if (splitMethod == SplitMethod.LINK) {
          t = timings.sub("deleteDocuments");
          t.resume();
          // apply deletions specific to this partition. As a side-effect on the first call this also populates
          // a cache of docsets to delete per leaf reader per partition, which is reused for subsequent partitions.
          iw.deleteDocuments(new SplittingQuery(partitionNumber, field, rangesArr, hashRouter, splitKey, docsToDeleteCache, currentPartition));
          t.pause();
        } else {
          // This removes deletions but optimize might still be needed because sub-shards will have the same number of segments as the parent shard.
          t = timings.sub("addIndexes");
          t.resume();
          for (int segmentNumber = 0; segmentNumber docsToDelete;
    private final AtomicInteger currentPartition;

    SplittingQuery(int partition, SchemaField field, DocRouter.Range[] rangesArr, HashBasedRouter hashRouter, String splitKey,
                   Map docsToDelete, AtomicInteger currentPartition) {
      this.partition = partition;
      this.field = field;
      this.rangesArr = rangesArr;
      this.hashRouter = hashRouter;
      this.splitKey = splitKey;
      this.docsToDelete = docsToDelete;
      this.currentPartition = currentPartition;
    }

    @Override
    public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
      return new ConstantScoreWeight(this, boost) {

        @Override
        public Scorer scorer(LeafReaderContext context) throws IOException {
          RTimerTree t = timings.sub("findDocsToDelete");
          t.resume();
          FixedBitSet set = findDocsToDelete(context);
          t.pause();
          log.info("### partition=" + partition + ", leaf=" + context + ", maxDoc=" + context.reader().maxDoc() +
          ", numDels=" + context.reader().numDeletedDocs() + ", setLen=" + set.length() + ", setCard=" + set.cardinality());
          Bits liveDocs = context.reader().getLiveDocs();
          if (liveDocs != null) {
            // check that we don't delete already deleted docs
            FixedBitSet dels = FixedBitSet.copyOf(liveDocs);
            dels.flip(0, dels.length());
            dels.and(set);
            if (dels.cardinality() > 0) {
              log.error("### INVALID DELS " + dels.cardinality());
            }
          }
          return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(set, set.length()));
        }

        @Override
        public boolean isCacheable(LeafReaderContext ctx) {
          return false;
        }

        @Override
        public String toString() {
          return "weight(shardSplittingQuery,part" + partition + ")";
        }
      };
    }

    private FixedBitSet findDocsToDelete(LeafReaderContext readerContext) throws IOException {
      // check whether a cached copy of bitsets already exists for this reader
      FixedBitSet[] perPartition = docsToDelete.get(readerContext.reader().getCoreCacheHelper().getKey());
      if (perPartition != null) {
        return perPartition[partition];
      }
      synchronized (docsToDelete) {
        perPartition = docsToDelete.get(readerContext.reader().getCoreCacheHelper().getKey());
        if (perPartition != null) {
          return perPartition[partition];
        }

        perPartition = split(readerContext, numPieces, field, rangesArr, splitKey, hashRouter, currentPartition, true);
        docsToDelete.put(readerContext.reader().getCoreCacheHelper().getKey(), perPartition);
        return perPartition[partition];
      }
    }

    @Override
    public String toString(String field) {
      return "shardSplittingQuery";
    }

    @Override
    public boolean equals(Object obj) {
      if (obj == null) {
        return false;
      }
      if (this == obj) {
        return true;
      }
      if (!(obj instanceof SplittingQuery)) {
        return false;
      }
      SplittingQuery q = (SplittingQuery)obj;
      return partition == q.partition;
    }

    @Override
    public int hashCode() {
      return partition;
    }

    @Override
    public void visit(QueryVisitor visitor) {
      visitor.visitLeaf(this);
    }
  }

  static FixedBitSet[] split(LeafReaderContext readerContext, int numPieces, SchemaField field, DocRouter.Range[] rangesArr,
                             String splitKey, HashBasedRouter hashRouter, AtomicInteger currentPartition, boolean delete) throws IOException {
    LeafReader reader = readerContext.reader();
    FixedBitSet[] docSets = new FixedBitSet[numPieces];
    for (int i=0; i 0 && commaIdx + 1 < part1.length())  {
      char ch = part1.charAt(commaIdx + 1);
      if (ch >= '0' && ch <= '9') {
        part1 = part1.substring(0, commaIdx);
      }
    }
    return part1;
  }


  // change livedocs on the reader to delete those docs we don't want
  static class LiveDocsReader extends FilterCodecReader {
    final FixedBitSet liveDocs;
    final int numDocs;

    public LiveDocsReader(CodecReader in, FixedBitSet liveDocs) {
      super(in);
      this.liveDocs = liveDocs;
      this.numDocs = liveDocs.cardinality();
    }

    @Override
    public int numDocs() {
      return numDocs;
    }

    @Override
    public Bits getLiveDocs() {
      return liveDocs;
    }

    @Override
    public CacheHelper getCoreCacheHelper() {
      return in.getCoreCacheHelper();
    }

    @Override
    public CacheHelper getReaderCacheHelper() {
      return null;
    }
  }

}
 















© 2015 - 2024 Weber Informatics LLC | Privacy Policy