• Home
• org.apache.solr
• solr-core
• 4.7.0
• source code
• CoreAdminHandler.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

org.apache.solr.handler.admin.CoreAdminHandler Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.solr.handler.admin;

import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Future;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.SyncStrategy;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.CloseHook;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrXMLCoresLocator;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.CommitUpdateCommand;
import org.apache.solr.update.MergeIndexesCommand;
import org.apache.solr.update.SplitIndexCommand;
import org.apache.solr.update.UpdateLog;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.util.NumberUtils;
import org.apache.solr.util.RefCounted;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;

/**
 *
 * @since solr 1.3
 */
public class CoreAdminHandler extends RequestHandlerBase {
  protected static Logger log = LoggerFactory.getLogger(CoreAdminHandler.class);
  protected final CoreContainer coreContainer;

  public CoreAdminHandler() {
    super();
    // Unlike most request handlers, CoreContainer initialization 
    // should happen in the constructor...  
    this.coreContainer = null;
  }


  /**
   * Overloaded ctor to inject CoreContainer into the handler.
   *
   * @param coreContainer Core Container of the solr webapp installed.
   */
  public CoreAdminHandler(final CoreContainer coreContainer) {
    this.coreContainer = coreContainer;
  }


  @Override
  final public void init(NamedList args) {
    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
            "CoreAdminHandler should not be configured in solrconf.xml\n" +
                    "it is a special Handler configured directly by the RequestDispatcher");
  }

  /**
   * The instance of CoreContainer this handler handles. This should be the CoreContainer instance that created this
   * handler.
   *
   * @return a CoreContainer instance
   */
  public CoreContainer getCoreContainer() {
    return this.coreContainer;
  }

  @Override
  public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
    // Make sure the cores is enabled
    CoreContainer cores = getCoreContainer();
    if (cores == null) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
              "Core container instance missing");
    }
    //boolean doPersist = false;

    // Pick the action
    SolrParams params = req.getParams();
    CoreAdminAction action = CoreAdminAction.STATUS;
    String a = params.get(CoreAdminParams.ACTION);
    if (a != null) {
      action = CoreAdminAction.get(a);
      if (action == null) {
        this.handleCustomAction(req, rsp);
      }
    }
    if (action != null) {
      switch (action) {
        case CREATE: {
          this.handleCreateAction(req, rsp);
          break;
        }

        case RENAME: {
          this.handleRenameAction(req, rsp);
          break;
        }

        case UNLOAD: {
          this.handleUnloadAction(req, rsp);
          break;
        }

        case STATUS: {
          this.handleStatusAction(req, rsp);
          break;

        }

        case PERSIST: {
          this.handlePersistAction(req, rsp);
          break;
        }

        case RELOAD: {
          this.handleReloadAction(req, rsp);
          break;
        }

        case SWAP: {
          this.handleSwapAction(req, rsp);
          break;
        }

        case MERGEINDEXES: {
          this.handleMergeAction(req, rsp);
          break;
        }

        case SPLIT: {
          this.handleSplitAction(req, rsp);
          break;
        }

        case PREPRECOVERY: {
          this.handleWaitForStateAction(req, rsp);
          break;
        }
        
        case REQUESTRECOVERY: {
          this.handleRequestRecoveryAction(req, rsp);
          break;
        }
        
        case REQUESTSYNCSHARD: {
          this.handleRequestSyncAction(req, rsp);
          break;
        }
        
        // todo : Can this be done by the regular RecoveryStrategy route?
        case REQUESTAPPLYUPDATES: {
          this.handleRequestApplyUpdatesAction(req, rsp);
          break;
        }
        case REQUESTBUFFERUPDATES:  {
          this.handleRequestBufferUpdatesAction(req, rsp);
          break;
        }
        case OVERSEEROP:{
          ZkController zkController = coreContainer.getZkController();
          if(zkController != null){
            String op = req.getParams().get("op");
            if("leader".equals(op)){
              zkController.forceOverSeer();
            } else if ("rejoin".equals(op)) zkController.rejoinOverseerElection();
          }
          break;
        }
        default: {
          this.handleCustomAction(req, rsp);
          break;
        }
        case LOAD:
          break;
      }
    }
    rsp.setHttpCaching(false);
  }

  
  /**
   * Handle the core admin SPLIT action.
   */
  protected void handleSplitAction(SolrQueryRequest adminReq, SolrQueryResponse rsp) throws IOException {
    SolrParams params = adminReq.getParams();
    List ranges = null;

    String[] pathsArr = params.getParams("path");
    String rangesStr = params.get(CoreAdminParams.RANGES);    // ranges=a-b,c-d,e-f
    if (rangesStr != null)  {
      String[] rangesArr = rangesStr.split(",");
      if (rangesArr.length == 0) {
        throw new SolrException(ErrorCode.BAD_REQUEST, "There must be at least one range specified to split an index");
      } else  {
        ranges = new ArrayList(rangesArr.length);
        for (String r : rangesArr) {
          try {
            ranges.add(DocRouter.DEFAULT.fromString(r));
          } catch (Exception e) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Exception parsing hexadecimal hash range: " + r, e);
          }
        }
      }
    }
    String splitKey = params.get("split.key");
    String[] newCoreNames = params.getParams("targetCore");
    String cname = params.get(CoreAdminParams.CORE, "");

    if ((pathsArr == null || pathsArr.length == 0) && (newCoreNames == null || newCoreNames.length == 0)) {
      throw new SolrException(ErrorCode.BAD_REQUEST, "Either path or targetCore param must be specified");
    }

    log.info("Invoked split action for core: " + cname);
    SolrCore core = coreContainer.getCore(cname);
    SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
    List newCores = null;

    try {
      // TODO: allow use of rangesStr in the future
      List paths = null;
      int partitions = pathsArr != null ? pathsArr.length : newCoreNames.length;

      DocRouter router = null;
      String routeFieldName = null;
      if (coreContainer.isZooKeeperAware()) {
        ClusterState clusterState = coreContainer.getZkController().getClusterState();
        String collectionName = req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName();
        DocCollection collection = clusterState.getCollection(collectionName);
        String sliceName = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
        Slice slice = clusterState.getSlice(collectionName, sliceName);
        router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;
        if (ranges == null) {
          DocRouter.Range currentRange = slice.getRange();
          ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null;
        }
        Object routerObj = collection.get(DOC_ROUTER); // for back-compat with Solr 4.4
        if (routerObj != null && routerObj instanceof Map) {
          Map routerProps = (Map) routerObj;
          routeFieldName = (String) routerProps.get("field");
        }
      }

      if (pathsArr == null) {
        newCores = new ArrayList(partitions);
        for (String newCoreName : newCoreNames) {
          SolrCore newcore = coreContainer.getCore(newCoreName);
          if (newcore != null) {
            newCores.add(newcore);
          } else {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Core with core name " + newCoreName + " expected but doesn't exist.");
          }
        }
      } else {
        paths = Arrays.asList(pathsArr);
      }


      SplitIndexCommand cmd = new SplitIndexCommand(req, paths, newCores, ranges, router, routeFieldName, splitKey);
      core.getUpdateHandler().split(cmd);

      // After the split has completed, someone (here?) should start the process of replaying the buffered updates.

    } catch (Exception e) {
      log.error("ERROR executing split:", e);
      throw new RuntimeException(e);

    } finally {
      if (req != null) req.close();
      if (core != null) core.close();
      if (newCores != null) {
        for (SolrCore newCore : newCores) {
          newCore.close();
        }
      }
    }

  }


  protected void handleMergeAction(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
    SolrParams params = req.getParams();
    String cname = params.required().get(CoreAdminParams.CORE);
    SolrCore core = coreContainer.getCore(cname);
    SolrQueryRequest wrappedReq = null;

    List sourceCores = Lists.newArrayList();
    List> searchers = Lists.newArrayList();
    // stores readers created from indexDir param values
    List readersToBeClosed = Lists.newArrayList();
    List dirsToBeReleased = Lists.newArrayList();
    if (core != null) {
      try {
        String[] dirNames = params.getParams(CoreAdminParams.INDEX_DIR);
        if (dirNames == null || dirNames.length == 0) {
          String[] sources = params.getParams("srcCore");
          if (sources == null || sources.length == 0)
            throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
                "At least one indexDir or srcCore must be specified");

          for (int i = 0; i < sources.length; i++) {
            String source = sources[i];
            SolrCore srcCore = coreContainer.getCore(source);
            if (srcCore == null)
              throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
                  "Core: " + source + " does not exist");
            sourceCores.add(srcCore);
          }
        } else  {
          DirectoryFactory dirFactory = core.getDirectoryFactory();
          for (int i = 0; i < dirNames.length; i++) {
            Directory dir = dirFactory.get(dirNames[i], DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
            dirsToBeReleased.add(dir);
            // TODO: why doesn't this use the IR factory? what is going on here?
            readersToBeClosed.add(DirectoryReader.open(dir));
          }
        }

        List readers = null;
        if (readersToBeClosed.size() > 0)  {
          readers = readersToBeClosed;
        } else {
          readers = Lists.newArrayList();
          for (SolrCore solrCore: sourceCores) {
            // record the searchers so that we can decref
            RefCounted searcher = solrCore.getSearcher();
            searchers.add(searcher);
            readers.add(searcher.get().getIndexReader());
          }
        }

        UpdateRequestProcessorChain processorChain =
                core.getUpdateProcessingChain(params.get(UpdateParams.UPDATE_CHAIN));
        wrappedReq = new LocalSolrQueryRequest(core, req.getParams());
        UpdateRequestProcessor processor =
                processorChain.createProcessor(wrappedReq, rsp);
        processor.processMergeIndexes(new MergeIndexesCommand(readers, req));
      } catch (Exception e) {
        // log and rethrow so that if the finally fails we don't lose the original problem
        log.error("ERROR executing merge:", e);
        throw e;
      } finally {
        for (RefCounted searcher : searchers) {
          if (searcher != null) searcher.decref();
        }
        for (SolrCore solrCore : sourceCores) {
          if (solrCore != null) solrCore.close();
        }
        IOUtils.closeWhileHandlingException(readersToBeClosed);
        for (Directory dir : dirsToBeReleased) {
          DirectoryFactory dirFactory = core.getDirectoryFactory();
          dirFactory.release(dir);
        }
        if (wrappedReq != null) wrappedReq.close();
        core.close();
      }
    }
  }

  /**
   * Handle Custom Action.
   * 

   * This method could be overridden by derived classes to handle custom actions. 
 By default - this method throws a
   * solr exception. Derived classes are free to write their derivation if necessary.
   */
  protected void handleCustomAction(SolrQueryRequest req, SolrQueryResponse rsp) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unsupported operation: " +
            req.getParams().get(CoreAdminParams.ACTION));
  }

  public static ImmutableMap paramToProp = ImmutableMap.builder()
      .put(CoreAdminParams.CONFIG, CoreDescriptor.CORE_CONFIG)
      .put(CoreAdminParams.SCHEMA, CoreDescriptor.CORE_SCHEMA)
      .put(CoreAdminParams.DATA_DIR, CoreDescriptor.CORE_DATADIR)
      .put(CoreAdminParams.ULOG_DIR, CoreDescriptor.CORE_ULOGDIR)
      .put(CoreAdminParams.LOAD_ON_STARTUP, CoreDescriptor.CORE_LOADONSTARTUP)
      .put(CoreAdminParams.TRANSIENT, CoreDescriptor.CORE_TRANSIENT)
      .put(CoreAdminParams.SHARD, CoreDescriptor.CORE_SHARD)
      .put(CoreAdminParams.COLLECTION, CoreDescriptor.CORE_COLLECTION)
      .put(CoreAdminParams.ROLES, CoreDescriptor.CORE_ROLES)
      .put(CoreAdminParams.CORE_NODE_NAME, CoreDescriptor.CORE_NODE_NAME)
      .put(ZkStateReader.NUM_SHARDS_PROP, CloudDescriptor.NUM_SHARDS)
      .build();

  public static ImmutableMap cloudParamToProp;

  protected static CoreDescriptor buildCoreDescriptor(SolrParams params, CoreContainer container) {

    String name = checkNotEmpty(params.get(CoreAdminParams.NAME),
        "Missing parameter [" + CoreAdminParams.NAME + "]");
    String instancedir = params.get(CoreAdminParams.INSTANCE_DIR);
    if (StringUtils.isEmpty(instancedir)) {
      instancedir = name; // will be resolved later against solr.home
      //instancedir = container.getSolrHome() + "/" + name;
    }

    Properties coreProps = new Properties();
    for (String param : paramToProp.keySet()) {
      String value = params.get(param, null);
      if (StringUtils.isNotEmpty(value)) {
        coreProps.setProperty(paramToProp.get(param), value);
      }
    }
    Iterator paramsIt = params.getParameterNamesIterator();
    while (paramsIt.hasNext()) {
      String param = paramsIt.next();
      if (!param.startsWith(CoreAdminParams.PROPERTY_PREFIX))
        continue;
      String propName = param.substring(CoreAdminParams.PROPERTY_PREFIX.length());
      String propValue = params.get(param);
      coreProps.setProperty(propName, propValue);
    }

    return new CoreDescriptor(container, name, instancedir, coreProps, params);
  }

  private static String checkNotEmpty(String value, String message) {
    if (StringUtils.isEmpty(value))
      throw new SolrException(ErrorCode.BAD_REQUEST, message);
    return value;
  }

  /**
   * Handle 'CREATE' action.
   *
   * @throws SolrException in case of a configuration error.
   */
  protected void handleCreateAction(SolrQueryRequest req, SolrQueryResponse rsp) throws SolrException {

    SolrParams params = req.getParams();
    CoreDescriptor dcore = buildCoreDescriptor(params, coreContainer);

    if (coreContainer.getAllCoreNames().contains(dcore.getName())) {
      log.warn("Creating a core with existing name is not allowed");
      throw new SolrException(ErrorCode.SERVER_ERROR,
          "Core with name '" + dcore.getName() + "' already exists.");
    }

    // TODO this should be moved into CoreContainer, really...
    try {
      if (coreContainer.getZkController() != null) {
        coreContainer.preRegisterInZk(dcore);
      }

      // make sure we can write out the descriptor first
      coreContainer.getCoresLocator().create(coreContainer, dcore);
      
      SolrCore core = coreContainer.create(dcore);
      
      coreContainer.register(core, false);
      
      if (coreContainer.getCoresLocator() instanceof SolrXMLCoresLocator) {
        // hack - in this case we persist once more because a core create race might
        // have dropped entries.
        coreContainer.getCoresLocator().create(coreContainer);
      }
      rsp.add("core", core.getName());
    }
    catch (Exception ex) {
      if (coreContainer.isZooKeeperAware() && dcore != null) {
        try {
          coreContainer.getZkController().unregister(dcore.getName(), dcore);
        } catch (InterruptedException e) {
          Thread.currentThread().interrupt();
          SolrException.log(log, null, e);
        } catch (KeeperException e) {
          SolrException.log(log, null, e);
        }
      }
      
      Throwable tc = ex;
      Throwable c = null;
      do {
        tc = tc.getCause();
        if (tc != null) {
          c = tc;
        }
      } while (tc != null);
      
      String rootMsg = "";
      if (c != null) {
        rootMsg = " Caused by: " + c.getMessage();
      }
      
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
          "Error CREATEing SolrCore '" + dcore.getName() + "': " +
          ex.getMessage() + rootMsg, ex);
    }
  }

  /**
   * Handle "RENAME" Action
   */
  protected void handleRenameAction(SolrQueryRequest req, SolrQueryResponse rsp) throws SolrException {
    SolrParams params = req.getParams();

    String name = params.get(CoreAdminParams.OTHER);
    String cname = params.get(CoreAdminParams.CORE);

    if (cname.equals(name)) return;

    coreContainer.rename(cname, name);

  }

  /**
   * Handle "ALIAS" action
   */
  @Deprecated
  protected void handleAliasAction(SolrQueryRequest req, SolrQueryResponse rsp) {
    SolrParams params = req.getParams();

    String name = params.get(CoreAdminParams.OTHER);
    String cname = params.get(CoreAdminParams.CORE);
    boolean doPersist = false;
    if (cname.equals(name)) return;

    SolrCore core = coreContainer.getCore(cname);
    if (core != null) {
      doPersist = coreContainer.isPersistent();
      coreContainer.register(name, core, false);
      // no core.close() since each entry in the cores map should increase the ref
    }
    return;
  }


  /**
   * Handle "UNLOAD" Action
   */
  protected void handleUnloadAction(SolrQueryRequest req,
      SolrQueryResponse rsp) throws SolrException {
    SolrParams params = req.getParams();
    String cname = params.get(CoreAdminParams.CORE);
    Boolean closeCore = true;
    if (!coreContainer.isLoadedNotPendingClose(cname)) {
      closeCore = false;
    }
    SolrCore core = coreContainer.remove(cname);
    try {
      if (core == null) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
            "No such core exists '" + cname + "'");
      } else {
        if (coreContainer.getZkController() != null) {
          // we are unloading, cancel any ongoing recovery
          if (core != null) {
            if (coreContainer.getZkController() != null) {
              core.getSolrCoreState().cancelRecovery();
            }
          }
        }
        
        if (params.getBool(CoreAdminParams.DELETE_INDEX, false)) {
          try {
            core.getDirectoryFactory().remove(core.getIndexDir());
          } catch (Exception e) {
            SolrException.log(log, "Failed to flag index dir for removal for core:"
                    + core.getName() + " dir:" + core.getIndexDir());
          }
        }
      }

      if (params.getBool(CoreAdminParams.DELETE_DATA_DIR, false)) {
        try {
          core.getDirectoryFactory().remove(core.getDataDir(), true);
        } catch (Exception e) {
          SolrException.log(log, "Failed to flag data dir for removal for core:"
                  + core.getName() + " dir:" + core.getDataDir());
        }
      }
      
      if (params.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, false)) {
        core.addCloseHook(new CloseHook() {
          @Override
          public void preClose(SolrCore core) {}
          
          @Override
          public void postClose(SolrCore core) {
            CoreDescriptor cd = core.getCoreDescriptor();
            if (cd != null) {
              File instanceDir = new File(cd.getInstanceDir());
              try {
                FileUtils.deleteDirectory(instanceDir);
              } catch (IOException e) {
                SolrException.log(log, "Failed to delete instance dir for core:"
                    + core.getName() + " dir:" + instanceDir.getAbsolutePath());
              }
            }
          }
        });
      }
    } finally {
      // it's important that we try and cancel recovery
      // before we close here - else we might close the
      // core *in* recovery and end up locked in recovery
      // waiting to for recovery to be cancelled
      if (core != null) {
        if (coreContainer.getZkController() != null) {
          core.getSolrCoreState().cancelRecovery();
        }
        if (closeCore) {
          core.close();
        }
        
        if (coreContainer.getZkController() != null) {
          log.info("Unregistering core " + core.getName() + " from cloudstate.");
          try {
            coreContainer.getZkController().unregister(cname,
                core.getCoreDescriptor());
          } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
                "Could not unregister core " + cname + " from cloudstate: "
                    + e.getMessage(), e);
          } catch (KeeperException e) {
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
                "Could not unregister core " + cname + " from cloudstate: "
                    + e.getMessage(), e);
          }
        }
      }
    }
    
  }

  /**
   * Handle "STATUS" action
   */
  protected void handleStatusAction(SolrQueryRequest req, SolrQueryResponse rsp)
          throws SolrException {
    SolrParams params = req.getParams();

    String cname = params.get(CoreAdminParams.CORE);
    String indexInfo = params.get(CoreAdminParams.INDEX_INFO);
    boolean isIndexInfoNeeded = Boolean.parseBoolean(null == indexInfo ? "true" : indexInfo);
    boolean doPersist = false;
    NamedList