Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.schema;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.Version;
import org.apache.solr.analysis.TokenizerChain;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.rest.schema.FieldTypeXmlAdapter;
import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.util.FileUtils;
import org.apache.solr.util.RTimer;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
/** Solr-managed schema - non-user-editable, but can be mutable via internal and external REST API requests. */
public final class ManagedIndexSchema extends IndexSchema {
private final boolean isMutable;
@Override public boolean isMutable() { return isMutable; }
final String managedSchemaResourceName;
int schemaZkVersion;
final Object schemaUpdateLock;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
/**
* Constructs a schema using the specified resource name and stream.
*
* By default, this follows the normal config path directory searching rules.
* @see org.apache.solr.core.SolrResourceLoader#openResource
*/
ManagedIndexSchema(SolrConfig solrConfig, String name, InputSource is, boolean isMutable,
String managedSchemaResourceName, int schemaZkVersion, Object schemaUpdateLock) {
super(name, is, solrConfig.luceneMatchVersion, solrConfig.getResourceLoader(), solrConfig.getSubstituteProperties());
this.isMutable = isMutable;
this.managedSchemaResourceName = managedSchemaResourceName;
this.schemaZkVersion = schemaZkVersion;
this.schemaUpdateLock = schemaUpdateLock;
}
/**
* Persist the schema to local storage or to ZooKeeper
* @param createOnly set to false to allow update of existing schema
*/
public boolean persistManagedSchema(boolean createOnly) {
if (loader instanceof ZkSolrResourceLoader) {
return persistManagedSchemaToZooKeeper(createOnly);
}
// Persist locally
File managedSchemaFile = new File(loader.getConfigDir(), managedSchemaResourceName);
OutputStreamWriter writer = null;
try {
File parentDir = managedSchemaFile.getParentFile();
if ( ! parentDir.isDirectory()) {
if ( ! parentDir.mkdirs()) {
final String msg = "Can't create managed schema directory " + parentDir.getAbsolutePath();
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
}
final FileOutputStream out = new FileOutputStream(managedSchemaFile);
writer = new OutputStreamWriter(out, StandardCharsets.UTF_8);
persist(writer);
if (log.isInfoEnabled()) {
log.info("Upgraded to managed schema at {}", managedSchemaFile.getPath());
}
} catch (IOException e) {
final String msg = "Error persisting managed schema " + managedSchemaFile;
log.error(msg, e);
throw new SolrException(ErrorCode.SERVER_ERROR, msg, e);
} finally {
IOUtils.closeQuietly(writer);
try {
FileUtils.sync(managedSchemaFile);
} catch (IOException e) {
final String msg = "Error syncing the managed schema file " + managedSchemaFile;
log.error(msg, e);
}
}
return true;
}
/**
* Persists the managed schema to ZooKeeper using optimistic concurrency.
*
* If createOnly is true, success is when the schema is created or if it previously existed.
*
* If createOnly is false, success is when the schema is persisted - this will only happen
* if schemaZkVersion matches the version in ZooKeeper.
*
* @return true on success
*/
boolean persistManagedSchemaToZooKeeper(boolean createOnly) {
final ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader)loader;
final ZkController zkController = zkLoader.getZkController();
final SolrZkClient zkClient = zkController.getZkClient();
final String managedSchemaPath = zkLoader.getConfigSetZkPath() + "/" + managedSchemaResourceName;
boolean success = true;
boolean schemaChangedInZk = false;
try {
// Persist the managed schema
StringWriter writer = new StringWriter();
persist(writer);
final byte[] data = writer.toString().getBytes(StandardCharsets.UTF_8);
if (createOnly) {
try {
zkClient.create(managedSchemaPath, data, CreateMode.PERSISTENT, true);
schemaZkVersion = 0;
log.info("Created and persisted managed schema znode at {}", managedSchemaPath);
} catch (KeeperException.NodeExistsException e) {
// This is okay - do nothing and fall through
log.info("Managed schema znode at {} already exists - no need to create it", managedSchemaPath);
}
} else {
try {
// Assumption: the path exists
Stat stat = zkClient.setData(managedSchemaPath, data, schemaZkVersion, true);
schemaZkVersion = stat.getVersion();
log.info("Persisted managed schema version {} at {}", schemaZkVersion, managedSchemaPath);
} catch (KeeperException.BadVersionException e) {
log.error("Bad version when trying to persist schema using {} due to: ", schemaZkVersion, e);
success = false;
schemaChangedInZk = true;
}
}
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt(); // Restore the interrupted status
}
final String msg = "Error persisting managed schema at " + managedSchemaPath;
log.error(msg, e);
throw new SolrException(ErrorCode.SERVER_ERROR, msg, e);
}
if (schemaChangedInZk) {
String msg = "Failed to persist managed schema at " + managedSchemaPath
+ " - version mismatch";
log.info(msg);
throw new SchemaChangedInZkException(ErrorCode.CONFLICT, msg + ", retry.");
}
return success;
}
/**
* Block up to a specified maximum time until we see agreement on the schema
* version in ZooKeeper across all replicas for a collection.
*/
public static void waitForSchemaZkVersionAgreement(String collection, String localCoreNodeName,
int schemaZkVersion, ZkController zkController, int maxWaitSecs)
{
RTimer timer = new RTimer();
// get a list of active replica cores to query for the schema zk version (skipping this core of course)
List concurrentTasks = new ArrayList<>();
for (String coreUrl : getActiveReplicaCoreUrls(zkController, collection, localCoreNodeName))
concurrentTasks.add(new GetZkSchemaVersionCallable(coreUrl, schemaZkVersion));
if (concurrentTasks.isEmpty())
return; // nothing to wait for ...
if (log.isInfoEnabled()) {
log.info("Waiting up to {} secs for {} replicas to apply schema update version {} for collection {}"
, maxWaitSecs, concurrentTasks.size(), schemaZkVersion, collection);
}
// use an executor service to invoke schema zk version requests in parallel with a max wait time
int poolSize = Math.min(concurrentTasks.size(), 10);
ExecutorService parallelExecutor =
ExecutorUtil.newMDCAwareFixedThreadPool(poolSize, new SolrNamedThreadFactory("managedSchemaExecutor"));
try {
List> results =
parallelExecutor.invokeAll(concurrentTasks, maxWaitSecs, TimeUnit.SECONDS);
// determine whether all replicas have the update
List failedList = null; // lazily init'd
for (int f=0; f < results.size(); f++) {
int vers = -1;
Future next = results.get(f);
if (next.isDone() && !next.isCancelled()) {
// looks to have finished, but need to check the version value too
try {
vers = next.get();
} catch (ExecutionException e) {
// shouldn't happen since we checked isCancelled
}
}
if (vers == -1) {
String coreUrl = concurrentTasks.get(f).coreUrl;
log.warn("Core {} version mismatch! Expected {} but got {}", coreUrl, schemaZkVersion, vers);
if (failedList == null) failedList = new ArrayList<>();
failedList.add(coreUrl);
}
}
// if any tasks haven't completed within the specified timeout, it's an error
if (failedList != null)
throw new SolrException(ErrorCode.SERVER_ERROR, failedList.size()+" out of "+(concurrentTasks.size() + 1)+
" replicas failed to update their schema to version "+schemaZkVersion+" within "+
maxWaitSecs+" seconds! Failed cores: "+failedList);
} catch (InterruptedException ie) {
log.warn("Core {} was interrupted waiting for schema version {} to propagate to {} replicas for collection {}"
, localCoreNodeName, schemaZkVersion, concurrentTasks.size(), collection);
Thread.currentThread().interrupt();
} finally {
if (!parallelExecutor.isShutdown())
parallelExecutor.shutdown();
}
if (log.isInfoEnabled()) {
log.info("Took {}ms for {} replicas to apply schema update version {} for collection {}",
timer.getTime(), concurrentTasks.size(), schemaZkVersion, collection);
}
}
protected static List getActiveReplicaCoreUrls(ZkController zkController, String collection, String localCoreNodeName) {
List activeReplicaCoreUrls = new ArrayList<>();
ZkStateReader zkStateReader = zkController.getZkStateReader();
ClusterState clusterState = zkStateReader.getClusterState();
Set liveNodes = clusterState.getLiveNodes();
final DocCollection docCollection = clusterState.getCollectionOrNull(collection);
if (docCollection != null && docCollection.getActiveSlicesArr().length > 0) {
final Slice[] activeSlices = docCollection.getActiveSlicesArr();
for (Slice next : activeSlices) {
Map replicasMap = next.getReplicasMap();
if (replicasMap != null) {
for (Map.Entry entry : replicasMap.entrySet()) {
Replica replica = entry.getValue();
if (!localCoreNodeName.equals(replica.getName()) &&
replica.getState() == Replica.State.ACTIVE &&
liveNodes.contains(replica.getNodeName())) {
ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(replica);
activeReplicaCoreUrls.add(replicaCoreProps.getCoreUrl());
}
}
}
}
}
return activeReplicaCoreUrls;
}
@SuppressWarnings({"rawtypes"})
private static class GetZkSchemaVersionCallable extends SolrRequest implements Callable {
private String coreUrl;
private int expectedZkVersion;
GetZkSchemaVersionCallable(String coreUrl, int expectedZkVersion) {
super(METHOD.GET, "/schema/zkversion");
this.coreUrl = coreUrl;
this.expectedZkVersion = expectedZkVersion;
}
@Override
public SolrParams getParams() {
ModifiableSolrParams wparams = new ModifiableSolrParams();
wparams.set("refreshIfBelowVersion", expectedZkVersion);
return wparams;
}
@Override
public Integer call() throws Exception {
int remoteVersion = -1;
try (HttpSolrClient solr = new HttpSolrClient.Builder(coreUrl).build()) {
// eventually, this loop will get killed by the ExecutorService's timeout
while (remoteVersion == -1 || remoteVersion < expectedZkVersion) {
try {
HttpSolrClient.HttpUriRequestResponse mrr = solr.httpUriRequest(this);
NamedList