Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.invoke.MethodHandles;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.collect.ImmutableList;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.util.Version;
import org.apache.solr.client.solrj.io.stream.expr.Expressible;
import org.apache.solr.cloud.RecoveryStrategy;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.ConfigNode;
import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.transform.TransformerFactory;
import org.apache.solr.rest.RestManager;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.IndexSchemaFactory;
import org.apache.solr.search.CacheConfig;
import org.apache.solr.search.FastLRUCache;
import org.apache.solr.search.QParserPlugin;
import org.apache.solr.search.SolrCache;
import org.apache.solr.search.ValueSourceParser;
import org.apache.solr.search.stats.StatsCache;
import org.apache.solr.servlet.SolrRequestParsers;
import org.apache.solr.spelling.QueryConverter;
import org.apache.solr.update.SolrIndexConfig;
import org.apache.solr.update.UpdateLog;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
import org.apache.solr.util.DOMConfigNode;
import org.apache.solr.util.DataConfigNode;
import org.apache.solr.util.circuitbreaker.CircuitBreakerManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.common.params.CommonParams.PATH;
import static org.apache.solr.common.util.Utils.fromJSON;
import static org.apache.solr.common.util.Utils.makeMap;
import static org.apache.solr.core.ConfigOverlay.ZNODEVER;
import static org.apache.solr.core.SolrConfig.PluginOpts.LAZY;
import static org.apache.solr.core.SolrConfig.PluginOpts.MULTI_OK;
import static org.apache.solr.core.SolrConfig.PluginOpts.NOOP;
import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_CLASS;
import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_NAME;
import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_NAME_IN_OVERLAY;
import static org.apache.solr.core.XmlConfigFile.assertWarnOrFail;
/**
* Provides a static reference to a Config object modeling the main
* configuration data for a a Solr instance -- typically found in
* "solrconfig.xml".
*/
public class SolrConfig implements MapSerializable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String DEFAULT_CONF_FILE = "solrconfig.xml";
private final String resourceName;
private int znodeVersion;
ConfigNode root;
private final SolrResourceLoader resourceLoader;
private Properties substituteProperties;
private RequestParams requestParams;
public enum PluginOpts {
MULTI_OK,
REQUIRE_NAME,
REQUIRE_NAME_IN_OVERLAY,
REQUIRE_CLASS,
LAZY,
// EnumSet.of and/or EnumSet.copyOf(Collection) are annoying
// because of type determination
NOOP
}
private int multipartUploadLimitKB;
private int formUploadLimitKB;
private boolean enableRemoteStreams;
private boolean enableStreamBody;
private boolean handleSelect;
private boolean addHttpRequestToContext;
private final SolrRequestParsers solrRequestParsers;
/**
* TEST-ONLY: Creates a configuration instance from an instance directory and file name
* @param instanceDir the directory used to create the resource loader
* @param name the configuration name used by the loader if the stream is null
*/
public SolrConfig(Path instanceDir, String name)
throws IOException {
this(new SolrResourceLoader(instanceDir), name, true, null);
}
public static SolrConfig readFromResourceLoader(SolrResourceLoader loader, String name, boolean isConfigsetTrusted, Properties substitutableProperties) {
try {
return new SolrConfig(loader, name, isConfigsetTrusted, substitutableProperties);
} catch (Exception e) {
String resource;
if (loader instanceof ZkSolrResourceLoader) {
resource = name;
} else {
resource = Paths.get(loader.getConfigDir()).resolve(name).toString();
}
throw new SolrException(ErrorCode.SERVER_ERROR, "Error loading solr config from " + resource, e);
}
}
private class ResourceProvider implements Function {
int zkVersion;
int hash = -1;
InputStream in;
String fileName;
ResourceProvider(InputStream in) {
this.in = in;
if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) {
ZkSolrResourceLoader.ZkByteArrayInputStream zkin = (ZkSolrResourceLoader.ZkByteArrayInputStream) in;
zkVersion = zkin.getStat().getVersion();
hash = Objects.hash(zkVersion, overlay.getZnodeVersion());
this.fileName = zkin.fileName;
}
}
@Override
public InputStream apply(String s) {
return in;
}
}
/**
* Creates a configuration instance from a resource loader, a configuration name and a stream.
* If the stream is null, the resource loader will open the configuration stream.
* If the stream is not null, no attempt to load the resource will occur (the name is not used).
* @param loader the resource loader
* @param name the configuration name
* @param isConfigsetTrusted false if configset was uploaded using unsecured configset upload API, true otherwise
* @param substitutableProperties optional properties to substitute into the XML
*/
@SuppressWarnings("unchecked")
private SolrConfig(SolrResourceLoader loader, String name, boolean isConfigsetTrusted, Properties substitutableProperties)
throws IOException {
this.resourceLoader = loader;
this.resourceName = name;
this.substituteProperties = substitutableProperties;
getOverlay();//just in case it is not initialized
// insist we have non-null substituteProperties; it might get overlaid
Map configCache =null;
if (loader.getCoreContainer() != null && loader.getCoreContainer().getObjectCache() != null) {
configCache = (Map) loader.getCoreContainer().getObjectCache()
.computeIfAbsent(ConfigSetService.ConfigResource.class.getName(), s -> new ConcurrentHashMap<>());
ResourceProvider rp = new ResourceProvider(loader.openResource(name));
IndexSchemaFactory.VersionedConfig cfg = rp.fileName == null ? null : configCache.get(rp.fileName);
if (cfg != null) {
if (rp.hash != -1) {
if (rp.hash == cfg.version) {
log.debug("LOADED_FROM_CACHE");
root = cfg.data;
} else {
readXml(loader, name, configCache, rp);
}
}
}
}
if(root == null) {
readXml(loader, name, configCache,new ResourceProvider(loader.openResource(name)) );
}
ConfigNode.SUBSTITUTES.set(key -> {
if(substitutableProperties== null || !substitutableProperties.containsKey(key)) {
Object o = overlay.getUserProps().get(key);
return o ==null? null: o.toString();
} else {
return substitutableProperties.getProperty(key);
}
});
try {
getRequestParams();
initLibs(loader, isConfigsetTrusted);
String val = root.child(IndexSchema.LUCENE_MATCH_VERSION_PARAM,
() -> new RuntimeException("Missing: " + IndexSchema.LUCENE_MATCH_VERSION_PARAM)).txt();
luceneMatchVersion = SolrConfig.parseLuceneVersionString(val);
log.info("Using Lucene MatchVersion: {}", luceneMatchVersion);
String indexConfigPrefix;
// Old indexDefaults and mainIndex sections are deprecated and fails fast for luceneMatchVersion=>LUCENE_4_0_0.
// For older solrconfig.xml's we allow the old sections, but never mixed with the new
boolean hasDeprecatedIndexConfig = get("indexDefaults").exists() || get("mainIndex").exists();
if (hasDeprecatedIndexConfig) {
throw new SolrException(ErrorCode.FORBIDDEN, " and configuration sections are discontinued. Use instead.");
} else {
indexConfigPrefix = "indexConfig";
}
assertWarnOrFail("The config has been discontinued and NRT mode is always used by Solr." +
" This config will be removed in future versions.", get(indexConfigPrefix).get("nrtMode").isNull(),
true
);
assertWarnOrFail("Solr no longer supports forceful unlocking via the 'unlockOnStartup' option. " +
"This is no longer necessary for the default lockType except in situations where " +
"it would be dangerous and should not be done. For other lockTypes and/or " +
"directoryFactory options it may also be dangerous and users must resolve " +
"problematic locks manually.",
!get(indexConfigPrefix).get("unlockOnStartup").exists(),
true // 'fail' in trunk
);
// Parse indexConfig section, using mainIndex as backup in case old config is used
indexConfig = new SolrIndexConfig(get("indexConfig"), null);
booleanQueryMaxClauseCount = get("query").get("maxBooleanClauses").intVal(BooleanQuery.getMaxClauseCount());
if (BooleanQuery.getMaxClauseCount() < booleanQueryMaxClauseCount) {
log.warn("solrconfig.xml: of {} is greater than global limit of {} and will have no effect {}"
, booleanQueryMaxClauseCount, BooleanQuery.getMaxClauseCount()
, "set 'maxBooleanClauses' in solr.xml to increase global limit");
}
// Warn about deprecated / discontinued parameters
// boolToFilterOptimizer has had no effect since 3.1
if (get("query").get("boolTofilterOptimizer").exists())
log.warn("solrconfig.xml: is currently not implemented and has no effect.");
if (get("query").get("HashDocSet").exists())
log.warn("solrconfig.xml: is deprecated and no longer used.");
// TODO: Old code - in case somebody wants to re-enable. Also see SolrIndexSearcher#search()
// filtOptEnabled = getBool("query/boolTofilterOptimizer/@enabled", false);
// filtOptCacheSize = getInt("query/boolTofilterOptimizer/@cacheSize",32);
// filtOptThreshold = getFloat("query/boolTofilterOptimizer/@threshold",.05f);
useFilterForSortedQuery = get("query").get("useFilterForSortedQuery").boolVal(false);
queryResultWindowSize = Math.max(1, get("query").get("queryResultWindowSize").intVal(1));
queryResultMaxDocsCached = get("query").get("queryResultMaxDocsCached").intVal(Integer.MAX_VALUE);
enableLazyFieldLoading = get("query").get("enableLazyFieldLoading").boolVal(false);
filterCacheConfig = CacheConfig.getConfig(this, get("query").get("filterCache"), "query/filterCache");
queryResultCacheConfig = CacheConfig.getConfig(this, get("query").get("queryResultCache"), "query/queryResultCache");
documentCacheConfig = CacheConfig.getConfig(this, get("query").get("documentCache"), "query/documentCache");
CacheConfig conf = CacheConfig.getConfig(this, get("query").get("fieldValueCache"), "query/fieldValueCache");
if (conf == null) {
Map args = new HashMap<>();
args.put(NAME, "fieldValueCache");
args.put("size", "10000");
args.put("initialSize", "10");
args.put("showItems", "-1");
conf = new CacheConfig(FastLRUCache.class, args, null);
}
fieldValueCacheConfig = conf;
useColdSearcher = get("query").get("useColdSearcher").boolVal(false);
dataDir = get("dataDir").txt();
if (dataDir != null && dataDir.length() == 0) dataDir = null;
org.apache.solr.search.SolrIndexSearcher.initRegenerators(this);
if (get("jmx").exists()) {
log.warn("solrconfig.xml: is no longer supported, use solr.xml:/metrics/reporter section instead");
}
httpCachingConfig = new HttpCachingConfig(this);
maxWarmingSearchers = get("query").get("maxWarmingSearchers").intVal(1);
slowQueryThresholdMillis = get("query").get("slowQueryThresholdMillis").intVal(-1);
for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin);
Map userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache",
get("query").getAll("cache"));
List caches = getPluginInfos(SolrCache.class.getName());
if (!caches.isEmpty()) {
for (PluginInfo c : caches) {
userCacheConfigs.put(c.name, CacheConfig.getConfig(this, "cache", c.attributes, null));
}
}
this.userCacheConfigs = Collections.unmodifiableMap(userCacheConfigs);
updateHandlerInfo = loadUpdatehandlerInfo();
multipartUploadLimitKB = get("requestDispatcher").get("requestParsers").intAttr("multipartUploadLimitInKB", Integer.MAX_VALUE);
if (multipartUploadLimitKB == -1) multipartUploadLimitKB = Integer.MAX_VALUE;
formUploadLimitKB = get("requestDispatcher").get("requestParsers").intAttr("formdataUploadLimitInKB", Integer.MAX_VALUE);
if (formUploadLimitKB == -1) formUploadLimitKB = Integer.MAX_VALUE;
enableRemoteStreams = get("requestDispatcher").get("requestParsers").boolAttr("enableRemoteStreaming", false);
enableStreamBody = get("requestDispatcher").get("requestParsers").boolAttr("enableStreamBody", false);
handleSelect = get("requestDispatcher").boolAttr("handleSelect", !luceneMatchVersion.onOrAfter(Version.LUCENE_7_0_0));
addHttpRequestToContext = get("requestDispatcher").get("requestParsers").boolAttr("addHttpRequestToContext", false);
List argsInfos = getPluginInfos(InitParams.class.getName());
if (argsInfos != null) {
Map argsMap = new HashMap<>();
for (PluginInfo p : argsInfos) {
InitParams args = new InitParams(p);
argsMap.put(args.name == null ? String.valueOf(args.hashCode()) : args.name, args);
}
this.initParams = Collections.unmodifiableMap(argsMap);
}
solrRequestParsers = new SolrRequestParsers(this);
log.debug("Loaded SolrConfig: {}", name);
} finally {
ConfigNode.SUBSTITUTES.remove();
}
}
private void readXml(SolrResourceLoader loader, String name, Map configCache, ResourceProvider rp) throws IOException {
XmlConfigFile xml = new XmlConfigFile(loader,rp, name, null, "/config/", null);
root = new DataConfigNode(new DOMConfigNode(xml.getDocument().getDocumentElement()));
this.znodeVersion = rp.zkVersion;
if(configCache != null && rp.fileName !=null) {
configCache.put(rp.fileName, new IndexSchemaFactory.VersionedConfig(rp.hash, root));
}
}
private static final AtomicBoolean versionWarningAlreadyLogged = new AtomicBoolean(false);
public static final Version parseLuceneVersionString(final String matchVersion) {
final Version version;
try {
version = Version.parseLeniently(matchVersion);
} catch (ParseException pe) {
throw new SolrException(ErrorCode.SERVER_ERROR,
"Invalid luceneMatchVersion. Should be of the form 'V.V.V' (e.g. 4.8.0)", pe);
}
if (version == Version.LATEST && !versionWarningAlreadyLogged.getAndSet(true)) {
log.warn("You should not use LATEST as luceneMatchVersion property: "
+ "if you use this setting, and then Solr upgrades to a newer release of Lucene, "
+ "sizable changes may happen. If precise back compatibility is important "
+ "then you should instead explicitly specify an actual Lucene version.");
}
return version;
}
public static final List plugins = ImmutableList.builder()
.add(new SolrPluginInfo(SolrRequestHandler.class, SolrRequestHandler.TYPE, REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK, LAZY))
.add(new SolrPluginInfo(QParserPlugin.class, "queryParser", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(Expressible.class, "expressible", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(QueryResponseWriter.class, "queryResponseWriter", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK, LAZY))
.add(new SolrPluginInfo(ValueSourceParser.class, "valueSourceParser", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(TransformerFactory.class, "transformer", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(SearchComponent.class, "searchComponent", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(UpdateRequestProcessorFactory.class, "updateProcessor", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(SolrCache.class, "cache", REQUIRE_NAME, MULTI_OK))
// TODO: WTF is up with queryConverter???
// it apparently *only* works as a singleton? - SOLR-4304
// and even then -- only if there is a single SpellCheckComponent
// because of queryConverter.setIndexAnalyzer
.add(new SolrPluginInfo(QueryConverter.class, "queryConverter", REQUIRE_NAME, REQUIRE_CLASS))
.add(new SolrPluginInfo(PluginBag.RuntimeLib.class, "runtimeLib", REQUIRE_NAME, MULTI_OK))
// this is hackish, since it picks up all SolrEventListeners,
// regardless of when/how/why they are used (or even if they are
// declared outside of the appropriate context) but there's no nice
// way around that in the PluginInfo framework
.add(new SolrPluginInfo(InitParams.class, InitParams.TYPE, MULTI_OK, REQUIRE_NAME_IN_OVERLAY))
.add(new SolrPluginInfo(it -> {
List result = new ArrayList<>();
result.addAll(it.get("query").getAll("listener"));
result.addAll( it.get("updateHandler").getAll("listener"));
return result;
}, SolrEventListener.class, "//listener", REQUIRE_CLASS, MULTI_OK, REQUIRE_NAME_IN_OVERLAY))
.add(new SolrPluginInfo(DirectoryFactory.class, "directoryFactory", REQUIRE_CLASS))
.add(new SolrPluginInfo(RecoveryStrategy.Builder.class, "recoveryStrategy"))
.add(new SolrPluginInfo(it -> it.get("indexConfig").getAll("deletionPolicy"), IndexDeletionPolicy.class, "indexConfig/deletionPolicy", REQUIRE_CLASS))
.add(new SolrPluginInfo(CodecFactory.class, "codecFactory", REQUIRE_CLASS))
.add(new SolrPluginInfo(IndexReaderFactory.class, "indexReaderFactory", REQUIRE_CLASS))
.add(new SolrPluginInfo(UpdateRequestProcessorChain.class, "updateRequestProcessorChain", MULTI_OK))
.add(new SolrPluginInfo(it -> it.get("updateHandler").getAll("updateLog"), UpdateLog.class, "updateHandler/updateLog"))
.add(new SolrPluginInfo(IndexSchemaFactory.class, "schemaFactory", REQUIRE_CLASS))
.add(new SolrPluginInfo(RestManager.class, "restManager"))
.add(new SolrPluginInfo(StatsCache.class, "statsCache", REQUIRE_CLASS))
.add(new SolrPluginInfo(CircuitBreakerManager.class, "circuitBreaker"))
.build();
public static final Map classVsSolrPluginInfo;
static {
Map map = new HashMap<>();
for (SolrPluginInfo plugin : plugins) map.put(plugin.clazz.getName(), plugin);
classVsSolrPluginInfo = Collections.unmodifiableMap(map);
}
public static class SolrPluginInfo {
@SuppressWarnings({"rawtypes"})
public final Class clazz;
public final String tag;
public final Set options;
final Function> configReader;
private SolrPluginInfo(Class clz, String tag, PluginOpts... opts) {
this(solrConfig -> solrConfig.root.getAll(null, tag), clz, tag, opts);
}
@SuppressWarnings({"unchecked", "rawtypes"})
private SolrPluginInfo(Function> configReader, Class clz, String tag, PluginOpts... opts) {
this.configReader = configReader;
this.clazz = clz;
this.tag = tag;
this.options = opts == null ? Collections.EMPTY_SET : EnumSet.of(NOOP, opts);
}
public String getCleanTag() {
return tag.replaceAll("/", "");
}
public String getTagCleanLower() {
return getCleanTag().toLowerCase(Locale.ROOT);
}
}
@SuppressWarnings({"unchecked", "rawtypes"})
public static ConfigOverlay getConfigOverlay(SolrResourceLoader loader) {
InputStream in = null;
InputStreamReader isr = null;
try {
try {
in = loader.openResource(ConfigOverlay.RESOURCE_NAME);
} catch (IOException e) {
// TODO: we should be explicitly looking for file not found exceptions
// and logging if it's not the expected IOException
// hopefully no problem, assume no overlay.json file
return new ConfigOverlay(Collections.EMPTY_MAP, -1);
}
int version = 0; // will be always 0 for file based resourceLoader
if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) {
version = ((ZkSolrResourceLoader.ZkByteArrayInputStream) in).getStat().getVersion();
log.debug("Config overlay loaded. version : {} ", version);
}
Map m = (Map) fromJSON(in);
return new ConfigOverlay(m, version);
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Error reading config overlay", e);
} finally {
IOUtils.closeQuietly(isr);
IOUtils.closeQuietly(in);
}
}
private Map initParams = Collections.emptyMap();
public Map getInitParams() {
return initParams;
}
protected UpdateHandlerInfo loadUpdatehandlerInfo() {
return new UpdateHandlerInfo( get("updateHandler").attr("class"),
get("updateHandler").get("autoCommit").get("maxDocs").intVal( -1),
get("updateHandler").get("autoCommit").get("maxTime").intVal( -1),
convertHeapOptionStyleConfigStringToBytes(get("updateHandler").get("autoCommit").get("maxSize").txt()),
get("updateHandler").get("indexWriter").get("closeWaitsForMerges").boolVal(true),
get("updateHandler").get("autoCommit").get("openSearcher").boolVal(true),
get("updateHandler").get("autoSoftCommit").get("maxDocs").intVal(-1),
get("updateHandler").get("autoSoftCommit").get("maxTime").intVal(-1),
get("updateHandler").get("commitWithin").get("softCommit").boolVal(true));
}
/**
* Converts a Java heap option-like config string to bytes. Valid suffixes are: 'k', 'm', 'g'
* (case insensitive). If there is no suffix, the default unit is bytes.
* For example, 50k = 50KB, 20m = 20MB, 4g = 4GB, 300 = 300 bytes
* @param configStr the config setting to parse
* @return the size, in bytes. -1 if the given config string is empty
*/
protected static long convertHeapOptionStyleConfigStringToBytes(String configStr) {
if (configStr== null || configStr.isEmpty()) {
return -1;
}
long multiplier = 1;
String numericValueStr = configStr;
char suffix = Character.toLowerCase(configStr.charAt(configStr.length() - 1));
if (Character.isLetter(suffix)) {
if (suffix == 'k') {
multiplier = FileUtils.ONE_KB;
}
else if (suffix == 'm') {
multiplier = FileUtils.ONE_MB;
}
else if (suffix == 'g') {
multiplier = FileUtils.ONE_GB;
} else {
throw new RuntimeException("Invalid suffix. Valid suffixes are 'k' (KB), 'm' (MB), 'g' (G). "
+ "No suffix means the amount is in bytes. ");
}
numericValueStr = configStr.substring(0, configStr.length() - 1);
}
try {
return Long.parseLong(numericValueStr) * multiplier;
} catch (NumberFormatException e) {
throw new RuntimeException("Invalid format. The config setting should be a long with an "
+ "optional letter suffix. Valid suffixes are 'k' (KB), 'm' (MB), 'g' (G). "
+ "No suffix means the amount is in bytes.");
}
}
private void loadPluginInfo(SolrPluginInfo pluginInfo) {
boolean requireName = pluginInfo.options.contains(REQUIRE_NAME);
boolean requireClass = pluginInfo.options.contains(REQUIRE_CLASS);
List result = readPluginInfos(pluginInfo, requireName, requireClass);
if (1 < result.size() && !pluginInfo.options.contains(MULTI_OK)) {
throw new SolrException
(SolrException.ErrorCode.SERVER_ERROR,
"Found " + result.size() + " configuration sections when at most "
+ "1 is allowed matching expression: " + pluginInfo.getCleanTag());
}
if (!result.isEmpty()) pluginStore.put(pluginInfo.clazz.getName(), result);
}
public List readPluginInfos(SolrPluginInfo info, boolean requireName, boolean requireClass) {
ArrayList result = new ArrayList<>();
for (ConfigNode node : info.configReader.apply(this)) {
PluginInfo pluginInfo = new PluginInfo(node, "[solrconfig.xml] " + info.tag, requireName, requireClass);
if (pluginInfo.isEnabled()) result.add(pluginInfo);
}
return result;
}
public SolrRequestParsers getRequestParsers() {
return solrRequestParsers;
}
/* The set of materialized parameters: */
public final int booleanQueryMaxClauseCount;
// SolrIndexSearcher - nutch optimizer -- Disabled since 3.1
// public final boolean filtOptEnabled;
// public final int filtOptCacheSize;
// public final float filtOptThreshold;
// SolrIndexSearcher - caches configurations
public final CacheConfig filterCacheConfig;
public final CacheConfig queryResultCacheConfig;
public final CacheConfig documentCacheConfig;
public final CacheConfig fieldValueCacheConfig;
public final Map userCacheConfigs;
// SolrIndexSearcher - more...
public final boolean useFilterForSortedQuery;
public final int queryResultWindowSize;
public final int queryResultMaxDocsCached;
public final boolean enableLazyFieldLoading;
// IndexConfig settings
public final SolrIndexConfig indexConfig;
protected UpdateHandlerInfo updateHandlerInfo;
private Map> pluginStore = new LinkedHashMap<>();
public final int maxWarmingSearchers;
public final boolean useColdSearcher;
public final Version luceneMatchVersion;
protected String dataDir;
public final int slowQueryThresholdMillis; // threshold above which a query is considered slow
private final HttpCachingConfig httpCachingConfig;
public HttpCachingConfig getHttpCachingConfig() {
return httpCachingConfig;
}
public static class HttpCachingConfig implements MapSerializable {
/**
* For extracting Expires "ttl" from config
*/
private final static Pattern MAX_AGE
= Pattern.compile("\\bmax-age=(\\d+)");
@Override
public Map toMap(Map map) {
return makeMap("never304", never304,
"etagSeed", etagSeed,
"lastModFrom", lastModFrom.name().toLowerCase(Locale.ROOT),
"cacheControl", cacheControlHeader);
}
public enum LastModFrom {
OPENTIME, DIRLASTMOD, BOGUS;
/**
* Input must not be null
*/
public static LastModFrom parse(final String s) {
try {
return valueOf(s.toUpperCase(Locale.ROOT));
} catch (Exception e) {
log.warn("Unrecognized value for lastModFrom: {}", s, e);
return BOGUS;
}
}
}
private final boolean never304;
private final String etagSeed;
private final String cacheControlHeader;
private final Long maxAge;
private final LastModFrom lastModFrom;
private ConfigNode configNode;
private HttpCachingConfig(SolrConfig conf) {
configNode = conf.root;
//"requestDispatcher/httpCaching/";
never304 = get("requestDispatcher").get("httpCaching").boolAttr("never304", false);
etagSeed = get("requestDispatcher").get("httpCaching").attr("etagSeed", "Solr");
lastModFrom = LastModFrom.parse(get("requestDispatcher").get("httpCaching").attr("lastModFrom","openTime"));
cacheControlHeader = get("requestDispatcher").get("httpCaching").get("cacheControl").txt();
Long tmp = null; // maxAge
if (null != cacheControlHeader) {
try {
final Matcher ttlMatcher = MAX_AGE.matcher(cacheControlHeader);
final String ttlStr = ttlMatcher.find() ? ttlMatcher.group(1) : null;
tmp = (null != ttlStr && !"".equals(ttlStr))
? Long.valueOf(ttlStr)
: null;
} catch (Exception e) {
log.warn("Ignoring exception while attempting to extract max-age from cacheControl config: {}"
, cacheControlHeader, e);
}
}
maxAge = tmp;
}
private ConfigNode get(String name){
return configNode.get(name);
}
public boolean isNever304() {
return never304;
}
public String getEtagSeed() {
return etagSeed;
}
/**
* null if no Cache-Control header
*/
public String getCacheControlHeader() {
return cacheControlHeader;
}
/**
* null if no max age limitation
*/
public Long getMaxAge() {
return maxAge;
}
public LastModFrom getLastModFrom() {
return lastModFrom;
}
}
public static class UpdateHandlerInfo implements MapSerializable {
public final String className;
public final int autoCommmitMaxDocs, autoCommmitMaxTime,
autoSoftCommmitMaxDocs, autoSoftCommmitMaxTime;
public final long autoCommitMaxSizeBytes;
public final boolean indexWriterCloseWaitsForMerges;
public final boolean openSearcher; // is opening a new searcher part of hard autocommit?
public final boolean commitWithinSoftCommit;
/**
* @param autoCommmitMaxDocs set -1 as default
* @param autoCommmitMaxTime set -1 as default
* @param autoCommitMaxSize set -1 as default
*/
public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, long autoCommitMaxSize, boolean indexWriterCloseWaitsForMerges, boolean openSearcher,
int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, boolean commitWithinSoftCommit) {
this.className = className;
this.autoCommmitMaxDocs = autoCommmitMaxDocs;
this.autoCommmitMaxTime = autoCommmitMaxTime;
this.autoCommitMaxSizeBytes = autoCommitMaxSize;
this.indexWriterCloseWaitsForMerges = indexWriterCloseWaitsForMerges;
this.openSearcher = openSearcher;
this.autoSoftCommmitMaxDocs = autoSoftCommmitMaxDocs;
this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime;
this.commitWithinSoftCommit = commitWithinSoftCommit;
}
@Override
@SuppressWarnings({"unchecked", "rawtypes"})
public Map toMap(Map map) {
LinkedHashMap result = new LinkedHashMap();
result.put("indexWriter", makeMap("closeWaitsForMerges", indexWriterCloseWaitsForMerges));
result.put("commitWithin", makeMap("softCommit", commitWithinSoftCommit));
result.put("autoCommit", makeMap(
"maxDocs", autoCommmitMaxDocs,
"maxTime", autoCommmitMaxTime,
"openSearcher", openSearcher
));
result.put("autoSoftCommit",
makeMap("maxDocs", autoSoftCommmitMaxDocs,
"maxTime", autoSoftCommmitMaxTime));
return result;
}
}
// public Map> getUpdateProcessorChainInfo() { return updateProcessorChainInfo; }
public UpdateHandlerInfo getUpdateHandlerInfo() {
return updateHandlerInfo;
}
public String getDataDir() {
return dataDir;
}
/**
* SolrConfig keeps a repository of plugins by the type. The known interfaces are the types.
*
* @param type The key is FQN of the plugin class there are a few known types : SolrFormatter, SolrFragmenter
* SolrRequestHandler,QParserPlugin, QueryResponseWriter,ValueSourceParser,
* SearchComponent, QueryConverter, SolrEventListener, DirectoryFactory,
* IndexDeletionPolicy, IndexReaderFactory, {@link TransformerFactory}
*/
@SuppressWarnings({"unchecked", "rawtypes"})
public List getPluginInfos(String type) {
List result = pluginStore.get(type);
SolrPluginInfo info = classVsSolrPluginInfo.get(type);
if (info != null &&
(info.options.contains(REQUIRE_NAME) || info.options.contains(REQUIRE_NAME_IN_OVERLAY))) {
Map infos = overlay.getNamedPlugins(info.getCleanTag());
if (!infos.isEmpty()) {
LinkedHashMap map = new LinkedHashMap<>();
if (result != null) for (PluginInfo pluginInfo : result) {
//just create a UUID for the time being so that map key is not null
String name = pluginInfo.name == null ?
UUID.randomUUID().toString().toLowerCase(Locale.ROOT) :
pluginInfo.name;
map.put(name, pluginInfo);
}
for (Map.Entry e : infos.entrySet()) {
map.put(e.getKey(), new PluginInfo(info.getCleanTag(), e.getValue()));
}
result = new ArrayList<>(map.values());
}
}
return result == null ? Collections.emptyList() : result;
}
public PluginInfo getPluginInfo(String type) {
List result = pluginStore.get(type);
if (result == null || result.isEmpty()) {
return null;
}
if (1 == result.size()) {
return result.get(0);
}
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Multiple plugins configured for type: " + type);
}
private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) {
// TODO Want to remove SolrResourceLoader.getInstancePath; it can be on a Standalone subclass.
// For Zk subclass, it's needed for the time being as well. We could remove that one if we remove two things
// in SolrCloud: (1) instancePath/lib and (2) solrconfig lib directives with relative paths. Can wait till 9.0.
Path instancePath = loader.getInstancePath();
List urls = new ArrayList<>();
Path libPath = instancePath.resolve("lib");
if (Files.exists(libPath)) {
try {
urls.addAll(SolrResourceLoader.getURLs(libPath));
} catch (IOException e) {
log.warn("Couldn't add files from {} to classpath: {}", libPath, e);
}
}
List nodes = root.getAll("lib");
if (nodes != null && nodes.size() > 0) {
if (!isConfigsetTrusted) {
throw new SolrException(ErrorCode.UNAUTHORIZED,
"The configset for this collection was uploaded without any authentication in place,"
+ " and use of is not available for collections with untrusted configsets. To use this component, re-upload the configset"
+ " after enabling authentication and authorization.");
}
for (int i = 0; i < nodes.size(); i++) {
ConfigNode node = nodes.get(i);
String baseDir = node.attr("dir");
String path = node.attr(PATH);
if (null != baseDir) {
// :TODO: add support for a simpler 'glob' mutually exclusive of regex
Path dir = instancePath.resolve(baseDir);
String regex = node.attr("regex");
try {
if (regex == null)
urls.addAll(SolrResourceLoader.getURLs(dir));
else
urls.addAll(SolrResourceLoader.getFilteredURLs(dir, regex));
} catch (IOException e) {
log.warn("Couldn't add files from {} filtered by {} to classpath: {}", dir, regex, e);
}
} else if (null != path) {
final Path dir = instancePath.resolve(path);
try {
urls.add(dir.toUri().toURL());
} catch (MalformedURLException e) {
log.warn("Couldn't add file {} to classpath: {}", dir, e);
}
} else {
throw new RuntimeException("lib: missing mandatory attributes: 'dir' or 'path'");
}
}
}
if (!urls.isEmpty()) {
loader.addToClassLoader(urls);
loader.reloadLuceneSPI();
}
}
public int getMultipartUploadLimitKB() {
return multipartUploadLimitKB;
}
public int getFormUploadLimitKB() {
return formUploadLimitKB;
}
public boolean isHandleSelect() {
return handleSelect;
}
public boolean isAddHttpRequestToContext() {
return addHttpRequestToContext;
}
public boolean isEnableRemoteStreams() {
return enableRemoteStreams;
}
public boolean isEnableStreamBody() {
return enableStreamBody;
}
@Override
@SuppressWarnings({"unchecked", "rawtypes"})
public Map toMap(Map result) {
if (znodeVersion > -1) result.put(ZNODEVER, znodeVersion);
if(luceneMatchVersion != null) result.put(IndexSchema.LUCENE_MATCH_VERSION_PARAM, luceneMatchVersion.toString());
result.put("updateHandler", getUpdateHandlerInfo());
Map m = new LinkedHashMap();
result.put("query", m);
m.put("useFilterForSortedQuery", useFilterForSortedQuery);
m.put("queryResultWindowSize", queryResultWindowSize);
m.put("queryResultMaxDocsCached", queryResultMaxDocsCached);
m.put("enableLazyFieldLoading", enableLazyFieldLoading);
m.put("maxBooleanClauses", booleanQueryMaxClauseCount);
for (SolrPluginInfo plugin : plugins) {
List infos = getPluginInfos(plugin.clazz.getName());
if (infos == null || infos.isEmpty()) continue;
String tag = plugin.getCleanTag();
tag = tag.replace("/", "");
if (plugin.options.contains(PluginOpts.REQUIRE_NAME)) {
LinkedHashMap items = new LinkedHashMap();
for (PluginInfo info : infos) {
//TODO remove after fixing https://issues.apache.org/jira/browse/SOLR-13706
if (info.type.equals("searchComponent") && info.name.equals("highlight")) continue;
items.put(info.name, info);
}
for (Map.Entry e : overlay.getNamedPlugins(plugin.tag).entrySet()) items.put(e.getKey(), e.getValue());
result.put(tag, items);
} else {
if (plugin.options.contains(MULTI_OK)) {
ArrayList l = new ArrayList<>();
for (PluginInfo info : infos) l.add(info);
result.put(tag, l);
} else {
result.put(tag, infos.get(0));
}
}
}
addCacheConfig(m, filterCacheConfig, queryResultCacheConfig, documentCacheConfig, fieldValueCacheConfig);
m = new LinkedHashMap();
result.put("requestDispatcher", m);
m.put("handleSelect", handleSelect);
if (httpCachingConfig != null) m.put("httpCaching", httpCachingConfig);
m.put("requestParsers", makeMap("multipartUploadLimitKB", multipartUploadLimitKB,
"formUploadLimitKB", formUploadLimitKB,
"addHttpRequestToContext", addHttpRequestToContext));
if (indexConfig != null) result.put("indexConfig", indexConfig);
//TODO there is more to add
return result;
}
@SuppressWarnings({"unchecked", "rawtypes"})
private void addCacheConfig(Map queryMap, CacheConfig... cache) {
if (cache == null) return;
for (CacheConfig config : cache) if (config != null) queryMap.put(config.getNodeName(), config);
}
public Properties getSubstituteProperties() {
Map p = getOverlay().getUserProps();
if (p == null || p.isEmpty()) return substituteProperties;
Properties result = new Properties(substituteProperties);
result.putAll(p);
return result;
}
private ConfigOverlay overlay;
public ConfigOverlay getOverlay() {
if (overlay == null) {
overlay = getConfigOverlay(resourceLoader);
}
return overlay;
}
public RequestParams getRequestParams() {
if (requestParams == null) {
return refreshRequestParams();
}
return requestParams;
}
/**
* The version of package that should be loaded for a given package name
* This information is stored in the params.json in the same configset
* If params.json is absent or there is no corresponding version specified for a given package,
* this returns a null and the latest is used by the caller
*/
public String maxPackageVersion(String pkg) {
RequestParams.ParamSet p = getRequestParams().getParams(PackageListeners.PACKAGE_VERSIONS);
if (p == null) {
return null;
}
Object o = p.get().get(pkg);
if (o == null || PackageLoader.LATEST.equals(o)) return null;
return o.toString();
}
public RequestParams refreshRequestParams() {
requestParams = RequestParams.getFreshRequestParams(resourceLoader, requestParams);
if (log.isDebugEnabled()) {
log.debug("current version of requestparams : {}", requestParams.getZnodeVersion());
}
return requestParams;
}
public SolrResourceLoader getResourceLoader() {
return resourceLoader;
}
public int getZnodeVersion() {
return znodeVersion;
}
public String getName() {
return resourceName;
}
public String getResourceName() {
return resourceName;
}
/**fetches a child node by name. An "empty node" is returned if the child does not exist
* This never returns a null
*
*
*/
public ConfigNode get(String name) {
if (!overlay.hasKey(name)) {
//there is no overlay
return root.get(name);
}
return new OverlaidConfigNode(overlay, name, null,root.get(name));
}
public ConfigNode get(String name, Predicate test) {
return root.get(name, test);
}
}
