Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.apache.solr.handler.designer.SchemaDesignerConfigSetHelper Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.designer;
import static org.apache.solr.common.params.CommonParams.VERSION_FIELD;
import static org.apache.solr.common.util.Utils.fromJSONString;
import static org.apache.solr.common.util.Utils.toJavabin;
import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME;
import static org.apache.solr.handler.designer.SchemaDesignerAPI.getConfigSetZkPath;
import static org.apache.solr.handler.designer.SchemaDesignerAPI.getMutableId;
import static org.apache.solr.schema.IndexSchema.NEST_PATH_FIELD_NAME;
import static org.apache.solr.schema.IndexSchema.ROOT_FIELD_NAME;
import static org.apache.solr.schema.ManagedIndexSchemaFactory.DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.file.PathUtils;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.util.EntityUtils;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudLegacySolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
import org.apache.solr.client.solrj.request.schema.SchemaRequest;
import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.apache.solr.cloud.ZkConfigSetService;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.schema.CopyField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.ManagedIndexSchema;
import org.apache.solr.schema.ManagedIndexSchemaFactory;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TextField;
import org.apache.solr.util.RTimer;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
class SchemaDesignerConfigSetHelper implements SchemaDesignerConstants {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final Set removeFieldProps =
new HashSet<>(Arrays.asList("href", "id", "copyDest"));
private static final List includeLangIds = Arrays.asList("ws", "general", "rev", "sort");
private static final String ZNODE_PATH_DELIM = "/";
private static final String MULTIVALUED = "multiValued";
private static final int TEXT_PREFIX_LEN = "text_".length();
private final CoreContainer cc;
private final SchemaSuggester schemaSuggester;
SchemaDesignerConfigSetHelper(CoreContainer cc, SchemaSuggester schemaSuggester) {
this.cc = cc;
this.schemaSuggester = schemaSuggester;
}
@SuppressWarnings("unchecked")
Map analyzeField(String configSet, String fieldName, String fieldText)
throws IOException {
final String mutableId = getMutableId(configSet);
final URI uri;
try {
uri =
collectionApiEndpoint(mutableId, "analysis", "field")
.setParameter(CommonParams.WT, CommonParams.JSON)
.setParameter("analysis.showmatch", "true")
.setParameter("analysis.fieldname", fieldName)
.setParameter("analysis.fieldvalue", "POST")
.build();
} catch (URISyntaxException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
Map analysis = Collections.emptyMap();
HttpPost httpPost = new HttpPost(uri);
httpPost.setHeader("Content-Type", "text/plain");
httpPost.setEntity(new ByteArrayEntity(fieldText.getBytes(StandardCharsets.UTF_8)));
try {
HttpResponse resp = ((CloudLegacySolrClient) cloudClient()).getHttpClient().execute(httpPost);
int statusCode = resp.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new SolrException(
SolrException.ErrorCode.getErrorCode(statusCode),
EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8));
}
Map response =
(Map)
fromJSONString(EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8));
if (response != null) {
analysis = (Map) response.get("analysis");
}
} finally {
httpPost.releaseConnection();
}
return analysis;
}
List listCollectionsForConfig(String configSet) {
final List collections = new ArrayList<>();
Map states =
zkStateReader().getClusterState().getCollectionStates();
for (Map.Entry e : states.entrySet()) {
final String coll = e.getKey();
if (coll.startsWith(DESIGNER_PREFIX)) {
continue; // ignore temp
}
try {
if (configSet.equals(e.getValue().get().getConfigName()) && e.getValue().get() != null) {
collections.add(coll);
}
} catch (Exception exc) {
log.warn("Failed to get config name for {}", coll, exc);
}
}
return collections;
}
@SuppressWarnings("unchecked")
public String addSchemaObject(String configSet, Map addJson)
throws IOException, SolrServerException {
String mutableId = getMutableId(configSet);
SchemaRequest.Update addAction;
String action;
String objectName = null;
if (addJson.containsKey("add-field")) {
action = "add-field";
Map fieldAttrs = (Map) addJson.get(action);
objectName = (String) fieldAttrs.get("name");
addAction = new SchemaRequest.AddField(fieldAttrs);
} else if (addJson.containsKey("add-dynamic-field")) {
action = "add-dynamic-field";
Map fieldAttrs = (Map) addJson.get(action);
objectName = (String) fieldAttrs.get("name");
addAction = new SchemaRequest.AddDynamicField(fieldAttrs);
} else if (addJson.containsKey("add-copy-field")) {
action = "add-copy-field";
Map map = (Map) addJson.get(action);
Object dest = map.get("dest");
List destFields = null;
if (dest instanceof String) {
destFields = Collections.singletonList((String) dest);
} else if (dest instanceof List) {
destFields = (List) dest;
} else if (dest instanceof Collection) {
Collection destColl = (Collection) dest;
destFields = new ArrayList<>(destColl);
}
addAction = new SchemaRequest.AddCopyField((String) map.get("source"), destFields);
} else if (addJson.containsKey("add-field-type")) {
action = "add-field-type";
Map fieldAttrs = (Map) addJson.get(action);
objectName = (String) fieldAttrs.get("name");
FieldTypeDefinition ftDef = new FieldTypeDefinition();
ftDef.setAttributes(fieldAttrs);
addAction = new SchemaRequest.AddFieldType(ftDef);
} else {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Unsupported action in request body! " + addJson);
}
// Using the SchemaAPI vs. working on the schema directly because SchemaField.create methods are
// package protected
log.info("Sending {} request for configSet {}: {}", action, mutableId, addJson);
SchemaResponse.UpdateResponse schemaResponse = addAction.process(cloudClient(), mutableId);
Exception exc = schemaResponse.getException();
if (exc instanceof SolrException) {
throw (SolrException) exc;
} else if (schemaResponse.getStatus() != 0) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, exc);
}
return objectName;
}
void reloadTempCollection(String mutableId, boolean delete)
throws IOException, SolrServerException {
if (delete) {
log.debug("Deleting and re-creating existing collection {} after schema update", mutableId);
CollectionAdminRequest.deleteCollection(mutableId).process(cloudClient());
try {
zkStateReader().waitForState(mutableId, 30, TimeUnit.SECONDS, Objects::isNull);
} catch (InterruptedException | TimeoutException e) {
throw new IOException(
"Failed to see deleted collection " + mutableId + " reflected in cluster state",
SolrZkClient.checkInterrupted(e));
}
createCollection(mutableId, mutableId);
log.debug("Deleted and re-created existing collection: {}", mutableId);
} else {
CollectionAdminRequest.reloadCollection(mutableId).process(cloudClient());
log.debug("Reloaded existing collection: {}", mutableId);
}
}
Map updateSchemaObject(
String configSet, Map updateJson, ManagedIndexSchema schemaBeforeUpdate)
throws IOException, SolrServerException {
String name = (String) updateJson.get("name");
String mutableId = getMutableId(configSet);
boolean needsRebuild = false;
SolrException solrExc = null;
String updateType;
String updateError = null;
if (updateJson.get("type") != null) {
updateType = schemaBeforeUpdate.isDynamicField(name) ? "dynamicField" : "field";
try {
needsRebuild = updateField(configSet, updateJson, schemaBeforeUpdate);
} catch (SolrException exc) {
if (exc.code() != 400) {
throw exc;
}
solrExc = exc;
updateError = solrExc.getMessage() + " Previous settings will be restored.";
}
} else {
updateType = "type";
needsRebuild = updateFieldType(configSet, name, updateJson, schemaBeforeUpdate);
}
// the update may have required a full rebuild of the index, otherwise, it's just a reload /
// re-index sample
reloadTempCollection(mutableId, needsRebuild);
Map results = new HashMap<>();
results.put("rebuild", needsRebuild);
results.put("updateType", updateType);
if (updateError != null) {
results.put("updateError", updateError);
}
if (solrExc != null) {
results.put("solrExc", solrExc);
}
return results;
}
protected boolean updateFieldType(
String configSet,
String typeName,
Map updateJson,
ManagedIndexSchema schemaBeforeUpdate) {
boolean needsRebuild = false;
Map typeAttrs =
updateJson.entrySet().stream()
.filter(e -> !removeFieldProps.contains(e.getKey()))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
FieldType fieldType = schemaBeforeUpdate.getFieldTypeByName(typeName);
// this is a field type
Object multiValued = typeAttrs.get(MULTIVALUED);
if (typeHasMultiValuedChange(multiValued, fieldType)) {
needsRebuild = true;
log.warn(
"Re-building the temp collection for {} after type {} updated to multi-valued {}",
configSet,
typeName,
multiValued);
}
// nice, the json for this field looks like
// "synonymQueryStyle":
// "org.apache.solr.parser.SolrQueryParserBase$SynonymQueryStyle:AS_SAME_TERM"
if (typeAttrs.get("synonymQueryStyle") instanceof String) {
String synonymQueryStyle = (String) typeAttrs.get("synonymQueryStyle");
if (synonymQueryStyle.lastIndexOf(':') != -1) {
typeAttrs.put(
"synonymQueryStyle",
synonymQueryStyle.substring(synonymQueryStyle.lastIndexOf(':') + 1));
}
}
ManagedIndexSchema updatedSchema =
schemaBeforeUpdate.replaceFieldType(
fieldType.getTypeName(), (String) typeAttrs.get("class"), typeAttrs);
updatedSchema.persistManagedSchema(false);
return needsRebuild;
}
boolean updateField(
String configSet, Map updateField, ManagedIndexSchema schemaBeforeUpdate)
throws IOException, SolrServerException {
String mutableId = getMutableId(configSet);
String name = (String) updateField.get("name");
String type = (String) updateField.get("type");
String copyDest = (String) updateField.get("copyDest");
Map fieldAttributes =
updateField.entrySet().stream()
.filter(e -> !removeFieldProps.contains(e.getKey()))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
boolean needsRebuild = false;
SchemaField schemaField = schemaBeforeUpdate.getField(name);
boolean isDynamic = schemaBeforeUpdate.isDynamicField(name);
String currentType = schemaField.getType().getTypeName();
SimpleOrderedMap fromTypeProps;
if (type.equals(currentType)) {
// no type change, so just pull the current type's props (with defaults) as we'll use these
// to determine which props get explicitly overridden on the field
fromTypeProps =
schemaBeforeUpdate.getFieldTypeByName(currentType).getNamedPropertyValues(true);
} else {
// validate type change
FieldType newType = schemaBeforeUpdate.getFieldTypeByName(type);
if (newType == null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Invalid update request for field "
+ name
+ "! Field type "
+ type
+ " doesn't exist!");
}
validateTypeChange(configSet, schemaField, newType);
// type change looks valid
fromTypeProps = newType.getNamedPropertyValues(true);
}
// the diff holds all the explicit properties not inherited from the type
Map diff = new HashMap<>();
for (Map.Entry e : fieldAttributes.entrySet()) {
String attr = e.getKey();
Object attrValue = e.getValue();
if ("name".equals(attr) || "type".equals(attr)) {
continue; // we don't want these in the diff map
}
if ("required".equals(attr)) {
diff.put(attr, attrValue != null ? attrValue : false);
} else {
Object fromType = fromTypeProps.get(attr);
if (fromType == null || !fromType.equals(attrValue)) {
diff.put(attr, attrValue);
}
}
}
// detect if they're trying to copy multi-valued fields into a single-valued field
Object multiValued = diff.get(MULTIVALUED);
if (multiValued == null) {
// mv not overridden explicitly, but we need the actual value, which will come from the new
// type (if that changed) or the current field
multiValued =
type.equals(currentType)
? schemaField.multiValued()
: schemaBeforeUpdate.getFieldTypeByName(type).isMultiValued();
}
if (!isDynamic && Boolean.FALSE.equals(multiValued)) {
// make sure there are no mv source fields if this is a copy dest
for (String src : schemaBeforeUpdate.getCopySources(name)) {
SchemaField srcField = schemaBeforeUpdate.getField(src);
if (srcField.multiValued()) {
log.warn(
"Cannot change multi-valued field {} to single-valued because it is a copy field destination for multi-valued field {}",
name,
src);
multiValued = Boolean.TRUE;
diff.put(MULTIVALUED, multiValued);
break;
}
}
}
if (Boolean.FALSE.equals(multiValued) && schemaField.multiValued()) {
// changing from multi- to single value ... verify the data agrees ...
validateMultiValuedChange(configSet, schemaField, Boolean.FALSE);
}
// switch from single-valued to multi-valued requires a full rebuild
// See SOLR-12185 ... if we're switching from single to multi-valued, then it's a big operation
if (fieldHasMultiValuedChange(multiValued, schemaField)) {
needsRebuild = true;
log.warn(
"Need to rebuild the temp collection for {} after field {} updated to multi-valued {}",
configSet,
name,
multiValued);
}
if (!needsRebuild) {
// check term vectors too
Boolean storeTermVector =
(Boolean) fieldAttributes.getOrDefault("termVectors", Boolean.FALSE);
if (schemaField.storeTermVector() != storeTermVector) {
// cannot change termVectors w/o a full-rebuild
needsRebuild = true;
}
}
log.info("For {}, replacing field {} with attributes: {}", configSet, name, diff);
final FieldType fieldType = schemaBeforeUpdate.getFieldTypeByName(type);
ManagedIndexSchema updatedSchema =
isDynamic
? schemaBeforeUpdate.replaceDynamicField(name, fieldType, diff)
: schemaBeforeUpdate.replaceField(name, fieldType, diff);
// persist the change before applying the copy-field updates
updatedSchema.persistManagedSchema(false);
if (!isDynamic) {
applyCopyFieldUpdates(mutableId, copyDest, name, updatedSchema);
}
return needsRebuild;
}
protected void validateMultiValuedChange(String configSet, SchemaField field, Boolean multiValued)
throws IOException {
List docs = getStoredSampleDocs(configSet);
if (!docs.isEmpty()) {
boolean isMV = schemaSuggester.isMultiValued(field.getName(), docs);
if (isMV && !multiValued) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Cannot change field "
+ field.getName()
+ " to single-valued as some sample docs have multiple values!");
}
}
}
protected void validateTypeChange(String configSet, SchemaField field, FieldType toType)
throws IOException {
if ("_version_".equals(field.getName())) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Cannot change type of the _version_ field; it must be a plong.");
}
List docs = getStoredSampleDocs(configSet);
if (!docs.isEmpty()) {
schemaSuggester.validateTypeChange(field, toType, docs);
}
}
void deleteStoredSampleDocs(String configSet) {
try {
cloudClient().deleteByQuery(BLOB_STORE_ID, "id:" + configSet + "_sample/*", 10);
} catch (IOException | SolrServerException | SolrException exc) {
final String excStr = exc.toString();
log.warn("Failed to delete sample docs from blob store for {} due to: {}", configSet, excStr);
}
}
@SuppressWarnings("unchecked")
List getStoredSampleDocs(final String configSet) throws IOException {
List docs = null;
final URI uri;
try {
uri =
collectionApiEndpoint(BLOB_STORE_ID, "blob", configSet + "_sample")
.setParameter(CommonParams.WT, "filestream")
.build();
} catch (URISyntaxException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
HttpGet httpGet = new HttpGet(uri);
try {
HttpResponse entity =
((CloudLegacySolrClient) cloudClient()).getHttpClient().execute(httpGet);
int statusCode = entity.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK) {
byte[] bytes = DefaultSampleDocumentsLoader.streamAsBytes(entity.getEntity().getContent());
if (bytes.length > 0) {
docs = (List) Utils.fromJavabin(bytes);
}
} else if (statusCode != HttpStatus.SC_NOT_FOUND) {
byte[] bytes = DefaultSampleDocumentsLoader.streamAsBytes(entity.getEntity().getContent());
throw new IOException(
"Failed to lookup stored docs for "
+ configSet
+ " due to: "
+ new String(bytes, StandardCharsets.UTF_8));
} // else not found is ok
} finally {
httpGet.releaseConnection();
}
return docs != null ? docs : Collections.emptyList();
}
void storeSampleDocs(final String configSet, List docs) throws IOException {
docs.forEach(d -> d.removeField(VERSION_FIELD)); // remove _version_ field before storing ...
postDataToBlobStore(
cloudClient(),
configSet + "_sample",
DefaultSampleDocumentsLoader.streamAsBytes(toJavabin(docs)));
}
protected void postDataToBlobStore(CloudSolrClient cloudClient, String blobName, byte[] bytes)
throws IOException {
final URI uri;
try {
uri = collectionApiEndpoint(BLOB_STORE_ID, "blob", blobName).build();
} catch (URISyntaxException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
HttpPost httpPost = new HttpPost(uri);
try {
httpPost.setHeader("Content-Type", "application/octet-stream");
httpPost.setEntity(new ByteArrayEntity(bytes));
HttpResponse resp = ((CloudLegacySolrClient) cloudClient).getHttpClient().execute(httpPost);
int statusCode = resp.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new SolrException(
SolrException.ErrorCode.getErrorCode(statusCode),
EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8));
}
} finally {
httpPost.releaseConnection();
}
}
private String getBaseUrl(final String collection) {
String baseUrl = null;
try {
Set liveNodes = zkStateReader().getClusterState().getLiveNodes();
DocCollection docColl = zkStateReader().getCollection(collection);
if (docColl != null && !liveNodes.isEmpty()) {
Optional maybeActive =
docColl.getReplicas().stream().filter(r -> r.isActive(liveNodes)).findAny();
if (maybeActive.isPresent()) {
baseUrl = maybeActive.get().getBaseUrl();
}
}
} catch (Exception exc) {
log.warn("Failed to lookup base URL for collection {}", collection, exc);
}
if (baseUrl == null) {
baseUrl = zkStateReader().getBaseUrlForNodeName(cc.getZkController().getNodeName());
}
return baseUrl;
}
private URIBuilder collectionApiEndpoint(
final String collection, final String... morePathSegments) throws URISyntaxException {
URI baseUrl = new URI(getBaseUrl(collection));
// build up a list of path segments including any path in the base URL, collection, and
// additional segments provided by caller
List path = new ArrayList<>(URLEncodedUtils.parsePathSegments(baseUrl.getPath()));
path.add(collection);
if (morePathSegments != null && morePathSegments.length > 0) {
path.addAll(Arrays.asList(morePathSegments));
}
return new URIBuilder(baseUrl).setPathSegments(path);
}
protected String getManagedSchemaZkPath(final String configSet) {
return getConfigSetZkPath(configSet, DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME);
}
ManagedIndexSchema toggleNestedDocsFields(ManagedIndexSchema schema, boolean enabled) {
return enabled
? enableNestedDocsFields(schema, true)
: deleteNestedDocsFieldsIfNeeded(schema, true);
}
ManagedIndexSchema enableNestedDocsFields(ManagedIndexSchema schema, boolean persist) {
boolean madeChanges = false;
if (!schema.hasExplicitField(ROOT_FIELD_NAME)) {
Map fieldAttrs = Map.of("docValues", false, "indexed", true, "stored", false);
schema =
(ManagedIndexSchema)
schema.addField(schema.newField(ROOT_FIELD_NAME, "string", fieldAttrs), false);
madeChanges = true;
}
if (!schema.hasExplicitField(NEST_PATH_FIELD_NAME)) {
schema =
(ManagedIndexSchema)
schema.addField(
schema.newField(
NEST_PATH_FIELD_NAME, NEST_PATH_FIELD_NAME, Collections.emptyMap()),
false);
madeChanges = true;
}
if (madeChanges && persist) {
schema.persistManagedSchema(false);
}
return schema;
}
ManagedIndexSchema deleteNestedDocsFieldsIfNeeded(ManagedIndexSchema schema, boolean persist) {
List toDelete = new ArrayList<>();
if (schema.hasExplicitField(ROOT_FIELD_NAME)) {
toDelete.add(ROOT_FIELD_NAME);
}
if (schema.hasExplicitField(NEST_PATH_FIELD_NAME)) {
toDelete.add(NEST_PATH_FIELD_NAME);
}
if (!toDelete.isEmpty()) {
schema = schema.deleteFields(toDelete);
if (persist) {
schema.persistManagedSchema(false);
}
}
return schema;
}
SolrConfig loadSolrConfig(String configSet) {
ZkSolrResourceLoader zkLoader = zkLoaderForConfigSet(configSet);
boolean trusted = isConfigSetTrusted(configSet);
return SolrConfig.readFromResourceLoader(zkLoader, SOLR_CONFIG_XML, trusted, null);
}
ManagedIndexSchema loadLatestSchema(String configSet) {
return loadLatestSchema(loadSolrConfig(configSet));
}
ManagedIndexSchema loadLatestSchema(SolrConfig solrConfig) {
ManagedIndexSchemaFactory factory = new ManagedIndexSchemaFactory();
factory.init(new NamedList<>());
return factory.create(DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME, solrConfig, null);
}
int getCurrentSchemaVersion(final String configSet) throws IOException {
int currentVersion = -1;
final String path = getManagedSchemaZkPath(configSet);
try {
Stat stat = cc.getZkController().getZkClient().exists(path, null, true);
if (stat != null) {
currentVersion = stat.getVersion();
}
} catch (KeeperException.NoNodeException notExists) {
// safe to ignore
} catch (KeeperException | InterruptedException e) {
throw new IOException(
"Error getting version for schema: " + configSet, SolrZkClient.checkInterrupted(e));
}
return currentVersion;
}
void createCollection(final String collection, final String configSet)
throws IOException, SolrServerException {
createCollection(collection, configSet, 1, 1);
}
void createCollection(
final String collection, final String configSet, int numShards, int numReplicas)
throws IOException, SolrServerException {
RTimer timer = new RTimer();
SolrResponse rsp =
CollectionAdminRequest.createCollection(collection, configSet, numShards, numReplicas)
.process(cloudClient());
try {
CollectionsHandler.waitForActiveCollection(collection, cc, rsp);
} catch (KeeperException | InterruptedException e) {
throw new IOException(
"Failed waiting for new collection " + collection + " to reach the active state",
SolrZkClient.checkInterrupted(e));
}
double tookMs = timer.getTime();
log.debug(
"Took {} ms to create new collection {} with configSet {}", tookMs, collection, configSet);
}
protected CloudSolrClient cloudClient() {
return cc.getSolrClientCache().getCloudSolrClient(cc.getZkController().getZkServerAddress());
}
protected ZkStateReader zkStateReader() {
return cc.getZkController().getZkStateReader();
}
boolean applyCopyFieldUpdates(
String mutableId, String copyDest, String fieldName, ManagedIndexSchema schema)
throws IOException, SolrServerException {
boolean updated = false;
if (copyDest == null || copyDest.trim().isEmpty()) {
// delete all the copy field directives for this field
List copyFieldsList = schema.getCopyFieldsList(fieldName);
if (!copyFieldsList.isEmpty()) {
List dests =
copyFieldsList.stream()
.map(cf -> cf.getDestination().getName())
.collect(Collectors.toList());
SchemaRequest.DeleteCopyField delAction =
new SchemaRequest.DeleteCopyField(fieldName, dests);
SchemaResponse.UpdateResponse schemaResponse = delAction.process(cloudClient(), mutableId);
if (schemaResponse.getStatus() != 0) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, schemaResponse.getException());
}
updated = true;
}
} else {
SchemaField field = schema.getField(fieldName);
Set desired = new HashSet<>();
for (String dest : copyDest.trim().split(",")) {
String toAdd = dest.trim();
if (toAdd.equals(fieldName)) {
continue; // cannot copy to self
}
// make sure the field exists and is multi-valued if this field is
SchemaField toAddField = schema.getFieldOrNull(toAdd);
if (toAddField != null) {
if (!field.multiValued() || toAddField.multiValued()) {
desired.add(toAdd);
} else {
log.warn(
"Skipping copy-field dest {} for {} because it is not multi-valued!",
toAdd,
fieldName);
}
} else {
log.warn(
"Skipping copy-field dest {} for {} because it doesn't exist!", toAdd, fieldName);
}
}
Set existing =
schema.getCopyFieldsList(fieldName).stream()
.map(cf -> cf.getDestination().getName())
.collect(Collectors.toSet());
Set add =
desired.stream()
.filter(e -> !existing.contains(e))
.collect(Collectors.toUnmodifiableSet());
if (!add.isEmpty()) {
SchemaRequest.AddCopyField addAction =
new SchemaRequest.AddCopyField(fieldName, new ArrayList<>(add));
SchemaResponse.UpdateResponse schemaResponse = addAction.process(cloudClient(), mutableId);
if (schemaResponse.getStatus() != 0) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, schemaResponse.getException());
}
updated = true;
} // no additions ...
Set del =
existing.stream()
.filter(e -> !desired.contains(e))
.collect(Collectors.toUnmodifiableSet());
if (!del.isEmpty()) {
SchemaRequest.DeleteCopyField delAction =
new SchemaRequest.DeleteCopyField(fieldName, new ArrayList<>(del));
SchemaResponse.UpdateResponse schemaResponse = delAction.process(cloudClient(), mutableId);
if (schemaResponse.getStatus() != 0) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, schemaResponse.getException());
}
updated = true;
} // no deletions ...
}
return updated;
}
protected boolean fieldHasMultiValuedChange(Object multiValued, SchemaField schemaField) {
return (multiValued == null
|| (Boolean.TRUE.equals(multiValued) && !schemaField.multiValued())
|| (Boolean.FALSE.equals(multiValued) && schemaField.multiValued()));
}
protected boolean typeHasMultiValuedChange(Object multiValued, FieldType fieldType) {
return (multiValued == null
|| (Boolean.TRUE.equals(multiValued) && !fieldType.isMultiValued())
|| (Boolean.FALSE.equals(multiValued) && fieldType.isMultiValued()));
}
ManagedIndexSchema syncLanguageSpecificObjectsAndFiles(
String configSet,
ManagedIndexSchema schema,
List langs,
boolean dynamicEnabled,
String copyFrom)
throws IOException {
if (!langs.isEmpty()) {
// there's a subset of languages applied, so remove all the other langs
schema = removeLanguageSpecificObjectsAndFiles(configSet, schema, langs);
}
// now restore any missing types / files for the languages we need, optionally adding back
// dynamic fields too
schema =
restoreLanguageSpecificObjectsAndFiles(configSet, schema, langs, dynamicEnabled, copyFrom);
schema.persistManagedSchema(false);
return schema;
}
protected ManagedIndexSchema removeLanguageSpecificObjectsAndFiles(
String configSet, ManagedIndexSchema schema, List langs) throws IOException {
final Set languages = new HashSet<>(includeLangIds);
languages.addAll(langs);
final Set usedTypes =
schema.getFields().values().stream()
.map(f -> f.getType().getTypeName())
.collect(Collectors.toSet());
final Set usedLangs =
schema.getFields().values().stream()
.filter(f -> isTextType(f.getType()))
.map(f -> f.getType().getTypeName().substring(TEXT_PREFIX_LEN))
.collect(Collectors.toSet());
// don't remove types / files for langs that are explicitly being used by a field
languages.addAll(usedLangs);
Map types = schema.getFieldTypes();
final Set toRemove =
types.values().stream()
.filter(this::isTextType)
.filter(t -> !languages.contains(t.getTypeName().substring(TEXT_PREFIX_LEN)))
.map(FieldType::getTypeName)
.filter(t -> !usedTypes.contains(t)) // not explicitly used by a field
.collect(Collectors.toSet());
// find dynamic fields that refer to the types we're removing ...
List toRemoveDF =
Arrays.stream(schema.getDynamicFields())
.filter(df -> toRemove.contains(df.getPrototype().getType().getTypeName()))
.map(df -> df.getPrototype().getName())
.collect(Collectors.toList());
schema = schema.deleteDynamicFields(toRemoveDF);
schema = schema.deleteFieldTypes(toRemove);
SolrZkClient zkClient = cc.getZkController().getZkClient();
final String configPathInZk = ZkConfigSetService.CONFIGS_ZKNODE + ZNODE_PATH_DELIM + configSet;
final Set toRemoveFiles = new HashSet<>();
final Set langExt = languages.stream().map(l -> "_" + l).collect(Collectors.toSet());
try {
ZkMaintenanceUtils.traverseZkTree(
zkClient,
configPathInZk,
ZkMaintenanceUtils.VISIT_ORDER.VISIT_POST,
path -> {
if (!isMatchingLangOrNonLangFile(path, langExt)) toRemoveFiles.add(path);
});
} catch (KeeperException.NoNodeException nne) {
// no-op
} catch (KeeperException | InterruptedException e) {
throw new IOException(
"Failed to traverse znode path: " + configPathInZk, SolrZkClient.checkInterrupted(e));
}
for (String path : toRemoveFiles) {
try {
zkClient.delete(path, -1, false);
} catch (KeeperException.NoNodeException nne) {
// no-op
} catch (KeeperException | InterruptedException e) {
throw new IOException("Failed to delete znode: " + path, SolrZkClient.checkInterrupted(e));
}
}
return schema;
}
protected ManagedIndexSchema restoreLanguageSpecificObjectsAndFiles(
String configSet,
ManagedIndexSchema schema,
List langs,
boolean dynamicEnabled,
String copyFrom)
throws IOException {
// pull the dynamic fields from the copyFrom schema
ManagedIndexSchema copyFromSchema = loadLatestSchema(copyFrom);
final Set langSet = new HashSet<>(includeLangIds);
langSet.addAll(langs);
boolean restoreAllLangs = langs.isEmpty();
final Set langFilesToRestore = new HashSet<>();
// Restore missing files
SolrZkClient zkClient = zkStateReader().getZkClient();
String configPathInZk = ZkConfigSetService.CONFIGS_ZKNODE + ZNODE_PATH_DELIM + copyFrom;
final Set langExt = langSet.stream().map(l -> "_" + l).collect(Collectors.toSet());
try {
ZkMaintenanceUtils.traverseZkTree(
zkClient,
configPathInZk,
ZkMaintenanceUtils.VISIT_ORDER.VISIT_POST,
path -> {
if (path.endsWith(".txt")) {
if (restoreAllLangs) {
langFilesToRestore.add(path);
return;
}
final String pathWoExt = path.substring(0, path.length() - 4);
for (String lang : langExt) {
if (pathWoExt.endsWith(lang)) {
langFilesToRestore.add(path);
break;
}
}
}
});
} catch (KeeperException.NoNodeException nne) {
// no-op
} catch (KeeperException | InterruptedException e) {
throw new IOException(
"Failed to traverse znode path: " + configPathInZk, SolrZkClient.checkInterrupted(e));
}
if (!langFilesToRestore.isEmpty()) {
final String replacePathDir = "/" + configSet;
final String origPathDir = "/" + copyFrom;
for (String path : langFilesToRestore) {
String copyToPath = path.replace(origPathDir, replacePathDir);
try {
if (!zkClient.exists(copyToPath, true)) {
zkClient.makePath(copyToPath, false, true);
zkClient.setData(copyToPath, zkClient.getData(path, null, null, true), true);
}
} catch (KeeperException | InterruptedException e) {
throw new IOException(
"Failed to restore file at znode path: " + copyToPath,
SolrZkClient.checkInterrupted(e));
}
}
}
// Restore field types
final Map existingTypes = schema.getFieldTypes();
List addTypes =
copyFromSchema.getFieldTypes().values().stream()
.filter(
t ->
isLangTextType(t, restoreAllLangs ? null : langSet)
&& !existingTypes.containsKey(t.getTypeName()))
.collect(Collectors.toList());
if (!addTypes.isEmpty()) {
schema = schema.addFieldTypes(addTypes, false);
}
if (dynamicEnabled) {
// restore language specific dynamic fields
final Set existingDynFields =
Arrays.stream(schema.getDynamicFieldPrototypes())
.map(SchemaField::getName)
.collect(Collectors.toSet());
final Set langFieldTypeNames =
schema.getFieldTypes().values().stream()
.filter(t -> isLangTextType(t, restoreAllLangs ? null : langSet))
.map(FieldType::getTypeName)
.collect(Collectors.toSet());
List addDynFields =
Arrays.stream(copyFromSchema.getDynamicFields())
.filter(df -> langFieldTypeNames.contains(df.getPrototype().getType().getTypeName()))
.filter(df -> !existingDynFields.contains(df.getPrototype().getName()))
.map(IndexSchema.DynamicField::getPrototype)
.collect(Collectors.toList());
if (!addDynFields.isEmpty()) {
schema = schema.addDynamicFields(addDynFields, null, false);
}
} else {
schema = removeDynamicFields(schema);
}
return schema;
}
private boolean isMatchingLangOrNonLangFile(final String path, final Set langs) {
if (!path.endsWith(".txt")) return true; // not a .txt file, always include
int slashAt = path.lastIndexOf('/');
String fileName = slashAt != -1 ? path.substring(slashAt + 1) : "";
if (!fileName.contains("_"))
return true; // looking for file names like stopwords_en.txt, not a match, so skip it
// remove the .txt extension
final String pathWoExt = fileName.substring(0, fileName.length() - 4);
for (String lang : langs) {
if (pathWoExt.endsWith(lang)) {
return true;
}
}
// if we fall thru to here, then the file should be excluded
return false;
}
private boolean isTextType(final FieldType t) {
return t.getTypeName().startsWith("text_") && TextField.class.equals(t.getClass());
}
private boolean isLangTextType(final FieldType t, final Set langSet) {
return isTextType(t)
&& (langSet == null || langSet.contains(t.getTypeName().substring("text_".length())));
}
protected ManagedIndexSchema removeDynamicFields(ManagedIndexSchema schema) {
List dynamicFieldNames =
Arrays.stream(schema.getDynamicFields())
.map(f -> f.getPrototype().getName())
.collect(Collectors.toList());
if (!dynamicFieldNames.isEmpty()) {
schema = schema.deleteDynamicFields(dynamicFieldNames);
}
return schema;
}
protected ManagedIndexSchema restoreDynamicFields(
ManagedIndexSchema schema, List langs, String copyFrom) {
// pull the dynamic fields from the copyFrom schema
ManagedIndexSchema copyFromSchema = loadLatestSchema(copyFrom);
IndexSchema.DynamicField[] dynamicFields = copyFromSchema.getDynamicFields();
if (dynamicFields.length == 0 && !DEFAULT_CONFIGSET_NAME.equals(copyFrom)) {
copyFromSchema = loadLatestSchema(DEFAULT_CONFIGSET_NAME);
dynamicFields = copyFromSchema.getDynamicFields();
}
if (dynamicFields.length == 0) {
return schema;
}
final Set existingDFNames =
Arrays.stream(schema.getDynamicFields())
.map(df -> df.getPrototype().getName())
.collect(Collectors.toSet());
List toAdd =
Arrays.stream(dynamicFields)
.filter(df -> !existingDFNames.contains(df.getPrototype().getName()))
.map(IndexSchema.DynamicField::getPrototype)
.collect(Collectors.toList());
// only restore language specific dynamic fields that match our langSet
if (!langs.isEmpty()) {
final Set langSet = new HashSet<>(includeLangIds);
langSet.addAll(langs);
toAdd =
toAdd.stream()
.filter(
df ->
!df.getName().startsWith("*_txt_")
|| langSet.contains(df.getName().substring("*_txt_".length())))
.collect(Collectors.toList());
}
if (!toAdd.isEmpty()) {
// grab any field types that need to be re-added
final Map fieldTypes = schema.getFieldTypes();
List addTypes =
toAdd.stream()
.map(SchemaField::getType)
.filter(t -> !fieldTypes.containsKey(t.getTypeName()))
.collect(Collectors.toList());
if (!addTypes.isEmpty()) {
schema = schema.addFieldTypes(addTypes, false);
}
schema = schema.addDynamicFields(toAdd, null, true);
}
return schema;
}
void checkSchemaVersion(String configSet, final int versionInRequest, int currentVersion)
throws IOException {
if (versionInRequest < 0) {
return; // don't enforce the version check
}
if (currentVersion == -1) {
currentVersion = getCurrentSchemaVersion(configSet);
}
if (currentVersion != versionInRequest) {
if (configSet.startsWith(DESIGNER_PREFIX)) {
configSet = configSet.substring(DESIGNER_PREFIX.length());
}
throw new SolrException(
SolrException.ErrorCode.CONFLICT,
"Your schema version "
+ versionInRequest
+ " for "
+ configSet
+ " is out-of-date; current version is: "
+ currentVersion
+ ". Perhaps another user also updated the schema while you were editing it? You'll need to retry your update after the schema is refreshed.");
}
}
List listConfigsInZk() throws IOException {
return cc.getConfigSetService().listConfigs();
}
byte[] downloadAndZipConfigSet(String configId) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Path tmpDirectory =
Files.createTempDirectory("schema-designer-" + FilenameUtils.getName(configId));
try {
cc.getConfigSetService().downloadConfig(configId, tmpDirectory);
try (ZipOutputStream zipOut = new ZipOutputStream(baos)) {
Files.walkFileTree(
tmpDirectory,
new SimpleFileVisitor<>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
throws IOException {
if (Files.isHidden(dir)) {
return FileVisitResult.SKIP_SUBTREE;
}
String dirName = tmpDirectory.relativize(dir).toString();
if (!dirName.endsWith("/")) {
dirName += "/";
}
zipOut.putNextEntry(new ZipEntry(dirName));
zipOut.closeEntry();
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
if (!Files.isHidden(file)) {
try (InputStream fis = Files.newInputStream(file)) {
ZipEntry zipEntry = new ZipEntry(tmpDirectory.relativize(file).toString());
zipOut.putNextEntry(zipEntry);
fis.transferTo(zipOut);
}
}
return FileVisitResult.CONTINUE;
}
});
}
} finally {
PathUtils.deleteDirectory(tmpDirectory);
}
return baos.toByteArray();
}
public boolean isConfigSetTrusted(String configSetName) {
try {
return cc.getConfigSetService().isConfigSetTrusted(configSetName);
} catch (IOException e) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"Could not load conf " + configSetName + ": " + e.getMessage(),
e);
}
}
public void removeConfigSetTrust(String configSetName) {
try {
Map metadata = Collections.singletonMap("trusted", false);
cc.getConfigSetService().setConfigMetadata(configSetName, metadata);
} catch (IOException e) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"Could not remove trusted flag for configSet " + configSetName + ": " + e.getMessage(),
e);
}
}
protected ZkSolrResourceLoader zkLoaderForConfigSet(final String configSet) {
SolrResourceLoader loader = cc.getResourceLoader();
return new ZkSolrResourceLoader(
loader.getInstancePath(), configSet, loader.getClassLoader(), cc.getZkController());
}
}
