org.apache.solr.update.processor.PreAnalyzedUpdateProcessorFactory Maven / Gradle / Ivy
Show all versions of solr-core Show documentation
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.update.processor;
import java.lang.invoke.MethodHandles;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexableField;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.JsonPreAnalyzedParser;
import org.apache.solr.schema.PreAnalyzedField;
import org.apache.solr.schema.PreAnalyzedField.PreAnalyzedParser;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.SimplePreAnalyzedParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* An update processor that parses configured fields of any document being added
* using {@link PreAnalyzedField} with the configured format parser.
*
* Fields are specified using the same patterns as in {@link FieldMutatingUpdateProcessorFactory}.
* They are then checked whether they follow a pre-analyzed format defined by parser.
* Valid fields are then parsed. The original {@link SchemaField} is used for the initial
* creation of {@link IndexableField}, which is then modified to add the results from
* parsing (token stream value and/or string value) and then it will be directly added to
* the final Lucene {@link Document} to be indexed.
* Fields that are declared in the patterns list but are not present
* in the current schema will be removed from the input document.
* Implementation details
* This update processor uses {@link PreAnalyzedParser}
* to parse the original field content (interpreted as a string value), and thus
* obtain the stored part and the token stream part. Then it creates the "template"
* {@link Field}-s using the original {@link SchemaField#createFields(Object)}
* as declared in the current schema. Finally it sets the pre-analyzed parts if
* available (string value and the token
* stream value) on the first field of these "template" fields. If the declared
* field type does not support stored or indexed parts then such parts are silently
* discarded. Finally the updated "template" {@link Field}-s are added to the resulting
* {@link SolrInputField}, and the original value of that field is removed.
* Example configuration
* In the example configuration below there are two update chains, one that
* uses the "simple" parser ({@link SimplePreAnalyzedParser}) and one that uses
* the "json" parser ({@link JsonPreAnalyzedParser}). Field "nonexistent" will be
* removed from input documents if not present in the schema. Other fields will be
* analyzed and if valid they will be converted to {@link IndexableField}-s or if
* they are not in a valid format that can be parsed with the selected parser they
* will be passed as-is. Assuming that ssto field is stored but not
* indexed, and sind field is indexed but not stored: if
* ssto input value contains the indexed part then this part will
* be discarded and only the stored value part will be retained. Similarly,
* if sind input value contains the stored part then it
* will be discarded and only the token stream part will be retained.
*
*
* <updateRequestProcessorChain name="pre-analyzed-simple">
* <processor class="solr.PreAnalyzedUpdateProcessorFactory">
* <str name="fieldName">title</str>
* <str name="fieldName">nonexistent</str>
* <str name="fieldName">ssto</str>
* <str name="fieldName">sind</str>
* <str name="parser">simple</str>
* </processor>
* <processor class="solr.RunUpdateProcessorFactory" />
* </updateRequestProcessorChain>
*
* <updateRequestProcessorChain name="pre-analyzed-json">
* <processor class="solr.PreAnalyzedUpdateProcessorFactory">
* <str name="fieldName">title</str>
* <str name="fieldName">nonexistent</str>
* <str name="fieldName">ssto</str>
* <str name="fieldName">sind</str>
* <str name="parser">json</str>
* </processor>
* <processor class="solr.RunUpdateProcessorFactory" />
* </updateRequestProcessorChain>
*
*
* @since 4.3.0
*/
public class PreAnalyzedUpdateProcessorFactory extends FieldMutatingUpdateProcessorFactory {
private PreAnalyzedField parser;
private String parserImpl;
@Override
public void init(final NamedList args) {
parserImpl = (String)args.get("parser");
args.remove("parser");
// initialize inclusion / exclusion patterns
super.init(args);
}
@Override
public UpdateRequestProcessor getInstance(SolrQueryRequest req,
SolrQueryResponse rsp, UpdateRequestProcessor next) {
return new PreAnalyzedUpdateProcessor(getSelector(), next, req.getSchema(), parser);
}
@Override
public void inform(SolrCore core) {
super.inform(core);
parser = new PreAnalyzedField();
Map args = new HashMap<>();
if (parserImpl != null) {
args.put(PreAnalyzedField.PARSER_IMPL, parserImpl);
}
parser.init(core.getLatestSchema(), args);
}
}
class PreAnalyzedUpdateProcessor extends FieldMutatingUpdateProcessor {
private PreAnalyzedField parser;
private IndexSchema schema;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public PreAnalyzedUpdateProcessor(FieldNameSelector sel, UpdateRequestProcessor next, IndexSchema schema, PreAnalyzedField parser) {
super(sel, next);
this.schema = schema;
this.parser = parser;
}
@Override
protected SolrInputField mutate(SolrInputField src) {
SchemaField sf = schema.getFieldOrNull(src.getName());
if (sf == null) { // remove this field
return null;
}
FieldType type = PreAnalyzedField.createFieldType(sf);
if (type == null) { // neither indexed nor stored - skip
return null;
}
SolrInputField res = new SolrInputField(src.getName());
for (Object o : src) {
if (o == null) {
continue;
}
Field pre = (Field)parser.createField(sf, o);
if (pre != null) {
res.addValue(pre);
} else { // restore the original value
log.warn("Could not parse field {} - using original value as is: {}", src.getName(), o);
res.addValue(o);
}
}
return res;
}
}