org.apache.solr.handler.loader.JsonLoader Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of solr-core Show documentation
Show all versions of solr-core Show documentation
Apache Solr (module: core)
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.loader;
import static java.util.stream.Collectors.toList;
import static org.apache.solr.common.params.CommonParams.ID;
import static org.apache.solr.common.params.CommonParams.JSON;
import static org.apache.solr.common.params.CommonParams.PATH;
import static org.apache.solr.common.params.CommonParams.VERSION_FIELD;
import static org.apache.solr.common.params.ShardParams._ROUTE_;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.JsonRecordReader;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.handler.RequestHandlerUtils;
import org.apache.solr.handler.UpdateRequestHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.CommitUpdateCommand;
import org.apache.solr.update.DeleteUpdateCommand;
import org.apache.solr.update.RollbackUpdateCommand;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.util.RecordingJSONParser;
import org.noggit.JSONParser;
import org.noggit.JSONParser.ParseException;
import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @since solr 4.0
*/
public class JsonLoader extends ContentStreamLoader {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final AtomicBoolean WARNED_ABOUT_INDEX_TIME_BOOSTS = new AtomicBoolean();
public static final String CHILD_DOC_KEY = "_childDocuments_";
@Override
public String getDefaultWT() {
return JSON;
}
@Override
public void load(
SolrQueryRequest req,
SolrQueryResponse rsp,
ContentStream stream,
UpdateRequestProcessor processor)
throws Exception {
new SingleThreadedJsonLoader(req, rsp, processor).load(req, rsp, stream, processor);
}
@SuppressWarnings("unchecked")
public static SolrInputDocument buildDoc(Map m) {
SolrInputDocument result = new SolrInputDocument();
for (Map.Entry e : m.entrySet()) {
if (mapEntryIsChildDoc(e.getValue())) { // parse child documents
if (e.getValue() instanceof List) {
List value = (List) e.getValue();
for (Object o : value) {
if (o instanceof Map) {
// retain the value as a list, even if the list contains a single value.
if (!result.containsKey(e.getKey())) {
result.setField(e.getKey(), new ArrayList<>(1));
}
result.addField(e.getKey(), buildDoc((Map) o));
}
}
} else if (e.getValue() instanceof Map) {
result.addField(e.getKey(), buildDoc((Map) e.getValue()));
}
} else {
result.setField(e.getKey(), e.getValue());
}
}
return result;
}
private static boolean mapEntryIsChildDoc(Object val) {
if (val instanceof List) {
List listVal = (List) val;
if (listVal.size() == 0) return false;
return listVal.get(0) instanceof Map;
}
return val instanceof Map;
}
static class SingleThreadedJsonLoader extends ContentStreamLoader {
protected final UpdateRequestProcessor processor;
protected final SolrQueryRequest req;
protected SolrQueryResponse rsp;
protected JSONParser parser;
protected final int commitWithin;
protected final boolean overwrite;
SingleThreadedJsonLoader(
SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor processor) {
this.processor = processor;
this.req = req;
this.rsp = rsp;
commitWithin = req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1);
overwrite = req.getParams().getBool(UpdateParams.OVERWRITE, true);
}
@Override
public void load(
SolrQueryRequest req,
SolrQueryResponse rsp,
ContentStream stream,
UpdateRequestProcessor processor)
throws Exception {
try (Reader reader = getReader(stream)) {
this.processUpdate(reader);
} catch (ParseException e) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Cannot parse provided JSON: " + e.getMessage());
}
}
private Reader getReader(ContentStream stream) throws IOException {
if (log.isTraceEnabled()) {
String body = StrUtils.stringFromReader(stream.getReader());
log.trace("body: {}", body);
return new StringReader(body);
}
return stream.getReader();
}
@SuppressWarnings("fallthrough")
void processUpdate(Reader reader) throws IOException {
String path = (String) req.getContext().get(PATH);
if (UpdateRequestHandler.DOC_PATH.equals(path)
|| "false".equals(req.getParams().get("json.command"))) {
String split = req.getParams().get("split");
String[] f = req.getParams().getParams("f");
handleSplitMode(split, f, reader);
return;
}
parser = new JSONParser(reader);
int ev = parser.nextEvent();
while (ev != JSONParser.EOF) {
switch (ev) {
case JSONParser.ARRAY_START:
handleAdds();
break;
case JSONParser.STRING:
if (parser.wasKey()) {
String v = parser.getString();
if (v.equals(UpdateRequestHandler.ADD)) {
int ev2 = parser.nextEvent();
if (ev2 == JSONParser.OBJECT_START) {
processor.processAdd(parseAdd());
} else if (ev2 == JSONParser.ARRAY_START) {
handleAdds();
} else {
assertEvent(ev2, JSONParser.OBJECT_START);
}
} else if (v.equals(UpdateRequestHandler.COMMIT)) {
CommitUpdateCommand cmd = new CommitUpdateCommand(req, false);
cmd.waitSearcher = true;
parseCommitOptions(cmd);
processor.processCommit(cmd);
} else if (v.equals(UpdateRequestHandler.OPTIMIZE)) {
CommitUpdateCommand cmd = new CommitUpdateCommand(req, true);
cmd.waitSearcher = true;
parseCommitOptions(cmd);
processor.processCommit(cmd);
} else if (v.equals(UpdateRequestHandler.DELETE)) {
handleDeleteCommand();
} else if (v.equals(UpdateRequestHandler.ROLLBACK)) {
processor.processRollback(parseRollback());
} else {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Unknown command '" + v + "' at [" + parser.getPosition() + "]");
}
break;
}
// fall through
case JSONParser.LONG:
case JSONParser.NUMBER:
case JSONParser.BIGNUMBER:
case JSONParser.BOOLEAN:
case JSONParser.NULL:
if (log.isInfoEnabled()) {
log.info(
"Can't have a value here. Unexpected {} at [{}]",
JSONParser.getEventString(ev),
parser.getPosition());
}
case JSONParser.OBJECT_START:
case JSONParser.OBJECT_END:
case JSONParser.ARRAY_END:
break;
default:
log.info("Noggit UNKNOWN_EVENT_ID: {}", ev);
break;
}
// read the next event
ev = parser.nextEvent();
}
}
private void handleSplitMode(String split, String[] fields, final Reader reader)
throws IOException {
if (split == null) split = "/";
if (fields == null || fields.length == 0) fields = new String[] {"$FQN:/**"};
final boolean echo = "true".equals(req.getParams().get("echo"));
final String srcField = req.getParams().get("srcField");
final boolean mapUniqueKeyOnly = req.getParams().getBool("mapUniqueKeyOnly", false);
if (srcField != null) {
if (!"/".equals(split))
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Raw data can be stored only if split=/");
parser = new RecordingJSONParser(reader);
} else {
parser = new JSONParser(reader);
}
JsonRecordReader jsonRecordReader = JsonRecordReader.getInst(split, Arrays.asList(fields));
jsonRecordReader.streamRecords(
parser,
new JsonRecordReader.Handler() {
ArrayList