org.apache.solr.update.processor.IgnoreLargeDocumentProcessorFactory Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of solr-core Show documentation
Apache Solr Core
There is a newer version: 9.6.1
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.solr.update.processor;

import java.io.IOException;
import java.util.Collection;
import java.util.IdentityHashMap;
import java.util.Map;

import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;

import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;

/**
 * 
 * Gives system administrators a way to ignore very large update from clients.
 * When an update goes through processors its size can change
 * therefore this processor should be the last processor of the chain.
 * 
 * @since 7.4.0
 */
public class IgnoreLargeDocumentProcessorFactory extends UpdateRequestProcessorFactory {
  public static final String LIMIT_SIZE_PARAM = "limit";

  // limit of a SolrInputDocument size (in kb)
  private long maxDocumentSize = 1024 * 1024;

  @Override
  public void init(NamedList args) {
    maxDocumentSize = args.toSolrParams().required().getLong(LIMIT_SIZE_PARAM);
    args.remove(LIMIT_SIZE_PARAM);

    if (args.size() > 0) {
      throw new SolrException(SERVER_ERROR,
          "Unexpected init param(s): '" +
              args.getName(0) + "'");
    }
  }

  @Override
  public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
    return new UpdateRequestProcessor(next) {
      @Override
      public void processAdd(AddUpdateCommand cmd) throws IOException {
        long docSize = ObjectSizeEstimator.estimate(cmd.getSolrInputDocument());
        if (docSize / 1024 > maxDocumentSize) {
          throw new SolrException(BAD_REQUEST, "Size of the document "+cmd.getPrintableId()+" is too large, around:"+docSize);
        }
        super.processAdd(cmd);
      }
    };
  }

  /**
   * Util class for quickly estimate size of a {@link org.apache.solr.common.SolrInputDocument}
   * Compare to {@link org.apache.lucene.util.RamUsageEstimator}, this class have some pros
   * 
   * does not use reflection
   * 
go as deep as needed to compute size of all {@link org.apache.solr.common.SolrInputField} and
   * all {@link org.apache.solr.common.SolrInputDocument} children
   * 
compute size of String based on its length
   * 
fast estimate size of a {@link java.util.Map} or a {@link java.util.Collection}
   * 
   */
  // package private for testing
  static class ObjectSizeEstimator {
    /**
     * Sizes of primitive classes.
     */
    private static final Map,Integer> primitiveSizes = new IdentityHashMap<>();
    static {
      primitiveSizes.put(boolean.class, 1);
      primitiveSizes.put(Boolean.class, 1);
      primitiveSizes.put(byte.class, 1);
      primitiveSizes.put(Byte.class, 1);
      primitiveSizes.put(char.class, Character.BYTES);
      primitiveSizes.put(Character.class, Character.BYTES);
      primitiveSizes.put(short.class, Short.BYTES);
      primitiveSizes.put(Short.class, Short.BYTES);
      primitiveSizes.put(int.class, Integer.BYTES);
      primitiveSizes.put(Integer.class, Integer.BYTES);
      primitiveSizes.put(float.class, Float.BYTES);
      primitiveSizes.put(Float.class, Float.BYTES);
      primitiveSizes.put(double.class, Double.BYTES);
      primitiveSizes.put(Double.class, Double.BYTES);
      primitiveSizes.put(long.class, Long.BYTES);
      primitiveSizes.put(Long.class, Long.BYTES);
    }

    static long estimate(SolrInputDocument doc) {
      if (doc == null) return 0L;
      long size = 0;
      for (SolrInputField inputField : doc.values()) {
        size += primitiveEstimate(inputField.getName(), 0L);
        size += estimate(inputField.getValue());
      }

      if (doc.hasChildDocuments()) {
        for (SolrInputDocument childDoc : doc.getChildDocuments()) {
          size += estimate(childDoc);
        }
      }
      return size;
    }

    static long estimate(Object obj) {
      if (obj instanceof SolrInputDocument) {
        return estimate((SolrInputDocument) obj);
      }

      if (obj instanceof Map) {
        return estimate((Map) obj);
      }

      if (obj instanceof Collection) {
        return estimate((Collection) obj);
      }

      return primitiveEstimate(obj, 0L);
    }

    private static long primitiveEstimate(Object obj, long def) {
      Class clazz = obj.getClass();
      if (clazz.isPrimitive()) {
        return primitiveSizes.get(clazz);
      }
      if (obj instanceof String) {
        return ((String) obj).length() * Character.BYTES;
      }
      return def;
    }

    private static long estimate(Map map) {
      if (map.isEmpty()) return 0;
      long size = 0;
      for (Map.Entry entry : map.entrySet()) {
        size += primitiveEstimate(entry.getKey(), 0L);
        size += estimate(entry.getValue());
      }
      return size;
    }

    private static long estimate(Collection collection) {
      if (collection.isEmpty()) return 0;
      long size = 0;
      for (Object obj : collection) {
        size += estimate(obj);
      }
      return size;
    }
  }

}