com.metamx.common.parsers.JSONPathParser Maven / Gradle / Ivy

/*
 * Licensed to Metamarkets Group Inc. (Metamarkets) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  Metamarkets licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package com.metamx.common.parsers;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.Option;
import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
import com.metamx.common.Pair;
import com.metamx.common.StringUtils;

import java.math.BigInteger;
import java.nio.charset.CharsetEncoder;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * JSON parser class that uses the JsonPath library to access fields via path expressions.
 */
public class JSONPathParser implements Parser
{
  private final Map> fieldPathMap;
  private final boolean useFieldDiscovery;
  private final ObjectMapper mapper;
  private final CharsetEncoder enc = Charsets.UTF_8.newEncoder();
  private final Configuration jsonPathConfig;

  /**
   * Constructor
   *
   * @param fieldSpecs        List of field specifications.
   * @param useFieldDiscovery If true, automatically add root fields seen in the JSON document to the parsed object Map.
   *                          Only fields that contain a singular value or flat list (list containing no subobjects or lists) are automatically added.
   * @param mapper            Optionally provide an ObjectMapper, used by the parser for reading the input JSON.
   */
  public JSONPathParser(List fieldSpecs, boolean useFieldDiscovery, ObjectMapper mapper)
  {
    this.fieldPathMap = generateFieldPaths(fieldSpecs);
    this.useFieldDiscovery = useFieldDiscovery;
    this.mapper = mapper == null ? new ObjectMapper() : mapper;

    // Avoid using defaultConfiguration, as this depends on json-smart which we are excluding.
    this.jsonPathConfig = Configuration.builder()
                                       .jsonProvider(new JacksonJsonProvider())
                                       .mappingProvider(new JacksonMappingProvider())
                                       .options(EnumSet.of(Option.SUPPRESS_EXCEPTIONS))
                                       .build();
  }

  @Override
  public List getFieldNames()
  {
    return null;
  }

  @Override
  public void setFieldNames(Iterable fieldNames)
  {
  }

  /**
   * @param input JSON string. The root must be a JSON object, not an array.
   *              e.g., {"valid": "true"} and {"valid":[1,2,3]} are supported
   *              but [{"invalid": "true"}] and [1,2,3] are not.
   *
   * @return A map of field names and values
   */
  @Override
  public Map parse(String input)
  {
    try {
      Map map = new LinkedHashMap<>();
      Map document = mapper.readValue(
          input,
          new TypeReference>()
          {
          }
      );
      for (Map.Entry> entry : fieldPathMap.entrySet()) {
        String fieldName = entry.getKey();
        Pair pair = entry.getValue();
        JsonPath path = pair.rhs;
        Object parsedVal;
        if (pair.lhs == FieldType.ROOT) {
          parsedVal = document.get(fieldName);
        } else {
          parsedVal = path.read(document, jsonPathConfig);
        }
        if (parsedVal == null) {
          continue;
        }
        parsedVal = valueConversionFunction(parsedVal);
        map.put(fieldName, parsedVal);
      }
      if (useFieldDiscovery) {
        discoverFields(map, document);
      }
      return map;
    }
    catch (Exception e) {
      throw new ParseException(e, "Unable to parse row [%s]", input);
    }
  }

  private Map> generateFieldPaths(List fieldSpecs)
  {
    Map> map = new LinkedHashMap<>();
    for (FieldSpec fieldSpec : fieldSpecs) {
      String fieldName = fieldSpec.getName();
      if (map.get(fieldName) != null) {
        throw new IllegalArgumentException("Cannot have duplicate field definition: " + fieldName);
      }
      JsonPath path = JsonPath.compile(fieldSpec.getExpr());
      Pair pair = new Pair<>(fieldSpec.getType(), path);
      map.put(fieldName, pair);
    }
    return map;
  }

  private void discoverFields(Map map, Map document)
  {
    for (Map.Entry e : document.entrySet()) {
      String field = e.getKey();
      if (!map.containsKey(field)) {
        Object val = e.getValue();
        if (val == null) {
          continue;
        }
        if (val instanceof Map) {
          continue;
        }
        if (val instanceof List) {
          if (!isFlatList((List) val)) {
            continue;
          }
        }
        val = valueConversionFunction(val);
        map.put(field, val);
      }
    }
  }

  private Object valueConversionFunction(Object val)
  {
    if (val instanceof Integer) {
      return Long.valueOf((Integer) val);
    }

    if (val instanceof BigInteger) {
      return Double.valueOf(((BigInteger) val).doubleValue());
    }

    if (val instanceof String) {
      return charsetFix((String) val);
    }

    if (val instanceof List) {
      List