org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseNumeric Maven / Gradle / Ivy

Go to download
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.ql.udf.generic;

import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.NoMatchingMethodException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hive.common.HiveCompat;
import org.apache.hive.common.HiveCompat.CompatLevel;

/**
 * GenericUDF Base Class for operations.
 */
@Description(name = "op", value = "a op b - Returns the result of operation")
public abstract class GenericUDFBaseNumeric extends GenericUDFBaseBinary {

  protected transient PrimitiveObjectInspector leftOI;
  protected transient PrimitiveObjectInspector rightOI;
  protected transient PrimitiveObjectInspector resultOI;

  protected transient Converter converterLeft;
  protected transient Converter  converterRight;

  protected ByteWritable byteWritable = new ByteWritable();
  protected ShortWritable shortWritable = new ShortWritable();
  protected IntWritable intWritable = new IntWritable();
  protected LongWritable longWritable = new LongWritable();
  protected FloatWritable floatWritable = new FloatWritable();
  protected DoubleWritable doubleWritable = new DoubleWritable();
  protected HiveDecimalWritable decimalWritable = new HiveDecimalWritable();

  protected boolean confLookupNeeded = true;
  protected boolean ansiSqlArithmetic = false;

  public GenericUDFBaseNumeric() {
  }

  @Override
  public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length != 2) {
      throw new UDFArgumentException(opName + " requires two arguments.");
    }

    for (int i = 0; i < 2; i++) {
      Category category = arguments[i].getCategory();
      if (category != Category.PRIMITIVE) {
        throw new UDFArgumentTypeException(i, "The "
            + GenericUDFUtils.getOrdinal(i + 1)
            + " argument of " + opName + "  is expected to a "
            + Category.PRIMITIVE.toString().toLowerCase() + " type, but "
            + category.toString().toLowerCase() + " is found");
      }
    }

    // During map/reduce tasks, there may not be a valid HiveConf from the SessionState.
    // So lookup and save any needed conf information during query compilation in the Hive conf
    // (where there should be valid HiveConf from SessionState).  Plan serialization will ensure
    // we have access to these values in the map/reduce tasks.
    if (confLookupNeeded) {
      CompatLevel compatLevel = HiveCompat.getCompatLevel(SessionState.get().getConf());
      ansiSqlArithmetic = compatLevel.ordinal() > CompatLevel.HIVE_0_12.ordinal();
      confLookupNeeded = false;
    }

    leftOI = (PrimitiveObjectInspector) arguments[0];
    rightOI = (PrimitiveObjectInspector) arguments[1];
    resultOI = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
        deriveResultTypeInfo());
    converterLeft = ObjectInspectorConverters.getConverter(leftOI, resultOI);
    converterRight = ObjectInspectorConverters.getConverter(rightOI, resultOI);

    return resultOI;
  }

  @Override
  public Object evaluate(DeferredObject[] arguments) throws HiveException {
    if (arguments[0] == null || arguments[1] == null) {
      return null;
    }

    Object left = arguments[0].get();
    Object right = arguments[1].get();
    if (left == null && right == null) {
      return null;
    }

    // Handle decimal separately.
    if (resultOI.getPrimitiveCategory() == PrimitiveCategory.DECIMAL) {
      HiveDecimal hdLeft = PrimitiveObjectInspectorUtils.getHiveDecimal(left, leftOI);
      HiveDecimal hdRight = PrimitiveObjectInspectorUtils.getHiveDecimal(right, rightOI);
      if (hdLeft == null || hdRight == null) {
        return null;
      }
      HiveDecimalWritable result = evaluate(hdLeft, hdRight);
      return resultOI.getPrimitiveWritableObject(result);
    }

    left = converterLeft.convert(left);
    if (left == null) {
      return null;
    }
    right = converterRight.convert(right);
    if (right == null) {
      return null;
    }

    switch (resultOI.getPrimitiveCategory()) {
    case BYTE:
      return evaluate((ByteWritable) left, (ByteWritable) right);
    case SHORT:
      return evaluate((ShortWritable) left, (ShortWritable) right);
    case INT:
      return evaluate((IntWritable) left, (IntWritable) right);
    case LONG:
      return evaluate((LongWritable) left, (LongWritable) right);
    case FLOAT:
      return evaluate((FloatWritable) left, (FloatWritable) right);
    case DOUBLE:
      return evaluate((DoubleWritable) left, (DoubleWritable) right);
    default:
      // Should never happen.
      throw new RuntimeException("Unexpected type in evaluating " + opName + ": " +
        resultOI.getPrimitiveCategory());
    }
  }

  protected ByteWritable evaluate(ByteWritable left, ByteWritable right) {
    return null;
  }

  protected ShortWritable evaluate(ShortWritable left, ShortWritable right) {
    return null;
  }

  protected IntWritable evaluate(IntWritable left, IntWritable right) {
    return null;
  }

  protected LongWritable evaluate(LongWritable left, LongWritable right) {
    return null;
  }

  protected FloatWritable evaluate(FloatWritable left, FloatWritable right) {
    return null;
  }

  protected DoubleWritable evaluate(DoubleWritable left, DoubleWritable right) {
    return null;
  }

  protected HiveDecimalWritable evaluate(HiveDecimal left, HiveDecimal right) {
    return null;
  }

  /**
   * Default implementation for deriving typeinfo instance for the operator result.
   *
   * @param leftOI TypeInfo instance of the left operand
   * @param rightOI TypeInfo instance of the right operand
   * @return
   * @throws UDFArgumentException
   */
  private PrimitiveTypeInfo deriveResultTypeInfo() throws UDFArgumentException {
    PrimitiveTypeInfo left = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(leftOI);
    PrimitiveTypeInfo right = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(rightOI);
    if (!FunctionRegistry.isNumericType(left) || !FunctionRegistry.isNumericType(right)) {
      List argTypeInfos = new ArrayList(2);
      argTypeInfos.add(left);
      argTypeInfos.add(right);
      throw new NoMatchingMethodException(this.getClass(), argTypeInfos, null);
    }

    // If any of the type isn't exact, double is chosen.
    if (!FunctionRegistry.isExactNumericType(left) || !FunctionRegistry.isExactNumericType(right)) {
      return deriveResultApproxTypeInfo();
    }

    return deriveResultExactTypeInfo();
  }

  /**
   * Default implementation for getting the approximate type info for the operator result.
   * Divide operator overrides this.
   * @return
   */
  protected PrimitiveTypeInfo deriveResultApproxTypeInfo() {
    PrimitiveTypeInfo left = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(leftOI);
    PrimitiveTypeInfo right = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(rightOI);

    // string types get converted to double
    if (PrimitiveObjectInspectorUtils.getPrimitiveGrouping(left.getPrimitiveCategory())
            == PrimitiveGrouping.STRING_GROUP) {
      left = TypeInfoFactory.doubleTypeInfo;
    }
    if (PrimitiveObjectInspectorUtils.getPrimitiveGrouping(right.getPrimitiveCategory())
        == PrimitiveGrouping.STRING_GROUP) {
      right = TypeInfoFactory.doubleTypeInfo;
    }    

    // Use type promotion
    PrimitiveCategory commonCat = FunctionRegistry.getCommonCategory(left, right);
    if (commonCat == PrimitiveCategory.DECIMAL) {
      // Hive 0.12 behavior where double * decimal -> decimal is gone.
      return TypeInfoFactory.doubleTypeInfo;
    } else if (commonCat == null) {
      return TypeInfoFactory.doubleTypeInfo;
    } else {
      return left.getPrimitiveCategory() == commonCat ? left : right;
    }
  }

  /**
   * Default implementation for getting the exact type info for the operator result. It worked for all
   * but divide operator.
   *
   * @return
   */
  protected PrimitiveTypeInfo deriveResultExactTypeInfo() {
    PrimitiveTypeInfo left = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(leftOI);
    PrimitiveTypeInfo right = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(rightOI);

    // Now we are handling exact types. Base implementation handles type promotion.
    PrimitiveCategory commonCat = FunctionRegistry.getCommonCategory(left, right);
    if (commonCat == PrimitiveCategory.DECIMAL) {
      return deriveResultDecimalTypeInfo();
    } else {
      return left.getPrimitiveCategory() == commonCat ? left : right;
    }
  }

  /**
   * Derive the object inspector instance for the decimal result of the operator.
   */
  protected DecimalTypeInfo deriveResultDecimalTypeInfo() {
    int prec1 = leftOI.precision();
    int prec2 = rightOI.precision();
    int scale1 = leftOI.scale();
    int scale2 = rightOI.scale();
    return deriveResultDecimalTypeInfo(prec1, scale1, prec2, scale2);
  }

  protected abstract DecimalTypeInfo deriveResultDecimalTypeInfo(int prec1, int scale1, int prec2, int scale2);

  public void copyToNewInstance(Object newInstance) throws UDFArgumentException {
    super.copyToNewInstance(newInstance);
    GenericUDFBaseNumeric other = (GenericUDFBaseNumeric) newInstance;
    other.confLookupNeeded = this.confLookupNeeded;
    other.ansiSqlArithmetic = this.ansiSqlArithmetic;
  }

  public boolean isConfLookupNeeded() {
    return confLookupNeeded;
  }

  public void setConfLookupNeeded(boolean confLookupNeeded) {
    this.confLookupNeeded = confLookupNeeded;
  }

  public boolean isAnsiSqlArithmetic() {
    return ansiSqlArithmetic;
  }

  public void setAnsiSqlArithmetic(boolean ansiSqlArithmetic) {
    this.ansiSqlArithmetic = ansiSqlArithmetic;
  }
}