com.qiniu.pipeline.sdk.plugin.Parser.scala Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of transform-plugin-sdk Show documentation
The newest version!
package com.qiniu.pipeline.sdk.plugin

import java.io.{Serializable => JSerializable}
import java.lang.{Long => JLong}
import java.lang.{Double => JDouble}
import java.lang.reflect.Field
import java.util
import java.util.{List => JList, Map => JMap}

import org.apache.spark.sql.Row
import org.apache.spark.sql.types._
import org.slf4j.LoggerFactory

abstract class Parser(order: Integer = 1,
                      pluginFields: JList[String],
                      schema: StructType,
                      configuration: JMap[String, JSerializable] = new util.HashMap[String, JSerializable]())
    extends Comparable[Parser] with JSerializable {

  private val scalaIntCls = classOf[Int]
  private val javaIntegerCls = classOf[java.lang.Integer]
  private val scalaLongCls = classOf[Long]
  private val javaLongCls = classOf[java.lang.Long]

  private val javaDoubleCls = classOf[java.lang.Double]
  private val scalaDoubleCls = classOf[Double]
  private val javaFloatCls = classOf[java.lang.Float]
  private val scalaFloatCls = classOf[Float]

  private val timeStampCls = classOf[java.sql.Timestamp]
  private val stringCls = classOf[String]

  protected final val parserLog = LoggerFactory.getLogger(this.getClass)

  def parse(originData: Row): AnyRef

  protected final def getOrder = order

  def getPluginFields = pluginFields

  def getSchema = schema

  protected def getFieldIndex(field: String) = schema.fieldIndex(field)

  protected final override def compareTo(that: Parser): Int = this.getOrder.compareTo(that.getOrder)

  protected final def setDefaultValue(field: Field): Any = {
    if (field == null) {
      null
    } else {
      val fieldCls = field.getType
      fieldCls match {
        case cls if cls == stringCls => ""
        case cls if cls == timeStampCls => new java.sql.Timestamp(0L)
        case cls if (cls == scalaIntCls || cls == javaIntegerCls || cls == scalaLongCls || cls == javaLongCls) => 0L
        case cls if (cls == javaDoubleCls || cls == scalaDoubleCls || cls == javaFloatCls || cls == scalaFloatCls) => 0D
        case _ =>
          parserLog.warn(s"The field: ${field.getName}'s " +
              s"type: ${fieldCls.getName} don't match long, float, date or string type.")
          null
      }
    }
  }

  // 根据row中field类型，转换为srcClass中各个Field相应的类型
  protected final def getValue(field: Field, curFieldValue: Any) = {
    if (field == null) {
      null
    } else {
      val fieldCls = field.getType
      fieldCls match {
        // 因为curFieldValue默认为String, timeStamp, long 和 double类型这些类型之一, 不需要适配.
        case cls if cls == stringCls => curFieldValue
        case cls if cls == timeStampCls => curFieldValue
        case cls if (cls == scalaLongCls || cls == javaLongCls) => curFieldValue
        case cls if (cls == javaDoubleCls || cls == scalaDoubleCls) => curFieldValue

        case cls if (cls == scalaIntCls) => // 当srcClass中存在 Int 类型字段
          curFieldValue.getClass match {
            case cls if cls == javaLongCls =>
              val value: Int = curFieldValue.asInstanceOf[JLong].intValue()
              value
            case cls if cls == scalaLongCls =>
              val value: Int = curFieldValue.asInstanceOf[Long].intValue()
              value
          }

        case cls if (cls == javaIntegerCls) => // 当srcClass中存在 java.lang.Integer 类型字段
          curFieldValue.getClass match {
            case cls if cls == javaLongCls =>
              val value: Int = curFieldValue.asInstanceOf[JLong].intValue()
              Int.box(value)
            case cls if cls == scalaLongCls =>
              val value: Int = curFieldValue.asInstanceOf[Long].intValue()
              Int.box(value)
          }

        case cls if (cls == javaFloatCls) => // 当srcClass中存在 java.lang.Float 类型字段
          curFieldValue.getClass match {
            case cls if cls == javaDoubleCls =>
              val value: Float = curFieldValue.asInstanceOf[JDouble].floatValue()
              Float.box(value)
            case cls if cls == scalaDoubleCls =>
              val value: Float = curFieldValue.asInstanceOf[Double].floatValue()
              Float.box(value)
          }

        case cls if (cls == scalaFloatCls) => // 当srcClass中存在 Float 类型字段
          curFieldValue.getClass match {
            case cls if cls == javaDoubleCls =>
              val value: Float = curFieldValue.asInstanceOf[JDouble].floatValue()
              value
            case cls if cls == scalaDoubleCls =>
              val value: Float = curFieldValue.asInstanceOf[Double].floatValue()
              value
          }

        case _ =>
          parserLog.warn(s"The field: ${field.getName}'s " +
              s"type: ${fieldCls.getName} don't match long, float, date or string type.")
          null
      }
    }
  }
}

case class PluginRuntimeException(message: String) extends RuntimeException(message)

object PluginRuntimeException {
  def create(msg: String): PluginRuntimeException = new PluginRuntimeException(msg)

  def create(msg: String, cause: Throwable): Throwable = new PluginRuntimeException(msg).initCause(cause)
}