com.qiniu.pipeline.sdk.plugin.AbstractScalaParser.scala Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of transform-plugin-sdk Show documentation
The newest version!
package com.qiniu.pipeline.sdk.plugin

import org.apache.spark.sql.types.StructType
import java.lang.reflect.Field

import org.apache.spark.sql.Row

import scala.util.{Failure, Success, Try}


/**
  * 构造方法，Scala编写Plugin时, 需要继承ScalaParser类
  *
  * @param pluginFields transform spec对应plugin中output所有字段
  * @param schema       用户打点/输入数据的schema
  */
abstract class AbstractScalaParser[Src, Out](pluginFields: Seq[String],
                                             schema: StructType) extends ScalaParser(pluginFields, schema) {

  /**
    * 设置srcClass(plugin输入)，即pipeline repo中所有字段
    *
    */
  def srcClass: Class[Src]

  /**
    * 设置outClass(plugin输出)，即transform spec对应plugin中output所有字段
    *
    */
  def outClass: Class[Out]

  /**
    *
    * @param src plugin输入，即pipeline repo中所有字段
    * @return 允许返回单个或多个Out类型实例
    */
  def parse(src: Src): List[Out]

  private final def srcInstance: Src = srcClass.newInstance

  override final def parse(row: Row): Seq[Row] = {
    // row transform to Src object
    val srcFields = schema.fieldNames
    val srcObject: Src = srcInstance
    srcFields.indices.foreach { index =>
      val name = srcFields(index)
      var field: Field = null
      Try {
        field = srcClass.getDeclaredField(name)
        field.setAccessible(true)
        field.set(srcObject, getValue(field, row.get(index)))
      } match {
        case Success(res) => res
        case Failure(ex) =>
          if (ex.isInstanceOf[IllegalAccessException]) {
            parserLog.warn(s"Access the field: ${name} in srcClass: ${srcClass.getCanonicalName()} failed.")
          } else if (ex.isInstanceOf[NoSuchFieldException]) {
            parserLog.warn(s"Set the field: ${name}'s value in srcClass: ${srcClass.getCanonicalName} failed.")
          } else {
            throw PluginRuntimeException.create(s"Set the field: ${name}" +
                s" in srcClass: ${srcClass.getCanonicalName} failed.", ex)
          }

      }
    }

    val outputs = parse(srcObject)

    // Out object transform to Row
    outputs.map { outInstance =>
      val arr: Array[Any] = pluginFields.indices.map { index =>
        val name = pluginFields(index)
        var field: Field = null
        Try {
          field = outClass.getDeclaredField(name)
          field.setAccessible(true)
          field.get(outInstance)
        } match {
          case Success(res) =>
            if (res == null) {
              // 判断为null，这里设置默认值
              setDefaultValue(field)
            } else {
              res
            }
          case Failure(ex) =>
            if (ex.isInstanceOf[IllegalAccessException]) {
              parserLog.warn(s"Access the field: ${name} in outClass: ${outClass.getCanonicalName()} failed.")
              setDefaultValue(field)
            } else if (ex.isInstanceOf[NoSuchFieldException]) {
              parserLog.warn(s"Get the field: ${name}'s value in outClass: ${outClass.getCanonicalName} failed.")
              setDefaultValue(field)
            } else {
              throw PluginRuntimeException.create(s"Get the field: ${name}'s value" +
                  s" in outClass: ${outClass.getCanonicalName} failed.", ex)
            }
        }
      }.toArray
      Row.fromSeq(arr)
    }
  }

}