com.qiniu.pipeline.sdk.plugin.AbstractJavaParser Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of transform-plugin-sdk Show documentation
The newest version!
package com.qiniu.pipeline.sdk.plugin;

import org.apache.spark.sql.Row;
import org.apache.spark.sql.catalyst.expressions.GenericRow;
import org.apache.spark.sql.types.StructType;

import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;


public abstract class AbstractJavaParser extends JavaParser {

  /**
   * 构造方法，Scala编写Plugin时, 需要继承ScalaParser类
   *
   * @param pluginFields transform spec对应plugin中output所有字段
   * @param schema       用户打点/输入数据的schema
   */
  public AbstractJavaParser(List pluginFields, StructType schema) {
    super(pluginFields, schema);
  }

  /**
   * 设置srcClass(plugin输入)，即pipeline repo中所有字段
   *
   */
  protected abstract Class srcClass();

  /**
   * 设置outClass(plugin输出)，即transform spec对应plugin中output所有字段
   *
   */
  protected abstract Class outClass();

  /**
   *
   * @param src plugin输入，即pipeline repo中所有字段
   * @return 允许返回单个或多个Out类型实例
   */
  protected abstract List parse(Src src);

  private final Src srcInstance() {
    try {
      return srcClass().newInstance();
    } catch (Exception ex) {
      throw new PluginRuntimeException(ex.getMessage());
    }
  }

  @Override
  public final List parse(Row row) {
    String[] names = getSchema().fieldNames();
    Src srcRow = srcInstance();
    for (int i = 0; i < names.length; i++) {
      String name = names[i];
      Field field = null;
      try {
        field = srcClass().getDeclaredField(name);
        field.setAccessible(true);
        field.set(srcRow, getValue(field, row.get(i)));
      } catch (NoSuchFieldException ex) {
        parserLog().warn(String.format("Set the field: %s's value in srcClass: %s failed.",
            name, srcClass().getCanonicalName()));
        continue;
      } catch (IllegalAccessException ex) {
        parserLog().warn(String.format("Access the field: %s in srcClass: %s failed.",
            name, srcClass().getCanonicalName()));
        continue;
      } catch (Exception ex) {
        throw new PluginRuntimeException(String.format("Set the field: %s in srcClass: %s failed, and the error is: %s.",
            name, srcClass().getCanonicalName(), ex.getMessage()));
      }
    }

    List outputs = parse(srcRow);

    List results = new ArrayList();
    for (Out out : outputs) {
      List pluginFields = getPluginFields();
      Object[] array = new Object[pluginFields.size()];
      for (int i = 0; i < pluginFields.size(); i++) {
        String name = pluginFields.get(i);
        Field field = null;
        try {
          field = outClass().getDeclaredField(name);
          field.setAccessible(true);
          Object tmp = field.get(out);
          if(tmp == null) { // 判断为null，这里设置默认值
            array[i] = setDefaultValue(field);
          }else {
            array[i] = tmp;
          }
        } catch (IllegalAccessException ex) {
          array[i] = setDefaultValue(field);
          parserLog().warn(String.format("Access the field: %s in outClass: %s failed.",
              name, outClass().getCanonicalName()));
        } catch (NoSuchFieldException ex) {
          array[i] = setDefaultValue(field);
          parserLog().warn(String.format("Get the field: %s's value in outClass: %s failed.",
              name, outClass().getCanonicalName()));
        } catch (Exception ex) {
          throw new PluginRuntimeException(String.format("Get the field: %s in outClass: %s failed, and the error is: %s.",
              names, outClass().getCanonicalName(), ex.getMessage()));
        }
      }
      results.add(new GenericRow(array));
    }
    return results;
  }
}