com.datastax.data.prepare.spark.dataset.SparkSqlOperator Maven / Gradle / Ivy

Go to download
package com.datastax.data.prepare.spark.dataset;

import com.datastax.insight.annonation.InsightComponent;
import com.datastax.insight.annonation.InsightComponentArg;
import com.datastax.insight.core.driver.SparkContextBuilder;
import com.datastax.insight.spec.Operator;
import com.google.common.base.Strings;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;

public class SparkSqlOperator implements Operator {
    @InsightComponent(name = "spark sql", description = "spark sql")
    public static  Dataset sparkSql(
            @InsightComponentArg(externalInput = true,name = "数据集", description = "数据集",request = true) Dataset data,
            @InsightComponentArg(name = "表名", description = "临时表的名字",request = true) String table,
            @InsightComponentArg(name = "sql语句", description = "sql语句,多条语句以分号分隔开") String sqls) {
        Dataset dataset = data;
        SparkSession spark = SparkContextBuilder.getSession();
        for (String sql : sqls.split(";")) {
            dataset.createOrReplaceTempView(table);
            dataset = (Dataset)spark.sql(sql);
            spark.catalog().dropTempView(table);
        }
        return dataset;
    }

    @InsightComponent(name = "spark sql", description = "spark sql")
    public static  Dataset sparkSql(
            @InsightComponentArg(externalInput = true,name = "占位符1", description = "占位符1",request = true) org.apache.spark.ml.feature.StringIndexer place1,
            @InsightComponentArg(externalInput = true,name = "占位符2", description = "占位符2",request = true) String place2,
            @InsightComponentArg(name = "sql语句", description = "sql语句",request = true) String sql) {
        SparkSession spark = SparkContextBuilder.getSession();
        if (!Strings.isNullOrEmpty(sql)) {
            return  (Dataset)spark.sql(sql);
        }
        return null;
    }

    @InsightComponent(name = "spark sql", description = "spark sql")
    public static  Dataset sparkSql(
            @InsightComponentArg(name = "sql语句", description = "sql语句",request = true) String sql) {
        SparkSession spark = SparkContextBuilder.getSession();
        if (!Strings.isNullOrEmpty(sql)) {
            return  (Dataset)spark.sql(sql);
        }
        return null;
    }

    @InsightComponent(name = "placeholder", description = "placeholder")
    public static void place() {
        System.out.println("placeholder");
    }

}