com.datastax.insight.agent.SparkAgent Maven / Gradle / Ivy

Go to download
package com.datastax.insight.agent;

import com.datastax.insight.core.entity.Cache;
import com.datastax.insight.core.entity.Context;
import com.datastax.insight.agent.dao.InsightDAO;
import com.datastax.insight.core.dag.DAG;
import com.datastax.insight.core.service.BigdataService;
import com.datastax.util.web.JSONUtil;
import com.google.common.base.Strings;
import org.apache.spark.sql.SparkSession;
import java.io.File;
import java.io.PrintStream;
import java.nio.file.Paths;
import java.util.Map;

public class SparkAgent {
    public static void main(String[] args) {
        String address = args[0];
        String schema = args[1];
        String username = args[2];
        String password = args[3];
        String userId = args[4];
        String projectId = args[5];
        String flowId=args[6];
        String batchId=args[7];
        String mode=args[8];
        String hits=args[9];
        String nodeId=args[10];
        String explorationFile=args[11];
        String runMode=args[12];
   /*     String address = "192.168.1.33:3306";
        String schema = "dataexa-insight-ccb";
        String username ="root";
        String password = "DataExa5528280";
        String userId = "100012";
        String projectId ="10160";
        String flowId="10025";
        String batchId="1563267601076";
        String mode="run_exploration";
        String hits="";
        String nodeId="";
        String explorationFile="";
        String runMode="1";*/
        Cache.addCache("address", address);
        Cache.addCache("schema", schema);
        Cache.addCache("username", username);
        Cache.addCache("password", password);
        Cache.addCache("userId", userId);


        Cache.addCache("projectId", projectId);
        Cache.addCache("flowId", flowId);
        Cache.addCache("batchId", batchId);
        Cache.addCache("mode", mode);
        Cache.addCache("hits", hits);
        Cache.addCache("nodeId", nodeId);
        Cache.addCache("explorationFile", explorationFile);
        Cache.addCache("runMode",runMode);
        //Cache.addCache("flowVersionId",getFlowVersionId(Long.valueOf(projectId),Long.valueOf(flowId)));
        Cache.addCache("flowVersionId","0");
        if (Integer.parseInt(runMode) == 1) {
            String localStdoutFile = getLocalStdoutFile(Long.parseLong(userId),flowId,batchId);
            try {
                System.setOut(new PrintStream(new File(localStdoutFile)));
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        System.out.println("===DataExa-Insight Flow Params Started===");
        System.out.println("##################################################");
//        System.out.println(StringUtils.join(args, " "));
        System.out.println("address: " + address);
        System.out.println("schema: " + schema);
//        System.out.println("username: " + username);
//        System.out.println("password: " + password);
        System.out.println("userId: " + userId);
        System.out.println("projectId: " + projectId);
        System.out.println("flowId: " + flowId);
        System.out.println("batchId: " + batchId);
        System.out.println("mode: " + mode);
        System.out.println("hits: " + hits);
        System.out.println("nodeId: " + nodeId);
        System.out.println("explorationFile: " + explorationFile);
        System.out.println("runMode: " + runMode);
        System.out.println("##################################################");
        System.out.println("===DataExa-Insight Flow Params Ended===");

        //manually init context
        ConfigManager.init();
        systemPropsInit(Long.parseLong(userId));
        SparkSession spark = init(Long.parseLong(userId));
//        spark.sparkContext()
//                .setCheckpointDir(FileSystem.get(spark.sparkContext().hadoopConfiguration())
//                        .getWorkingDirectory().toString() + "checkpoint/");

        try {
            runCmdId(projectId, flowId, mode, hits, nodeId, explorationFile);
        } finally {
            spark.stop();
        }
    }

    private static SparkSession init(long userId) {
        SparkSession.Builder builder = SparkSession
                .builder()
                //.master("local[2]")
                .appName("spark-insight");

        InsightDAO dao = new InsightDAO();
    /*    Map settings = dao.getSystemSettings(userId);

        String warehouse = settings.getOrDefault("spark.sql.warehouse.dir", "");
        String metastore = settings.getOrDefault("hive.metastore.uris", "");*/
        String warehouse ="";
        String metastore = "";

        if(!Strings.isNullOrEmpty(warehouse) && !Strings.isNullOrEmpty(metastore)) {
            builder.config("spark.sql.warehouse.dir", warehouse)
                    .config("hive.metastore.uris", metastore)
                    .enableHiveSupport();
        }

        return builder.getOrCreate();
    }

    private static Object runCmdId(String projectId, String flowId, String mode, String hits, String nodeId, String explorationFile){
        String dagJson=new InsightDAO().getDAGJson(Long.parseLong(projectId), Long.parseLong(flowId));
        if(dagJson==null){
            System.out.println("没找到对应的流程！");
            return null;
        }
        return runCmd(dagJson, mode, hits, nodeId, explorationFile);
    }

    private static Object runCmd(String dagJson, String mode, String hits, String nodeId, String explorationFile){
        DAG dag= JSONUtil.getEntity(dagJson,DAG.class);
        //todo dag
        Object ret= BigdataService.invokeDag(dag, mode, hits, nodeId, explorationFile);
        if(ret!=null) {
            System.out.println("Execution result : " + ret.toString());
        }
        return ret;
    }

    private static void systemPropsInit(Long userId) {
       /* InsightDAO dao = new InsightDAO();
        Map settings = dao.getSystemSettings(userId);
        String store_path = settings.get("store.path");
        Context.SYSTEM_PROPERTIES.put(Context.MAIN_FOLDER,parsePath(store_path));
        Context.SYSTEM_PROPERTIES.put(Context.COMPONENT_FOLDER,parsePath(store_path + settings.get("store.component")));
        Context.SYSTEM_PROPERTIES.put(Context.DATASET_FOLDER,parsePath(store_path + settings.get("store.dataset")));
        Context.SYSTEM_PROPERTIES.put(Context.MODEL_FOLDER,parsePath(store_path + settings.get("store.model")));
        Context.SYSTEM_PROPERTIES.put(Context.TASK_FOLDER,parsePath(store_path + settings.get("store.task")));
        Context.SYSTEM_PROPERTIES.put(Context.MISC_FOLDER,parsePath(store_path + settings.get("store.misc")));
        Context.SYSTEM_PROPERTIES.put(Context.DRIVER_FOLDER,parsePath(store_path + settings.get("store.driver")));
        Context.SYSTEM_PROPERTIES.put(Context.LOG_FOLDER,parsePath(settings.get("LOG_HOME")));
        Context.SYSTEM_PROPERTIES.put(Context.TEMP_FOLDER,parsePath(settings.get("temp.path")));*/


        String store_path = "/tmp";
        Context.SYSTEM_PROPERTIES.put(Context.MAIN_FOLDER,parsePath(store_path));
        Context.SYSTEM_PROPERTIES.put(Context.COMPONENT_FOLDER,parsePath(store_path + "/component"));
        Context.SYSTEM_PROPERTIES.put(Context.DATASET_FOLDER,parsePath(store_path + "/dataset"));
        Context.SYSTEM_PROPERTIES.put(Context.MODEL_FOLDER,parsePath(store_path + "/model"));
        Context.SYSTEM_PROPERTIES.put(Context.TASK_FOLDER,parsePath(store_path + "/task"));
        Context.SYSTEM_PROPERTIES.put(Context.MISC_FOLDER,parsePath(store_path + "/misc"));
        Context.SYSTEM_PROPERTIES.put(Context.DRIVER_FOLDER,parsePath(store_path +"/driver"));
        Context.SYSTEM_PROPERTIES.put(Context.LOG_FOLDER,parsePath("/tmp/LOG_HOME"));
        Context.SYSTEM_PROPERTIES.put(Context.TEMP_FOLDER,parsePath("/tmp/path"));



    }

    private static String parsePath(String path) {
        String rightPath = path;
        if (!path.startsWith("hdfs")) {
            rightPath = "file://" + path;
        }
        return rightPath;
    }

    private static String getLocalStdoutFile(Long userId,String flowId,String batchId) {
        InsightDAO dao = new InsightDAO();
        //Map settings = dao.getSystemSettings(userId);
        String logId = "flow-" + flowId + "-" + batchId;
      //  String logHome = settings.get("LOG_HOME");
        String logHome = "/tmp/LOG_HOME";
        return Paths.get(logHome, logId + ".stdout").toString();
    }

    private static Long getFlowVersionId(Long projectId,Long flowId) {
        if (projectId == 0L) {
            return 0L;
        }
        InsightDAO dao = new InsightDAO();
        return dao.getFlowVersionId(flowId);
    }
}