com.datastax.insight.core.driver.SparkContextBuilder Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of commons-io Show documentation

commons-io

The newest version!

package com.datastax.insight.core.driver;


import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession;

import java.util.Map;

public class SparkContextBuilder {

    private static SparkConfig action;
    private static JavaSparkContext jsc;
    private static SparkContext sc;
    private static SparkSession session;

    //// TODO: 16-9-12  load from conf file
    static {
        action=new SparkConfig();
    }

    public static JavaSparkContext getJContext(){
        if(jsc ==null) {
            jsc = JavaSparkContext.fromSparkContext(SparkContext.getOrCreate(
                    new SparkConf().setMaster(action.getMaster()).setAppName(action.getAppName())));
        }
        return jsc;
    }

    public static SparkContext getContext(){
        if(sc ==null) {
            sc = SparkContext.getOrCreate(
                    new SparkConf().setMaster(action.getMaster()).setAppName(action.getAppName()));
        }
        return sc;
    }

    public static SparkSession getSession(){
        if(session==null){
            session=SparkSession.builder().master(action.getMaster()).appName(action.getAppName()).getOrCreate();
        }
        return session;
    }

    public static SparkSession getSession(Map configs){
        if(session==null){
            SparkSession.Builder builder=SparkSession.builder().master(action.getMaster()).appName(action.getAppName());
            for(String key : configs.keySet()){
                builder=builder.config(key,configs.get(key));
            }
            session=builder.getOrCreate();
        }
        return session;
    }
}