es.accenture.flink.Job.JobBatchSink Maven / Gradle / Ivy

Go to download
package es.accenture.flink.Job;

import es.accenture.flink.Sink.KuduOutputFormat;
import es.accenture.flink.Utils.RowSerializable;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;

/**
 * A job which reads a line of elements, split the line by spaces to generate the rows,
 * and writes the result on another Kudu database.
 * (This example split the line by spaces and concatenate string 'NEW' to each row)
 */
public class JobBatchSink {

    public static void main(String[] args) throws Exception {

        /********Only for test, delete once finished*******/
        args[0] = "TableSink";
        args[1] = "create";
        args[2] = "localhost";
        /**************************************************/

        if(args.length!=3){
            System.out.println( "JobBatchSink params: [TableWrite] [Mode] [Master Address]\n");
            return;
        }

        final String TABLE_NAME = args[0];
        final Integer MODE;
        if (args[1].equalsIgnoreCase("create")){
            MODE = KuduOutputFormat.CREATE;
        } else if (args[1].equalsIgnoreCase("append")){
            MODE = KuduOutputFormat.APPEND;
        } else if (args[1].equalsIgnoreCase("override")){
            MODE = KuduOutputFormat.OVERRIDE;
        } else {
            System.out.println("Error in param [Mode]. Only create, append or override allowed.");
            return;
        }
        final String KUDU_MASTER = args[2];

        System.out.println("-----------------------------------------------");
        System.out.println("1. Creates a new data set from elements.\n" +
                "2. Concat string 'NEW' to each field.\n" +
                "3. Write back in a new Kudu DB (" + args[1] + ").");
        System.out.println("-----------------------------------------------");

        // Schema of the table to create
        String[] columnNames = new String[3];
        columnNames[0] = "col1";
        columnNames[1] = "col2";
        columnNames[2] = "col3";

        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

        DataSet input = env.fromElements("value1 value2 value3");

        DataSet out = input.map(new MyMapFunction());

        out.output(new KuduOutputFormat(KUDU_MASTER, TABLE_NAME, columnNames, MODE));

        env.execute();

    }

    /**
     * Map function which receives a String, splits it, and creates as many row as word has the string
     * This row contains two fields, first field is a serial generated automatically starting in 0,
     * second field is the substring generated by the split function.
     *
     */
    private static class MyMapFunction implements MapFunction {
        @Override
        public RowSerializable map(String inputs) throws Exception {
            RowSerializable r = new RowSerializable(3);
            Integer i = 0;
            for (String s : inputs.split(" ")) {
                r.setField(i, s.concat("NEW"));
                i++;
            }
            return r;
        }
    }

}