com.clickzetta.platform.example.UpsertExample Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of clickzetta-java Show documentation
The java SDK for clickzetta's Lakehouse
There is a newer version: 2.0.0
package com.clickzetta.platform.example;

import com.clickzetta.platform.client.api.Client;
import com.clickzetta.platform.client.api.Options;
import com.clickzetta.platform.client.api.Row;
import com.clickzetta.platform.client.api.Stream;
import com.clickzetta.platform.client.api.FlushMode;
import io.grpc.CallOptions;
import java.io.*;
import java.math.BigDecimal;
import java.sql.Date;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

public class UpsertExample {
        //example:java -cp /tmp/igs-java-sdk-0.0.1-shaded.jar -Dlog4j.configuration=file:///tmp/log4j.properties com.clickzetta.platform.example.UpsertExample 172.24.124.5 10086 lakehouse_forpd_tpch2g test_2 /tmp/11 , sync 4 10 col1:string,col2:int,col3:double,col4:float,col5:string
    public static void main(String[] args) throws IOException {
        String host = args[0];
        Integer port = Integer.parseInt(args[1]);
        String dbname = args[2];
        String tablename = args[3];
        String fileName = args[4];
        String splitsign = args[5];
        String type = args[6];
        int tabletnum = Integer.parseInt(args[7]);
        int BufferLinesNum = Integer.parseInt(args[8]);
        String columandtype = args[9];
        String[] strs = columandtype.split(",");
        Map map = new LinkedHashMap<>();
        for (String string : strs) {
            String key = string.split(":")[0].trim();
            String value = string.split(":")[1].trim();
            map.put(key,value);
        }
        Client client = Client.getBuilder()
                .authenticate(false)
                .crlAddr(host, port)
                .build();
        Options options;
        if (type.equals("sync")){
            options = Options.builder()
                .withFlushMode(FlushMode.AUTO_FLUSH_SYNC)
                .withFlushInterval(10000 * 1000)
                .withMutationBufferMaxNum(20)
                .withMutationBufferLinesNum(BufferLinesNum)
                .withRpcCallOptions(CallOptions.DEFAULT
                    .withMaxOutboundMessageSize(8 * 1024 * 1024)
                    .withMaxInboundMessageSize(8 * 1024 * 1024))
                .build();
        }else {
            options = Options.builder()
                .withFlushMode(FlushMode.AUTO_FLUSH_BACKGROUND)
                .withFlushInterval(10000 * 1000)
                .withMutationBufferMaxNum(20)
                .withMutationBufferLinesNum(BufferLinesNum)
                .withRpcCallOptions(CallOptions.DEFAULT
                    .withMaxOutboundMessageSize(8 * 1024 * 1024)
                    .withMaxInboundMessageSize(8 * 1024 * 1024))
                .build();
        }

        Stream stream = client.createStream(dbname, tablename, options);

        BufferedReader br;
        try {
            br = new BufferedReader(new FileReader(fileName));
            String line;
            while((line = br.readLine()) != null){
                String[] split= line.split(splitsign);
                Row row = stream.createUpsertRow();
                int num = 0;
                for (Map.Entry entry : map.entrySet()) {
                    if (split.length < num + 1) {
                        row.setValue(entry.getKey(), changeForType(entry.getValue(), ""));
                    } else {
                        row.setValue(entry.getKey(), changeForType(entry.getValue(), split[num]));
                    }
                    num++;
                }
                stream.apply(row);
            }
            stream.close();
            client.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
        }
    }

    static Object changeForType(String type, String value){
        switch (type.toLowerCase()){
            case "int":
            case "integer":
                return value.equals("NULL") ? null : Integer.parseInt(value);
            case "bigint":
                return value.equals("NULL") ? null : Long.parseLong(value);
            case "decimal":
                return value.equals("NULL") ? null : new BigDecimal(value);
            case "double":
                return value.equals("NULL") ? null : Double.parseDouble(value);
            case "float":
                return value.equals("NULL") ? null : Float.parseFloat(value);
            case "string":
            case "varchar":
            case "char":
                return value.equals("NULL") ? null : value;
            case "date":
                return value.equals("NULL") ? null : Date.valueOf(value);
            case "datetime":
                SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                try {
                    if (value.equals("NULL")){
                        return null;
                    }
                    java.util.Date date = simpleDateFormat.parse(value);
                    return date.getTime();
                } catch (ParseException e) {
                    e.printStackTrace();
                }
            default:
                throw new ClassCastException(type+"字段类型不支持");
        }
    }
    public static List fileReadToArray(String filePath) {
        // 使用ArrayList来存储每行读取到的字符串
        List arrayList = new ArrayList<>();
        try {
            BufferedReader b = new BufferedReader(new FileReader(filePath));
            String s;
            //判断是否到一行字符串
            while ((s = b.readLine()) != null) {
                arrayList.add(s);
            }
            b.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return arrayList;
    }
}