com.clickzetta.platform.example.ParallelBulkLoadExample Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of clickzetta-java Show documentation

The java SDK for clickzetta's Lakehouse

There is a newer version: 2.0.0

package com.clickzetta.platform.example;

import com.clickzetta.platform.client.api.BulkLoadCommitOptions;
import com.clickzetta.platform.client.api.BulkLoadOperation;
import com.clickzetta.platform.client.api.BulkLoadOptions;
import com.clickzetta.platform.client.api.BulkLoadState;
import com.clickzetta.platform.client.api.BulkLoadStream;
import com.clickzetta.platform.client.api.BulkLoadWriter;
import com.clickzetta.platform.client.api.Client;
import com.clickzetta.platform.client.api.Row;

import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class ParallelBulkLoadExample {

  public static void main(String[] args) throws IOException, InterruptedException {

    BulkLoadOptions bulkLoadOptions = BulkLoadOptions.newBuilder()
      .withOperation(BulkLoadOperation.APPEND)
      .build();

    Client client = Client.getBuilder()
      .authenticate(true)
      .streamUrl("xxx")
      .build();

    BulkLoadStream stream = client.createBulkLoadStream(
      "schemaName", "tableName", bulkLoadOptions);
    String streamId = stream.getStreamId();

    ExecutorService executor = Executors.newFixedThreadPool(4);

    for (int i = 0; i < 4; ++i) {
      final int partitionId = i;
      executor.submit(new Runnable() {
        @Override
        public void run() {
          BulkLoadWriter writer = null;
          try {
            writer = stream.openWriter(partitionId);
            Row row = writer.createRow();
            for (int t = 0; t < 100; t++) {
              row.setValue("col1", t);
              row.setValue("col2", null);
              row.setValue("col3", "test" + t);
              writer.write(row);
            }
            writer.close();
          } catch (IOException e) {
            throw new RuntimeException(e);
          }
        }
      });
    }

    stream.commit(BulkLoadCommitOptions.newBuilder().withWorkspace("ws").withVc("vc").build());

    boolean successful = false;
    long remainingRetries = 10;
    do {
      BulkLoadState state = stream.getStreamState();
      switch (stream.getStreamState()) {
        case COMMIT_SUCCESS:
          successful = true;
          break;
        case COMMIT_FAILED:
          successful = false;
          break;
        default:
          remainingRetries--;
      }
      Thread.sleep(5_000);
      remainingRetries--;
    } while (remainingRetries > 0);

    if (!successful) {
      System.out.println("Bulk load failed: " + stream.getSqlErrorMsg());
    }

    stream.close();
    client.close();
  }

}