com.clickzetta.platform.tools.IngestCmd Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of clickzetta-java Show documentation
The java SDK for clickzetta's Lakehouse
There is a newer version: 2.0.0
package com.clickzetta.platform.tools;

import com.clickzetta.platform.arrow.ArrowIGSTableMeta;
import com.clickzetta.platform.arrow.ArrowTable;
import com.clickzetta.platform.client.CZClient;
import com.clickzetta.platform.client.api.Client;
import com.clickzetta.platform.client.api.ClientBuilder;
import com.clickzetta.platform.client.api.ClientContext;
import com.clickzetta.platform.util.JsonParser;
import com.google.common.base.Preconditions;
import org.apache.commons.cli.*;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Tuple2;

import java.io.IOException;
import java.util.List;
import java.util.Locale;
import java.util.Properties;

public class IngestCmd {

  private static final Logger LOG = LoggerFactory.getLogger(IngestData.class);

  public static WrapConf parameterParser(String[] args) throws IOException {
    Options options = new Options();
    options.addOption("h", "help", false, "Print help.");
    options.addOption(Option.builder("c").longOpt("conf").type(String.class)
        .hasArg(true).required().desc("The json conf user define.")
        .build());

    options.addOption(Option.builder("s").longOpt("host").type(String.class)
        .hasArg(true).desc("The hostname/ip of the ingestion server")
        .build());

    options.addOption(Option.builder("p").longOpt("port").type(Integer.class)
        .hasArg(true).desc("The tcp listening port of ingestion server.")
        .build());

    options.addOption(Option.builder("u").longOpt("url").type(String.class)
        .hasArg(true).desc("The stream url for gateway conf.")
        .build());

    options.addOption(Option.builder("w").longOpt("workspace").type(String.class)
        .hasArg(true).desc("If your workspace is passed from script,you can specify it here.")
        .build());

    options.addOption(Option.builder("sn").longOpt("schemaName").type(String.class)
        .hasArg(true).desc("If your schemaName is passed from script,you can specify it here.")
        .build());

    options.addOption(Option.builder("tn").longOpt("tableName").type(String.class)
        .hasArg(true).desc("If your tableName is passed from script,you can specify it here.")
        .build());

    HelpFormatter hf = new HelpFormatter();
    hf.setWidth(110);
    CommandLineParser parser = new DefaultParser();
    try {
      CommandLine commandLine = parser.parse(options, args);
      if (commandLine.hasOption('h')) {
        hf.printHelp("WrapExample", options, true);
        System.exit(1);
      }

      boolean exist = commandLine.hasOption("c");
      if (exist) {
        String confPath = commandLine.getOptionValue("c");
        WrapConf conf = new JsonParser().parserWrapConf(confPath);
        conf.configurePath = confPath;

        if (commandLine.hasOption("s")) {
          conf.crlHost = commandLine.getOptionValue("s");
        }
        if (commandLine.hasOption("p")) {
          conf.crlPort = Integer.parseInt(commandLine.getOptionValue("p"));
        }
        if (commandLine.hasOption("u")) {
          conf.streamUrl = commandLine.getOptionValue("u");
        }
        if (commandLine.hasOption("w")) {
          conf.workspace = commandLine.getOptionValue("w");
        }
        if (commandLine.hasOption("sn")) {
          conf.schemaName = commandLine.getOptionValue("sn");
        }
        if (commandLine.hasOption("tn")) {
          conf.tableName = commandLine.getOptionValue("tn");
        }
        return conf;
      }
      // not reach here if all right.
      throw new ParseException("The Conf File parse fail or not set.");
    } catch (ParseException e) {
      hf.printHelp("WrapExample", options, true);
      throw new IOException(e);
    }
  }

  public static void main(String[] args) throws IOException {
    WrapConf conf = parameterParser(args);
    WrapCmd command = conf.command;
    Preconditions.checkArgument(!StringUtils.isEmpty(conf.crlHost) || !StringUtils.isEmpty(conf.streamUrl),
        "crlHost or streamUrl can not be empty");
    if (!StringUtils.isEmpty(conf.crlHost)) {
      Preconditions.checkArgument(conf.crlPort > 0, "crlPort can not less than 0");
    }
    Preconditions.checkArgument(!StringUtils.isEmpty(conf.schemaName), "schemaName can not be empty");
    Preconditions.checkArgument(!StringUtils.isEmpty(conf.tableName), "tableName can not be empty");
    Preconditions.checkArgument(command != null, "command can not be empty");
    Preconditions.checkArgument(!CollectionUtils.isEmpty(command.rpcMethod), "command list can not be empty");

    ClientBuilder builder = Client.getBuilder();
    if (!StringUtils.isEmpty(conf.crlHost) && conf.crlPort != 0) {
      builder.crlAddr(conf.crlHost, conf.crlPort);
    }
    if (conf.instanceId != null) {
      builder.instanceId(conf.getInstanceId());
    }
    if (conf.workspace != null) {
      builder.workspace(conf.getWorkspace());
    }
    if (!StringUtils.isEmpty(conf.getStreamUrl())) {
      builder.streamUrl(conf.getStreamUrl());
    }
    if (conf.getProperties() != null) {
      builder.properties(new Properties() {{
        putAll(conf.getProperties());
      }});
    }

    //
    CZClient client = (CZClient) builder.build();
    ClientContext clientContext = client.getClientContext();

    try {
      LOG.info("Start to run command: " + command.rpcMethod);
      for (String method : command.rpcMethod) {
        switch (method.toLowerCase(Locale.ROOT)) {
          case "createtablet":
            // first we call gettablemeta to init all meta info.
            ArrowIGSTableMeta arrowIGSTableMeta = client.createOrGetStreamV2(clientContext.instanceId(),
                clientContext.workspace(), conf.schemaName, conf.tableName, 1);
            ArrowTable arrowTable = new ArrowTable(arrowIGSTableMeta);
            LOG.info("get arrow table success. {}", arrowTable);
            LOG.info("method {} targetV2method {} response {}", method, "createOrGetStreamV2", "success");
            break;
          case "getmutateworkers":
            List> workerTuples = client.getRouteWorkers(clientContext.instanceId(),
                clientContext.workspace(), conf.schemaName, conf.tableName, clientContext.igsConnectMode());
            LOG.info("method {} targetV2method{} response {}", method, "getRouteWorkers", workerTuples);
            break;
          case "flushtablets":
            client.commitTablets(clientContext.instanceId(), clientContext.workspace(),
                conf.schemaName, conf.tableName, -1L, command.tabletIds);
            LOG.info("method {} response {}", method, "success");
            break;
          case "droptablets":
            client.closeStreamV2(clientContext.instanceId(), clientContext.workspace(),
                conf.schemaName, conf.tableName);
            LOG.info("method {} targetV2method {} response {}", method, "closeStreamV2", "success");
            break;
          case "dummyvalid":
            String str = "dummy-test";
            String response = client.dummyValid(clientContext.instanceId(), clientContext.workspace(), str);
            if (!str.equals(response)) {
              throw new IOException("dummyvalid response check failed. {} {}");
            }
            LOG.info("method {} response {}", method, "success");
            break;
          default:
            throw new UnsupportedOperationException("Unsupported method type: " + method);
        }
      }
      LOG.info("End to run command.");
    } finally {
      client.close();
    }
  }
}