org.apache.pulsar.admin.cli.CmdFunctions Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.pulsar.admin.cli;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.commons.lang.StringUtils.isNotBlank;
import static org.apache.pulsar.common.naming.TopicName.DEFAULT_NAMESPACE;
import static org.apache.pulsar.common.naming.TopicName.PUBLIC_TENANT;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.converters.StringConverter;
import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import java.io.File;
import java.lang.reflect.Field;
import java.lang.reflect.Type;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.text.WordUtils;
import org.apache.pulsar.admin.cli.utils.CmdUtils;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import org.apache.pulsar.common.functions.ConsumerConfig;
import org.apache.pulsar.common.functions.FunctionConfig;
import org.apache.pulsar.common.functions.FunctionState;
import org.apache.pulsar.common.functions.ProducerConfig;
import org.apache.pulsar.common.functions.Resources;
import org.apache.pulsar.common.functions.UpdateOptionsImpl;
import org.apache.pulsar.common.functions.Utils;
import org.apache.pulsar.common.functions.WindowConfig;
import org.apache.pulsar.common.util.ObjectMapperFactory;
@Slf4j
@Parameters(commandDescription = "Interface for managing Pulsar Functions "
+ "(lightweight, Lambda-style compute processes that work with Pulsar)")
public class CmdFunctions extends CmdBase {
private final LocalRunner localRunner;
private final CreateFunction creater;
private final DeleteFunction deleter;
private final UpdateFunction updater;
private final GetFunction getter;
private final GetFunctionStatus functionStatus;
@Getter
private final GetFunctionStats functionStats;
private final RestartFunction restart;
private final StopFunction stop;
private final StartFunction start;
private final ListFunctions lister;
private final StateGetter stateGetter;
private final StatePutter statePutter;
private final TriggerFunction triggerer;
private final UploadFunction uploader;
private final DownloadFunction downloader;
/**
* Base command.
*/
@Getter
abstract class BaseCommand extends CliCommand {
@Override
void run() throws Exception {
try {
processArguments();
} catch (Exception e) {
System.err.println(e.getMessage());
System.err.println();
String chosenCommand = jcommander.getParsedCommand();
getUsageFormatter().usage(chosenCommand);
return;
}
runCmd();
}
void processArguments() throws Exception {}
abstract void runCmd() throws Exception;
}
/**
* Namespace level command.
*/
@Getter
abstract class NamespaceCommand extends BaseCommand {
@Parameter(names = "--tenant", description = "The tenant of a Pulsar Function")
protected String tenant;
@Parameter(names = "--namespace", description = "The namespace of a Pulsar Function")
protected String namespace;
@Override
public void processArguments() {
if (tenant == null) {
tenant = PUBLIC_TENANT;
}
if (namespace == null) {
namespace = DEFAULT_NAMESPACE;
}
}
}
/**
* Function level command.
*/
@Getter
abstract class FunctionCommand extends BaseCommand {
@Parameter(names = "--fqfn", description = "The Fully Qualified Function Name (FQFN) for the function")
protected String fqfn;
@Parameter(names = "--tenant", description = "The tenant of a Pulsar Function")
protected String tenant;
@Parameter(names = "--namespace", description = "The namespace of a Pulsar Function")
protected String namespace;
@Parameter(names = "--name", description = "The name of a Pulsar Function")
protected String functionName;
@Override
void processArguments() throws Exception {
super.processArguments();
boolean usesSetters = (null != tenant || null != namespace || null != functionName);
boolean usesFqfn = (null != fqfn);
// Throw an exception if --fqfn is set alongside any combination of --tenant, --namespace, and --name
if (usesFqfn && usesSetters) {
throw new RuntimeException("You must specify either a Fully Qualified Function Name (FQFN) "
+ "or tenant, namespace, and function name");
} else if (usesFqfn) {
// If the --fqfn flag is used, parse tenant, namespace, and name using that flag
String[] fqfnParts = fqfn.split("/");
if (fqfnParts.length != 3) {
throw new RuntimeException(
"Fully qualified function names (FQFNs) must be of the form tenant/namespace/name");
}
tenant = fqfnParts[0];
namespace = fqfnParts[1];
functionName = fqfnParts[2];
} else {
if (tenant == null) {
tenant = PUBLIC_TENANT;
}
if (namespace == null) {
namespace = DEFAULT_NAMESPACE;
}
if (null == functionName) {
throw new RuntimeException(
"You must specify a name for the function or a Fully Qualified Function Name (FQFN)");
}
}
}
}
/**
* Commands that require a function config.
*/
@Getter
abstract class FunctionDetailsCommand extends BaseCommand {
@Parameter(names = "--fqfn", description = "The Fully Qualified Function Name (FQFN) for the function"
+ " #Java, Python")
protected String fqfn;
@Parameter(names = "--tenant", description = "The tenant of a Pulsar Function #Java, Python, Go")
protected String tenant;
@Parameter(names = "--namespace", description = "The namespace of a Pulsar Function #Java, Python, Go")
protected String namespace;
@Parameter(names = "--name", description = "The name of a Pulsar Function #Java, Python, Go")
protected String functionName;
// for backwards compatibility purposes
@Parameter(names = "--className", description = "The class name of a Pulsar Function", hidden = true)
protected String deprecatedClassName;
@Parameter(names = "--classname", description = "The class name of a Pulsar Function #Java, Python")
protected String className;
@Parameter(names = { "-t", "--function-type" }, description = "The built-in Pulsar Function type")
protected String functionType;
@Parameter(names = "--jar", description = "Path to the JAR file for the function "
+ "(if the function is written in Java). It also supports URL path [http/https/file "
+ "(file protocol assumes that file already exists on worker host)/function "
+ "(package URL from packages management service)] from which worker can download the package. #Java",
listConverter = StringConverter.class)
protected String jarFile;
@Parameter(names = "--py", description = "Path to the main Python file/Python Wheel file for the function "
+ "(if the function is written in Python). It also supports URL path [http/https/file "
+ "(file protocol assumes that file already exists on worker host)/function "
+ "(package URL from packages management service)] from which worker can download the package. #Python",
listConverter = StringConverter.class)
protected String pyFile;
@Parameter(names = "--go", description = "Path to the main Go executable binary for the function "
+ "(if the function is written in Go). It also supports URL path [http/https/file "
+ "(file protocol assumes that file already exists on worker host)/function "
+ "(package URL from packages management service)] from which worker can download the package. #Go")
protected String goFile;
@Parameter(names = {"-i", "--inputs"}, description = "The input topic or "
+ "topics (multiple topics can be specified as a comma-separated list) of a Pulsar Function"
+ " #Java, Python, Go")
protected String inputs;
// for backwards compatibility purposes
@Parameter(names = "--topicsPattern", description = "TopicsPattern to consume from list of topics "
+ "under a namespace that match the pattern. [--input] and [--topic-pattern] are mutually exclusive. "
+ "Add SerDe class name for a pattern in --custom-serde-inputs (supported for java fun only)",
hidden = true)
protected String deprecatedTopicsPattern;
@Parameter(names = "--topics-pattern", description = "The topic pattern to consume from a list of topics "
+ "under a namespace that matches the pattern. [--input] and [--topics-pattern] are mutually "
+ "exclusive. Add SerDe class name for a pattern in --custom-serde-inputs (supported for java "
+ "functions only) #Java, Python")
protected String topicsPattern;
@Parameter(names = {"-o", "--output"},
description = "The output topic of a Pulsar Function (If none is specified, no output is written)"
+ " #Java, Python, Go")
protected String output;
@Parameter(names = "--producer-config", description = "The custom producer configuration (as a JSON string)"
+ " #Java")
protected String producerConfig;
// for backwards compatibility purposes
@Parameter(names = "--logTopic",
description = "The topic to which the logs of a Pulsar Function are produced", hidden = true)
protected String deprecatedLogTopic;
@Parameter(names = "--log-topic", description = "The topic to which the logs of a Pulsar Function are produced"
+ " #Java, Python, Go")
protected String logTopic;
@Parameter(names = {"-st", "--schema-type"}, description = "The builtin schema type or "
+ "custom schema class name to be used for messages output by the function #Java")
protected String schemaType = "";
// for backwards compatibility purposes
@Parameter(names = "--customSerdeInputs",
description = "The map of input topics to SerDe class names (as a JSON string)", hidden = true)
protected String deprecatedCustomSerdeInputString;
@Parameter(names = "--custom-serde-inputs",
description = "The map of input topics to SerDe class names (as a JSON string) #Java, Python")
protected String customSerdeInputString;
@Parameter(names = "--custom-schema-inputs",
description = "The map of input topics to Schema properties (as a JSON string) #Java, Python")
protected String customSchemaInputString;
@Parameter(names = "--custom-schema-outputs",
description = "The map of input topics to Schema properties (as a JSON string) #Java")
protected String customSchemaOutputString;
@Parameter(names = "--input-specs",
description = "The map of inputs to custom configuration (as a JSON string) #Java, Python, Go")
protected String inputSpecs;
@Parameter(names = "--input-type-class-name",
description = "The class name of input type class #Java, Python, Go")
protected String inputTypeClassName;
// for backwards compatibility purposes
@Parameter(names = "--outputSerdeClassName",
description = "The SerDe class to be used for messages output by the function", hidden = true)
protected String deprecatedOutputSerdeClassName;
@Parameter(names = "--output-serde-classname",
description = "The SerDe class to be used for messages output by the function #Java, Python")
protected String outputSerdeClassName;
@Parameter(names = "--output-type-class-name",
description = "The class name of output type class #Java, Python, Go")
protected String outputTypeClassName;
// for backwards compatibility purposes
@Parameter(names = "--functionConfigFile", description = "The path to a YAML config file that specifies "
+ "the configuration of a Pulsar Function", hidden = true)
protected String deprecatedFnConfigFile;
@Parameter(names = "--function-config-file",
description = "The path to a YAML config file that specifies the configuration of a Pulsar Function"
+ " #Java, Python, Go")
protected String fnConfigFile;
// for backwards compatibility purposes
@Parameter(names = "--processingGuarantees", description = "The processing guarantees (aka delivery semantics) "
+ "applied to the function", hidden = true)
protected FunctionConfig.ProcessingGuarantees deprecatedProcessingGuarantees;
@Parameter(names = "--processing-guarantees",
description = "The processing guarantees (as known as delivery semantics) applied to the function."
+ " Available values are: `ATLEAST_ONCE`, `ATMOST_ONCE`, `EFFECTIVELY_ONCE`."
+ " If it is not specified, the `ATLEAST_ONCE` delivery guarantee is used."
+ " #Java, Python, Go")
protected FunctionConfig.ProcessingGuarantees processingGuarantees;
// for backwards compatibility purposes
@Parameter(names = "--userConfig", description = "User-defined config key/values", hidden = true)
protected String deprecatedUserConfigString;
@Parameter(names = "--user-config", description = "User-defined config key/values #Java, Python, Go")
protected String userConfigString;
@Parameter(names = "--retainOrdering",
description = "Function consumes and processes messages in order", hidden = true)
protected Boolean deprecatedRetainOrdering;
@Parameter(names = "--retain-ordering", description = "Function consumes and processes messages in order #Java")
protected Boolean retainOrdering;
@Parameter(names = "--retain-key-ordering",
description = "Function consumes and processes messages in key order #Java")
protected Boolean retainKeyOrdering;
@Parameter(names = "--batch-builder", description = "BatcherBuilder provides two types of "
+ "batch construction methods, DEFAULT and KEY_BASED. The default value is: DEFAULT")
protected String batchBuilder;
@Parameter(names = "--forward-source-message-property", description = "Forwarding input message's properties "
+ "to output topic when processing (use false to disable it) #Java", arity = 1)
protected Boolean forwardSourceMessageProperty = true;
@Parameter(names = "--subs-name", description = "Pulsar source subscription name if user wants a specific "
+ "subscription-name for input-topic consumer #Java, Python, Go")
protected String subsName;
@Parameter(names = "--subs-position", description = "Pulsar source subscription position if user wants to "
+ "consume messages from the specified location #Java")
protected SubscriptionInitialPosition subsPosition;
@Parameter(names = "--skip-to-latest", description = "Whether or not the consumer skip to latest message "
+ "upon function instance restart", arity = 1)
protected Boolean skipToLatest;
@Parameter(names = "--parallelism", description = "The parallelism factor of a Pulsar Function "
+ "(i.e. the number of function instances to run) #Java")
protected Integer parallelism;
@Parameter(names = "--cpu", description = "The cpu in cores that need to be allocated "
+ "per function instance(applicable only to docker runtime) #Java(Process & K8s),Python(K8s),Go(K8s)")
protected Double cpu;
@Parameter(names = "--ram", description = "The ram in bytes that need to be allocated "
+ "per function instance(applicable only to process/docker runtime)"
+ " #Java(Process & K8s),Python(K8s),Go(K8s)")
protected Long ram;
@Parameter(names = "--disk", description = "The disk in bytes that need to be allocated "
+ "per function instance(applicable only to docker runtime) #Java(Process & K8s),Python(K8s),Go(K8s)")
protected Long disk;
// for backwards compatibility purposes
@Parameter(names = "--windowLengthCount", description = "The number of messages per window", hidden = true)
protected Integer deprecatedWindowLengthCount;
@Parameter(names = "--window-length-count", description = "The number of messages per window #Java")
protected Integer windowLengthCount;
// for backwards compatibility purposes
@Parameter(names = "--windowLengthDurationMs",
description = "The time duration of the window in milliseconds", hidden = true)
protected Long deprecatedWindowLengthDurationMs;
@Parameter(names = "--window-length-duration-ms",
description = "The time duration of the window in milliseconds #Java")
protected Long windowLengthDurationMs;
// for backwards compatibility purposes
@Parameter(names = "--slidingIntervalCount",
description = "The number of messages after which the window slides", hidden = true)
protected Integer deprecatedSlidingIntervalCount;
@Parameter(names = "--sliding-interval-count",
description = "The number of messages after which the window slides #Java")
protected Integer slidingIntervalCount;
// for backwards compatibility purposes
@Parameter(names = "--slidingIntervalDurationMs",
description = "The time duration after which the window slides", hidden = true)
protected Long deprecatedSlidingIntervalDurationMs;
@Parameter(names = "--sliding-interval-duration-ms",
description = "The time duration after which the window slides #Java")
protected Long slidingIntervalDurationMs;
// for backwards compatibility purposes
@Parameter(names = "--autoAck",
description = "Whether or not the framework acknowledges messages automatically", hidden = true)
protected Boolean deprecatedAutoAck = null;
@Parameter(names = "--auto-ack",
description = "Whether or not the framework acknowledges messages automatically"
+ " #Java, Python, Go", arity = 1)
protected Boolean autoAck;
// for backwards compatibility purposes
@Parameter(names = "--timeoutMs", description = "The message timeout in milliseconds", hidden = true)
protected Long deprecatedTimeoutMs;
@Parameter(names = "--timeout-ms", description = "The message timeout in milliseconds #Java, Python")
protected Long timeoutMs;
@Parameter(names = "--max-message-retries",
description = "How many times should we try to process a message before giving up #Java")
protected Integer maxMessageRetries;
@Parameter(names = "--custom-runtime-options", description = "A string that encodes options to "
+ "customize the runtime, see docs for configured runtime for details #Java")
protected String customRuntimeOptions;
@Parameter(names = "--secrets", description = "The map of secretName to an object that encapsulates "
+ "how the secret is fetched by the underlying secrets provider #Java, Python")
protected String secretsString;
@Parameter(names = "--dead-letter-topic",
description = "The topic where messages that are not processed successfully are sent to #Java")
protected String deadLetterTopic;
protected FunctionConfig functionConfig;
protected String userCodeFile;
private void mergeArgs() {
if (isBlank(className) && !isBlank(deprecatedClassName)) {
className = deprecatedClassName;
}
if (isBlank(topicsPattern) && !isBlank(deprecatedTopicsPattern)) {
topicsPattern = deprecatedTopicsPattern;
}
if (isBlank(logTopic) && !isBlank(deprecatedLogTopic)) {
logTopic = deprecatedLogTopic;
}
if (isBlank(outputSerdeClassName) && !isBlank(deprecatedOutputSerdeClassName)) {
outputSerdeClassName = deprecatedOutputSerdeClassName;
}
if (isBlank(customSerdeInputString) && !isBlank(deprecatedCustomSerdeInputString)) {
customSerdeInputString = deprecatedCustomSerdeInputString;
}
if (isBlank(fnConfigFile) && !isBlank(deprecatedFnConfigFile)) {
fnConfigFile = deprecatedFnConfigFile;
}
if (processingGuarantees == null && deprecatedProcessingGuarantees != null) {
processingGuarantees = deprecatedProcessingGuarantees;
}
if (isBlank(userConfigString) && !isBlank(deprecatedUserConfigString)) {
userConfigString = deprecatedUserConfigString;
}
if (retainOrdering == null && deprecatedRetainOrdering != null) {
retainOrdering = deprecatedRetainOrdering;
}
if (windowLengthCount == null && deprecatedWindowLengthCount != null) {
windowLengthCount = deprecatedWindowLengthCount;
}
if (windowLengthDurationMs == null && deprecatedWindowLengthDurationMs != null) {
windowLengthDurationMs = deprecatedWindowLengthDurationMs;
}
if (slidingIntervalCount == null && deprecatedSlidingIntervalCount != null) {
slidingIntervalCount = deprecatedSlidingIntervalCount;
}
if (slidingIntervalDurationMs == null && deprecatedSlidingIntervalDurationMs != null) {
slidingIntervalDurationMs = deprecatedSlidingIntervalDurationMs;
}
if (autoAck == null && deprecatedAutoAck != null) {
autoAck = deprecatedAutoAck;
}
if (timeoutMs == null && deprecatedTimeoutMs != null) {
timeoutMs = deprecatedTimeoutMs;
}
}
@Override
void processArguments() throws Exception {
super.processArguments();
// merge deprecated args with new args
mergeArgs();
// Initialize config builder either from a supplied YAML config file or from scratch
if (null != fnConfigFile) {
functionConfig = CmdUtils.loadConfig(fnConfigFile, FunctionConfig.class);
} else {
functionConfig = new FunctionConfig();
}
if (null != fqfn) {
parseFullyQualifiedFunctionName(fqfn, functionConfig);
} else {
if (null != tenant) {
functionConfig.setTenant(tenant);
}
if (null != namespace) {
functionConfig.setNamespace(namespace);
}
if (null != functionName) {
functionConfig.setName(functionName);
}
}
if (null != inputs) {
List inputTopics = Arrays.asList(inputs.split(","));
functionConfig.setInputs(inputTopics);
}
if (null != customSerdeInputString) {
Type type = new TypeToken
© 2015 - 2024 Weber Informatics LLC | Privacy Policy