Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.apache.pulsar.functions.runtime.thread.ThreadRuntime Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.pulsar.functions.runtime.thread;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.broker.PulsarServerException;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.api.ClientBuilder;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.common.nar.FileUtils;
import org.apache.pulsar.functions.instance.InstanceConfig;
import org.apache.pulsar.functions.instance.InstanceUtils;
import org.apache.pulsar.functions.instance.JavaInstanceRunnable;
import org.apache.pulsar.functions.instance.stats.FunctionCollectorRegistry;
import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.InstanceCommunication;
import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatus;
import org.apache.pulsar.functions.runtime.Runtime;
import org.apache.pulsar.functions.secretsprovider.SecretsProvider;
import org.apache.pulsar.functions.utils.FunctionCommon;
import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManager;
import org.apache.pulsar.functions.worker.ConnectorsManager;
import org.apache.pulsar.functions.worker.FunctionsManager;
/**
* A function container implemented using java thread.
*/
@Slf4j
public class ThreadRuntime implements Runtime {
// The thread that invokes the function
private Thread fnThread;
private static final int THREAD_SHUTDOWN_TIMEOUT_MILLIS = 10_000;
@Getter
private final InstanceConfig instanceConfig;
private JavaInstanceRunnable javaInstanceRunnable;
private final ThreadGroup threadGroup;
private final FunctionCacheManager fnCache;
private final String jarFile;
private final String transformFunctionFile;
private final ClientBuilder clientBuilder;
private final PulsarClient pulsarClient;
private final PulsarAdmin pulsarAdmin;
private final String stateStorageImplClass;
private final String stateStorageServiceUrl;
private final SecretsProvider secretsProvider;
private final FunctionCollectorRegistry collectorRegistry;
private final String narExtractionDirectory;
private final Optional connectorsManager;
private final Optional functionsManager;
ThreadRuntime(InstanceConfig instanceConfig,
FunctionCacheManager fnCache,
ThreadGroup threadGroup,
String jarFile,
String transformFunctionFile,
PulsarClient client,
ClientBuilder clientBuilder,
PulsarAdmin pulsarAdmin,
String stateStorageImplClass,
String stateStorageServiceUrl,
SecretsProvider secretsProvider,
FunctionCollectorRegistry collectorRegistry,
String narExtractionDirectory,
Optional connectorsManager,
Optional functionsManager) {
this.instanceConfig = instanceConfig;
if (instanceConfig.getFunctionDetails().getRuntime() != Function.FunctionDetails.Runtime.JAVA) {
throw new RuntimeException("Thread Container only supports Java Runtime");
}
this.threadGroup = threadGroup;
this.fnCache = fnCache;
this.jarFile = jarFile;
this.transformFunctionFile = transformFunctionFile;
this.clientBuilder = clientBuilder;
this.pulsarClient = client;
this.pulsarAdmin = pulsarAdmin;
this.stateStorageImplClass = stateStorageImplClass;
this.stateStorageServiceUrl = stateStorageServiceUrl;
this.secretsProvider = secretsProvider;
this.collectorRegistry = collectorRegistry;
this.narExtractionDirectory = narExtractionDirectory;
this.connectorsManager = connectorsManager;
this.functionsManager = functionsManager;
}
private static ClassLoader getFunctionClassLoader(InstanceConfig instanceConfig,
String functionId,
String jarFile,
String narExtractionDirectory,
FunctionCacheManager fnCache,
Optional connectorsManager,
Optional functionsManager,
Function.FunctionDetails.ComponentType componentType)
throws Exception {
if (FunctionCommon.isFunctionCodeBuiltin(instanceConfig.getFunctionDetails(), componentType)) {
if (componentType == Function.FunctionDetails.ComponentType.FUNCTION && functionsManager.isPresent()) {
return functionsManager.get()
.getFunction(instanceConfig.getFunctionDetails().getBuiltin())
.getFunctionPackage().getClassLoader();
}
if (componentType == Function.FunctionDetails.ComponentType.SOURCE && connectorsManager.isPresent()) {
return connectorsManager.get()
.getConnector(instanceConfig.getFunctionDetails().getSource().getBuiltin())
.getConnectorFunctionPackage().getClassLoader();
}
if (componentType == Function.FunctionDetails.ComponentType.SINK && connectorsManager.isPresent()) {
return connectorsManager.get()
.getConnector(instanceConfig.getFunctionDetails().getSink().getBuiltin())
.getConnectorFunctionPackage().getClassLoader();
}
}
return loadJars(jarFile, instanceConfig, functionId, instanceConfig.getFunctionDetails().getName(),
narExtractionDirectory, fnCache);
}
public static ClassLoader loadJars(String jarFile,
InstanceConfig instanceConfig,
String functionId,
String functionName,
String narExtractionDirectory,
FunctionCacheManager fnCache) throws Exception {
if (jarFile == null) {
return Thread.currentThread().getContextClassLoader();
}
ClassLoader fnClassLoader;
boolean loadedAsNar = false;
if (FileUtils.mayBeANarArchive(new File(jarFile))) {
try {
log.info("Trying Loading file as NAR file: {}", jarFile);
// Let's first try to treat it as a nar archive
fnCache.registerFunctionInstanceWithArchive(
functionId,
instanceConfig.getInstanceName(),
jarFile, narExtractionDirectory);
loadedAsNar = true;
} catch (FileNotFoundException e) {
// this is usually like
// java.io.FileNotFoundException: /tmp/pulsar-nar/xxx.jar-unpacked/xxxxx/META-INF/MANIFEST.MF'
log.error("The file {} does not look like a .nar file {}", jarFile, e.toString());
}
}
if (!loadedAsNar) {
log.info("Load file as simple JAR file: {}", jarFile);
// create the function class loader
fnCache.registerFunctionInstance(
functionId,
instanceConfig.getInstanceName(),
Arrays.asList(jarFile),
Collections.emptyList());
}
log.info(
"Initialize function class loader for function {} at function cache manager, functionClassLoader: {}",
functionName, fnCache.getClassLoader(functionId));
fnClassLoader = fnCache.getClassLoader(functionId);
if (null == fnClassLoader) {
throw new Exception("No function class loader available.");
}
return fnClassLoader;
}
/**
* The core logic that initialize the thread container and executes the function.
*/
@Override
public void start() throws Exception {
// extract class loader for function
ClassLoader functionClassLoader =
getFunctionClassLoader(instanceConfig, instanceConfig.getFunctionId(), jarFile, narExtractionDirectory,
fnCache, connectorsManager, functionsManager,
InstanceUtils.calculateSubjectType(instanceConfig.getFunctionDetails()));
ClassLoader transformFunctionClassLoader = transformFunctionFile == null ? null : getFunctionClassLoader(
instanceConfig, instanceConfig.getTransformFunctionId(), transformFunctionFile, narExtractionDirectory,
fnCache, connectorsManager, functionsManager, Function.FunctionDetails.ComponentType.FUNCTION);
// re-initialize JavaInstanceRunnable so that variables in constructor can be re-initialized
this.javaInstanceRunnable = new JavaInstanceRunnable(
instanceConfig,
clientBuilder,
pulsarClient,
pulsarAdmin,
stateStorageImplClass,
stateStorageServiceUrl,
secretsProvider,
collectorRegistry,
functionClassLoader,
transformFunctionClassLoader);
log.info("ThreadContainer starting function with instanceId {} functionId {} namespace {}",
instanceConfig.getInstanceId(),
instanceConfig.getFunctionId(),
instanceConfig.getFunctionDetails().getNamespace());
this.fnThread = new Thread(threadGroup, javaInstanceRunnable,
String.format("%s-%s",
FunctionCommon.getFullyQualifiedName(instanceConfig.getFunctionDetails()),
instanceConfig.getInstanceId()));
this.fnThread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
@Override
public void uncaughtException(Thread t, Throwable e) {
log.error("Uncaught exception in thread {}", t, e);
}
});
this.fnThread.start();
}
@Override
public void join() throws Exception {
if (this.fnThread != null) {
this.fnThread.join();
}
}
@Override
public void stop() {
if (fnThread != null) {
// interrupt the instance thread
fnThread.interrupt();
try {
// If the instance thread doesn't respond within some time, attempt to
// kill the thread
fnThread.join(THREAD_SHUTDOWN_TIMEOUT_MILLIS, 0);
if (fnThread.isAlive()) {
log.warn("The function instance thread is still alive after {} milliseconds. "
+ "Giving up waiting and moving forward to close function.",
THREAD_SHUTDOWN_TIMEOUT_MILLIS);
}
} catch (InterruptedException e) {
// ignore this
}
// make sure JavaInstanceRunnable is closed
this.javaInstanceRunnable.close();
log.info("Unloading JAR files for instanceId {} functionId {} namespace {}",
instanceConfig.getInstanceId(),
instanceConfig.getFunctionId(),
instanceConfig.getFunctionDetails().getNamespace());
// once the thread quits, clean up the instance
fnCache.unregisterFunctionInstance(
instanceConfig.getFunctionId(),
instanceConfig.getInstanceName());
}
}
@Override
public CompletableFuture getFunctionStatus(int instanceId) {
CompletableFuture statsFuture = new CompletableFuture<>();
if (!isAlive()) {
FunctionStatus.Builder functionStatusBuilder = FunctionStatus.newBuilder();
functionStatusBuilder.setRunning(false);
Throwable ex = getDeathException();
if (ex != null && ex.getMessage() != null) {
functionStatusBuilder.setFailureException(ex.getMessage());
}
statsFuture.complete(functionStatusBuilder.build());
return statsFuture;
}
FunctionStatus.Builder functionStatusBuilder = javaInstanceRunnable.getFunctionStatus();
functionStatusBuilder.setRunning(true);
statsFuture.complete(functionStatusBuilder.build());
return statsFuture;
}
@Override
public CompletableFuture getAndResetMetrics() {
return CompletableFuture.completedFuture(javaInstanceRunnable.getAndResetMetrics());
}
@Override
public CompletableFuture getMetrics(int instanceId) {
return CompletableFuture.completedFuture(javaInstanceRunnable.getMetrics());
}
@Override
public String getPrometheusMetrics() throws IOException {
if (javaInstanceRunnable == null) {
throw new PulsarServerException("javaInstanceRunnable is not initialized");
}
return javaInstanceRunnable.getStatsAsString();
}
@Override
public CompletableFuture resetMetrics() {
javaInstanceRunnable.resetMetrics();
return CompletableFuture.completedFuture(null);
}
@Override
public boolean isAlive() {
if (this.fnThread != null) {
return this.fnThread.isAlive();
} else {
return false;
}
}
@Override
public Throwable getDeathException() {
if (isAlive()) {
return null;
} else if (null != javaInstanceRunnable) {
return javaInstanceRunnable.getDeathException();
} else {
return null;
}
}
}
