org.conqat.lib.commons.io.ProcessUtils Maven / Gradle / Ivy
Show all versions of teamscale-lib-commons Show documentation
/*
* Copyright (c) CQSE GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.conqat.lib.commons.io;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.conqat.lib.commons.assertion.CCSMAssert;
import org.conqat.lib.commons.string.StringUtils;
import org.conqat.lib.commons.system.SystemUtils;
/**
* Starting point to execute system processes and should be used in favor over
* {@link ProcessBuilder#start()} and {@link Runtime#exec(String)}.
*
* Takes care of reading stdout and stderr of the process in separate threads to avoid blocking. In
* addition, this class enforces a global {@link #CONCURRENT_EXTERNAL_PROCESSES limit} of system
* processes running at once in order to not exhaust the available RAM.
*/
public class ProcessUtils {
/**
* Denotes the maximum number of external processes to run at once. Defaults to unlimited.
*
* We provide the possibility to limit external processes, as they are out of the JVMs control and
* may consume a huge amount of memory. This may lead to not enough memory being available for
* Teamscale, in turn leading to OOM errors (TS-33478)
*/
private static final int CONCURRENT_EXTERNAL_PROCESSES = Integer
.getInteger("com.teamscale.external-process.maximum-concurrency", Integer.MAX_VALUE);
/**
* Provides a limitation for concurrent process executions.
*
* @see #executeWithConcurrencyLimit(ProcessBuilder, String, long, IStreamConsumer, IStreamConsumer)
*/
private static final Semaphore CONCURRENCY_LIMIT = new Semaphore(CONCURRENT_EXTERNAL_PROCESSES, true);
/** Character set used for process I/O. Important on Windows. */
private static final Charset CONSOLE_CHARSET = determineConsoleCharset();
private static Charset determineConsoleCharset() {
if (SystemUtils.isWindows()) {
try {
// Can't use CONSOLE_CHARSET here, because this is the method to determine it
Charset temporaryCharset = StandardCharsets.UTF_8;
DefaultStreamConsumer stdoutConsumer = new DefaultStreamConsumer(temporaryCharset, true);
executeWithoutConcurrencyLimit(new ProcessBuilder("chcp.com"), null, -1, temporaryCharset,
stdoutConsumer, new DefaultStreamConsumer(temporaryCharset, false));
Matcher matcher = Pattern.compile("\\d+").matcher(stdoutConsumer.getContent());
if (matcher.find()) {
@SuppressWarnings("InjectedReferences") // If the charset is unknown, we fall back to UTF-8
String charsetName = "Cp" + matcher.group();
return Charset.forName(charsetName);
}
} catch (IOException | IllegalArgumentException e) {
// Keep default.
}
}
return StandardCharsets.UTF_8;
}
/**
* Executes a process in a thread-safe way.
*
* @param completeArguments
* Array of command line arguments to start the process
*
* @return result of the execution
*/
public static ExecutionResult execute(String[] completeArguments) throws IOException {
return execute(completeArguments, null);
}
/**
* Executes a process in the given directory in a thread-safe way.
*
* @param completeArguments
* Array of command line arguments to start the process
* @param directory
* The directory the process should be executed in
*
* @return result of the execution
*/
public static ExecutionResult executeInDirectory(String[] completeArguments, File directory) throws IOException {
ProcessBuilder builder = new ProcessBuilder(completeArguments);
builder.directory(directory);
return execute(builder, null);
}
/**
* Executes a process in a thread-safe way.
*
* @param completeArguments
* Array of command line arguments to start the process
* @param input
* String that gets written to stdin
*
* @return result of the execution
*/
public static ExecutionResult execute(String[] completeArguments, String input) throws IOException {
ProcessBuilder builder = new ProcessBuilder(completeArguments);
return execute(builder, input);
}
/**
* Executes a process in a thread-safe way.
*
* @param builder
* builder that gets executed
* @return result of the execution
*/
public static ExecutionResult execute(ProcessBuilder builder) throws IOException {
return execute(builder, null);
}
/**
* Executes a process in a thread-safe way.
*
* @param builder
* builder that gets executed
* @param input
* String that gets written to stdin
* @return result of the execution
*/
public static ExecutionResult execute(ProcessBuilder builder, String input) throws IOException {
return execute(builder, input, -1);
}
/**
* Executes a process in a thread-safe way.
*
* @param builder
* builder that gets executed
* @param input
* String that gets written to stdin (may be null).
* @param timeout
* the number of seconds to wait for the process. If this runs longer, the process is
* killed. Passing a value of 0 or less makes the method wait forever (until the process
* finishes normally). To find out whether the process was killed, query
* {@link ExecutionResult#terminatedByTimeoutOrInterruption()}.
*
* @return result of the execution
*/
public static ExecutionResult execute(ProcessBuilder builder, String input, long timeout) throws IOException {
return execute(builder, input, timeout, true);
}
/**
* Executes a process in a thread-safe way.
*
* @param builder
* builder that gets executed
* @param timeout
* {@link Duration} to wait for the process. If this runs longer, the process is killed.
* Passing a value of 0 or less makes the method wait forever (until the process finishes
* normally). To find out whether the process was killed, query
* {@link ExecutionResult#terminatedByTimeoutOrInterruption()}.
*
* @return result of the execution
*/
public static ExecutionResult execute(ProcessBuilder builder, String input, Duration timeout) throws IOException {
return execute(builder, input, timeout.toSeconds());
}
/**
* Executes a process in a thread-safe way.
*
* @param builder
* builder that gets executed
* @param input
* String that gets written to stdin (may be null).
* @param timeout
* the number of seconds to wait for the process. If this runs longer, the process is
* killed. Passing a value of 0 or less makes the method wait forever (until the process
* finishes normally). To find out whether the process was killed, query
* {@link ExecutionResult#terminatedByTimeoutOrInterruption()}.
* @param collectOutputStreamContent
* boolean that indicates if the content from the stderr and stdout shall be collected.
* False is useful if the content is not needed and / or if the content would cause an
* OutOfMemoryException.
*
* @return result of the execution
*/
public static ExecutionResult execute(ProcessBuilder builder, String input, long timeout,
boolean collectOutputStreamContent) throws IOException {
DefaultStreamConsumer stdoutConsumer = new DefaultStreamConsumer(CONSOLE_CHARSET, collectOutputStreamContent);
DefaultStreamConsumer stderrConsumer = new DefaultStreamConsumer(CONSOLE_CHARSET, collectOutputStreamContent);
int exitCode = execute(builder, input, timeout, stdoutConsumer, stderrConsumer);
return new ExecutionResult(stdoutConsumer.getContent(), stderrConsumer.getContent(), exitCode, exitCode == -1);
}
/**
* Executes a process in a thread-safe way.
*
* @param builder
* builder that gets executed
* @param input
* String that gets written to stdin (may be null).
* @param timeout
* the number of seconds to wait for the process. If this runs longer, the process is
* killed. Passing a value of 0 or less makes the method wait forever (until the process
* finishes normally).
* @param stdoutConsumer
* {@link IStreamConsumer} responsible for consuming STDOUT of the process.
* @param stderrConsumer
* {@link IStreamConsumer} responsible for consuming STDERR of the process.
*
* @return exit code of the execution. {@code -1} if timeout was reached.
*/
public static int execute(ProcessBuilder builder, String input, long timeout, IStreamConsumer stdoutConsumer,
IStreamConsumer stderrConsumer) throws IOException {
return executeWithConcurrencyLimit(builder, input, timeout, stdoutConsumer, stderrConsumer);
}
/**
* Returns the full path of the given executable (e.g. 'node') as resolved by the OS. If the
* executable is not on the PATH or there is an error, {@code null} is returned.
*
* @implNote This uses the 'which' (Unix) or 'where' (Windows) utilities provided by the OS.
*/
public static @Nullable String getLocationForExecutableOnPath(String executable) {
ProcessBuilder builder = new ProcessBuilder();
if (SystemUtils.isWindows()) {
builder.command("where", executable);
} else {
builder.command("which", executable);
}
try {
ExecutionResult result = execute(builder);
if (result.returnCode == 0) {
return result.getStdout().trim();
}
} catch (IOException e) {
// Ignored
}
return null;
}
private static int executeWithConcurrencyLimit(ProcessBuilder builder, String input, long timeout,
IStreamConsumer stdoutConsumer, IStreamConsumer stderrConsumer) throws IOException {
try {
CONCURRENCY_LIMIT.acquire();
} catch (InterruptedException e) {
// Re-set interrupt flag
Thread.currentThread().interrupt();
return -1;
}
try {
return executeWithoutConcurrencyLimit(builder, input, timeout, ProcessUtils.CONSOLE_CHARSET, stdoutConsumer,
stderrConsumer);
} finally {
CONCURRENCY_LIMIT.release();
}
}
private static int executeWithoutConcurrencyLimit(ProcessBuilder builder, String input, long timeout,
Charset consoleCharset, IStreamConsumer stdoutConsumer, IStreamConsumer stderrConsumer) throws IOException {
// start process
Process process = builder.start();
// Read output streams of the process in their own threads
StreamConsumingThread stderrReader = new StreamConsumingThread(process.getErrorStream(), stderrConsumer);
StreamConsumingThread stdoutReader = new StreamConsumingThread(process.getInputStream(), stdoutConsumer);
// write input to process
if (input != null) {
Writer stdIn = new OutputStreamWriter(process.getOutputStream(), consoleCharset);
stdIn.write(input);
stdIn.close();
}
// wait for process
boolean processTimeoutOrInterruption = !waitForProcess(process, timeout);
int exitValue = -1;
if (!processTimeoutOrInterruption) {
exitValue = process.exitValue();
}
try {
// It is important to wait for the threads, so the output is
// completely stored.
stderrReader.join();
stdoutReader.join();
} catch (InterruptedException e) {
// ignore this one
}
return exitValue;
}
/**
* Waits for the process to end or terminates it if it hits the timeout. The return value indicated
* whether the process terminated (true) or was killed by the timeout (false).
*
* @param maxRuntimeSeconds
* is this is non-positive, this method waits until the process terminates (without
* timeout).
*/
private static boolean waitForProcess(Process process, long maxRuntimeSeconds) {
boolean processFinished;
try {
long timeout = maxRuntimeSeconds;
if (timeout <= 0) {
// waitFor handles zero/negative timeout by returning the current
// state immediately.
// But we want to wait indefinitely for this case and cannot use waitFor(void)
// because this may fail with IllegalThreadStateException (see TS-29795)
timeout = Long.MAX_VALUE;
}
processFinished = process.waitFor(timeout, TimeUnit.SECONDS);
} catch (InterruptedException ignored) {
// Got interrupted while waiting for the process to finish
processFinished = false;
}
if (!processFinished) {
process.destroy();
}
return processFinished;
}
/**
* Parameter object that encapsulates the result of a process execution. This object is immutable.
*/
public static class ExecutionResult {
/** Output on stdout of the process */
private final String stdout;
/** Output on stderr of the process */
private final String stderr;
/** Return code of the process */
private final int returnCode;
/**
* Whether the process was terminated by a timeout or a process interruption.
*/
private final boolean processTimeoutOrInterruption;
/** Constructor */
private ExecutionResult(String stdout, String stderr, int returnCode, boolean processTimeoutOrInterruption) {
this.stdout = stdout;
this.stderr = stderr;
this.returnCode = returnCode;
this.processTimeoutOrInterruption = processTimeoutOrInterruption;
}
/** Returns stdout. */
public String getStdout() {
return stdout;
}
/** Returns stderr. */
public String getStderr() {
return stderr;
}
/** Returns returnCode. */
public int getReturnCode() {
return returnCode;
}
/**
* Returns whether the process was terminated by a timeout or a process interruption.
*/
public boolean terminatedByTimeoutOrInterruption() {
return processTimeoutOrInterruption;
}
}
/**
* Runs the given process builder with the given input. If creating the process fails, the process
* times out or the process exits with a non-zero exit code, an exception is thrown
*
* @param builder
* The process builder to execute.
* @param input
* The input to the program.
* @param timeout
* How many seconds to wait before terminating the process. If this is less than 1, no
* timeout is applied.
* @param exceptionConstructor
* Constructs the exception that is thrown in case something goes wrong. The first
* argument is a descriptive message, the second is either null (in case of
* a timeout or non-zero exit code) or the exception that caused the failure. This
* parameter is compatible with most exception constructors, e.g.
* ConQATException::new.
* @return the execution result of the successfully run process.
* @throws T
* An exception constructed with the given constructor. Thrown if the process fails to
* run and return a zero exit code.
*/
public static ExecutionResult executeOrThrow(ProcessBuilder builder, String input,
int timeout, BiFunction exceptionConstructor) throws T {
String commandString = StringUtils.concat(builder.command(), StringUtils.SPACE);
try {
ExecutionResult result = execute(builder, input, timeout);
if (result.terminatedByTimeoutOrInterruption()) {
throw exceptionConstructor.apply("Process " + commandString + " timed out.", null);
}
if (result.getReturnCode() != 0) {
throw exceptionConstructor.apply("Process " + commandString + " failed with non-zero exit code "
+ result.getReturnCode() + ". Standard output: '" + result.getStdout() + "', Error output: '"
+ result.getStderr() + "'", null);
}
return result;
} catch (IOException e) {
throw exceptionConstructor.apply("Failed to execute " + commandString, e);
}
}
/**
* Responsible for consuming the stdout/stderr of any executed process. Delegates the actual
* consumption the a {@link IStreamConsumer}.
*/
private static class StreamConsumingThread extends Thread {
private static final Logger LOGGER = LogManager.getLogger();
private final InputStream inputStream;
private final IStreamConsumer streamConsumer;
private StreamConsumingThread(InputStream inputStream, IStreamConsumer streamConsumer) {
this.inputStream = inputStream;
this.streamConsumer = streamConsumer;
start();
}
@Override
public synchronized void run() {
try {
streamConsumer.consume(inputStream);
} catch (IOException e) {
LOGGER.warn("Encountered IOException during stream consumption", e);
}
}
}
/**
* Provides the possibility to consume STDOUT and/or STDERR of an executing process.
*
* @implSpec Implementations must be thread-safe
* @see #execute(ProcessBuilder, String, long, IStreamConsumer, IStreamConsumer)
* @see #defaultStreamConsumer(boolean)
*/
public interface IStreamConsumer {
/**
* Consumes the provided {@code stream}. This operation must block until the {@link InputStream} is
* completely consumed.
*/
void consume(InputStream stream) throws IOException;
}
/**
* Provides a default {@link IStreamConsumer} suitable for most use-cases.
*
* @see DefaultStreamConsumer
*/
public static DefaultStreamConsumer defaultStreamConsumer(boolean storeContent) {
return new DefaultStreamConsumer(CONSOLE_CHARSET, storeContent);
}
/**
* {@link IStreamConsumer} which completely reads the provided {@link InputStream}. If
* {@link #storeContent} is set, the content is stored and can be retrieved using
* {@link #getContent()}.
*/
public static final class DefaultStreamConsumer implements IStreamConsumer {
private final Charset charset;
private final boolean storeContent;
private final StringBuilder content = new StringBuilder();
private DefaultStreamConsumer(@NonNull Charset charset, boolean storeContent) {
CCSMAssert.isNotNull(charset, () -> String.format("Expected \"%s\" to be not null", "charset"));
this.storeContent = storeContent;
this.charset = charset;
}
@Override
public synchronized void consume(InputStream stream) throws IOException {
BufferedReader reader = new BufferedReader(new InputStreamReader(stream, charset));
char[] buffer = new char[1024];
int read;
while ((read = reader.read(buffer)) != -1) {
if (storeContent) {
content.append(buffer, 0, read);
}
}
}
/**
* @return The stored content. If the consumer was constructed to not store any content, the result
* is empty.
*/
public synchronized String getContent() {
return content.toString();
}
}
}