io.trino.tests.product.launcher.cli.SuiteRun Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of trino-product-tests-launcher Show documentation
Show all versions of trino-product-tests-launcher Show documentation
Trino - Product tests launcher
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.tests.product.launcher.cli;
import com.google.common.base.Joiner;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import com.google.inject.Module;
import io.airlift.log.Logger;
import io.airlift.units.Duration;
import io.trino.jvm.Threads;
import io.trino.tests.product.launcher.Extensions;
import io.trino.tests.product.launcher.env.EnvironmentConfig;
import io.trino.tests.product.launcher.env.EnvironmentConfigFactory;
import io.trino.tests.product.launcher.env.EnvironmentFactory;
import io.trino.tests.product.launcher.env.EnvironmentModule;
import io.trino.tests.product.launcher.env.EnvironmentOptions;
import io.trino.tests.product.launcher.env.jdk.JdkProviderFactory;
import io.trino.tests.product.launcher.suite.Suite;
import io.trino.tests.product.launcher.suite.SuiteFactory;
import io.trino.tests.product.launcher.suite.SuiteModule;
import io.trino.tests.product.launcher.suite.SuiteTestRun;
import io.trino.tests.product.launcher.util.ConsoleTable;
import picocli.CommandLine.Command;
import picocli.CommandLine.ExitCode;
import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
import java.io.File;
import java.io.OutputStream;
import java.io.PrintStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import static com.google.common.base.MoreObjects.toStringHelper;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.util.concurrent.Futures.immediateFuture;
import static io.airlift.concurrent.Threads.daemonThreadsNamed;
import static io.airlift.units.Duration.nanosSince;
import static io.airlift.units.Duration.succinctNanos;
import static io.trino.tests.product.launcher.cli.SuiteRun.TestRunResult.HEADER;
import static io.trino.tests.product.launcher.cli.TestRun.Execution.ENVIRONMENT_SKIPPED_EXIT_CODE;
import static java.lang.Math.max;
import static java.lang.String.format;
import static java.lang.management.ManagementFactory.getThreadMXBean;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.Executors.newScheduledThreadPool;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.joining;
@Command(
name = "run",
description = "Run suite tests",
usageHelpAutoWidth = true)
public class SuiteRun
extends LauncherCommand
{
private static final Logger log = Logger.get(SuiteRun.class);
private static final ScheduledExecutorService diagnosticExecutor = newScheduledThreadPool(2, daemonThreadsNamed("TestRun-diagnostic"));
@Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message and exit")
public boolean usageHelpRequested;
@Mixin
public SuiteRunOptions suiteRunOptions = new SuiteRunOptions();
@Mixin
public EnvironmentOptions environmentOptions = new EnvironmentOptions();
public SuiteRun(OutputStream outputStream, Extensions extensions)
{
super(SuiteRun.Execution.class, outputStream, extensions);
}
@Override
List getCommandModules()
{
return ImmutableList.of(
new SuiteModule(extensions.getAdditionalSuites()),
new EnvironmentModule(environmentOptions, extensions.getAdditionalEnvironments()),
suiteRunOptions.toModule());
}
public static class SuiteRunOptions
{
private static final String DEFAULT_VALUE = "(default: ${DEFAULT-VALUE})";
@Option(names = "--suite", paramLabel = "", description = "Name of the suite(s) to run (comma separated)", required = true, split = ",")
public List suites;
@Option(names = "--test-jar", paramLabel = "", description = "Path to test JAR " + DEFAULT_VALUE, defaultValue = "${product-tests.module}/target/${product-tests.name}-${project.version}-executable.jar")
public File testJar;
@Option(names = "--cli-executable", paramLabel = "", description = "Path to CLI executable " + DEFAULT_VALUE, defaultValue = "${cli.bin}")
public File cliJar;
@Option(names = "--impacted-features", paramLabel = "", description = "Only run tests in environments with these features " + DEFAULT_VALUE)
public Optional impactedFeatures;
@Option(names = "--logs-dir", paramLabel = "", description = "Location of the exported logs directory " + DEFAULT_VALUE)
public Optional logsDirBase;
@Option(names = "--timeout", paramLabel = "", description = "Maximum duration of suite execution " + DEFAULT_VALUE, defaultValue = "999d")
public Duration timeout;
public Module toModule()
{
return binder -> binder.bind(SuiteRunOptions.class).toInstance(this);
}
}
public static class Execution
implements Callable
{
// TODO do not store mutable state
private final SuiteRunOptions suiteRunOptions;
// TODO do not store mutable state
private final EnvironmentOptions environmentOptions;
private final SuiteFactory suiteFactory;
private final JdkProviderFactory jdkProviderFactory;
private final EnvironmentFactory environmentFactory;
private final EnvironmentConfigFactory configFactory;
private final PrintStream printStream;
private final long suiteStartTime;
@Inject
public Execution(
SuiteRunOptions suiteRunOptions,
EnvironmentOptions environmentOptions,
SuiteFactory suiteFactory,
JdkProviderFactory jdkProviderFactory,
EnvironmentFactory environmentFactory,
EnvironmentConfigFactory configFactory,
PrintStream printStream)
{
this.suiteRunOptions = requireNonNull(suiteRunOptions, "suiteRunOptions is null");
this.environmentOptions = requireNonNull(environmentOptions, "environmentOptions is null");
this.suiteFactory = requireNonNull(suiteFactory, "suiteFactory is null");
this.jdkProviderFactory = requireNonNull(jdkProviderFactory, "jdkProviderFactory is null");
this.environmentFactory = requireNonNull(environmentFactory, "environmentFactory is null");
this.configFactory = requireNonNull(configFactory, "configFactory is null");
this.printStream = requireNonNull(printStream, "printStream is null");
this.suiteStartTime = System.nanoTime();
}
@Override
public Integer call()
{
AtomicBoolean finished = new AtomicBoolean();
Future diagnosticFlow = immediateFuture(null);
try {
long timeoutMillis = suiteRunOptions.timeout.toMillis();
long marginMillis = SECONDS.toMillis(10);
if (timeoutMillis < marginMillis) {
log.error("Unsupported small timeout value: %s ms", timeoutMillis);
return ExitCode.SOFTWARE;
}
diagnosticFlow = diagnosticExecutor.schedule(() -> reportSuspectedTimeout(finished), timeoutMillis - marginMillis, MILLISECONDS);
return runSuites();
}
finally {
finished.set(true);
diagnosticFlow.cancel(true);
}
}
private int runSuites()
{
List suiteNames = requireNonNull(suiteRunOptions.suites, "suiteRunOptions.suites is null");
EnvironmentConfig environmentConfig = configFactory.getConfig(environmentOptions.config);
ImmutableList.Builder suiteResults = ImmutableList.builder();
suiteNames.forEach(suiteName -> {
Suite suite = suiteFactory.getSuite(suiteName);
List suiteTestRuns = suite.getTestRuns(environmentConfig).stream()
.map(suiteRun -> suiteRun.withConfigApplied(environmentConfig))
.collect(toImmutableList());
log.info("Running suite '%s' with config '%s' and test runs:\n%s",
suiteName,
environmentConfig.getConfigName(),
formatSuiteTestRuns(suiteTestRuns));
List testRunsResults = suiteTestRuns.stream()
.map(testRun -> executeSuiteTestRun(suiteName, testRun, environmentConfig))
.collect(toImmutableList());
suiteResults.addAll(testRunsResults);
});
List results = suiteResults.build();
printTestRunsSummary(results);
if (getFailedCount(results) > 0) {
return ExitCode.SOFTWARE;
}
return ExitCode.OK;
}
private String formatSuiteTestRuns(List suiteTestRuns)
{
Joiner joiner = Joiner.on("\n");
ConsoleTable table = new ConsoleTable();
table.addHeader("environment", "options", "groups", "excluded groups", "tests", "excluded tests");
suiteTestRuns.forEach(testRun -> table.addRow(
testRun.getEnvironmentName(),
testRun.getExtraOptions(),
joiner.join(testRun.getGroups()),
joiner.join(testRun.getExcludedGroups()),
joiner.join(testRun.getTests()),
joiner.join(testRun.getExcludedTests())).addSeparator());
return table.render();
}
private void printTestRunsSummary(List results)
{
ConsoleTable table = new ConsoleTable();
table.addHeader(HEADER.toArray());
results.forEach(result -> table.addRow(result.toRow()));
table.addSeparator();
log.info("Suite tests results:\n%s", table.render());
}
private static long getFailedCount(List results)
{
return results.stream()
.filter(TestRunResult::hasFailed)
.count();
}
public TestRunResult executeSuiteTestRun(String suiteName, SuiteTestRun suiteTestRun, EnvironmentConfig environmentConfig)
{
String runId = generateRandomRunId();
TestRun.TestRunOptions testRunOptions = createTestRunOptions(runId, suiteName, suiteTestRun, environmentConfig, suiteRunOptions.logsDirBase);
if (testRunOptions.timeout.toMillis() == 0) {
return new TestRunResult(
suiteName,
runId,
suiteTestRun,
environmentConfig,
new Duration(0, MILLISECONDS),
OptionalInt.empty(),
Optional.of(new Exception("Test execution not attempted because suite total running time limit was exhausted")));
}
log.info("Starting test run %s with config %s and remaining timeout %s", suiteTestRun, environmentConfig, testRunOptions.timeout);
log.info("Execute this test run using:\n%s test run %s", environmentOptions.launcherBin, OptionsPrinter.format(environmentOptions, testRunOptions));
Stopwatch stopwatch = Stopwatch.createStarted();
try {
int exitCode = runTest(runId, environmentConfig, testRunOptions);
Optional exception = Optional.empty();
if (exitCode != 0 && exitCode != ENVIRONMENT_SKIPPED_EXIT_CODE) {
exception = Optional.of(new RuntimeException(format("Tests exited with code %d", exitCode)));
}
return new TestRunResult(suiteName, runId, suiteTestRun, environmentConfig, succinctNanos(stopwatch.stop().elapsed(NANOSECONDS)), OptionalInt.of(exitCode), exception);
}
catch (RuntimeException e) {
return new TestRunResult(suiteName, runId, suiteTestRun, environmentConfig, succinctNanos(stopwatch.stop().elapsed(NANOSECONDS)), OptionalInt.empty(), Optional.of(e));
}
}
private static String generateRandomRunId()
{
return UUID.randomUUID().toString().replace("-", "");
}
private int runTest(String runId, EnvironmentConfig environmentConfig, TestRun.TestRunOptions testRunOptions)
{
TestRun.Execution execution = new TestRun.Execution(environmentFactory, jdkProviderFactory, environmentOptions, environmentConfig, testRunOptions, printStream);
log.info("Test run %s started", runId);
int exitCode = execution.call();
log.info("Test run %s finished", runId);
return exitCode;
}
private TestRun.TestRunOptions createTestRunOptions(String runId, String suiteName, SuiteTestRun suiteTestRun, EnvironmentConfig environmentConfig, Optional logsDirBase)
{
TestRun.TestRunOptions testRunOptions = new TestRun.TestRunOptions();
testRunOptions.environment = suiteTestRun.getEnvironmentName();
testRunOptions.extraOptions = suiteTestRun.getExtraOptions();
testRunOptions.testArguments = suiteTestRun.getTemptoRunArguments();
testRunOptions.testJar = suiteRunOptions.testJar;
testRunOptions.cliJar = suiteRunOptions.cliJar;
testRunOptions.impactedFeatures = suiteRunOptions.impactedFeatures;
String suiteRunId = suiteRunId(runId, suiteName, suiteTestRun, environmentConfig);
testRunOptions.reportsDir = Paths.get("testing/trino-product-tests/target/reports/" + suiteRunId);
testRunOptions.logsDirBase = logsDirBase.map(dir -> dir.resolve(suiteRunId));
// Calculate remaining time
testRunOptions.timeout = remainingTimeout();
return testRunOptions;
}
private Duration remainingTimeout()
{
long remainingNanos = suiteRunOptions.timeout.roundTo(NANOSECONDS) - nanosSince(suiteStartTime).roundTo(NANOSECONDS);
return succinctNanos(max(remainingNanos, 0));
}
private void reportSuspectedTimeout(AtomicBoolean finished)
{
if (finished.get()) {
return;
}
// Test may not complete on time because they take too long (legitimate from Launcher's perspective), or because Launcher hangs.
// In the latter case it's worth reporting Launcher's threads.
// Log something immediately before getting thread information
log.warn("Test execution is not finished yet, a deadlock or hang is suspected. Thread dump will follow.");
log.warn(
"Full Thread Dump:\n%s",
Arrays.stream(getThreadMXBean().dumpAllThreads(true, true))
.map(Threads::fullToString)
.collect(joining("\n")));
}
private static String suiteRunId(String runId, String suiteName, SuiteTestRun suiteTestRun, EnvironmentConfig environmentConfig)
{
return format("%s-%s-%s-%s", suiteName, suiteTestRun.getEnvironmentName(), environmentConfig.getConfigName(), runId);
}
}
static class TestRunResult
{
public static final List HEADER = List.of("id", "suite", "environment", "config", "options", "status", "elapsed", "error");
private final String runId;
private final SuiteTestRun suiteRun;
private final EnvironmentConfig environmentConfig;
private final Duration duration;
private final Optional throwable;
private final OptionalInt exitCode;
private final String suiteName;
public TestRunResult(String suiteName, String runId, SuiteTestRun suiteRun, EnvironmentConfig environmentConfig, Duration duration, OptionalInt exitCode, Optional throwable)
{
this.suiteName = suiteName;
this.runId = runId;
this.suiteRun = requireNonNull(suiteRun, "suiteRun is null");
this.environmentConfig = requireNonNull(environmentConfig, "environmentConfig is null");
this.duration = requireNonNull(duration, "duration is null");
this.exitCode = exitCode;
this.throwable = requireNonNull(throwable, "throwable is null");
}
public boolean hasFailed()
{
return this.throwable.isPresent();
}
public boolean wasSkipped()
{
return this.exitCode.orElse(0) == ENVIRONMENT_SKIPPED_EXIT_CODE;
}
@Override
public String toString()
{
return toStringHelper(this)
.add("suiteName", suiteName)
.add("runId", runId)
.add("suiteRun", suiteRun)
.add("suiteConfig", environmentConfig)
.add("duration", duration)
.add("throwable", throwable)
.toString();
}
public Object[] toRow()
{
return new Object[] {
runId,
suiteName,
suiteRun.getEnvironmentName(),
environmentConfig.getConfigName(),
suiteRun.getExtraOptions(),
getStatusString(),
duration,
throwable.map(Throwable::getMessage).orElse("-")};
}
private String getStatusString()
{
if (wasSkipped()) {
return "SKIPPED";
}
return hasFailed() ? "FAILED" : "SUCCESS";
}
}
}