com.spotify.helios.testing.TemporaryJobs Maven / Gradle / Ivy
/*-
* -\-\-
* Helios Testing Library
* --
* Copyright (C) 2016 Spotify AB
* --
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* -/-/-
*/
package com.spotify.helios.testing;
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.base.Optional.fromNullable;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Strings.isNullOrEmpty;
import static com.google.common.collect.Lists.newArrayList;
import static com.spotify.helios.testing.Jobs.undeploy;
import static java.lang.String.format;
import static java.lang.System.getProperty;
import static java.util.Arrays.asList;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Splitter;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.spotify.helios.client.HeliosClient;
import com.spotify.helios.common.Json;
import com.spotify.helios.common.descriptors.Job;
import com.spotify.helios.common.descriptors.JobId;
import com.spotify.helios.common.descriptors.JobStatus;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import com.typesafe.config.ConfigList;
import com.typesafe.config.ConfigValue;
import com.typesafe.config.ConfigValueFactory;
import com.typesafe.config.ConfigValueType;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.MultipleFailureException;
import org.junit.runners.model.Statement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TemporaryJobs implements TestRule {
private static final Logger log = LoggerFactory.getLogger(TemporaryJobs.class);
static final String HELIOS_TESTING_PROFILE = "helios.testing.profile";
private static final String HELIOS_TESTING_PROFILES = "helios.testing.profiles.";
private static final String DEFAULT_USER = getProperty("user.name");
private static final Prober DEFAULT_PROBER = new DefaultProber();
private static final String DEFAULT_LOCAL_HOST_FILTER = ".+";
private static final String DEFAULT_PREFIX_DIRECTORY = "/tmp/helios-temp-jobs";
private static final String DEFAULT_TEST_REPORT_DIRECTORY = "target/helios-reports/test";
private static final long JOB_HEALTH_CHECK_INTERVAL_MILLIS = SECONDS.toMillis(5);
private static final long DEFAULT_DEPLOY_TIMEOUT_MILLIS = MINUTES.toMillis(10);
private final HeliosClient client;
private final Prober prober;
private final String defaultHostFilter;
private final JobPrefixFile jobPrefixFile;
private final Config config;
private final Map env;
private final List jobs = Lists.newCopyOnWriteArrayList();
private final Deployer deployer;
private final TemporaryJobReports reports;
private final ThreadLocal reportWriter;
private boolean removedOldJobs = false;
private final ExecutorService executor = MoreExecutors.getExitingExecutorService(
(ThreadPoolExecutor) Executors.newFixedThreadPool(
1, new ThreadFactoryBuilder()
.setNameFormat("helios-test-runner-%d")
.setDaemon(true)
.build()),
0, SECONDS);
private final Path prefixDirectory;
TemporaryJobs(final Builder builder, final Config config) {
this.client = checkNotNull(builder.client, "client");
this.prober = checkNotNull(builder.prober, "prober");
this.defaultHostFilter = checkNotNull(builder.hostFilter, "hostFilter");
this.env = checkNotNull(builder.env, "env");
checkArgument(builder.deployTimeoutMillis >= 0, "deployTimeoutMillis");
this.deployer = fromNullable(builder.deployer).or(
new DefaultDeployer(client, jobs, builder.hostPickingStrategy,
builder.jobDeployedMessageFormat, builder.deployTimeoutMillis));
prefixDirectory = Paths.get(fromNullable(builder.prefixDirectory)
.or(DEFAULT_PREFIX_DIRECTORY));
try {
if (isNullOrEmpty(builder.jobPrefix)) {
this.jobPrefixFile = JobPrefixFile.create(prefixDirectory);
} else {
this.jobPrefixFile = JobPrefixFile.create(builder.jobPrefix, prefixDirectory);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
if (builder.reports == null) {
final Path testReportDirectory = Paths.get(
fromNullable(builder.testReportDirectory).or(DEFAULT_TEST_REPORT_DIRECTORY));
this.reports = new TemporaryJobJsonReports(testReportDirectory);
} else {
this.reports = builder.reports;
}
this.reportWriter = new ThreadLocal() {
@Override
protected TemporaryJobReports.ReportWriter initialValue() {
log.warn("unable to determine test context, writing to default event log");
return TemporaryJobs.this.reports.getDefaultWriter();
}
};
// Load in the prefix so it can be used in the config
final Config configWithPrefix = ConfigFactory.empty()
.withValue("prefix", ConfigValueFactory.fromAnyRef(prefix()));
this.config = config.withFallback(configWithPrefix).resolve();
}
/**
* Perform setup. This is normally called by JUnit when TemporaryJobs is used with @Rule.
* If @Rule cannot be used, call this method before calling {@link #job()}.
*
* Note: When not being used as a @Rule, jobs will not be monitored during test run.
*/
public void before() {
deployer.readyToDeploy();
}
/**
* Perform teardown. This is normally called by JUnit when TemporaryJobs is used with @Rule.
* If @Rule cannot be used, call this method after running tests.
*/
public void after() {
after(Optional.absent());
}
void after(final Optional writer) {
final Optional undeploy = writer
.transform(new Function() {
@Override
public TemporaryJobReports.Step apply(final TemporaryJobReports.ReportWriter writer) {
return writer.step("undeploy");
}
});
final List jobIds = Lists.newArrayListWithCapacity(jobs.size());
// Stop the test runner thread
executor.shutdownNow();
try {
final boolean terminated = executor.awaitTermination(30, SECONDS);
if (!terminated) {
log.warn("Failed to stop test runner thread");
}
} catch (InterruptedException ignore) {
// ignored
}
final List errors = newArrayList();
for (final TemporaryJob job : jobs) {
jobIds.add(job.job().getId());
job.undeploy(errors);
}
for (final TemporaryJobReports.Step step : undeploy.asSet()) {
step.tag("jobs", jobIds);
}
for (final AssertionError error : errors) {
log.error(error.getMessage());
}
// Don't delete the prefix file if any errors occurred during undeployment, so that we'll
// try to undeploy them the next time TemporaryJobs is run.
if (errors.isEmpty()) {
jobPrefixFile.delete();
for (final TemporaryJobReports.Step step : undeploy.asSet()) {
step.markSuccess();
}
}
for (final TemporaryJobReports.Step step : undeploy.asSet()) {
step.finish();
}
}
public TemporaryJobBuilder job() {
return this.job(Job.newBuilder());
}
private TemporaryJobBuilder job(final Job.Builder jobBuilder) {
final TemporaryJobBuilder builder = new TemporaryJobBuilder(
deployer, jobPrefixFile.prefix(), prober, env, reportWriter.get(), jobBuilder);
if (config.hasPath("env")) {
final Config env = config.getConfig("env");
for (final Entry entry : env.entrySet()) {
builder.env(entry.getKey(), entry.getValue().unwrapped());
}
}
if (config.hasPath("version")) {
builder.version(config.getString("version"));
}
if (config.hasPath("image")) {
builder.image(config.getString("image"));
}
if (config.hasPath("command")) {
builder.command(getListByKey("command", config));
}
if (config.hasPath("host")) {
builder.host(config.getString("host"));
}
if (config.hasPath("deploy")) {
builder.deploy(getListByKey("deploy", config));
}
if (config.hasPath("imageInfoFile")) {
builder.imageFromInfoFile(config.getString("imageInfoFile"));
}
if (config.hasPath("registrationDomain")) {
builder.registrationDomain(config.getString("registrationDomain"));
}
// port and expires intentionally left out -- since expires is a specific point in time, I
// cannot imagine a config-file use for it, additionally for ports, I'm thinking that port
// allocations are not likely to be common -- but PR's welcome if I'm wrong. - [email protected]
builder.hostFilter(defaultHostFilter);
return builder;
}
public TemporaryJobBuilder jobWithConfig(final String configFile) throws IOException {
checkNotNull(configFile);
final Path configPath = Paths.get(configFile);
final File file = configPath.toFile();
if (!file.exists() || !file.isFile() || !file.canRead()) {
throw new IllegalArgumentException("Cannot read file " + file);
}
final byte[] bytes = Files.readAllBytes(configPath);
final String config = new String(bytes, UTF_8);
final Job job = Json.read(config, Job.class);
return this.job(job.toBuilder());
}
private static List getListByKey(final String key, final Config config) {
final ConfigList endpointList = config.getList(key);
final List stringList = Lists.newArrayList();
for (final ConfigValue v : endpointList) {
if (v.valueType() != ConfigValueType.STRING) {
throw new RuntimeException("Item in " + key + " list [" + v + "] is not a string");
}
stringList.add((String) v.unwrapped());
}
return stringList;
}
/**
* Creates a new instance of TemporaryJobs. Will attempt to connect to a helios master according
* to the following factors, where the order of precedence is top to bottom.
*
* - HELIOS_DOMAIN - If set, use a helios master running in this domain.
* - HELIOS_ENDPOINTS - If set, use one of the endpoints, which are specified as a comma
* separated list.
* - Testing Profile - If a testing profile can be loaded, use either {@code domain} or
* {@code endpoints} if present. If both are specified, {@code domain} takes precedence.
* - DOCKER_HOST - If set, assume a helios master is running on this host, so connect to it on
* port {@code 5801}.
* - Use {@code http://localhost:5801}
*
*
* @return an instance of TemporaryJobs
*
* @see Helios Testing Framework - Configuration By File
*/
public static TemporaryJobs create() {
return builder().build();
}
public static TemporaryJobs create(final HeliosClient client) {
return builder().client(client).build();
}
public static TemporaryJobs create(final String domain) {
return builder().domain(domain).build();
}
public static TemporaryJobs createFromProfile(final String profile) {
return builder(profile).build();
}
@Override
public Statement apply(final Statement base, final Description description) {
return new Statement() {
@Override
public void evaluate() throws Throwable {
final TemporaryJobReports.ReportWriter writer = reports.getWriterForTest(description);
reportWriter.set(writer);
final TemporaryJobReports.Step test = writer.step("test");
before();
removeOldJobs();
try {
perform(base, writer);
test.markSuccess();
} finally {
after(Optional.of(writer));
test.finish();
writer.close();
reportWriter.set(null);
}
}
};
}
private void perform(final Statement base, final TemporaryJobReports.ReportWriter writer)
throws InterruptedException {
// Run the actual test on a thread
final Future