Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.glowroot.central.SyntheticMonitorService Maven / Gradle / Ivy
/*
* Copyright 2017-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.glowroot.central;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeoutException;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.concurrent.GuardedBy;
import com.google.common.base.MoreObjects;
import com.google.common.base.Ticker;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.machinepublishers.jbrowserdriver.JBrowserDriver;
import com.machinepublishers.jbrowserdriver.ProxyConfig;
import com.machinepublishers.jbrowserdriver.RequestHeaders;
import com.machinepublishers.jbrowserdriver.Settings;
import com.machinepublishers.jbrowserdriver.UserAgent;
import com.machinepublishers.jbrowserdriver.UserAgent.Family;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AUTH;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.concurrent.FutureCallback;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.ssl.SSLContextBuilder;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.immutables.value.Value;
import org.openqa.selenium.WebDriver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.glowroot.agent.api.Instrumentation;
import org.glowroot.agent.api.Instrumentation.AlreadyInTransactionBehavior;
import org.glowroot.central.RollupService.AgentRollupConsumer;
import org.glowroot.central.repo.ActiveAgentDao;
import org.glowroot.central.repo.AlertingDisabledDao;
import org.glowroot.central.repo.ConfigRepositoryImpl;
import org.glowroot.central.repo.IncidentDao;
import org.glowroot.central.repo.SyntheticResultDao;
import org.glowroot.central.repo.SyntheticResultDao.SyntheticResultRollup0;
import org.glowroot.central.util.ClusterManager;
import org.glowroot.central.util.MoreExecutors2;
import org.glowroot.common.util.Clock;
import org.glowroot.common.util.Styles;
import org.glowroot.common.util.Throwables;
import org.glowroot.common.util.Version;
import org.glowroot.common2.config.HttpProxyConfig;
import org.glowroot.common2.config.MoreConfigDefaults;
import org.glowroot.common2.repo.ActiveAgentRepository.AgentRollup;
import org.glowroot.common2.repo.ConfigRepository.AgentConfigNotFoundException;
import org.glowroot.common2.repo.IncidentRepository.OpenIncident;
import org.glowroot.common2.repo.util.AlertingService;
import org.glowroot.common2.repo.util.Compilations;
import org.glowroot.common2.repo.util.Encryption;
import org.glowroot.wire.api.model.AgentConfigOuterClass.AgentConfig.AlertConfig;
import org.glowroot.wire.api.model.AgentConfigOuterClass.AgentConfig.AlertConfig.AlertCondition;
import org.glowroot.wire.api.model.AgentConfigOuterClass.AgentConfig.AlertConfig.AlertCondition.SyntheticMonitorCondition;
import org.glowroot.wire.api.model.AgentConfigOuterClass.AgentConfig.SyntheticMonitorConfig;
import org.glowroot.wire.api.model.AgentConfigOuterClass.AgentConfig.SyntheticMonitorConfig.SyntheticMonitorKind;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
class SyntheticMonitorService implements Runnable {
private static final Logger logger = LoggerFactory.getLogger(SyntheticMonitorService.class);
private static final Pattern encryptedPattern = Pattern.compile("\"ENCRYPTED:([^\"]*)\"");
private static final int PING_TIMEOUT_MILLIS = 60000;
private static final long PING_TIMEOUT_NANOS = MILLISECONDS.toNanos(PING_TIMEOUT_MILLIS);
private static final RequestHeaders REQUEST_HEADERS;
static {
// this list is from com.machinepublishers.jbrowserdriver.RequestHeaders.CHROME,
// with added Glowroot-Transaction-Type header
LinkedHashMap headersTmp = new LinkedHashMap<>();
headersTmp.put("Host", RequestHeaders.DYNAMIC_HEADER);
headersTmp.put("Connection", "keep-alive");
headersTmp.put("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
headersTmp.put("Upgrade-Insecure-Requests", "1");
headersTmp.put("User-Agent", RequestHeaders.DYNAMIC_HEADER);
headersTmp.put("Referer", RequestHeaders.DYNAMIC_HEADER);
headersTmp.put("Accept-Encoding", "gzip, deflate, sdch");
headersTmp.put("Accept-Language", "en-US,en;q=0.8");
headersTmp.put("Cookie", RequestHeaders.DYNAMIC_HEADER);
headersTmp.put("Glowroot-Transaction-Type", "Synthetic");
REQUEST_HEADERS = new RequestHeaders(headersTmp);
}
private final ActiveAgentDao activeAgentDao;
private final ConfigRepositoryImpl configRepository;
private final AlertingDisabledDao alertingDisabledDao;
private final IncidentDao incidentDao;
private final AlertingService alertingService;
private final SyntheticResultDao syntheticResponseDao;
private final Ticker ticker;
private final Clock clock;
private final ConcurrentMap executionRateLimiter;
private final CloseableHttpAsyncClient asyncHttpClient;
@GuardedBy("syncHttpClientHolder")
private volatile SyncHttpClientHolder syncHttpClientHolder;
private final Object syncHttpClientHolderLock = new Object();
private final String shortVersion;
private final UserAgent userAgent;
private final ExecutorService mainLoopExecutor;
private final ExecutorService workerExecutor;
private final ListeningExecutorService subWorkerExecutor;
private final Set activeSyntheticMonitors =
Sets.newConcurrentHashSet();
private volatile boolean closed;
SyntheticMonitorService(ActiveAgentDao activeAgentDao, ConfigRepositoryImpl configRepository,
AlertingDisabledDao alertingDisabledDao, IncidentDao incidentDao,
AlertingService alertingService, SyntheticResultDao syntheticResponseDao,
ClusterManager clusterManager, Ticker ticker, Clock clock, String version)
throws Exception {
this.activeAgentDao = activeAgentDao;
this.configRepository = configRepository;
this.alertingDisabledDao = alertingDisabledDao;
this.incidentDao = incidentDao;
this.alertingService = alertingService;
this.syntheticResponseDao = syntheticResponseDao;
this.ticker = ticker;
this.clock = clock;
executionRateLimiter = clusterManager
.createReplicatedMap("syntheticMonitorExecutionRateLimiter", 30, SECONDS);
if (version.equals(Version.UNKNOWN_VERSION)) {
shortVersion = "";
} else {
int index = version.indexOf(", built ");
if (index == -1) {
shortVersion = "/" + version;
} else {
shortVersion = "/" + version.substring(0, index);
}
}
// asyncHttpClient is only used for pings, so safe to ignore cert errors
asyncHttpClient = HttpAsyncClients.custom()
.setUserAgent("GlowrootCentral" + shortVersion)
.setDefaultHeaders(
Arrays.asList(new BasicHeader("Glowroot-Transaction-Type", "Synthetic")))
.setMaxConnPerRoute(10) // increasing from default 2
.setMaxConnTotal(1000) // increasing from default 20
.setSSLContext(SSLContextBuilder.create()
.loadTrustMaterial(new TrustSelfSignedStrategy())
.build())
.setSSLHostnameVerifier(new NoopHostnameVerifier())
.build();
asyncHttpClient.start();
syncHttpClientHolder =
createSyncHttpClientHolder(shortVersion, configRepository.getHttpProxyConfig());
// these parameters are from com.machinepublishers.jbrowserdriver.UserAgent.CHROME
// with added GlowrootCentral/ for identification purposes
userAgent = new UserAgent(Family.WEBKIT, "Google Inc.", "Win32", "Windows NT 6.1",
"5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)"
+ " Chrome/45.0.2454.85 Safari/537.36 GlowrootCentral" + shortVersion,
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)"
+ " Chrome/45.0.2454.85 Safari/537.36 GlowrootCentral" + shortVersion);
// there is one subworker per worker, so using same max
subWorkerExecutor = MoreExecutors.listeningDecorator(
MoreExecutors2.newCachedThreadPool("Synthetic-Monitor-Sub-Worker-%d"));
workerExecutor = MoreExecutors2.newCachedThreadPool("Synthetic-Monitor-Worker-%d");
mainLoopExecutor = MoreExecutors2.newSingleThreadExecutor("Synthetic-Monitor-Main-Loop");
mainLoopExecutor.execute(castInitialized(this));
}
@Override
public void run() {
while (!closed) {
try {
long currMillis = clock.currentTimeMillis();
long nextMillis = (long) Math.ceil(currMillis / 60000.0) * 60000;
// scheduling for 5 seconds after the minute (just to avoid exactly on the minute)
MILLISECONDS.sleep(nextMillis - currMillis + 5000);
runInternal();
} catch (InterruptedException e) {
// probably shutdown requested (see close method below)
logger.debug(e.getMessage(), e);
continue;
} catch (Throwable t) {
logger.error(t.getMessage(), t);
}
}
}
void close() throws Exception {
closed = true;
// shutdownNow() is needed to send interrupt to SyntheticMonitorService user test threads
subWorkerExecutor.shutdownNow();
if (!subWorkerExecutor.awaitTermination(10, SECONDS)) {
throw new IllegalStateException(
"Timed out waiting for synthetic user test threads to terminate");
}
// shutdownNow() is needed to send interrupt to SyntheticMonitorService check threads
workerExecutor.shutdownNow();
if (!workerExecutor.awaitTermination(10, SECONDS)) {
throw new IllegalStateException(
"Timed out waiting for synthetic monitor check threads to terminate");
}
// shutdownNow() is needed to send interrupt to SyntheticMonitorService main thread
mainLoopExecutor.shutdownNow();
if (!mainLoopExecutor.awaitTermination(10, SECONDS)) {
throw new IllegalStateException(
"Timed out waiting for synthetic monitor loop thread to terminate");
}
syncHttpClientHolder.syncHttpClient().close();
asyncHttpClient.close();
}
@Instrumentation.Transaction(transactionType = "Background",
transactionName = "Outer synthetic monitor loop",
traceHeadline = "Outer synthetic monitor loop",
timer = "outer synthetic monitor loop")
private void runInternal() throws Exception {
for (AgentRollup agentRollup : activeAgentDao
.readRecentlyActiveAgentRollups(DAYS.toMillis(7))) {
consumeAgentRollups(agentRollup, this::runSyntheticMonitors);
}
}
private void runSyntheticMonitors(AgentRollup agentRollup) throws InterruptedException {
List syntheticMonitorConfigs;
try {
syntheticMonitorConfigs = configRepository.getSyntheticMonitorConfigs(agentRollup.id());
} catch (InterruptedException e) {
// probably shutdown requested
throw e;
} catch (AgentConfigNotFoundException e) {
// be lenient if agent_config table is messed up
logger.debug(e.getMessage(), e);
return;
} catch (Exception e) {
logger.error("{} - {}", agentRollup.id(), e.getMessage(), e);
return;
}
if (syntheticMonitorConfigs.isEmpty()) {
return;
}
for (SyntheticMonitorConfig syntheticMonitorConfig : syntheticMonitorConfigs) {
List alertConfigs;
try {
alertConfigs = configRepository.getAlertConfigsForSyntheticMonitorId(
agentRollup.id(), syntheticMonitorConfig.getId());
} catch (InterruptedException e) {
// probably shutdown requested
throw e;
} catch (AgentConfigNotFoundException e) {
// be lenient if agent_config table is messed up
logger.debug(e.getMessage(), e);
return;
} catch (Exception e) {
logger.error(e.getMessage(), e);
continue;
}
String uniqueId = syntheticMonitorConfig.getId() + agentRollup.id();
if (executionRateLimiter.putIfAbsent(uniqueId, true) != null) {
// was run in the last 30 seconds (probably on a different cluster node)
continue;
}
workerExecutor.execute(() -> {
try {
switch (syntheticMonitorConfig.getKind()) {
case PING:
runPing(agentRollup, syntheticMonitorConfig, alertConfigs);
break;
case JAVA:
runJava(agentRollup, syntheticMonitorConfig, alertConfigs);
break;
default:
throw new IllegalStateException("Unexpected synthetic kind: "
+ syntheticMonitorConfig.getKind());
}
} catch (InterruptedException e) {
// probably shutdown requested (see close method above)
logger.debug(e.getMessage(), e);
} catch (Throwable t) {
logger.error("{} - {}", agentRollup.id(), t.getMessage(), t);
}
});
}
}
@Instrumentation.Transaction(transactionType = "Background",
transactionName = "Synthetic monitor", traceHeadline = "Synthetic monitor: {{0.id}}",
timer = "synthetic monitor",
alreadyInTransactionBehavior = AlreadyInTransactionBehavior.CAPTURE_NEW_TRANSACTION)
private void runPing(AgentRollup agentRollup, SyntheticMonitorConfig syntheticMonitorConfig,
List alertConfigs) throws Exception {
runSyntheticMonitor(agentRollup, syntheticMonitorConfig, alertConfigs,
() -> runPing(syntheticMonitorConfig.getPingUrl()));
}
@Instrumentation.Transaction(transactionType = "Background",
transactionName = "Synthetic monitor", traceHeadline = "Synthetic monitor: {{0.id}}",
timer = "synthetic monitor")
private void runJava(AgentRollup agentRollup, SyntheticMonitorConfig syntheticMonitorConfig,
List alertConfigs) throws Exception {
Matcher matcher = encryptedPattern.matcher(syntheticMonitorConfig.getJavaSource());
StringBuffer sb = new StringBuffer();
while (matcher.find()) {
String encryptedPassword = checkNotNull(matcher.group(1));
matcher.appendReplacement(sb, "\""
+ Encryption.decrypt(encryptedPassword, configRepository.getLazySecretKey())
+ "\"");
}
matcher.appendTail(sb);
runSyntheticMonitor(agentRollup, syntheticMonitorConfig, alertConfigs,
() -> runJava(sb.toString()));
}
private FutureWithStartTick runJava(String javaSource) throws Exception {
Class syntheticUserTestClass = Compilations.compile(javaSource);
// validation for default constructor and test method occurs on save
Constructor defaultConstructor = syntheticUserTestClass.getConstructor();
Method testMethod;
try {
testMethod = syntheticUserTestClass.getMethod("test", CloseableHttpClient.class);
} catch (NoSuchMethodException e) {
testMethod = syntheticUserTestClass.getMethod("test", WebDriver.class);
}
if (testMethod.getParameterTypes()[0] == CloseableHttpClient.class) {
synchronized (syncHttpClientHolderLock) {
HttpProxyConfig httpProxyConfig = configRepository.getHttpProxyConfig();
if (!syncHttpClientHolder.httpProxyConfig().equals(httpProxyConfig)) {
syncHttpClientHolder =
createSyncHttpClientHolder(shortVersion, httpProxyConfig);
// don't close the old http client since other threads may still be using it
// (it will be garbage collected when no longer in use)
}
}
return runJava(defaultConstructor, testMethod, syncHttpClientHolder.syncHttpClient());
} else {
return runWebDriverJava(defaultConstructor, testMethod);
}
}
private FutureWithStartTick runWebDriverJava(Constructor defaultConstructor,
Method testMethod) throws Exception {
Settings.Builder settings = Settings.builder()
.requestHeaders(REQUEST_HEADERS)
.userAgent(userAgent);
HttpProxyConfig httpProxyConfig = configRepository.getHttpProxyConfig();
if (!httpProxyConfig.host().isEmpty()) {
int proxyPort = MoreObjects.firstNonNull(httpProxyConfig.port(), 80);
settings.proxy(new ProxyConfig(ProxyConfig.Type.HTTP, httpProxyConfig.host(),
proxyPort, httpProxyConfig.username(),
httpProxyConfig.encryptedPassword()));
}
JBrowserDriver driver = new JBrowserDriver(settings.build());
return runJava(defaultConstructor, testMethod, driver);
}
private FutureWithStartTick runJava(Constructor defaultConstructor, Method testMethod,
Object testArg) {
long startTick = ticker.read();
FutureWithStartTick future = new FutureWithStartTick(startTick);
subWorkerExecutor.execute(() -> {
try {
future.complete(runJava(defaultConstructor, testMethod, testArg, startTick));
} catch (Throwable t) {
// unexpected exception
future.completeExceptionally(t);
}
});
return future;
}
public SyntheticRunResult runJava(Constructor defaultConstructor, Method testMethod,
Object testArg, long startTick) throws Exception {
try {
return runJavaInternal(defaultConstructor, testMethod, testArg, startTick);
} catch (InterruptedException e) {
// probably shutdown requested (see close method above)
logger.debug(e.getMessage(), e);
throw e;
} catch (Throwable t) {
// unexpected exception
logger.error(t.getMessage(), t);
throw t;
}
}
private SyntheticRunResult runJavaInternal(Constructor defaultConstructor, Method testMethod,
Object testArg, long startTick) throws Exception {
long captureTime;
long durationNanos;
try {
testMethod.invoke(defaultConstructor.newInstance(), testArg);
// capture time and duration before calling driver.quit()
captureTime = clock.currentTimeMillis();
durationNanos = ticker.read() - startTick;
} catch (InvocationTargetException e) {
logger.debug(e.getMessage(), e);
Throwable throwable = e.getTargetException();
if (throwable instanceof InterruptedException) {
// probably shutdown requested (see close method above)
throw (InterruptedException) throwable;
}
return ImmutableSyntheticRunResult.builder()
.captureTime(clock.currentTimeMillis())
.durationNanos(ticker.read() - startTick)
.throwable(throwable)
.build();
} finally {
if (testArg instanceof JBrowserDriver) {
// quit() can hang (easily reproducible on test that doesn't use testArg at all)
((JBrowserDriver) testArg).kill();
}
}
return ImmutableSyntheticRunResult.builder()
.captureTime(captureTime)
.durationNanos(durationNanos)
.build();
}
private FutureWithStartTick runPing(String url) throws Exception {
HttpGet httpGet = new HttpGet(url);
RequestConfig.Builder config = RequestConfig.custom()
// wait an extra second to make sure no edge case where
// SocketTimeoutException occurs with elapsed time < PING_TIMEOUT_MILLIS
.setSocketTimeout(PING_TIMEOUT_MILLIS + 1000);
HttpProxyConfig httpProxyConfig = configRepository.getHttpProxyConfig();
if (!httpProxyConfig.host().isEmpty()) {
int proxyPort = MoreObjects.firstNonNull(httpProxyConfig.port(), 80);
config.setProxy(new HttpHost(httpProxyConfig.host(), proxyPort));
}
httpGet.setConfig(config.build());
long startTick = ticker.read();
FutureWithStartTick future = new FutureWithStartTick(startTick);
subWorkerExecutor.execute(() -> {
try {
asyncHttpClient.execute(httpGet, getHttpClientContext(),
new CompletingFutureCallback(future));
} catch (Throwable t) {
logger.debug(t.getMessage(), t);
future.complete(ImmutableSyntheticRunResult.builder()
.captureTime(clock.currentTimeMillis())
.durationNanos(ticker.read() - startTick)
.throwable(t)
.build());
}
});
return future;
}
private void runSyntheticMonitor(AgentRollup agentRollup,
SyntheticMonitorConfig syntheticMonitorConfig, List alertConfigs,
Callable callable) throws Exception {
SyntheticMonitorUniqueKey uniqueKey = ImmutableSyntheticMonitorUniqueKey
.of(agentRollup.id(), syntheticMonitorConfig.getId());
if (!activeSyntheticMonitors.add(uniqueKey)) {
return;
}
FutureWithStartTick future = callable.call();
// important that uniqueKey is always removed on completion even on unexpected errors
future.whenComplete((v, t) -> activeSyntheticMonitors.remove(uniqueKey));
OnRunComplete onRunComplete = new OnRunComplete(agentRollup, syntheticMonitorConfig);
if (alertConfigs.isEmpty()) {
future.thenAccept(onRunComplete);
return;
}
int maxAlertThresholdMillis = 0;
for (AlertConfig alertConfig : alertConfigs) {
maxAlertThresholdMillis = Math.max(maxAlertThresholdMillis,
alertConfig.getCondition().getSyntheticMonitorCondition().getThresholdMillis());
}
long captureTime;
long durationNanos;
boolean success;
String errorMessage;
try {
// wait an extra second to make sure no edge case where TimeoutException occurs with
// stopwatch.elapsed(MILLISECONDS) < maxAlertThresholdMillis
SyntheticRunResult result = future.get(maxAlertThresholdMillis + 1000L, MILLISECONDS);
captureTime = result.captureTime();
durationNanos = result.durationNanos();
Throwable throwable = result.throwable();
if (throwable == null) {
success = true;
errorMessage = null;
} else {
success = false;
errorMessage = getBestMessageForSyntheticFailure(throwable);
}
} catch (TimeoutException e) {
logger.debug(e.getMessage(), e);
captureTime = clock.currentTimeMillis();
durationNanos = 0; // durationNanos is only used below when success is true
success = false;
errorMessage = null;
}
if (!isCurrentlyDisabled(agentRollup.id())) {
if (success) {
for (AlertConfig alertConfig : alertConfigs) {
AlertCondition alertCondition = alertConfig.getCondition();
SyntheticMonitorCondition condition =
alertCondition.getSyntheticMonitorCondition();
boolean currentlyTriggered =
durationNanos >= MILLISECONDS.toNanos(condition.getThresholdMillis());
sendAlertIfStatusChanged(agentRollup, syntheticMonitorConfig, alertConfig,
condition, captureTime, currentlyTriggered, null);
}
} else {
sendAlertOnErrorIfStatusChanged(agentRollup, syntheticMonitorConfig, alertConfigs,
errorMessage, captureTime);
}
}
// need to run at end to ensure new synthetic response doesn't get stored before consecutive
// count is checked in sendAlertOnErrorIfStatusChanged()
future.thenAccept(onRunComplete);
}
private boolean isCurrentlyDisabled(String agentRollupId) throws Exception {
Long disabledUntilTime = alertingDisabledDao.getAlertingDisabledUntilTime(agentRollupId);
return disabledUntilTime != null && disabledUntilTime > clock.currentTimeMillis();
}
private static String getBestMessageForSyntheticFailure(Throwable throwable) {
String message = throwable.getMessage();
if (throwable.getClass() == Exception.class && throwable.getCause() == null
&& message != null) {
// special case so synthetic monitors can display a simple error message without the
// exception class name clutter
return message;
} else {
return Throwables.getBestMessage(throwable);
}
}
private class OnRunComplete implements Consumer {
private final AgentRollup agentRollup;
private final SyntheticMonitorConfig syntheticMonitorConfig;
private OnRunComplete(AgentRollup agentRollup,
SyntheticMonitorConfig syntheticMonitorConfig) {
this.agentRollup = agentRollup;
this.syntheticMonitorConfig = syntheticMonitorConfig;
}
@Override
public void accept(SyntheticRunResult syntheticRunResult) {
String errorMessage = null;
long durationNanos = syntheticRunResult.durationNanos();
Throwable t = syntheticRunResult.throwable();
if (syntheticMonitorConfig.getKind() == SyntheticMonitorKind.PING
&& durationNanos >= PING_TIMEOUT_NANOS) {
durationNanos = PING_TIMEOUT_NANOS;
errorMessage = "Timeout";
} else if (t != null) {
errorMessage = getBestMessageForSyntheticFailure(t);
}
try {
syntheticResponseDao.store(agentRollup.id(), syntheticMonitorConfig.getId(),
MoreConfigDefaults.getDisplayOrDefault(syntheticMonitorConfig),
syntheticRunResult.captureTime(), durationNanos, errorMessage);
} catch (InterruptedException e) {
// probably shutdown requested (see close method above)
logger.debug(e.getMessage(), e);
return;
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}
private void sendAlertOnErrorIfStatusChanged(AgentRollup agentRollup,
SyntheticMonitorConfig syntheticMonitorConfig, List alertConfigs,
@Nullable String errorMessage, long captureTime) throws Exception {
for (AlertConfig alertConfig : alertConfigs) {
AlertCondition alertCondition = alertConfig.getCondition();
SyntheticMonitorCondition condition = alertCondition.getSyntheticMonitorCondition();
sendAlertIfStatusChanged(agentRollup, syntheticMonitorConfig, alertConfig, condition,
captureTime, true, errorMessage);
}
}
private void sendAlertIfStatusChanged(AgentRollup agentRollup,
SyntheticMonitorConfig syntheticMonitorConfig, AlertConfig alertConfig,
SyntheticMonitorCondition condition, long endTime, boolean currentlyTriggered,
@Nullable String errorMessage) throws Exception {
AlertCondition alertCondition = alertConfig.getCondition();
OpenIncident openIncident = incidentDao.readOpenIncident(agentRollup.id(), alertCondition,
alertConfig.getSeverity());
if (openIncident != null && !currentlyTriggered) {
incidentDao.resolveIncident(openIncident, endTime);
sendAlert(agentRollup.id(), agentRollup.display(), syntheticMonitorConfig,
alertConfig, condition, endTime, true, null);
} else if (openIncident == null && currentlyTriggered
&& consecutiveCountHit(agentRollup.id(), syntheticMonitorConfig, condition)) {
// the start time for the incident is the end time of the interval evaluated above
incidentDao.insertOpenIncident(agentRollup.id(), alertCondition,
alertConfig.getSeverity(), alertConfig.getNotification(), endTime);
sendAlert(agentRollup.id(), agentRollup.display(), syntheticMonitorConfig,
alertConfig, condition, endTime, false, errorMessage);
}
}
private boolean consecutiveCountHit(String agentRollupId,
SyntheticMonitorConfig syntheticMonitorConfig, SyntheticMonitorCondition condition)
throws Exception {
int consecutiveCount = condition.getConsecutiveCount();
if (consecutiveCount == 1) {
return true;
}
List syntheticResults = syntheticResponseDao.readLastFromRollup0(
agentRollupId, syntheticMonitorConfig.getId(), consecutiveCount - 1);
if (syntheticResults.size() < consecutiveCount - 1) {
return false;
}
for (SyntheticResultRollup0 syntheticResult : syntheticResults) {
if (!syntheticResult.error() && syntheticResult.totalDurationNanos() < MILLISECONDS
.toNanos(condition.getThresholdMillis())) {
return false;
}
}
return true;
}
private void sendAlert(String agentRollupId, String agentRollupDisplay,
SyntheticMonitorConfig syntheticMonitorConfig, AlertConfig alertConfig,
SyntheticMonitorCondition condition, long endTime, boolean ok,
@Nullable String errorMessage) throws Exception {
// subject is the same between initial and ok messages so they will be threaded by gmail
String subject = MoreConfigDefaults.getDisplayOrDefault(syntheticMonitorConfig);
StringBuilder sb = new StringBuilder();
sb.append(MoreConfigDefaults.getDisplayOrDefault(syntheticMonitorConfig));
if (errorMessage == null) {
sb.append(" time");
sb.append(AlertingService.getPreUpperBoundText(ok));
sb.append(AlertingService.getWithUnit(condition.getThresholdMillis(), "millisecond"));
sb.append(".");
} else {
sb.append(" resulted in error: ");
sb.append(errorMessage);
}
alertingService.sendNotification(
configRepository.getCentralAdminGeneralConfig().centralDisplayName(), agentRollupId,
agentRollupDisplay, alertConfig, endTime, subject, sb.toString(), ok);
}
private HttpClientContext getHttpClientContext() throws Exception {
HttpProxyConfig httpProxyConfig = configRepository.getHttpProxyConfig();
if (httpProxyConfig.host().isEmpty() || httpProxyConfig.username().isEmpty()) {
return HttpClientContext.create();
}
// perform preemptive proxy authentication
int proxyPort = MoreObjects.firstNonNull(httpProxyConfig.port(), 80);
HttpHost proxyHost = new HttpHost(httpProxyConfig.host(), proxyPort);
BasicScheme basicScheme = new BasicScheme();
basicScheme.processChallenge(new BasicHeader(AUTH.PROXY_AUTH, "BASIC realm="));
BasicAuthCache authCache = new BasicAuthCache();
authCache.put(proxyHost, basicScheme);
String password = httpProxyConfig.encryptedPassword();
if (!password.isEmpty()) {
password = Encryption.decrypt(password, configRepository.getLazySecretKey());
}
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(new AuthScope(proxyHost),
new UsernamePasswordCredentials(httpProxyConfig.username(), password));
HttpClientContext context = HttpClientContext.create();
context.setAuthCache(authCache);
context.setCredentialsProvider(credentialsProvider);
return context;
}
private static SyncHttpClientHolder createSyncHttpClientHolder(String shortVersion,
HttpProxyConfig httpProxyConfig) {
HttpClientBuilder builder = HttpClients.custom()
.setUserAgent("GlowrootCentral" + shortVersion)
.setDefaultHeaders(
Arrays.asList(new BasicHeader("Glowroot-Transaction-Type", "Synthetic")));
if (!httpProxyConfig.host().isEmpty()) {
int proxyPort = MoreObjects.firstNonNull(httpProxyConfig.port(), 80);
builder.setProxy(new HttpHost(httpProxyConfig.host(), proxyPort));
}
CloseableHttpClient httpClient = builder.setMaxConnPerRoute(10) // increasing from default 2
.setMaxConnTotal(1000) // increasing from default 20
.build();
return ImmutableSyncHttpClientHolder.of(httpClient, httpProxyConfig);
}
private static void consumeAgentRollups(AgentRollup agentRollup,
AgentRollupConsumer agentRollupConsumer) throws Exception {
for (AgentRollup childAgentRollup : agentRollup.children()) {
consumeAgentRollups(childAgentRollup, agentRollupConsumer);
}
agentRollupConsumer.accept(agentRollup);
}
@SuppressWarnings("return.type.incompatible")
private static /*@Initialized*/ T castInitialized(/*@UnderInitialization*/ T obj) {
return obj;
}
@Value.Immutable
@Styles.AllParameters
interface SyncHttpClientHolder {
CloseableHttpClient syncHttpClient();
HttpProxyConfig httpProxyConfig();
}
@Value.Immutable
@Styles.AllParameters
interface SyntheticMonitorUniqueKey {
String agentRollupId();
String syntheticMonitorId();
}
private static class FutureWithStartTick extends CompletableFuture {
private final long startTick;
private FutureWithStartTick(long startTick) {
this.startTick = startTick;
}
}
@Value.Immutable
interface SyntheticRunResult {
long captureTime();
long durationNanos();
@Nullable
Throwable throwable();
}
private class CompletingFutureCallback implements FutureCallback {
private final FutureWithStartTick future;
private CompletingFutureCallback(FutureWithStartTick future) {
this.future = future;
}
@Override
public void completed(HttpResponse response) {
ImmutableSyntheticRunResult.Builder builder = ImmutableSyntheticRunResult.builder()
.captureTime(clock.currentTimeMillis())
.durationNanos(ticker.read() - future.startTick);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode < 400) {
future.complete(builder.build());
} else {
future.complete(builder
.throwable(new Exception("Unexpected response status code: " + statusCode))
.build());
}
}
@Override
public void failed(Exception ex) {
future.complete(ImmutableSyntheticRunResult.builder()
.captureTime(clock.currentTimeMillis())
.durationNanos(ticker.read() - future.startTick)
.throwable(ex)
.build());
}
@Override
public void cancelled() {
future.complete(ImmutableSyntheticRunResult.builder()
.captureTime(clock.currentTimeMillis())
.durationNanos(ticker.read() - future.startTick)
.throwable(new RuntimeException("Unexpected cancellation"))
.build());
}
}
}
