org.glowroot.central.repo.SyntheticResultDaoImpl Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2017-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.glowroot.central.repo;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executor;
import java.util.concurrent.Future;
import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.utils.UUIDs;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.ListenableFuture;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.glowroot.central.repo.Common.NeedsRollup;
import org.glowroot.central.repo.proto.Stored;
import org.glowroot.central.util.Messages;
import org.glowroot.central.util.MoreFutures;
import org.glowroot.central.util.MoreFutures.DoRollup;
import org.glowroot.central.util.Session;
import org.glowroot.common.util.CaptureTimes;
import org.glowroot.common.util.Clock;
import org.glowroot.common.util.OnlyUsedByTests;
import org.glowroot.common2.repo.ConfigRepository.RollupConfig;
import org.glowroot.common2.repo.ErrorIntervalCollector;
import org.glowroot.common2.repo.ImmutableErrorInterval;
import org.glowroot.common2.repo.ImmutableSyntheticResult;
import org.glowroot.common2.repo.SyntheticResult;
import org.glowroot.common2.repo.SyntheticResult.ErrorInterval;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.concurrent.TimeUnit.HOURS;
public class SyntheticResultDaoImpl implements SyntheticResultDao {
private final Session session;
private final ConfigRepositoryImpl configRepository;
private final Executor asyncExecutor;
private final Clock clock;
private final SyntheticMonitorIdDao syntheticMonitorIdDao;
// index is rollupLevel
private final ImmutableList insertResultPS;
private final ImmutableList readResultPS;
private final ImmutableList readResultForRollupPS;
private final List insertNeedsRollup;
private final List readNeedsRollup;
private final List deleteNeedsRollup;
private final PreparedStatement readLastFromRollup0;
SyntheticResultDaoImpl(Session session, ConfigRepositoryImpl configRepository,
Executor asyncExecutor, int cassandraGcGraceSeconds, Clock clock) throws Exception {
this.session = session;
this.configRepository = configRepository;
this.asyncExecutor = asyncExecutor;
this.clock = clock;
syntheticMonitorIdDao = new SyntheticMonitorIdDao(session, configRepository, clock);
int count = configRepository.getRollupConfigs().size();
List rollupExpirationHours =
configRepository.getCentralStorageConfig().rollupExpirationHours();
List insertResultPS = new ArrayList<>();
List readResultPS = new ArrayList<>();
List readResultForRollupPS = new ArrayList<>();
for (int i = 0; i < count; i++) {
// total_duration_nanos and execution_count only represent successful results
session.createTableWithTWCS("create table if not exists synthetic_result_rollup_" + i
+ " (agent_rollup_id varchar, synthetic_config_id varchar, capture_time"
+ " timestamp, total_duration_nanos double, execution_count bigint,"
+ " error_intervals blob, primary key ((agent_rollup_id, synthetic_config_id),"
+ " capture_time))", rollupExpirationHours.get(i));
insertResultPS.add(session.prepare("insert into synthetic_result_rollup_" + i
+ " (agent_rollup_id, synthetic_config_id, capture_time, total_duration_nanos,"
+ " execution_count, error_intervals) values (?, ?, ?, ?, ?, ?) using ttl ?"));
readResultPS.add(session.prepare("select capture_time, total_duration_nanos,"
+ " execution_count, error_intervals from synthetic_result_rollup_" + i
+ " where agent_rollup_id = ? and synthetic_config_id = ? and capture_time >= ?"
+ " and capture_time <= ?"));
readResultForRollupPS.add(session.prepare("select total_duration_nanos,"
+ " execution_count, error_intervals from synthetic_result_rollup_" + i
+ " where agent_rollup_id = ? and synthetic_config_id = ? and capture_time > ?"
+ " and capture_time <= ?"));
}
this.insertResultPS = ImmutableList.copyOf(insertResultPS);
this.readResultPS = ImmutableList.copyOf(readResultPS);
this.readResultForRollupPS = ImmutableList.copyOf(readResultForRollupPS);
List insertNeedsRollup = new ArrayList<>();
List readNeedsRollup = new ArrayList<>();
List deleteNeedsRollup = new ArrayList<>();
for (int i = 1; i < count; i++) {
session.createTableWithLCS("create table if not exists synthetic_needs_rollup_" + i
+ " (agent_rollup_id varchar, capture_time timestamp, uniqueness timeuuid,"
+ " synthetic_config_ids set, primary key (agent_rollup_id,"
+ " capture_time, uniqueness)) with gc_grace_seconds = "
+ cassandraGcGraceSeconds, true);
insertNeedsRollup.add(session.prepare("insert into synthetic_needs_rollup_" + i
+ " (agent_rollup_id, capture_time, uniqueness, synthetic_config_ids) values"
+ " (?, ?, ?, ?) using TTL ?"));
readNeedsRollup.add(session.prepare("select capture_time, uniqueness,"
+ " synthetic_config_ids from synthetic_needs_rollup_" + i + " where"
+ " agent_rollup_id = ?"));
deleteNeedsRollup.add(session.prepare("delete from synthetic_needs_rollup_" + i
+ " where agent_rollup_id = ? and capture_time = ? and uniqueness = ?"));
}
this.insertNeedsRollup = insertNeedsRollup;
this.readNeedsRollup = readNeedsRollup;
this.deleteNeedsRollup = deleteNeedsRollup;
readLastFromRollup0 = session.prepare("select capture_time, total_duration_nanos,"
+ " error_intervals from synthetic_result_rollup_0 where agent_rollup_id = ? and"
+ " synthetic_config_id = ? order by capture_time desc limit ?");
}
// synthetic result records are not rolled up to their parent, but are stored directly for
// rollups that have their own synthetic monitors defined
@Override
public void store(String agentRollupId, String syntheticMonitorId,
String syntheticMonitorDisplay, long captureTime, long durationNanos,
@Nullable String errorMessage) throws Exception {
int ttl = getTTLs().get(0);
long maxCaptureTime = 0;
BoundStatement boundStatement = insertResultPS.get(0).bind();
maxCaptureTime = Math.max(captureTime, maxCaptureTime);
int adjustedTTL = Common.getAdjustedTTL(ttl, captureTime, clock);
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, syntheticMonitorId);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setDouble(i++, durationNanos);
boundStatement.setLong(i++, 1);
if (errorMessage == null) {
boundStatement.setToNull(i++);
} else {
Stored.ErrorInterval errorInterval = Stored.ErrorInterval.newBuilder()
.setFrom(captureTime)
.setTo(captureTime)
.setCount(1)
.setMessage(errorMessage)
.setDoNotMergeToTheLeft(false)
.setDoNotMergeToTheRight(false)
.build();
boundStatement.setBytes(i++, Messages.toByteBuffer(ImmutableList.of(errorInterval)));
}
boundStatement.setInt(i++, adjustedTTL);
List> futures = new ArrayList<>();
futures.add(session.writeAsync(boundStatement));
futures.addAll(syntheticMonitorIdDao.insert(agentRollupId, captureTime, syntheticMonitorId,
syntheticMonitorDisplay));
// wait for success before inserting "needs rollup" records
MoreFutures.waitForAll(futures);
// insert into synthetic_needs_rollup_1
List rollupConfigs = configRepository.getRollupConfigs();
long intervalMillis = rollupConfigs.get(1).intervalMillis();
long rollupCaptureTime = CaptureTimes.getRollup(captureTime, intervalMillis);
int needsRollupAdjustedTTL =
Common.getNeedsRollupAdjustedTTL(adjustedTTL, configRepository.getRollupConfigs());
boundStatement = insertNeedsRollup.get(0).bind();
i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setTimestamp(i++, new Date(rollupCaptureTime));
boundStatement.setUUID(i++, UUIDs.timeBased());
boundStatement.setSet(i++, ImmutableSet.of(syntheticMonitorId));
boundStatement.setInt(i++, needsRollupAdjustedTTL);
session.write(boundStatement);
}
@Override
public Map getSyntheticMonitorIds(String agentRollupId, long from, long to)
throws Exception {
return syntheticMonitorIdDao.getSyntheticMonitorIds(agentRollupId, from, to);
}
// from is INCLUSIVE
@Override
public List readSyntheticResults(String agentRollupId,
String syntheticMonitorId, long from, long to, int rollupLevel) throws Exception {
BoundStatement boundStatement = readResultPS.get(rollupLevel).bind();
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, syntheticMonitorId);
boundStatement.setTimestamp(i++, new Date(from));
boundStatement.setTimestamp(i++, new Date(to));
ResultSet results = session.read(boundStatement);
List syntheticResults = new ArrayList<>();
for (Row row : results) {
i = 0;
long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
double totalDurationNanos = row.getDouble(i++);
long executionCount = row.getLong(i++);
ByteBuffer errorIntervalsBytes = row.getBytes(i++);
List errorIntervals = new ArrayList<>();
if (errorIntervalsBytes != null) {
List storedErrorIntervals = Messages
.parseDelimitedFrom(errorIntervalsBytes, Stored.ErrorInterval.parser());
for (Stored.ErrorInterval storedErrorInterval : storedErrorIntervals) {
errorIntervals.add(ImmutableErrorInterval.builder()
.from(storedErrorInterval.getFrom())
.to(storedErrorInterval.getTo())
.count(storedErrorInterval.getCount())
.message(storedErrorInterval.getMessage())
.doNotMergeToTheLeft(storedErrorInterval.getDoNotMergeToTheLeft())
.doNotMergeToTheRight(storedErrorInterval.getDoNotMergeToTheRight())
.build());
}
}
syntheticResults.add(ImmutableSyntheticResult.builder()
.captureTime(captureTime)
.totalDurationNanos(totalDurationNanos)
.executionCount(executionCount)
.addAllErrorIntervals(errorIntervals)
.build());
}
return syntheticResults;
}
@Override
public List readLastFromRollup0(String agentRollupId,
String syntheticMonitorId, int x) throws Exception {
BoundStatement boundStatement = readLastFromRollup0.bind();
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, syntheticMonitorId);
boundStatement.setInt(i++, x);
ResultSet results = session.read(boundStatement);
List syntheticResults = new ArrayList<>();
for (Row row : results) {
i = 0;
long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
double totalDurationNanos = row.getDouble(i++);
ByteBuffer errorIntervalsBytes = row.getBytes(i++);
syntheticResults.add(ImmutableSyntheticResultRollup0.builder()
.captureTime(captureTime)
.totalDurationNanos(totalDurationNanos)
.error(errorIntervalsBytes != null)
.build());
}
return syntheticResults;
}
@Override
public void rollup(String agentRollupId) throws Exception {
List ttls = getTTLs();
int rollupLevel = 1;
while (rollupLevel < configRepository.getRollupConfigs().size()) {
int ttl = ttls.get(rollupLevel);
rollup(agentRollupId, rollupLevel, ttl);
rollupLevel++;
}
}
private void rollup(String agentRollupId, int rollupLevel, int ttl) throws Exception {
List rollupConfigs = configRepository.getRollupConfigs();
long rollupIntervalMillis = rollupConfigs.get(rollupLevel).intervalMillis();
Collection needsRollupList = Common.getNeedsRollupList(agentRollupId,
rollupLevel, rollupIntervalMillis, readNeedsRollup, session, clock);
Long nextRollupIntervalMillis = null;
if (rollupLevel + 1 < rollupConfigs.size()) {
nextRollupIntervalMillis = rollupConfigs.get(rollupLevel + 1).intervalMillis();
}
for (NeedsRollup needsRollup : needsRollupList) {
long captureTime = needsRollup.getCaptureTime();
long from = captureTime - rollupIntervalMillis;
int adjustedTTL = Common.getAdjustedTTL(ttl, captureTime, clock);
Set syntheticMonitorIds = needsRollup.getKeys();
List> futures = new ArrayList<>();
for (String syntheticMonitorId : syntheticMonitorIds) {
futures.add(rollupOne(rollupLevel, agentRollupId, syntheticMonitorId, from,
captureTime, adjustedTTL));
}
// wait for above async work to ensure rollup complete before proceeding
MoreFutures.waitForAll(futures);
int needsRollupAdjustedTTL =
Common.getNeedsRollupAdjustedTTL(adjustedTTL, rollupConfigs);
PreparedStatement insertNeedsRollup = nextRollupIntervalMillis == null ? null
: this.insertNeedsRollup.get(rollupLevel);
PreparedStatement deleteNeedsRollup = this.deleteNeedsRollup.get(rollupLevel - 1);
Common.postRollup(agentRollupId, needsRollup.getCaptureTime(), syntheticMonitorIds,
needsRollup.getUniquenessKeysForDeletion(), nextRollupIntervalMillis,
insertNeedsRollup, deleteNeedsRollup, needsRollupAdjustedTTL, session);
}
}
// from is non-inclusive
private ListenableFuture rollupOne(int rollupLevel, String agentRollupId,
String syntheticMonitorId, long from, long to, int adjustedTTL) throws Exception {
BoundStatement boundStatement = readResultForRollupPS.get(rollupLevel - 1).bind();
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, syntheticMonitorId);
boundStatement.setTimestamp(i++, new Date(from));
boundStatement.setTimestamp(i++, new Date(to));
ListenableFuture future = session.readAsyncWarnIfNoRows(boundStatement,
"no synthetic result table records found for agentRollupId={},"
+ " syntheticMonitorId={}, from={}, to={}, level={}",
agentRollupId, syntheticMonitorId, from, to, rollupLevel);
return MoreFutures.rollupAsync(future, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupOneFromRows(rollupLevel, agentRollupId, syntheticMonitorId, to,
adjustedTTL, rows);
}
});
}
private ListenableFuture rollupOneFromRows(int rollupLevel, String agentRollupId,
String syntheticMonitorId, long to, int adjustedTTL, Iterable rows)
throws Exception {
double totalDurationNanos = 0;
long executionCount = 0;
ErrorIntervalCollector errorIntervalCollector = new ErrorIntervalCollector();
for (Row row : rows) {
int i = 0;
totalDurationNanos += row.getDouble(i++);
executionCount += row.getLong(i++);
ByteBuffer errorIntervalsBytes = row.getBytes(i++);
if (errorIntervalsBytes == null) {
errorIntervalCollector.addGap();
} else {
List errorIntervals = Messages
.parseDelimitedFrom(errorIntervalsBytes, Stored.ErrorInterval.parser());
errorIntervalCollector.addErrorIntervals(fromProto(errorIntervals));
}
}
BoundStatement boundStatement = insertResultPS.get(rollupLevel).bind();
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, syntheticMonitorId);
boundStatement.setTimestamp(i++, new Date(to));
boundStatement.setDouble(i++, totalDurationNanos);
boundStatement.setLong(i++, executionCount);
List mergedErrorIntervals = errorIntervalCollector.getMergedErrorIntervals();
if (mergedErrorIntervals.isEmpty()) {
boundStatement.setToNull(i++);
} else {
boundStatement.setBytes(i++, Messages.toByteBuffer(toProto(mergedErrorIntervals)));
}
boundStatement.setInt(i++, adjustedTTL);
return session.writeAsync(boundStatement);
}
private List getTTLs() throws Exception {
List ttls = new ArrayList<>();
List rollupExpirationHours =
configRepository.getCentralStorageConfig().rollupExpirationHours();
for (long expirationHours : rollupExpirationHours) {
ttls.add(Ints.saturatedCast(HOURS.toSeconds(expirationHours)));
}
return ttls;
}
@OnlyUsedByTests
void truncateAll() throws Exception {
for (int i = 0; i < configRepository.getRollupConfigs().size(); i++) {
session.updateSchemaWithRetry("truncate synthetic_result_rollup_" + i);
}
for (int i = 1; i < configRepository.getRollupConfigs().size(); i++) {
session.updateSchemaWithRetry("truncate synthetic_needs_rollup_" + i);
}
}
private static List fromProto(List storedErrorIntervals) {
List errorIntervals = new ArrayList<>();
for (Stored.ErrorInterval storedErrorInterval : storedErrorIntervals) {
errorIntervals.add(ImmutableErrorInterval.builder()
.from(storedErrorInterval.getFrom())
.to(storedErrorInterval.getTo())
.count(storedErrorInterval.getCount())
.message(storedErrorInterval.getMessage())
.doNotMergeToTheLeft(storedErrorInterval.getDoNotMergeToTheLeft())
.doNotMergeToTheRight(storedErrorInterval.getDoNotMergeToTheRight())
.build());
}
return errorIntervals;
}
private static List toProto(List errorIntervals) {
List storedErrorIntervals = new ArrayList<>();
for (ErrorInterval errorInterval : errorIntervals) {
storedErrorIntervals.add(Stored.ErrorInterval.newBuilder()
.setFrom(errorInterval.from())
.setTo(errorInterval.to())
.setCount(errorInterval.count())
.setMessage(errorInterval.message())
.setDoNotMergeToTheLeft(errorInterval.doNotMergeToTheLeft())
.setDoNotMergeToTheRight(errorInterval.doNotMergeToTheRight())
.build());
}
return storedErrorIntervals;
}
}