Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.glowroot.central.repo.AggregateDaoImpl Maven / Gradle / Ivy
/*
* Copyright 2015-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.glowroot.central.repo;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executor;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.utils.UUIDs;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Multimap;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.protobuf.AbstractMessage;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.immutables.value.Value;
import org.glowroot.central.repo.Common.NeedsRollup;
import org.glowroot.central.repo.Common.NeedsRollupFromChildren;
import org.glowroot.central.util.Messages;
import org.glowroot.central.util.MoreFutures;
import org.glowroot.central.util.MoreFutures.DoRollup;
import org.glowroot.central.util.Session;
import org.glowroot.common.ConfigDefaults;
import org.glowroot.common.Constants;
import org.glowroot.common.live.ImmutableAggregateQuery;
import org.glowroot.common.live.ImmutableOverviewAggregate;
import org.glowroot.common.live.ImmutablePercentileAggregate;
import org.glowroot.common.live.ImmutableThroughputAggregate;
import org.glowroot.common.live.LiveAggregateRepository.AggregateQuery;
import org.glowroot.common.live.LiveAggregateRepository.OverviewAggregate;
import org.glowroot.common.live.LiveAggregateRepository.PercentileAggregate;
import org.glowroot.common.live.LiveAggregateRepository.SummaryQuery;
import org.glowroot.common.live.LiveAggregateRepository.ThroughputAggregate;
import org.glowroot.common.model.LazyHistogram;
import org.glowroot.common.model.LazyHistogram.ScratchBuffer;
import org.glowroot.common.model.MutableProfile;
import org.glowroot.common.model.MutableQuery;
import org.glowroot.common.model.MutableServiceCall;
import org.glowroot.common.model.OverallErrorSummaryCollector;
import org.glowroot.common.model.OverallSummaryCollector;
import org.glowroot.common.model.ProfileCollector;
import org.glowroot.common.model.QueryCollector;
import org.glowroot.common.model.ServiceCallCollector;
import org.glowroot.common.model.TransactionNameErrorSummaryCollector;
import org.glowroot.common.model.TransactionNameErrorSummaryCollector.ErrorSummarySortOrder;
import org.glowroot.common.model.TransactionNameSummaryCollector;
import org.glowroot.common.model.TransactionNameSummaryCollector.SummarySortOrder;
import org.glowroot.common.util.CaptureTimes;
import org.glowroot.common.util.Clock;
import org.glowroot.common.util.NotAvailableAware;
import org.glowroot.common.util.OnlyUsedByTests;
import org.glowroot.common.util.Styles;
import org.glowroot.common2.repo.ConfigRepository.AgentConfigNotFoundException;
import org.glowroot.common2.repo.ConfigRepository.RollupConfig;
import org.glowroot.common2.repo.MutableAggregate;
import org.glowroot.common2.repo.MutableThreadStats;
import org.glowroot.common2.repo.MutableTimer;
import org.glowroot.wire.api.model.AgentConfigOuterClass.AgentConfig.AdvancedConfig;
import org.glowroot.wire.api.model.AggregateOuterClass.Aggregate;
import org.glowroot.wire.api.model.AggregateOuterClass.OldAggregatesByType;
import org.glowroot.wire.api.model.AggregateOuterClass.OldTransactionAggregate;
import org.glowroot.wire.api.model.ProfileOuterClass.Profile;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.concurrent.TimeUnit.HOURS;
public class AggregateDaoImpl implements AggregateDao {
@SuppressWarnings("deprecation")
private static final HashFunction SHA_1 = Hashing.sha1();
private static final Table summaryTable = ImmutableTable.builder()
.partialName("summary")
.addColumns(ImmutableColumn.of("total_duration_nanos", "double"))
.addColumns(ImmutableColumn.of("transaction_count", "bigint"))
.summary(true)
.fromInclusive(false)
.build();
private static final Table errorSummaryTable = ImmutableTable.builder()
.partialName("error_summary")
.addColumns(ImmutableColumn.of("error_count", "bigint"))
.addColumns(ImmutableColumn.of("transaction_count", "bigint"))
.summary(true)
.fromInclusive(false)
.build();
private static final Table overviewTable = ImmutableTable.builder()
.partialName("overview")
.addColumns(ImmutableColumn.of("total_duration_nanos", "double"))
.addColumns(ImmutableColumn.of("transaction_count", "bigint"))
.addColumns(ImmutableColumn.of("async_transactions", "boolean"))
.addColumns(ImmutableColumn.of("main_thread_root_timers", "blob"))
.addColumns(ImmutableColumn.of("main_thread_total_cpu_nanos", "double"))
.addColumns(ImmutableColumn.of("main_thread_total_blocked_nanos", "double"))
.addColumns(ImmutableColumn.of("main_thread_total_waited_nanos", "double"))
.addColumns(ImmutableColumn.of("main_thread_total_allocated_bytes", "double"))
// ideally this would be named aux_thread_root_timer (as there is a single root)
.addColumns(ImmutableColumn.of("aux_thread_root_timers", "blob"))
.addColumns(ImmutableColumn.of("aux_thread_total_cpu_nanos", "double"))
.addColumns(ImmutableColumn.of("aux_thread_total_blocked_nanos", "double"))
.addColumns(ImmutableColumn.of("aux_thread_total_waited_nanos", "double"))
.addColumns(ImmutableColumn.of("aux_thread_total_allocated_bytes", "double"))
// ideally this would be named async_timers (as they are all root)
.addColumns(ImmutableColumn.of("async_root_timers", "blob"))
.summary(false)
.fromInclusive(true)
.build();
private static final Table histogramTable = ImmutableTable.builder()
.partialName("histogram")
.addColumns(ImmutableColumn.of("total_duration_nanos", "double"))
.addColumns(ImmutableColumn.of("transaction_count", "bigint"))
.addColumns(ImmutableColumn.of("duration_nanos_histogram", "blob"))
.summary(false)
.fromInclusive(true)
.build();
private static final Table throughputTable = ImmutableTable.builder()
.partialName("throughput")
.addColumns(ImmutableColumn.of("transaction_count", "bigint"))
.addColumns(ImmutableColumn.of("error_count", "bigint"))
.summary(false)
.fromInclusive(true)
.build();
private static final Table queryTable = ImmutableTable.builder()
.partialName("query")
.addColumns(ImmutableColumn.of("query_type", "varchar"))
.addColumns(ImmutableColumn.of("truncated_query_text", "varchar"))
// empty when truncated_query_text is really full query text
// (not null since this column must be used in clustering key)
.addColumns(ImmutableColumn.of("full_query_text_sha1", "varchar"))
.addColumns(ImmutableColumn.of("total_duration_nanos", "double"))
.addColumns(ImmutableColumn.of("execution_count", "bigint"))
.addColumns(ImmutableColumn.of("total_rows", "bigint"))
.addClusterKey("query_type")
.addClusterKey("truncated_query_text")
.addClusterKey("full_query_text_sha1") // need this for uniqueness
.summary(false)
.fromInclusive(false)
.build();
private static final Table serviceCallTable = ImmutableTable.builder()
.partialName("service_call")
.addColumns(ImmutableColumn.of("service_call_type", "varchar"))
.addColumns(ImmutableColumn.of("service_call_text", "varchar"))
.addColumns(ImmutableColumn.of("total_duration_nanos", "double"))
.addColumns(ImmutableColumn.of("execution_count", "bigint"))
.addClusterKey("service_call_type")
.addClusterKey("service_call_text")
.summary(false)
.fromInclusive(false)
.build();
private static final Table mainThreadProfileTable = ImmutableTable.builder()
.partialName("main_thread_profile")
.addColumns(ImmutableColumn.of("main_thread_profile", "blob"))
.summary(false)
.fromInclusive(false)
.build();
private static final Table auxThreadProfileTable = ImmutableTable.builder()
.partialName("aux_thread_profile")
.addColumns(ImmutableColumn.of("aux_thread_profile", "blob"))
.summary(false)
.fromInclusive(false)
.build();
private final Session session;
private final ActiveAgentDao activeAgentDao;
private final TransactionTypeDao transactionTypeDao;
private final FullQueryTextDao fullQueryTextDao;
private final ConfigRepositoryImpl configRepository;
private final Executor asyncExecutor;
private final Clock clock;
// list index is rollupLevel
private final Map> insertOverallPS;
private final Map> insertTransactionPS;
private final Map> readOverallPS;
private final Map> readOverallForRollupPS;
private final Map readOverallForRollupFromChildPS;
private final Map> readTransactionPS;
private final Map> readTransactionForRollupPS;
private final Map readTransactionForRollupFromChildPS;
private final List existsMainThreadProfileOverallPS;
private final List existsMainThreadProfileTransactionPS;
private final List existsAuxThreadProfileOverallPS;
private final List existsAuxThreadProfileTransactionPS;
private final List insertNeedsRollup;
private final List readNeedsRollup;
private final List deleteNeedsRollup;
private final PreparedStatement insertNeedsRollupFromChild;
private final PreparedStatement readNeedsRollupFromChild;
private final PreparedStatement deleteNeedsRollupFromChild;
private final ImmutableList allTables;
AggregateDaoImpl(Session session, ActiveAgentDao activeAgentDao,
TransactionTypeDao transactionTypeDao, FullQueryTextDao fullQueryTextDao,
ConfigRepositoryImpl configRepository, Executor asyncExecutor, Clock clock)
throws Exception {
this.session = session;
this.activeAgentDao = activeAgentDao;
this.transactionTypeDao = transactionTypeDao;
this.fullQueryTextDao = fullQueryTextDao;
this.configRepository = configRepository;
this.asyncExecutor = asyncExecutor;
this.clock = clock;
int count = configRepository.getRollupConfigs().size();
List rollupExpirationHours =
configRepository.getCentralStorageConfig().rollupExpirationHours();
List queryAndServiceCallRollupExpirationHours =
configRepository.getCentralStorageConfig()
.queryAndServiceCallRollupExpirationHours();
List profileRollupExpirationHours =
configRepository.getCentralStorageConfig().profileRollupExpirationHours();
allTables = ImmutableList.of(summaryTable, errorSummaryTable, overviewTable,
histogramTable, throughputTable, queryTable, serviceCallTable,
mainThreadProfileTable, auxThreadProfileTable);
Map> insertOverallMap = new HashMap<>();
Map> insertTransactionMap = new HashMap<>();
Map> readOverallMap = new HashMap<>();
Map> readOverallForRollupMap = new HashMap<>();
Map readOverallForRollupFromChildMap = new HashMap<>();
Map> readTransactionMap = new HashMap<>();
Map> readTransactionForRollupMap = new HashMap<>();
Map readTransactionForRollupFromChildMap = new HashMap<>();
for (Table table : allTables) {
List insertOverallList = new ArrayList<>();
List insertTransactionList = new ArrayList<>();
List readOverallList = new ArrayList<>();
List readOverallForRollupList = new ArrayList<>();
List readTransactionList = new ArrayList<>();
List readTransactionForRollupList = new ArrayList<>();
for (int i = 0; i < count; i++) {
int expirationHours;
if (table.partialName().equals("query")
|| table.partialName().equals("service_call")) {
expirationHours = queryAndServiceCallRollupExpirationHours.get(i);
} else if (table.partialName().equals("main_thread_profile")
|| table.partialName().equals("aux_thread_profile")) {
expirationHours = profileRollupExpirationHours.get(i);
} else {
expirationHours = rollupExpirationHours.get(i);
}
if (table.summary()) {
session.createTableWithTWCS(createSummaryTableQuery(table, false, i),
expirationHours);
session.createTableWithTWCS(createSummaryTableQuery(table, true, i),
expirationHours);
insertOverallList.add(session.prepare(insertSummaryPS(table, false, i)));
insertTransactionList.add(session.prepare(insertSummaryPS(table, true, i)));
readOverallList.add(session.prepare(readSummaryPS(table, false, i)));
readOverallForRollupList
.add(session.prepare(readSummaryForRollupPS(table, false, i)));
readTransactionList.add(session.prepare(readSummaryPS(table, true, i)));
readTransactionForRollupList
.add(session.prepare(readSummaryForRollupPS(table, true, i)));
} else {
session.createTableWithTWCS(createTableQuery(table, false, i), expirationHours);
session.createTableWithTWCS(createTableQuery(table, true, i), expirationHours);
insertOverallList.add(session.prepare(insertPS(table, false, i)));
insertTransactionList.add(session.prepare(insertPS(table, true, i)));
readOverallList.add(session.prepare(readPS(table, false, i)));
readOverallForRollupList.add(session.prepare(readForRollupPS(table, false, i)));
readTransactionList.add(session.prepare(readPS(table, true, i)));
readTransactionForRollupList
.add(session.prepare(readForRollupPS(table, true, i)));
}
}
insertOverallMap.put(table, ImmutableList.copyOf(insertOverallList));
insertTransactionMap.put(table, ImmutableList.copyOf(insertTransactionList));
readOverallMap.put(table, ImmutableList.copyOf(readOverallList));
readOverallForRollupMap.put(table, ImmutableList.copyOf(readOverallForRollupList));
if (table.summary()) {
readOverallForRollupFromChildMap.put(table,
session.prepare(readSummaryForRollupFromChildPS(table, false, 0)));
} else {
readOverallForRollupFromChildMap.put(table,
session.prepare(readForRollupFromChildPS(table, false, 0)));
}
readTransactionMap.put(table, ImmutableList.copyOf(readTransactionList));
readTransactionForRollupMap.put(table,
ImmutableList.copyOf(readTransactionForRollupList));
if (table.summary()) {
readTransactionForRollupFromChildMap.put(table,
session.prepare(readSummaryForRollupFromChildPS(table, true, 0)));
} else {
readTransactionForRollupFromChildMap.put(table,
session.prepare(readForRollupFromChildPS(table, true, 0)));
}
}
this.insertOverallPS = ImmutableMap.copyOf(insertOverallMap);
this.insertTransactionPS = ImmutableMap.copyOf(insertTransactionMap);
this.readOverallPS = ImmutableMap.copyOf(readOverallMap);
this.readOverallForRollupPS = ImmutableMap.copyOf(readOverallForRollupMap);
this.readOverallForRollupFromChildPS =
ImmutableMap.copyOf(readOverallForRollupFromChildMap);
this.readTransactionPS = ImmutableMap.copyOf(readTransactionMap);
this.readTransactionForRollupPS = ImmutableMap.copyOf(readTransactionForRollupMap);
this.readTransactionForRollupFromChildPS =
ImmutableMap.copyOf(readTransactionForRollupFromChildMap);
List existsMainThreadProfileOverallPS = new ArrayList<>();
List existsMainThreadProfileTransactionPS = new ArrayList<>();
List existsAuxThreadProfileOverallPS = new ArrayList<>();
List existsAuxThreadProfileTransactionPS = new ArrayList<>();
for (int i = 0; i < count; i++) {
existsMainThreadProfileOverallPS
.add(session.prepare(existsPS(mainThreadProfileTable, false, i)));
existsMainThreadProfileTransactionPS
.add(session.prepare(existsPS(mainThreadProfileTable, true, i)));
existsAuxThreadProfileOverallPS
.add(session.prepare(existsPS(auxThreadProfileTable, false, i)));
existsAuxThreadProfileTransactionPS
.add(session.prepare(existsPS(auxThreadProfileTable, true, i)));
}
this.existsMainThreadProfileOverallPS = existsMainThreadProfileOverallPS;
this.existsMainThreadProfileTransactionPS = existsMainThreadProfileTransactionPS;
this.existsAuxThreadProfileOverallPS = existsAuxThreadProfileOverallPS;
this.existsAuxThreadProfileTransactionPS = existsAuxThreadProfileTransactionPS;
// since rollup operations are idempotent, any records resurrected after gc_grace_seconds
// would just create extra work, but not have any other effect
//
// not using gc_grace_seconds of 0 since that disables hinted handoff
// (http://www.uberobert.com/cassandra_gc_grace_disables_hinted_handoff)
//
// it seems any value over max_hint_window_in_ms (which defaults to 3 hours) is good
long needsRollupGcGraceSeconds = HOURS.toSeconds(4);
List insertNeedsRollup = new ArrayList<>();
List readNeedsRollup = new ArrayList<>();
List deleteNeedsRollup = new ArrayList<>();
for (int i = 1; i < count; i++) {
session.createTableWithLCS("create table if not exists aggregate_needs_rollup_" + i
+ " (agent_rollup varchar, capture_time timestamp, uniqueness timeuuid,"
+ " transaction_types set, primary key (agent_rollup, capture_time,"
+ " uniqueness)) with gc_grace_seconds = " + needsRollupGcGraceSeconds, true);
// TTL is used to prevent non-idempotent rolling up of partially expired aggregates
// (e.g. "needs rollup" record resurrecting due to small gc_grace_seconds)
insertNeedsRollup.add(session.prepare("insert into aggregate_needs_rollup_" + i
+ " (agent_rollup, capture_time, uniqueness, transaction_types) values"
+ " (?, ?, ?, ?) using TTL ?"));
readNeedsRollup.add(session.prepare("select capture_time, uniqueness, transaction_types"
+ " from aggregate_needs_rollup_" + i + " where agent_rollup = ?"));
deleteNeedsRollup.add(session.prepare("delete from aggregate_needs_rollup_" + i
+ " where agent_rollup = ? and capture_time = ? and uniqueness = ?"));
}
this.insertNeedsRollup = insertNeedsRollup;
this.readNeedsRollup = readNeedsRollup;
this.deleteNeedsRollup = deleteNeedsRollup;
session.createTableWithLCS("create table if not exists aggregate_needs_rollup_from_child"
+ " (agent_rollup varchar, capture_time timestamp, uniqueness timeuuid,"
+ " child_agent_rollup varchar, transaction_types set, primary key"
+ " (agent_rollup, capture_time, uniqueness)) with gc_grace_seconds = "
+ needsRollupGcGraceSeconds, true);
// TTL is used to prevent non-idempotent rolling up of partially expired aggregates
// (e.g. "needs rollup" record resurrecting due to small gc_grace_seconds)
insertNeedsRollupFromChild = session.prepare("insert into aggregate_needs_rollup_from_child"
+ " (agent_rollup, capture_time, uniqueness, child_agent_rollup, transaction_types)"
+ " values (?, ?, ?, ?, ?) using TTL ?");
readNeedsRollupFromChild = session.prepare("select capture_time, uniqueness,"
+ " child_agent_rollup, transaction_types from aggregate_needs_rollup_from_child"
+ " where agent_rollup = ?");
deleteNeedsRollupFromChild = session.prepare("delete from aggregate_needs_rollup_from_child"
+ " where agent_rollup = ? and capture_time = ? and uniqueness = ?");
}
@Override
public void store(String agentId, long captureTime,
List aggregatesByTypeList,
List initialSharedQueryTexts) throws Exception {
List agentRollupIds = AgentRollupIds.getAgentRollupIds(agentId);
store(agentId, agentRollupIds, agentId, agentRollupIds, captureTime, aggregatesByTypeList,
initialSharedQueryTexts);
}
public void store(String agentId, List agentRollupIds,
String agentIdForMeta, List agentRollupIdsForMeta, long captureTime,
List aggregatesByTypeList,
List initialSharedQueryTexts) throws Exception {
if (aggregatesByTypeList.isEmpty()) {
MoreFutures.waitForAll(activeAgentDao.insert(agentIdForMeta, captureTime));
return;
}
TTL adjustedTTL = getAdjustedTTL(getTTLs().get(0), captureTime, clock);
List> futures = new ArrayList<>();
List sharedQueryTexts = new ArrayList<>();
for (Aggregate.SharedQueryText sharedQueryText : initialSharedQueryTexts) {
String fullTextSha1 = sharedQueryText.getFullTextSha1();
if (fullTextSha1.isEmpty()) {
String fullText = sharedQueryText.getFullText();
if (fullText.length() > Constants.AGGREGATE_QUERY_TEXT_TRUNCATE) {
// relying on agent side to rate limit (re-)sending the same full text
fullTextSha1 = SHA_1.hashString(fullText, UTF_8).toString();
futures.addAll(fullQueryTextDao.store(agentRollupIds, fullTextSha1, fullText));
sharedQueryText = Aggregate.SharedQueryText.newBuilder()
.setTruncatedText(fullText.substring(0,
Constants.AGGREGATE_QUERY_TEXT_TRUNCATE))
.setFullTextSha1(fullTextSha1)
.build();
}
}
sharedQueryTexts.add(sharedQueryText);
}
// wait for success before proceeding in order to ensure cannot end up with orphaned
// fullTextSha1
MoreFutures.waitForAll(futures);
futures.clear();
for (OldAggregatesByType aggregatesByType : aggregatesByTypeList) {
String transactionType = aggregatesByType.getTransactionType();
Aggregate overallAggregate = aggregatesByType.getOverallAggregate();
futures.addAll(storeOverallAggregate(agentId, transactionType, captureTime,
overallAggregate, sharedQueryTexts, adjustedTTL));
for (OldTransactionAggregate transactionAggregate : aggregatesByType
.getTransactionAggregateList()) {
futures.addAll(storeTransactionAggregate(agentId, transactionType,
transactionAggregate.getTransactionName(), captureTime,
transactionAggregate.getAggregate(), sharedQueryTexts, adjustedTTL));
}
// wait for success before proceeding in order to ensure cannot end up with
// "no overview table records found" during a transactionName rollup, since
// transactionName rollups are based on finding transactionName in summary table
MoreFutures.waitForAll(futures);
futures.clear();
for (OldTransactionAggregate transactionAggregate : aggregatesByType
.getTransactionAggregateList()) {
futures.addAll(storeTransactionNameSummary(agentId, transactionType,
transactionAggregate.getTransactionName(), captureTime,
transactionAggregate.getAggregate(), adjustedTTL));
}
futures.addAll(transactionTypeDao.store(agentRollupIdsForMeta, transactionType));
}
futures.addAll(activeAgentDao.insert(agentIdForMeta, captureTime));
// wait for success before inserting "needs rollup" records
MoreFutures.waitForAll(futures);
futures.clear();
List rollupConfigs = configRepository.getRollupConfigs();
// TODO report checker framework issue that occurs without this suppression
@SuppressWarnings("assignment.type.incompatible")
Set transactionTypes = aggregatesByTypeList.stream()
.map(OldAggregatesByType::getTransactionType).collect(Collectors.toSet());
int needsRollupAdjustedTTL =
Common.getNeedsRollupAdjustedTTL(adjustedTTL.generalTTL(), rollupConfigs);
if (agentRollupIds.size() > 1) {
BoundStatement boundStatement = insertNeedsRollupFromChild.bind();
int i = 0;
boundStatement.setString(i++, agentRollupIds.get(1));
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setUUID(i++, UUIDs.timeBased());
boundStatement.setString(i++, agentId);
boundStatement.setSet(i++, transactionTypes);
boundStatement.setInt(i++, needsRollupAdjustedTTL);
futures.add(session.writeAsync(boundStatement));
}
// insert into aggregate_needs_rollup_1
long intervalMillis = rollupConfigs.get(1).intervalMillis();
long rollupCaptureTime = CaptureTimes.getRollup(captureTime, intervalMillis);
BoundStatement boundStatement = insertNeedsRollup.get(0).bind();
int i = 0;
boundStatement.setString(i++, agentId);
boundStatement.setTimestamp(i++, new Date(rollupCaptureTime));
boundStatement.setUUID(i++, UUIDs.timeBased());
boundStatement.setSet(i++, transactionTypes);
boundStatement.setInt(i++, needsRollupAdjustedTTL);
futures.add(session.writeAsync(boundStatement));
MoreFutures.waitForAll(futures);
}
// query.from() is non-inclusive
@Override
public void mergeOverallSummaryInto(String agentRollupId, SummaryQuery query,
OverallSummaryCollector collector) throws Exception {
// currently have to do aggregation client-site (don't want to require Cassandra 2.2 yet)
ResultSet results = executeQuery(agentRollupId, query, summaryTable);
for (Row row : results) {
int i = 0;
// results are ordered by capture time so Math.max() is not needed here
long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
double totalDurationNanos = row.getDouble(i++);
long transactionCount = row.getLong(i++);
collector.mergeSummary(totalDurationNanos, transactionCount, captureTime);
}
}
// sortOrder and limit are only used by embedded H2 repository, while the central cassandra
// repository which currently has to pull in all records anyways just delegates ordering and
// limit to TransactionNameSummaryCollector
//
// query.from() is non-inclusive
@Override
public void mergeTransactionNameSummariesInto(String agentRollupId, SummaryQuery query,
SummarySortOrder sortOrder, int limit, TransactionNameSummaryCollector collector)
throws Exception {
// currently have to do group by / sort / limit client-side
BoundStatement boundStatement =
checkNotNull(readTransactionPS.get(summaryTable)).get(query.rollupLevel()).bind();
bindQuery(boundStatement, agentRollupId, query);
ResultSet results = session.read(boundStatement);
for (Row row : results) {
int i = 0;
long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
String transactionName = checkNotNull(row.getString(i++));
double totalDurationNanos = row.getDouble(i++);
long transactionCount = row.getLong(i++);
collector.collect(transactionName, totalDurationNanos, transactionCount, captureTime);
}
}
// query.from() is non-inclusive
@Override
public void mergeOverallErrorSummaryInto(String agentRollupId, SummaryQuery query,
OverallErrorSummaryCollector collector) throws Exception {
// currently have to do aggregation client-site (don't want to require Cassandra 2.2 yet)
ResultSet results = executeQuery(agentRollupId, query, errorSummaryTable);
for (Row row : results) {
int i = 0;
// results are ordered by capture time so Math.max() is not needed here
long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
long errorCount = row.getLong(i++);
long transactionCount = row.getLong(i++);
collector.mergeErrorSummary(errorCount, transactionCount, captureTime);
}
}
// sortOrder and limit are only used by embedded H2 repository, while the central cassandra
// repository which currently has to pull in all records anyways just delegates ordering and
// limit to TransactionNameErrorSummaryCollector
//
// query.from() is non-inclusive
@Override
public void mergeTransactionNameErrorSummariesInto(String agentRollupId, SummaryQuery query,
ErrorSummarySortOrder sortOrder, int limit,
TransactionNameErrorSummaryCollector collector) throws Exception {
// currently have to do group by / sort / limit client-side
BoundStatement boundStatement = checkNotNull(readTransactionPS.get(errorSummaryTable))
.get(query.rollupLevel()).bind();
bindQuery(boundStatement, agentRollupId, query);
ResultSet results = session.read(boundStatement);
for (Row row : results) {
int i = 0;
long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
String transactionName = checkNotNull(row.getString(i++));
long errorCount = row.getLong(i++);
long transactionCount = row.getLong(i++);
collector.collect(transactionName, errorCount, transactionCount, captureTime);
}
}
// query.from() is INCLUSIVE
@Override
public List readOverviewAggregates(String agentRollupId,
AggregateQuery query) throws Exception {
ResultSet results = executeQuery(agentRollupId, query, overviewTable);
List overviewAggregates = new ArrayList<>();
for (Row row : results) {
int i = 0;
long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
double totalDurationNanos = row.getDouble(i++);
long transactionCount = row.getLong(i++);
boolean asyncTransactions = row.getBool(i++);
List mainThreadRootTimers =
Messages.parseDelimitedFrom(row.getBytes(i++), Aggregate.Timer.parser());
Aggregate.ThreadStats mainThreadStats = Aggregate.ThreadStats.newBuilder()
.setTotalCpuNanos(getNextThreadStat(row, i++))
.setTotalBlockedNanos(getNextThreadStat(row, i++))
.setTotalWaitedNanos(getNextThreadStat(row, i++))
.setTotalAllocatedBytes(getNextThreadStat(row, i++))
.build();
// reading delimited singleton list for backwards compatibility with data written
// prior to 0.12.0
List list =
Messages.parseDelimitedFrom(row.getBytes(i++), Aggregate.Timer.parser());
Aggregate.Timer auxThreadRootTimer = list.isEmpty() ? null : list.get(0);
Aggregate.ThreadStats auxThreadStats;
if (auxThreadRootTimer == null) {
auxThreadStats = null;
i += 4;
} else {
auxThreadStats = Aggregate.ThreadStats.newBuilder()
.setTotalCpuNanos(getNextThreadStat(row, i++))
.setTotalBlockedNanos(getNextThreadStat(row, i++))
.setTotalWaitedNanos(getNextThreadStat(row, i++))
.setTotalAllocatedBytes(getNextThreadStat(row, i++))
.build();
}
List asyncTimers =
Messages.parseDelimitedFrom(row.getBytes(i++), Aggregate.Timer.parser());
ImmutableOverviewAggregate.Builder builder = ImmutableOverviewAggregate.builder()
.captureTime(captureTime)
.totalDurationNanos(totalDurationNanos)
.transactionCount(transactionCount)
.asyncTransactions(asyncTransactions)
.addAllMainThreadRootTimers(mainThreadRootTimers)
.mainThreadStats(mainThreadStats)
.auxThreadRootTimer(auxThreadRootTimer)
.auxThreadStats(auxThreadStats)
.addAllAsyncTimers(asyncTimers);
overviewAggregates.add(builder.build());
}
return overviewAggregates;
}
// query.from() is INCLUSIVE
@Override
public List readPercentileAggregates(String agentRollupId,
AggregateQuery query) throws Exception {
ResultSet results = executeQuery(agentRollupId, query, histogramTable);
List percentileAggregates = new ArrayList<>();
for (Row row : results) {
int i = 0;
long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
double totalDurationNanos = row.getDouble(i++);
long transactionCount = row.getLong(i++);
ByteBuffer bytes = checkNotNull(row.getBytes(i++));
Aggregate.Histogram durationNanosHistogram = Aggregate.Histogram.parseFrom(bytes);
percentileAggregates.add(ImmutablePercentileAggregate.builder()
.captureTime(captureTime)
.totalDurationNanos(totalDurationNanos)
.transactionCount(transactionCount)
.durationNanosHistogram(durationNanosHistogram)
.build());
}
return percentileAggregates;
}
// query.from() is INCLUSIVE
@Override
public List readThroughputAggregates(String agentRollupId,
AggregateQuery query) throws Exception {
ResultSet results = executeQuery(agentRollupId, query, throughputTable);
List throughputAggregates = new ArrayList<>();
for (Row row : results) {
int i = 0;
long captureTime = checkNotNull(row.getTimestamp(i++)).getTime();
long transactionCount = row.getLong(i++);
boolean hasErrorCount = !row.isNull(i);
long errorCount = row.getLong(i++);
throughputAggregates.add(ImmutableThroughputAggregate.builder()
.captureTime(captureTime)
.transactionCount(transactionCount)
.errorCount(hasErrorCount ? errorCount : null)
.build());
}
return throughputAggregates;
}
// query.from() is non-inclusive
@Override
public void mergeQueriesInto(String agentRollupId, AggregateQuery query,
QueryCollector collector) throws Exception {
ResultSet results = executeQuery(agentRollupId, query, queryTable);
long captureTime = Long.MIN_VALUE;
for (Row row : results) {
int i = 0;
captureTime = Math.max(captureTime, checkNotNull(row.getTimestamp(i++)).getTime());
String queryType = checkNotNull(row.getString(i++));
String truncatedText = checkNotNull(row.getString(i++));
// full_query_text_sha1 cannot be null since it is used in clustering key
String fullTextSha1 = Strings.emptyToNull(row.getString(i++));
double totalDurationNanos = row.getDouble(i++);
long executionCount = row.getLong(i++);
boolean hasTotalRows = !row.isNull(i);
long totalRows = row.getLong(i++);
collector.mergeQuery(queryType, truncatedText, fullTextSha1, totalDurationNanos,
executionCount, hasTotalRows, totalRows);
collector.updateLastCaptureTime(captureTime);
}
}
// query.from() is non-inclusive
@Override
public void mergeServiceCallsInto(String agentRollupId, AggregateQuery query,
ServiceCallCollector collector) throws Exception {
ResultSet results = executeQuery(agentRollupId, query, serviceCallTable);
long captureTime = Long.MIN_VALUE;
for (Row row : results) {
int i = 0;
captureTime = Math.max(captureTime, checkNotNull(row.getTimestamp(i++)).getTime());
String serviceCallType = checkNotNull(row.getString(i++));
String serviceCallText = checkNotNull(row.getString(i++));
double totalDurationNanos = row.getDouble(i++);
long executionCount = row.getLong(i++);
collector.mergeServiceCall(serviceCallType, serviceCallText, totalDurationNanos,
executionCount);
collector.updateLastCaptureTime(captureTime);
}
}
// query.from() is non-inclusive
@Override
public void mergeMainThreadProfilesInto(String agentRollupId, AggregateQuery query,
ProfileCollector collector) throws Exception {
mergeProfilesInto(agentRollupId, query, mainThreadProfileTable, collector);
}
// query.from() is non-inclusive
@Override
public void mergeAuxThreadProfilesInto(String agentRollupId, AggregateQuery query,
ProfileCollector collector) throws Exception {
mergeProfilesInto(agentRollupId, query, auxThreadProfileTable, collector);
}
@Override
public @Nullable String readFullQueryText(String agentRollupId, String fullQueryTextSha1)
throws Exception {
return fullQueryTextDao.getFullText(agentRollupId, fullQueryTextSha1);
}
// query.from() is non-inclusive
@Override
public boolean hasMainThreadProfile(String agentRollupId, AggregateQuery query)
throws Exception {
BoundStatement boundStatement = query.transactionName() == null
? existsMainThreadProfileOverallPS.get(query.rollupLevel()).bind()
: existsMainThreadProfileTransactionPS.get(query.rollupLevel()).bind();
bindQuery(boundStatement, agentRollupId, query);
ResultSet results = session.read(boundStatement);
return results.one() != null;
}
// query.from() is non-inclusive
@Override
public boolean hasAuxThreadProfile(String agentRollupId, AggregateQuery query)
throws Exception {
BoundStatement boundStatement = query.transactionName() == null
? existsAuxThreadProfileOverallPS.get(query.rollupLevel()).bind()
: existsAuxThreadProfileTransactionPS.get(query.rollupLevel()).bind();
bindQuery(boundStatement, agentRollupId, query);
ResultSet results = session.read(boundStatement);
return results.one() != null;
}
// query.from() is non-inclusive
@Override
public boolean shouldHaveQueries(String agentRollupId, AggregateQuery query) {
// TODO (this is only used for displaying data expired message)
return false;
}
// query.from() is non-inclusive
@Override
public boolean shouldHaveServiceCalls(String agentRollupId, AggregateQuery query) {
// TODO (this is only used for displaying data expired message)
return false;
}
// query.from() is non-inclusive
@Override
public boolean shouldHaveMainThreadProfile(String agentRollupId, AggregateQuery query) {
// TODO (this is only used for displaying data expired message)
return false;
}
// query.from() is non-inclusive
@Override
public boolean shouldHaveAuxThreadProfile(String agentRollupId, AggregateQuery query) {
// TODO (this is only used for displaying data expired message)
return false;
}
@Override
public void rollup(String agentRollupId) throws Exception {
rollup(agentRollupId, agentRollupId, AgentRollupIds.getParent(agentRollupId),
!agentRollupId.endsWith("::"));
}
public void rollup(String agentRollupId, String agentRollupIdForMeta,
@Nullable String parentAgentRollupId, boolean leaf) throws Exception {
List ttls = getTTLs();
if (!leaf) {
rollupFromChildren(agentRollupId, agentRollupIdForMeta, parentAgentRollupId,
ttls.get(0));
}
int rollupLevel = 1;
while (rollupLevel < configRepository.getRollupConfigs().size()) {
TTL ttl = ttls.get(rollupLevel);
rollup(agentRollupId, agentRollupIdForMeta, rollupLevel, ttl);
rollupLevel++;
}
}
@Override
@OnlyUsedByTests
public void truncateAll() throws Exception {
for (Table table : allTables) {
for (int i = 0; i < configRepository.getRollupConfigs().size(); i++) {
session.updateSchemaWithRetry(
"truncate " + getTableName(table.partialName(), false, i));
session.updateSchemaWithRetry(
"truncate " + getTableName(table.partialName(), true, i));
}
}
for (int i = 1; i < configRepository.getRollupConfigs().size(); i++) {
session.updateSchemaWithRetry("truncate aggregate_needs_rollup_" + i);
}
session.updateSchemaWithRetry("truncate aggregate_needs_rollup_from_child");
}
private void rollupFromChildren(String agentRollupId, String agentRollupIdForMeta,
@Nullable String parentAgentRollupId, TTL ttl) throws Exception {
final int rollupLevel = 0;
List needsRollupFromChildrenList = Common
.getNeedsRollupFromChildrenList(agentRollupId, readNeedsRollupFromChild, session);
List rollupConfigs = configRepository.getRollupConfigs();
long nextRollupIntervalMillis = rollupConfigs.get(rollupLevel + 1).intervalMillis();
for (NeedsRollupFromChildren needsRollupFromChildren : needsRollupFromChildrenList) {
long captureTime = needsRollupFromChildren.getCaptureTime();
TTL adjustedTTL = getAdjustedTTL(ttl, captureTime, clock);
RollupParams rollupParams =
getRollupParams(agentRollupId, agentRollupIdForMeta, rollupLevel, adjustedTTL);
List> futures = new ArrayList<>();
for (Map.Entry> entry : needsRollupFromChildren.getKeys()
.asMap()
.entrySet()) {
String transactionType = entry.getKey();
Collection childAgentRollupIds = entry.getValue();
futures.addAll(rollupOneFromChildren(rollupParams, transactionType,
childAgentRollupIds, captureTime));
}
// wait for above async work to ensure rollup complete before proceeding
MoreFutures.waitForAll(futures);
int needsRollupAdjustedTTL =
Common.getNeedsRollupAdjustedTTL(adjustedTTL.generalTTL(), rollupConfigs);
if (parentAgentRollupId != null) {
// insert needs to happen first before call to postRollup(), see method-level
// comment on postRollup
Common.insertNeedsRollupFromChild(agentRollupId, parentAgentRollupId,
insertNeedsRollupFromChild, needsRollupFromChildren, captureTime,
needsRollupAdjustedTTL, session);
}
Common.postRollup(agentRollupId, needsRollupFromChildren.getCaptureTime(),
needsRollupFromChildren.getKeys().keySet(),
needsRollupFromChildren.getUniquenessKeysForDeletion(),
nextRollupIntervalMillis, insertNeedsRollup.get(rollupLevel),
deleteNeedsRollupFromChild, needsRollupAdjustedTTL, session);
}
}
private void rollup(String agentRollupId, String agentRollupIdForMeta, int rollupLevel, TTL ttl)
throws Exception {
List rollupConfigs = configRepository.getRollupConfigs();
long rollupIntervalMillis = rollupConfigs.get(rollupLevel).intervalMillis();
Collection needsRollupList = Common.getNeedsRollupList(agentRollupId,
rollupLevel, rollupIntervalMillis, readNeedsRollup, session, clock);
Long nextRollupIntervalMillis = null;
if (rollupLevel + 1 < rollupConfigs.size()) {
nextRollupIntervalMillis = rollupConfigs.get(rollupLevel + 1).intervalMillis();
}
for (NeedsRollup needsRollup : needsRollupList) {
long captureTime = needsRollup.getCaptureTime();
TTL adjustedTTL = getAdjustedTTL(ttl, captureTime, clock);
int needsRollupAdjustedTTL =
Common.getNeedsRollupAdjustedTTL(adjustedTTL.generalTTL(), rollupConfigs);
RollupParams rollupParams =
getRollupParams(agentRollupId, agentRollupIdForMeta, rollupLevel, adjustedTTL);
long from = captureTime - rollupIntervalMillis;
Set transactionTypes = needsRollup.getKeys();
List> futures = new ArrayList<>();
for (String transactionType : transactionTypes) {
futures.addAll(rollupOne(rollupParams, transactionType, from, captureTime));
}
if (futures.isEmpty()) {
// no rollups occurred, warning already logged inside rollupOne() above
// this can happen there is an old "needs rollup" record that was created prior to
// TTL was introduced in 0.9.6, and when the "last needs rollup" record wasn't
// processed (also prior to 0.9.6), and when the corresponding old data has expired
Common.postRollup(agentRollupId, needsRollup.getCaptureTime(), transactionTypes,
needsRollup.getUniquenessKeysForDeletion(), null, null,
deleteNeedsRollup.get(rollupLevel - 1), -1, session);
continue;
}
// wait for above async work to ensure rollup complete before proceeding
MoreFutures.waitForAll(futures);
PreparedStatement insertNeedsRollup = nextRollupIntervalMillis == null ? null
: this.insertNeedsRollup.get(rollupLevel);
PreparedStatement deleteNeedsRollup = this.deleteNeedsRollup.get(rollupLevel - 1);
Common.postRollup(agentRollupId, needsRollup.getCaptureTime(), transactionTypes,
needsRollup.getUniquenessKeysForDeletion(), nextRollupIntervalMillis,
insertNeedsRollup, deleteNeedsRollup, needsRollupAdjustedTTL, session);
}
}
private List> rollupOneFromChildren(RollupParams rollup, String transactionType,
Collection childAgentRollupIds, long captureTime) throws Exception {
ImmutableAggregateQuery query = ImmutableAggregateQuery.builder()
.transactionType(transactionType)
.from(captureTime)
.to(captureTime)
.rollupLevel(rollup.rollupLevel()) // rolling up from same level (which is always 0)
.build();
List> futures = new ArrayList<>();
futures.add(rollupOverallSummaryFromChildren(rollup, query, childAgentRollupIds));
futures.add(rollupErrorSummaryFromChildren(rollup, query, childAgentRollupIds));
// key is transaction name, value child agent rollup id
Multimap transactionNames = ArrayListMultimap.create();
futures.add(rollupTransactionSummaryFromChildren(rollup, query, childAgentRollupIds,
transactionNames));
futures.add(rollupTransactionErrorSummaryFromChildren(rollup, query, childAgentRollupIds));
ScratchBuffer scratchBuffer = new ScratchBuffer();
futures.addAll(
rollupOtherPartsFromChildren(rollup, query, childAgentRollupIds, scratchBuffer));
for (Map.Entry> entry : transactionNames.asMap().entrySet()) {
futures.addAll(rollupOtherPartsFromChildren(rollup,
query.withTransactionName(entry.getKey()), entry.getValue(), scratchBuffer));
}
return futures;
}
private List> rollupOne(RollupParams rollup, String transactionType, long from,
long to) throws Exception {
ImmutableAggregateQuery query = ImmutableAggregateQuery.builder()
.transactionType(transactionType)
.from(from)
.to(to)
.rollupLevel(rollup.rollupLevel() - 1)
.build();
List> futures = new ArrayList<>();
futures.add(rollupOverallSummary(rollup, query));
futures.add(rollupErrorSummary(rollup, query));
Set transactionNames = new HashSet<>();
futures.add(rollupTransactionSummary(rollup, query, transactionNames));
futures.add(rollupTransactionErrorSummary(rollup, query));
ScratchBuffer scratchBuffer = new ScratchBuffer();
futures.addAll(rollupOtherParts(rollup, query, scratchBuffer));
for (String transactionName : transactionNames) {
futures.addAll(rollupOtherParts(rollup, query.withTransactionName(transactionName),
scratchBuffer));
}
return futures;
}
private List> rollupOtherParts(RollupParams rollup, AggregateQuery query,
ScratchBuffer scratchBuffer) throws Exception {
List> futures = new ArrayList<>();
futures.add(rollupOverview(rollup, query));
futures.add(rollupHistogram(rollup, query, scratchBuffer));
futures.add(rollupThroughput(rollup, query));
futures.add(rollupQueries(rollup, query));
futures.add(rollupServiceCalls(rollup, query));
futures.add(rollupThreadProfile(rollup, query, mainThreadProfileTable));
futures.add(rollupThreadProfile(rollup, query, auxThreadProfileTable));
return futures;
}
private List> rollupOtherPartsFromChildren(RollupParams rollup,
AggregateQuery query, Collection childAgentRollupIds,
ScratchBuffer scratchBuffer) throws Exception {
List> futures = new ArrayList<>();
futures.add(rollupOverviewFromChildren(rollup, query, childAgentRollupIds));
futures.add(rollupHistogramFromChildren(rollup, query, childAgentRollupIds, scratchBuffer));
futures.add(rollupThroughputFromChildren(rollup, query, childAgentRollupIds));
futures.add(rollupQueriesFromChildren(rollup, query, childAgentRollupIds));
futures.add(rollupServiceCallsFromChildren(rollup, query, childAgentRollupIds));
futures.add(rollupThreadProfileFromChildren(rollup, query, childAgentRollupIds,
mainThreadProfileTable));
futures.add(rollupThreadProfileFromChildren(rollup, query, childAgentRollupIds,
auxThreadProfileTable));
return futures;
}
private ListenableFuture rollupOverallSummary(RollupParams rollup, AggregateQuery query)
throws Exception {
ListenableFuture future =
executeQueryForRollup(rollup.agentRollupId(), query, summaryTable, true);
return MoreFutures.rollupAsync(future, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupOverallSummaryFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupOverallSummaryFromChildren(RollupParams rollup,
AggregateQuery query, Collection childAgentRollupIds) throws Exception {
List> futures =
getRowsForRollupFromChildren(query, childAgentRollupIds, summaryTable, true);
return MoreFutures.rollupAsync(futures, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupOverallSummaryFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupOverallSummaryFromRows(RollupParams rollup,
AggregateQuery query, Iterable rows) throws Exception {
double totalDurationNanos = 0;
long transactionCount = 0;
for (Row row : rows) {
totalDurationNanos += row.getDouble(0);
transactionCount += row.getLong(1);
}
BoundStatement boundStatement =
getInsertOverallPS(summaryTable, rollup.rollupLevel()).bind();
int i = 0;
boundStatement.setString(i++, rollup.agentRollupId());
boundStatement.setString(i++, query.transactionType());
boundStatement.setTimestamp(i++, new Date(query.to()));
boundStatement.setDouble(i++, totalDurationNanos);
boundStatement.setLong(i++, transactionCount);
boundStatement.setInt(i++, rollup.adjustedTTL().generalTTL());
return session.writeAsync(boundStatement);
}
private ListenableFuture rollupErrorSummary(RollupParams rollup, AggregateQuery query)
throws Exception {
ListenableFuture future =
executeQueryForRollup(rollup.agentRollupId(), query, errorSummaryTable, false);
return MoreFutures.rollupAsync(future, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupErrorSummaryFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupErrorSummaryFromChildren(RollupParams rollup,
AggregateQuery query, Collection childAgentRollupIds) throws Exception {
List> futures =
getRowsForRollupFromChildren(query, childAgentRollupIds, errorSummaryTable, false);
return MoreFutures.rollupAsync(futures, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupErrorSummaryFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupErrorSummaryFromRows(RollupParams rollup,
AggregateQuery query, Iterable rows) throws Exception {
long errorCount = 0;
long transactionCount = 0;
for (Row row : rows) {
errorCount += row.getLong(0);
transactionCount += row.getLong(1);
}
BoundStatement boundStatement =
getInsertOverallPS(errorSummaryTable, rollup.rollupLevel()).bind();
int i = 0;
boundStatement.setString(i++, rollup.agentRollupId());
boundStatement.setString(i++, query.transactionType());
boundStatement.setTimestamp(i++, new Date(query.to()));
boundStatement.setLong(i++, errorCount);
boundStatement.setLong(i++, transactionCount);
boundStatement.setInt(i++, rollup.adjustedTTL().generalTTL());
return session.writeAsync(boundStatement);
}
// transactionNames is passed in empty, and populated synchronously by this method
private ListenableFuture rollupTransactionSummary(RollupParams rollup,
AggregateQuery query, Set transactionNames) throws Exception {
BoundStatement boundStatement = checkNotNull(readTransactionForRollupPS.get(summaryTable))
.get(query.rollupLevel()).bind();
bindQuery(boundStatement, rollup.agentRollupId(), query);
// need to populate transactionNames synchronously before returning from this method
ResultSet results = session.read(boundStatement);
Map summaries = new HashMap<>();
for (Row row : results) {
String transactionName = mergeRowIntoSummaries(row, summaries);
transactionNames.add(transactionName);
}
return insertTransactionSummaries(rollup, query, summaries);
}
// transactionNames is passed in empty, and populated synchronously by this method
private ListenableFuture rollupTransactionSummaryFromChildren(RollupParams rollup,
AggregateQuery query, Collection childAgentRollupIds,
Multimap transactionNames) throws Exception {
Map> futures =
getRowsForSummaryRollupFromChildren(query, childAgentRollupIds, summaryTable, true);
Map summaries = new HashMap<>();
for (Map.Entry> entry : futures.entrySet()) {
String childAgentRollupId = entry.getKey();
ResultSet results = entry.getValue().get();
for (Row row : results) {
String transactionName = mergeRowIntoSummaries(row, summaries);
transactionNames.put(transactionName, childAgentRollupId);
}
}
return insertTransactionSummaries(rollup, query, summaries);
}
private ListenableFuture insertTransactionSummaries(RollupParams rollup,
AggregateQuery query, Map summaries) throws Exception {
BoundStatement boundStatement;
List> futures = new ArrayList<>();
PreparedStatement preparedStatement =
getInsertTransactionPS(summaryTable, rollup.rollupLevel());
for (Map.Entry entry : summaries.entrySet()) {
MutableSummary summary = entry.getValue();
boundStatement = preparedStatement.bind();
int i = 0;
boundStatement.setString(i++, rollup.agentRollupId());
boundStatement.setString(i++, query.transactionType());
boundStatement.setTimestamp(i++, new Date(query.to()));
boundStatement.setString(i++, entry.getKey());
boundStatement.setDouble(i++, summary.totalDurationNanos);
boundStatement.setLong(i++, summary.transactionCount);
boundStatement.setInt(i++, rollup.adjustedTTL().generalTTL());
futures.add(session.writeAsync(boundStatement));
}
return Futures.allAsList(futures);
}
private ListenableFuture rollupTransactionErrorSummary(RollupParams rollup,
AggregateQuery query) throws Exception {
BoundStatement boundStatement =
checkNotNull(readTransactionForRollupPS.get(errorSummaryTable))
.get(query.rollupLevel()).bind();
bindQuery(boundStatement, rollup.agentRollupId(), query);
ListenableFuture future = session.readAsync(boundStatement);
return MoreFutures.rollupAsync(future, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupTransactionErrorSummaryFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupTransactionErrorSummaryFromChildren(RollupParams rollup,
AggregateQuery query, Collection childAgentRollupIds) throws Exception {
Map> futures = getRowsForSummaryRollupFromChildren(
query, childAgentRollupIds, errorSummaryTable, false);
return MoreFutures.rollupAsync(futures.values(), asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupTransactionErrorSummaryFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupTransactionErrorSummaryFromRows(RollupParams rollup,
AggregateQuery query, Iterable rows) throws Exception {
BoundStatement boundStatement;
Map summaries = new HashMap<>();
for (Row row : rows) {
int i = 0;
String transactionName = checkNotNull(row.getString(i++));
MutableErrorSummary summary = summaries.get(transactionName);
if (summary == null) {
summary = new MutableErrorSummary();
summaries.put(transactionName, summary);
}
summary.errorCount += row.getLong(i++);
summary.transactionCount += row.getLong(i++);
}
PreparedStatement preparedStatement =
getInsertTransactionPS(errorSummaryTable, rollup.rollupLevel());
List> futures = new ArrayList<>();
for (Map.Entry entry : summaries.entrySet()) {
MutableErrorSummary summary = entry.getValue();
boundStatement = preparedStatement.bind();
int i = 0;
boundStatement.setString(i++, rollup.agentRollupId());
boundStatement.setString(i++, query.transactionType());
boundStatement.setTimestamp(i++, new Date(query.to()));
boundStatement.setString(i++, entry.getKey());
boundStatement.setLong(i++, summary.errorCount);
boundStatement.setLong(i++, summary.transactionCount);
boundStatement.setInt(i++, rollup.adjustedTTL().generalTTL());
futures.add(session.writeAsync(boundStatement));
}
return Futures.allAsList(futures);
}
private ListenableFuture rollupOverview(RollupParams rollup, AggregateQuery query)
throws Exception {
ListenableFuture future =
executeQueryForRollup(rollup.agentRollupId(), query, overviewTable, true);
return MoreFutures.rollupAsync(future, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupOverviewFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupOverviewFromChildren(RollupParams rollup,
AggregateQuery query, Collection childAgentRollupIds) throws Exception {
List> futures =
getRowsForRollupFromChildren(query, childAgentRollupIds, overviewTable, true);
return MoreFutures.rollupAsync(futures, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupOverviewFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupOverviewFromRows(RollupParams rollup, AggregateQuery query,
Iterable rows) throws Exception {
double totalDurationNanos = 0;
long transactionCount = 0;
boolean asyncTransactions = false;
List mainThreadRootTimers = new ArrayList<>();
MutableThreadStats mainThreadStats = new MutableThreadStats();
MutableTimer auxThreadRootTimer = MutableTimer.createAuxThreadRootTimer();
MutableThreadStats auxThreadStats = new MutableThreadStats();
List asyncTimers = new ArrayList<>();
for (Row row : rows) {
int i = 0;
totalDurationNanos += row.getDouble(i++);
transactionCount += row.getLong(i++);
if (row.getBool(i++)) {
asyncTransactions = true;
}
List toBeMergedMainThreadRootTimers =
Messages.parseDelimitedFrom(row.getBytes(i++), Aggregate.Timer.parser());
MutableAggregate.mergeRootTimers(toBeMergedMainThreadRootTimers, mainThreadRootTimers);
mainThreadStats.addTotalCpuNanos(getNextThreadStat(row, i++));
mainThreadStats.addTotalBlockedNanos(getNextThreadStat(row, i++));
mainThreadStats.addTotalWaitedNanos(getNextThreadStat(row, i++));
mainThreadStats.addTotalAllocatedBytes(getNextThreadStat(row, i++));
// reading delimited singleton list for backwards compatibility with data written
// prior to 0.12.0
List list =
Messages.parseDelimitedFrom(row.getBytes(i++), Aggregate.Timer.parser());
Aggregate.Timer toBeMergedAuxThreadRootTimer = list.isEmpty() ? null : list.get(0);
if (toBeMergedAuxThreadRootTimer == null) {
i += 4;
} else {
auxThreadRootTimer.merge(toBeMergedAuxThreadRootTimer);
auxThreadStats.addTotalCpuNanos(getNextThreadStat(row, i++));
auxThreadStats.addTotalBlockedNanos(getNextThreadStat(row, i++));
auxThreadStats.addTotalWaitedNanos(getNextThreadStat(row, i++));
auxThreadStats.addTotalAllocatedBytes(getNextThreadStat(row, i++));
}
List toBeMergedAsyncTimers =
Messages.parseDelimitedFrom(row.getBytes(i++), Aggregate.Timer.parser());
MutableAggregate.mergeRootTimers(toBeMergedAsyncTimers, asyncTimers);
}
BoundStatement boundStatement;
if (query.transactionName() == null) {
boundStatement = getInsertOverallPS(overviewTable, rollup.rollupLevel()).bind();
} else {
boundStatement = getInsertTransactionPS(overviewTable, rollup.rollupLevel()).bind();
}
int i = 0;
boundStatement.setString(i++, rollup.agentRollupId());
boundStatement.setString(i++, query.transactionType());
if (query.transactionName() != null) {
boundStatement.setString(i++, query.transactionName());
}
boundStatement.setTimestamp(i++, new Date(query.to()));
boundStatement.setDouble(i++, totalDurationNanos);
boundStatement.setLong(i++, transactionCount);
boundStatement.setBool(i++, asyncTransactions);
boundStatement.setBytes(i++,
Messages.toByteBuffer(MutableAggregate.toProto(mainThreadRootTimers)));
boundStatement.setDouble(i++, mainThreadStats.getTotalCpuNanos());
boundStatement.setDouble(i++, mainThreadStats.getTotalBlockedNanos());
boundStatement.setDouble(i++, mainThreadStats.getTotalWaitedNanos());
boundStatement.setDouble(i++, mainThreadStats.getTotalAllocatedBytes());
if (auxThreadRootTimer.getCount() == 0) {
boundStatement.setToNull(i++);
boundStatement.setToNull(i++);
boundStatement.setToNull(i++);
boundStatement.setToNull(i++);
boundStatement.setToNull(i++);
} else {
// writing as delimited singleton list for backwards compatibility with data written
// prior to 0.12.0
boundStatement.setBytes(i++, Messages
.toByteBuffer(MutableAggregate.toProto(ImmutableList.of(auxThreadRootTimer))));
boundStatement.setDouble(i++, auxThreadStats.getTotalCpuNanos());
boundStatement.setDouble(i++, auxThreadStats.getTotalBlockedNanos());
boundStatement.setDouble(i++, auxThreadStats.getTotalWaitedNanos());
boundStatement.setDouble(i++, auxThreadStats.getTotalAllocatedBytes());
}
if (asyncTimers.isEmpty()) {
boundStatement.setToNull(i++);
} else {
boundStatement.setBytes(i++,
Messages.toByteBuffer(MutableAggregate.toProto(asyncTimers)));
}
boundStatement.setInt(i++, rollup.adjustedTTL().generalTTL());
return session.writeAsync(boundStatement);
}
private ListenableFuture rollupHistogram(RollupParams rollup, AggregateQuery query,
ScratchBuffer scratchBuffer) throws Exception {
ListenableFuture future =
executeQueryForRollup(rollup.agentRollupId(), query, histogramTable, true);
return MoreFutures.rollupAsync(future, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupHistogramFromRows(rollup, query, rows, scratchBuffer);
}
});
}
private ListenableFuture rollupHistogramFromChildren(RollupParams rollup,
AggregateQuery query, Collection childAgentRollupIds,
ScratchBuffer scratchBuffer) throws Exception {
List> futures =
getRowsForRollupFromChildren(query, childAgentRollupIds, histogramTable, true);
return MoreFutures.rollupAsync(futures, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupHistogramFromRows(rollup, query, rows, scratchBuffer);
}
});
}
private ListenableFuture rollupHistogramFromRows(RollupParams rollup, AggregateQuery query,
Iterable rows, ScratchBuffer scratchBuffer) throws Exception {
double totalDurationNanos = 0;
long transactionCount = 0;
LazyHistogram durationNanosHistogram = new LazyHistogram();
for (Row row : rows) {
int i = 0;
totalDurationNanos += row.getDouble(i++);
transactionCount += row.getLong(i++);
ByteBuffer bytes = checkNotNull(row.getBytes(i++));
durationNanosHistogram.merge(Aggregate.Histogram.parseFrom(bytes));
}
BoundStatement boundStatement;
if (query.transactionName() == null) {
boundStatement = getInsertOverallPS(histogramTable, rollup.rollupLevel()).bind();
} else {
boundStatement = getInsertTransactionPS(histogramTable, rollup.rollupLevel()).bind();
}
int i = 0;
boundStatement.setString(i++, rollup.agentRollupId());
boundStatement.setString(i++, query.transactionType());
if (query.transactionName() != null) {
boundStatement.setString(i++, query.transactionName());
}
boundStatement.setTimestamp(i++, new Date(query.to()));
boundStatement.setDouble(i++, totalDurationNanos);
boundStatement.setLong(i++, transactionCount);
boundStatement.setBytes(i++, toByteBuffer(durationNanosHistogram.toProto(scratchBuffer)));
boundStatement.setInt(i++, rollup.adjustedTTL().generalTTL());
return session.writeAsync(boundStatement);
}
private ListenableFuture rollupThroughput(RollupParams rollup, AggregateQuery query)
throws Exception {
ListenableFuture future =
executeQueryForRollup(rollup.agentRollupId(), query, throughputTable, true);
return MoreFutures.rollupAsync(future, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupThroughputFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupThroughputFromChildren(RollupParams rollup,
AggregateQuery query, Collection childAgentRollupIds) throws Exception {
List> futures =
getRowsForRollupFromChildren(query, childAgentRollupIds, throughputTable, true);
return MoreFutures.rollupAsync(futures, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupThroughputFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupThroughputFromRows(RollupParams rollup, AggregateQuery query,
Iterable rows) throws Exception {
long transactionCount = 0;
// error_count is null for data inserted prior to glowroot central 0.9.18
// rolling up any interval with null error_count should result in null error_count
boolean hasMissingErrorCount = false;
long errorCount = 0;
for (Row row : rows) {
transactionCount += row.getLong(0);
if (row.isNull(1)) {
hasMissingErrorCount = true;
} else {
errorCount += row.getLong(1);
}
}
BoundStatement boundStatement;
if (query.transactionName() == null) {
boundStatement = getInsertOverallPS(throughputTable, rollup.rollupLevel()).bind();
} else {
boundStatement = getInsertTransactionPS(throughputTable, rollup.rollupLevel()).bind();
}
int i = 0;
boundStatement.setString(i++, rollup.agentRollupId());
boundStatement.setString(i++, query.transactionType());
if (query.transactionName() != null) {
boundStatement.setString(i++, query.transactionName());
}
boundStatement.setTimestamp(i++, new Date(query.to()));
boundStatement.setLong(i++, transactionCount);
if (hasMissingErrorCount) {
boundStatement.setToNull(i++);
} else {
boundStatement.setLong(i++, errorCount);
}
boundStatement.setInt(i++, rollup.adjustedTTL().generalTTL());
return session.writeAsync(boundStatement);
}
private ListenableFuture rollupQueries(RollupParams rollup, AggregateQuery query)
throws Exception {
ListenableFuture future =
executeQueryForRollup(rollup.agentRollupId(), query, queryTable, false);
return MoreFutures.rollupAsync(future, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupQueriesFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupQueriesFromChildren(RollupParams rollup,
AggregateQuery query, Collection childAgentRollupIds) throws Exception {
List> futures =
getRowsForRollupFromChildren(query, childAgentRollupIds, queryTable, false);
return MoreFutures.rollupAsync(futures, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupQueriesFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupQueriesFromRows(RollupParams rollup, AggregateQuery query,
Iterable rows) throws Exception {
QueryCollector collector =
new QueryCollector(rollup.maxQueryAggregatesPerTransactionAggregate());
for (Row row : rows) {
int i = 0;
String queryType = checkNotNull(row.getString(i++));
String truncatedText = checkNotNull(row.getString(i++));
// full_query_text_sha1 cannot be null since it is used in clustering key
String fullTextSha1 = Strings.emptyToNull(row.getString(i++));
double totalDurationNanos = row.getDouble(i++);
long executionCount = row.getLong(i++);
boolean hasTotalRows = !row.isNull(i);
long totalRows = row.getLong(i++);
collector.mergeQuery(queryType, truncatedText, fullTextSha1, totalDurationNanos,
executionCount, hasTotalRows, totalRows);
}
return insertQueries(collector.getSortedAndTruncatedQueries(), rollup.rollupLevel(),
rollup.agentRollupId(), query.transactionType(), query.transactionName(),
query.to(), rollup.adjustedTTL());
}
private ListenableFuture rollupServiceCalls(RollupParams rollup, AggregateQuery query)
throws Exception {
ListenableFuture future =
executeQueryForRollup(rollup.agentRollupId(), query, serviceCallTable, false);
return MoreFutures.rollupAsync(future, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupServiceCallsFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupServiceCallsFromChildren(RollupParams rollup,
AggregateQuery query, Collection childAgentRollupIds) throws Exception {
List> futures =
getRowsForRollupFromChildren(query, childAgentRollupIds, serviceCallTable, false);
return MoreFutures.rollupAsync(futures, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupServiceCallsFromRows(rollup, query, rows);
}
});
}
private ListenableFuture rollupServiceCallsFromRows(RollupParams rollup,
AggregateQuery query, Iterable rows) throws Exception {
ServiceCallCollector collector =
new ServiceCallCollector(rollup.maxServiceCallAggregatesPerTransactionAggregate());
for (Row row : rows) {
int i = 0;
String serviceCallType = checkNotNull(row.getString(i++));
String serviceCallText = checkNotNull(row.getString(i++));
double totalDurationNanos = row.getDouble(i++);
long executionCount = row.getLong(i++);
collector.mergeServiceCall(serviceCallType, serviceCallText, totalDurationNanos,
executionCount);
}
return insertServiceCalls(collector.getSortedAndTruncatedServiceCalls(),
rollup.rollupLevel(), rollup.agentRollupId(), query.transactionType(),
query.transactionName(), query.to(), rollup.adjustedTTL());
}
private ListenableFuture rollupThreadProfile(RollupParams rollup, AggregateQuery query,
Table table) throws Exception {
ListenableFuture future =
executeQueryForRollup(rollup.agentRollupId(), query, table, false);
return MoreFutures.rollupAsync(future, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupThreadProfileFromRows(rollup, query, rows, table);
}
});
}
private ListenableFuture rollupThreadProfileFromChildren(RollupParams rollup,
AggregateQuery query, Collection childAgentRollupIds, Table table)
throws Exception {
List> futures =
getRowsForRollupFromChildren(query, childAgentRollupIds, table, false);
return MoreFutures.rollupAsync(futures, asyncExecutor, new DoRollup() {
@Override
public ListenableFuture execute(Iterable rows) throws Exception {
return rollupThreadProfileFromRows(rollup, query, rows, table);
}
});
}
private ListenableFuture rollupThreadProfileFromRows(RollupParams rollup,
AggregateQuery query, Iterable rows, Table table) throws Exception {
MutableProfile profile = new MutableProfile();
for (Row row : rows) {
ByteBuffer bytes = checkNotNull(row.getBytes(0));
profile.merge(Profile.parseFrom(bytes));
}
BoundStatement boundStatement;
if (query.transactionName() == null) {
boundStatement = getInsertOverallPS(table, rollup.rollupLevel()).bind();
} else {
boundStatement = getInsertTransactionPS(table, rollup.rollupLevel()).bind();
}
int i = 0;
boundStatement.setString(i++, rollup.agentRollupId());
boundStatement.setString(i++, query.transactionType());
if (query.transactionName() != null) {
boundStatement.setString(i++, query.transactionName());
}
boundStatement.setTimestamp(i++, new Date(query.to()));
boundStatement.setBytes(i++, toByteBuffer(profile.toProto()));
boundStatement.setInt(i++, rollup.adjustedTTL().profileTTL());
return session.writeAsync(boundStatement);
}
private Map> getRowsForSummaryRollupFromChildren(
AggregateQuery query, Collection childAgentRollupIds, Table table,
boolean warnIfNoResults) throws Exception {
Map> futures = new HashMap<>();
for (String childAgentRollupId : childAgentRollupIds) {
BoundStatement boundStatement =
checkNotNull(readTransactionForRollupFromChildPS.get(table)).bind();
bindQueryForRollupFromChild(boundStatement, childAgentRollupId, query);
if (warnIfNoResults) {
futures.put(childAgentRollupId, session.readAsyncWarnIfNoRows(boundStatement,
"no summary table records found for agentRollupId={}, query={}",
childAgentRollupId, query));
} else {
futures.put(childAgentRollupId, session.readAsync(boundStatement));
}
}
return futures;
}
private List> getRowsForRollupFromChildren(AggregateQuery query,
Collection childAgentRollupIds, Table table, boolean warnIfNoResults)
throws Exception {
List> futures = new ArrayList<>();
for (String childAgentRollupId : childAgentRollupIds) {
futures.add(executeQueryForRollupFromChild(childAgentRollupId, query, table,
warnIfNoResults));
}
return futures;
}
private List> storeOverallAggregate(String agentRollupId,
String transactionType, long captureTime, Aggregate aggregate,
List sharedQueryTexts, TTL adjustedTTL) throws Exception {
final int rollupLevel = 0;
List> futures = new ArrayList<>();
BoundStatement boundStatement = getInsertOverallPS(summaryTable, rollupLevel).bind();
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setDouble(i++, aggregate.getTotalDurationNanos());
boundStatement.setLong(i++, aggregate.getTransactionCount());
boundStatement.setInt(i++, adjustedTTL.generalTTL());
futures.add(session.writeAsync(boundStatement));
if (aggregate.getErrorCount() > 0) {
boundStatement = getInsertOverallPS(errorSummaryTable, rollupLevel).bind();
i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setLong(i++, aggregate.getErrorCount());
boundStatement.setLong(i++, aggregate.getTransactionCount());
boundStatement.setInt(i++, adjustedTTL.generalTTL());
futures.add(session.writeAsync(boundStatement));
}
boundStatement = getInsertOverallPS(overviewTable, rollupLevel).bind();
i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setTimestamp(i++, new Date(captureTime));
bindAggregate(boundStatement, aggregate, i++, adjustedTTL);
futures.add(session.writeAsync(boundStatement));
boundStatement = getInsertOverallPS(histogramTable, rollupLevel).bind();
i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setDouble(i++, aggregate.getTotalDurationNanos());
boundStatement.setLong(i++, aggregate.getTransactionCount());
boundStatement.setBytes(i++, toByteBuffer(aggregate.getDurationNanosHistogram()));
boundStatement.setInt(i++, adjustedTTL.generalTTL());
futures.add(session.writeAsync(boundStatement));
boundStatement = getInsertOverallPS(throughputTable, rollupLevel).bind();
i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setLong(i++, aggregate.getTransactionCount());
boundStatement.setLong(i++, aggregate.getErrorCount());
boundStatement.setInt(i++, adjustedTTL.generalTTL());
futures.add(session.writeAsync(boundStatement));
if (aggregate.hasMainThreadProfile()) {
Profile profile = aggregate.getMainThreadProfile();
boundStatement = getInsertOverallPS(mainThreadProfileTable, rollupLevel).bind();
i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setBytes(i++, toByteBuffer(profile));
boundStatement.setInt(i++, adjustedTTL.profileTTL());
futures.add(session.writeAsync(boundStatement));
}
if (aggregate.hasAuxThreadProfile()) {
Profile profile = aggregate.getAuxThreadProfile();
boundStatement = getInsertOverallPS(auxThreadProfileTable, rollupLevel).bind();
i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setBytes(i++, toByteBuffer(profile));
boundStatement.setInt(i++, adjustedTTL.profileTTL());
futures.add(session.writeAsync(boundStatement));
}
futures.addAll(insertQueries(getQueries(aggregate), sharedQueryTexts, rollupLevel,
agentRollupId, transactionType, null, captureTime, adjustedTTL));
futures.addAll(insertServiceCallsProto(getServiceCalls(aggregate), rollupLevel,
agentRollupId, transactionType, null, captureTime, adjustedTTL));
return futures;
}
private List> storeTransactionAggregate(String agentRollupId, String transactionType,
String transactionName, long captureTime, Aggregate aggregate,
List sharedQueryTexts, TTL adjustedTTL) throws Exception {
final int rollupLevel = 0;
List> futures = new ArrayList<>();
BoundStatement boundStatement = getInsertTransactionPS(overviewTable, rollupLevel).bind();
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setString(i++, transactionName);
boundStatement.setTimestamp(i++, new Date(captureTime));
bindAggregate(boundStatement, aggregate, i++, adjustedTTL);
futures.add(session.writeAsync(boundStatement));
boundStatement = getInsertTransactionPS(histogramTable, rollupLevel).bind();
i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setString(i++, transactionName);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setDouble(i++, aggregate.getTotalDurationNanos());
boundStatement.setLong(i++, aggregate.getTransactionCount());
boundStatement.setBytes(i++, toByteBuffer(aggregate.getDurationNanosHistogram()));
boundStatement.setInt(i++, adjustedTTL.generalTTL());
futures.add(session.writeAsync(boundStatement));
boundStatement = getInsertTransactionPS(throughputTable, rollupLevel).bind();
i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setString(i++, transactionName);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setLong(i++, aggregate.getTransactionCount());
boundStatement.setLong(i++, aggregate.getErrorCount());
boundStatement.setInt(i++, adjustedTTL.generalTTL());
futures.add(session.writeAsync(boundStatement));
if (aggregate.hasMainThreadProfile()) {
Profile profile = aggregate.getMainThreadProfile();
boundStatement = getInsertTransactionPS(mainThreadProfileTable, rollupLevel).bind();
i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setString(i++, transactionName);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setBytes(i++, toByteBuffer(profile));
boundStatement.setInt(i++, adjustedTTL.profileTTL());
futures.add(session.writeAsync(boundStatement));
}
if (aggregate.hasAuxThreadProfile()) {
Profile profile = aggregate.getAuxThreadProfile();
boundStatement = getInsertTransactionPS(auxThreadProfileTable, rollupLevel).bind();
i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setString(i++, transactionName);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setBytes(i++, toByteBuffer(profile));
boundStatement.setInt(i++, adjustedTTL.profileTTL());
futures.add(session.writeAsync(boundStatement));
}
futures.addAll(insertQueries(getQueries(aggregate), sharedQueryTexts, rollupLevel,
agentRollupId, transactionType, transactionName, captureTime, adjustedTTL));
futures.addAll(insertServiceCallsProto(getServiceCalls(aggregate), rollupLevel,
agentRollupId, transactionType, transactionName, captureTime, adjustedTTL));
return futures;
}
private List> storeTransactionNameSummary(String agentRollupId,
String transactionType, String transactionName, long captureTime, Aggregate aggregate,
TTL adjustedTTL) throws Exception {
final int rollupLevel = 0;
List> futures = new ArrayList<>();
BoundStatement boundStatement = getInsertTransactionPS(summaryTable, rollupLevel).bind();
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setString(i++, transactionName);
boundStatement.setDouble(i++, aggregate.getTotalDurationNanos());
boundStatement.setLong(i++, aggregate.getTransactionCount());
boundStatement.setInt(i++, adjustedTTL.generalTTL());
futures.add(session.writeAsync(boundStatement));
if (aggregate.getErrorCount() > 0) {
boundStatement = getInsertTransactionPS(errorSummaryTable, rollupLevel).bind();
i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setString(i++, transactionName);
boundStatement.setLong(i++, aggregate.getErrorCount());
boundStatement.setLong(i++, aggregate.getTransactionCount());
boundStatement.setInt(i++, adjustedTTL.generalTTL());
futures.add(session.writeAsync(boundStatement));
}
return futures;
}
private List> insertQueries(List queries,
List sharedQueryTexts, int rollupLevel, String agentRollupId,
String transactionType, @Nullable String transactionName, long captureTime,
TTL adjustedTTL) throws Exception {
List> futures = new ArrayList<>();
for (Aggregate.Query query : queries) {
Aggregate.SharedQueryText sharedQueryText =
sharedQueryTexts.get(query.getSharedQueryTextIndex());
BoundStatement boundStatement;
if (transactionName == null) {
boundStatement = getInsertOverallPS(queryTable, rollupLevel).bind();
} else {
boundStatement = getInsertTransactionPS(queryTable, rollupLevel).bind();
}
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
if (transactionName != null) {
boundStatement.setString(i++, transactionName);
}
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setString(i++, query.getType());
String fullTextSha1 = sharedQueryText.getFullTextSha1();
if (fullTextSha1.isEmpty()) {
boundStatement.setString(i++, sharedQueryText.getFullText());
// full_query_text_sha1 cannot be null since it is used in clustering key
boundStatement.setString(i++, "");
} else {
boundStatement.setString(i++, sharedQueryText.getTruncatedText());
boundStatement.setString(i++, fullTextSha1);
}
boundStatement.setDouble(i++, query.getTotalDurationNanos());
boundStatement.setLong(i++, query.getExecutionCount());
if (query.hasTotalRows()) {
boundStatement.setLong(i++, query.getTotalRows().getValue());
} else {
boundStatement.setToNull(i++);
}
boundStatement.setInt(i++, adjustedTTL.queryTTL());
futures.add(session.writeAsync(boundStatement));
}
return futures;
}
private ListenableFuture insertQueries(List queries, int rollupLevel,
String agentRollupId, String transactionType, @Nullable String transactionName,
long captureTime, TTL adjustedTTL) throws Exception {
List> futures = new ArrayList<>();
for (MutableQuery query : queries) {
BoundStatement boundStatement;
if (transactionName == null) {
boundStatement = getInsertOverallPS(queryTable, rollupLevel).bind();
} else {
boundStatement = getInsertTransactionPS(queryTable, rollupLevel).bind();
}
String fullTextSha1 = query.getFullTextSha1();
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
if (transactionName != null) {
boundStatement.setString(i++, transactionName);
}
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setString(i++, query.getType());
boundStatement.setString(i++, query.getTruncatedText());
// full_query_text_sha1 cannot be null since it is used in clustering key
boundStatement.setString(i++, Strings.nullToEmpty(fullTextSha1));
boundStatement.setDouble(i++, query.getTotalDurationNanos());
boundStatement.setLong(i++, query.getExecutionCount());
if (query.hasTotalRows()) {
boundStatement.setLong(i++, query.getTotalRows());
} else {
boundStatement.setToNull(i++);
}
boundStatement.setInt(i++, adjustedTTL.queryTTL());
futures.add(session.writeAsync(boundStatement));
}
return Futures.allAsList(futures);
}
private List> insertServiceCallsProto(
List serviceCalls, int rollupLevel, String agentRollupId,
String transactionType, @Nullable String transactionName, long captureTime,
TTL adjustedTTL) throws Exception {
List> futures = new ArrayList<>();
for (Aggregate.ServiceCall serviceCall : serviceCalls) {
BoundStatement boundStatement;
if (transactionName == null) {
boundStatement = getInsertOverallPS(serviceCallTable, rollupLevel).bind();
} else {
boundStatement = getInsertTransactionPS(serviceCallTable, rollupLevel).bind();
}
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
if (transactionName != null) {
boundStatement.setString(i++, transactionName);
}
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setString(i++, serviceCall.getType());
boundStatement.setString(i++, serviceCall.getText());
boundStatement.setDouble(i++, serviceCall.getTotalDurationNanos());
boundStatement.setLong(i++, serviceCall.getExecutionCount());
boundStatement.setInt(i++, adjustedTTL.serviceCallTTL());
futures.add(session.writeAsync(boundStatement));
}
return futures;
}
private ListenableFuture insertServiceCalls(List serviceCalls,
int rollupLevel, String agentRollupId, String transactionType,
@Nullable String transactionName, long captureTime, TTL adjustedTTL) throws Exception {
List> futures = new ArrayList<>();
for (MutableServiceCall serviceCall : serviceCalls) {
BoundStatement boundStatement;
if (transactionName == null) {
boundStatement = getInsertOverallPS(serviceCallTable, rollupLevel).bind();
} else {
boundStatement = getInsertTransactionPS(serviceCallTable, rollupLevel).bind();
}
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, transactionType);
if (transactionName != null) {
boundStatement.setString(i++, transactionName);
}
boundStatement.setTimestamp(i++, new Date(captureTime));
boundStatement.setString(i++, serviceCall.getType());
boundStatement.setString(i++, serviceCall.getText());
boundStatement.setDouble(i++, serviceCall.getTotalDurationNanos());
boundStatement.setLong(i++, serviceCall.getExecutionCount());
boundStatement.setInt(i++, adjustedTTL.serviceCallTTL());
futures.add(session.writeAsync(boundStatement));
}
return Futures.allAsList(futures);
}
private PreparedStatement getInsertOverallPS(Table table, int rollupLevel) {
return checkNotNull(insertOverallPS.get(table)).get(rollupLevel);
}
private PreparedStatement getInsertTransactionPS(Table table, int rollupLevel) {
return checkNotNull(insertTransactionPS.get(table)).get(rollupLevel);
}
private ResultSet executeQuery(String agentRollupId, SummaryQuery query, Table table)
throws Exception {
BoundStatement boundStatement =
checkNotNull(readOverallPS.get(table)).get(query.rollupLevel()).bind();
bindQuery(boundStatement, agentRollupId, query);
return session.read(boundStatement);
}
private ResultSet executeQuery(String agentRollupId, AggregateQuery query, Table table)
throws Exception {
BoundStatement boundStatement;
if (query.transactionName() == null) {
boundStatement = checkNotNull(readOverallPS.get(table)).get(query.rollupLevel()).bind();
} else {
boundStatement =
checkNotNull(readTransactionPS.get(table)).get(query.rollupLevel()).bind();
}
bindQuery(boundStatement, agentRollupId, query);
return session.read(boundStatement);
}
private ListenableFuture executeQueryForRollup(String agentRollupId,
AggregateQuery query, Table table, boolean warnIfNoResults) throws Exception {
BoundStatement boundStatement;
if (query.transactionName() == null) {
boundStatement =
checkNotNull(readOverallForRollupPS.get(table)).get(query.rollupLevel()).bind();
} else {
boundStatement = checkNotNull(readTransactionForRollupPS.get(table))
.get(query.rollupLevel()).bind();
}
bindQuery(boundStatement, agentRollupId, query);
if (warnIfNoResults) {
return session.readAsyncWarnIfNoRows(boundStatement,
"no {} records found for agentRollupId={}, query={}", table.partialName(),
agentRollupId, query);
} else {
return session.readAsync(boundStatement);
}
}
private ListenableFuture executeQueryForRollupFromChild(String childAgentRollupId,
AggregateQuery query, Table table, boolean warnIfNoResults) throws Exception {
BoundStatement boundStatement;
if (query.transactionName() == null) {
boundStatement = checkNotNull(readOverallForRollupFromChildPS.get(table)).bind();
} else {
boundStatement = checkNotNull(readTransactionForRollupFromChildPS.get(table)).bind();
}
bindQueryForRollupFromChild(boundStatement, childAgentRollupId, query);
if (warnIfNoResults) {
return session.readAsyncWarnIfNoRows(boundStatement,
"no {} records found for agentRollupId={}, query={}", table.partialName(),
childAgentRollupId, query);
} else {
return session.readAsync(boundStatement);
}
}
private void mergeProfilesInto(String agentRollupId, AggregateQuery query, Table profileTable,
ProfileCollector collector) throws Exception {
ResultSet results = executeQuery(agentRollupId, query, profileTable);
long captureTime = Long.MIN_VALUE;
for (Row row : results) {
captureTime = Math.max(captureTime, checkNotNull(row.getTimestamp(0)).getTime());
ByteBuffer bytes = checkNotNull(row.getBytes(1));
// TODO optimize this byte copying
Profile profile = Profile.parseFrom(bytes);
collector.mergeProfile(profile);
collector.updateLastCaptureTime(captureTime);
}
}
private List getTTLs() throws Exception {
List rollupExpirationHours =
configRepository.getCentralStorageConfig().rollupExpirationHours();
List queryAndServiceCallRollupExpirationHours =
configRepository.getCentralStorageConfig()
.queryAndServiceCallRollupExpirationHours();
List profileRollupExpirationHours =
configRepository.getCentralStorageConfig().profileRollupExpirationHours();
List ttls = new ArrayList<>();
for (int i = 0; i < rollupExpirationHours.size(); i++) {
ttls.add(ImmutableTTL.builder()
.generalTTL(Ints.saturatedCast(HOURS.toSeconds(rollupExpirationHours.get(i))))
.queryTTL(Ints.saturatedCast(
HOURS.toSeconds(queryAndServiceCallRollupExpirationHours.get(i))))
.serviceCallTTL(Ints.saturatedCast(
HOURS.toSeconds(queryAndServiceCallRollupExpirationHours.get(i))))
.profileTTL(Ints
.saturatedCast(HOURS.toSeconds(profileRollupExpirationHours.get(i))))
.build());
}
return ttls;
}
private RollupParams getRollupParams(String agentRollupId, String agentRollupIdForMeta,
int rollupLevel, TTL adjustedTTL) throws Exception {
return ImmutableRollupParams.builder()
.agentRollupId(agentRollupId)
.rollupLevel(rollupLevel)
.adjustedTTL(adjustedTTL)
.maxQueryAggregatesPerTransactionAggregate(
getMaxQueryAggregatesPerTransactionAggregate(agentRollupIdForMeta))
.maxServiceCallAggregatesPerTransactionAggregate(
getMaxServiceCallAggregatesPerTransactionAggregate(agentRollupIdForMeta))
.build();
}
private static String mergeRowIntoSummaries(Row row, Map summaries) {
int i = 0;
String transactionName = checkNotNull(row.getString(i++));
MutableSummary summary = summaries.get(transactionName);
if (summary == null) {
summary = new MutableSummary();
summaries.put(transactionName, summary);
}
summary.totalDurationNanos += row.getDouble(i++);
summary.transactionCount += row.getLong(i++);
return transactionName;
}
private static void bindAggregate(BoundStatement boundStatement, Aggregate aggregate,
int startIndex, TTL adjustedTTL) throws IOException {
int i = startIndex;
boundStatement.setDouble(i++, aggregate.getTotalDurationNanos());
boundStatement.setLong(i++, aggregate.getTransactionCount());
boundStatement.setBool(i++, aggregate.getAsyncTransactions());
boundStatement.setBytes(i++, Messages.toByteBuffer(aggregate.getMainThreadRootTimerList()));
if (aggregate.hasOldMainThreadStats()) {
// data from agent prior to 0.10.9
Aggregate.OldThreadStats mainThreadStats = aggregate.getOldMainThreadStats();
boundStatement.setDouble(i++, mainThreadStats.getTotalCpuNanos().getValue());
boundStatement.setDouble(i++, mainThreadStats.getTotalBlockedNanos().getValue());
boundStatement.setDouble(i++, mainThreadStats.getTotalWaitedNanos().getValue());
boundStatement.setDouble(i++, mainThreadStats.getTotalAllocatedBytes().getValue());
} else {
Aggregate.ThreadStats mainThreadStats = aggregate.getMainThreadStats();
boundStatement.setDouble(i++, mainThreadStats.getTotalCpuNanos());
boundStatement.setDouble(i++, mainThreadStats.getTotalBlockedNanos());
boundStatement.setDouble(i++, mainThreadStats.getTotalWaitedNanos());
boundStatement.setDouble(i++, mainThreadStats.getTotalAllocatedBytes());
}
if (aggregate.hasAuxThreadRootTimer()) {
// writing as delimited singleton list for backwards compatibility with data written
// prior to 0.12.0
boundStatement.setBytes(i++,
Messages.toByteBuffer(ImmutableList.of(aggregate.getAuxThreadRootTimer())));
if (aggregate.hasOldAuxThreadStats()) {
Aggregate.OldThreadStats auxThreadStats = aggregate.getOldAuxThreadStats();
boundStatement.setDouble(i++, auxThreadStats.getTotalCpuNanos().getValue());
boundStatement.setDouble(i++, auxThreadStats.getTotalBlockedNanos().getValue());
boundStatement.setDouble(i++, auxThreadStats.getTotalWaitedNanos().getValue());
boundStatement.setDouble(i++, auxThreadStats.getTotalAllocatedBytes().getValue());
} else {
Aggregate.ThreadStats auxThreadStats = aggregate.getAuxThreadStats();
boundStatement.setDouble(i++, auxThreadStats.getTotalCpuNanos());
boundStatement.setDouble(i++, auxThreadStats.getTotalBlockedNanos());
boundStatement.setDouble(i++, auxThreadStats.getTotalWaitedNanos());
boundStatement.setDouble(i++, auxThreadStats.getTotalAllocatedBytes());
}
} else {
boundStatement.setToNull(i++);
boundStatement.setToNull(i++);
boundStatement.setToNull(i++);
boundStatement.setToNull(i++);
boundStatement.setToNull(i++);
}
List asyncTimers = aggregate.getAsyncTimerList();
if (asyncTimers.isEmpty()) {
boundStatement.setToNull(i++);
} else {
boundStatement.setBytes(i++, Messages.toByteBuffer(asyncTimers));
}
boundStatement.setInt(i++, adjustedTTL.generalTTL());
}
private static void bindQuery(BoundStatement boundStatement, String agentRollupId,
SummaryQuery query) {
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, query.transactionType());
boundStatement.setTimestamp(i++, new Date(query.from()));
boundStatement.setTimestamp(i++, new Date(query.to()));
}
private static void bindQuery(BoundStatement boundStatement, String agentRollupId,
AggregateQuery query) {
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, query.transactionType());
String transactionName = query.transactionName();
if (transactionName != null) {
boundStatement.setString(i++, transactionName);
}
boundStatement.setTimestamp(i++, new Date(query.from()));
boundStatement.setTimestamp(i++, new Date(query.to()));
}
private static void bindQueryForRollupFromChild(BoundStatement boundStatement,
String agentRollupId, AggregateQuery query) {
int i = 0;
boundStatement.setString(i++, agentRollupId);
boundStatement.setString(i++, query.transactionType());
String transactionName = query.transactionName();
if (transactionName != null) {
boundStatement.setString(i++, transactionName);
}
boundStatement.setTimestamp(i++, new Date(query.to()));
}
private static List getQueries(Aggregate aggregate) {
List queriesByTypeList = aggregate.getOldQueriesByTypeList();
if (queriesByTypeList.isEmpty()) {
return aggregate.getQueryList();
}
List queries = new ArrayList<>();
for (Aggregate.OldQueriesByType queriesByType : queriesByTypeList) {
for (Aggregate.OldQuery query : queriesByType.getQueryList()) {
queries.add(Aggregate.Query.newBuilder()
.setType(queriesByType.getType())
.setSharedQueryTextIndex(query.getSharedQueryTextIndex())
.setTotalDurationNanos(query.getTotalDurationNanos())
.setExecutionCount(query.getExecutionCount())
.setTotalRows(query.getTotalRows())
.build());
}
}
return queries;
}
private static List getServiceCalls(Aggregate aggregate) {
List serviceCallsByTypeList =
aggregate.getOldServiceCallsByTypeList();
if (serviceCallsByTypeList.isEmpty()) {
return aggregate.getServiceCallList();
}
List serviceCalls = new ArrayList<>();
for (Aggregate.OldServiceCallsByType serviceCallsByType : serviceCallsByTypeList) {
for (Aggregate.OldServiceCall serviceCall : serviceCallsByType.getServiceCallList()) {
serviceCalls.add(Aggregate.ServiceCall.newBuilder()
.setType(serviceCallsByType.getType())
.setText(serviceCall.getText())
.setTotalDurationNanos(serviceCall.getTotalDurationNanos())
.setExecutionCount(serviceCall.getExecutionCount())
.build());
}
}
return serviceCalls;
}
private static double getNextThreadStat(Row row, int columnIndex) {
Double threadStat = row.get(columnIndex, Double.class);
if (threadStat == null) {
// old data stored prior to 0.10.9
return NotAvailableAware.NA;
} else {
return threadStat;
}
}
private static String createTableQuery(Table table, boolean transaction, int i) {
StringBuilder sb = new StringBuilder();
sb.append("create table if not exists ");
sb.append(getTableName(table.partialName(), transaction, i));
sb.append(" (agent_rollup varchar, transaction_type varchar");
if (transaction) {
sb.append(", transaction_name varchar");
}
sb.append(", capture_time timestamp");
for (Column column : table.columns()) {
sb.append(", ");
sb.append(column.name());
sb.append(" ");
sb.append(column.type());
}
sb.append(", primary key ((agent_rollup, transaction_type");
if (transaction) {
sb.append(", transaction_name");
}
sb.append("), capture_time");
for (String clusterKey : table.clusterKey()) {
sb.append(", ");
sb.append(clusterKey);
}
sb.append("))");
return sb.toString();
}
private static String insertPS(Table table, boolean transaction, int i) {
StringBuilder sb = new StringBuilder();
sb.append("insert into ");
sb.append(getTableName(table.partialName(), transaction, i));
sb.append(" (agent_rollup, transaction_type");
if (transaction) {
sb.append(", transaction_name");
}
sb.append(", capture_time");
for (Column column : table.columns()) {
sb.append(", ");
sb.append(column.name());
}
sb.append(") values (?, ?, ?");
if (transaction) {
sb.append(", ?");
}
sb.append(Strings.repeat(", ?", table.columns().size()));
sb.append(") using TTL ?");
return sb.toString();
}
private static String readPS(Table table, boolean transaction, int i) {
StringBuilder sb = new StringBuilder();
sb.append("select capture_time");
for (Column column : table.columns()) {
sb.append(", ");
sb.append(column.name());
}
sb.append(" from ");
sb.append(getTableName(table.partialName(), transaction, i));
sb.append(" where agent_rollup = ? and transaction_type = ?");
if (transaction) {
sb.append(" and transaction_name = ?");
}
sb.append(" and capture_time >");
if (table.fromInclusive()) {
sb.append("=");
}
sb.append(" ? and capture_time <= ?");
return sb.toString();
}
private static String readForRollupPS(Table table, boolean transaction, int i) {
StringBuilder sb = new StringBuilder();
sb.append("select ");
appendColumnNames(sb, table.columns());
sb.append(" from ");
sb.append(getTableName(table.partialName(), transaction, i));
sb.append(" where agent_rollup = ? and transaction_type = ?");
if (transaction) {
sb.append(" and transaction_name = ?");
}
sb.append(" and capture_time > ? and capture_time <= ?");
return sb.toString();
}
private static String readForRollupFromChildPS(Table table, boolean transaction, int i) {
StringBuilder sb = new StringBuilder();
sb.append("select ");
appendColumnNames(sb, table.columns());
sb.append(" from ");
sb.append(getTableName(table.partialName(), transaction, i));
sb.append(" where agent_rollup = ? and transaction_type = ?");
if (transaction) {
sb.append(" and transaction_name = ?");
}
sb.append(" and capture_time = ?");
return sb.toString();
}
private static String existsPS(Table table, boolean transaction, int i) {
StringBuilder sb = new StringBuilder();
sb.append("select agent_rollup");
sb.append(" from ");
sb.append(getTableName(table.partialName(), transaction, i));
sb.append(" where agent_rollup = ? and transaction_type = ?");
if (transaction) {
sb.append(" and transaction_name = ?");
}
sb.append(" and capture_time >");
if (table.fromInclusive()) {
sb.append("=");
}
sb.append(" ? and capture_time <= ? limit 1");
return sb.toString();
}
private static String createSummaryTableQuery(Table table, boolean transaction, int i) {
StringBuilder sb = new StringBuilder();
sb.append("create table if not exists ");
sb.append(getTableName(table.partialName(), transaction, i));
sb.append(" (agent_rollup varchar, transaction_type varchar, capture_time timestamp");
if (transaction) {
sb.append(", transaction_name varchar");
}
for (Column column : table.columns()) {
sb.append(", ");
sb.append(column.name());
sb.append(" ");
sb.append(column.type());
}
sb.append(", primary key ((agent_rollup, transaction_type), capture_time");
if (transaction) {
sb.append(", transaction_name");
}
sb.append("))");
return sb.toString();
}
private static String insertSummaryPS(Table table, boolean transaction, int i) {
StringBuilder sb = new StringBuilder();
sb.append("insert into ");
sb.append(getTableName(table.partialName(), transaction, i));
sb.append(" (agent_rollup, transaction_type, capture_time");
if (transaction) {
sb.append(", transaction_name");
}
for (Column column : table.columns()) {
sb.append(", ");
sb.append(column.name());
}
sb.append(") values (?, ?, ?");
if (transaction) {
sb.append(", ?");
}
sb.append(Strings.repeat(", ?", table.columns().size()));
sb.append(") using TTL ?");
return sb.toString();
}
// currently have to do group by / sort / limit client-side, even on overall_summary
// because sum(double) requires Cassandra 2.2+
private static String readSummaryPS(Table table, boolean transaction, int i) {
StringBuilder sb = new StringBuilder();
// capture_time is needed to keep track of lastCaptureTime for rollup level when merging
// recent non-rolled up data
sb.append("select capture_time");
if (transaction) {
sb.append(", transaction_name");
}
for (Column column : table.columns()) {
sb.append(", ");
sb.append(column.name());
}
sb.append(" from ");
sb.append(getTableName(table.partialName(), transaction, i));
sb.append(" where agent_rollup = ? and transaction_type = ? and capture_time >");
if (table.fromInclusive()) {
sb.append("=");
}
sb.append(" ? and capture_time <= ?");
return sb.toString();
}
private static String readSummaryForRollupPS(Table table, boolean transaction, int i) {
StringBuilder sb = new StringBuilder();
sb.append("select ");
if (transaction) {
sb.append("transaction_name, ");
}
appendColumnNames(sb, table.columns());
sb.append(" from ");
sb.append(getTableName(table.partialName(), transaction, i));
sb.append(" where agent_rollup = ? and transaction_type = ? and capture_time > ?");
sb.append(" and capture_time <= ?");
return sb.toString();
}
private static String readSummaryForRollupFromChildPS(Table table, boolean transaction,
int i) {
StringBuilder sb = new StringBuilder();
sb.append("select ");
if (transaction) {
sb.append("transaction_name, ");
}
appendColumnNames(sb, table.columns());
sb.append(" from ");
sb.append(getTableName(table.partialName(), transaction, i));
sb.append(" where agent_rollup = ? and transaction_type = ? and capture_time = ?");
return sb.toString();
}
private static String getTableName(String partialName, boolean transaction, int i) {
StringBuilder sb = new StringBuilder();
sb.append("aggregate_");
if (transaction) {
sb.append("tn_");
} else {
sb.append("tt_");
}
sb.append(partialName);
sb.append("_rollup_");
sb.append(i);
return sb.toString();
}
private static void appendColumnNames(StringBuilder sb, List columns) {
boolean addSeparator = false;
for (Column column : columns) {
if (addSeparator) {
sb.append(", ");
}
sb.append(column.name());
addSeparator = true;
}
}
private static ByteBuffer toByteBuffer(AbstractMessage message) {
return ByteBuffer.wrap(message.toByteArray());
}
private int getMaxQueryAggregatesPerTransactionAggregate(String agentRollupId)
throws Exception {
AdvancedConfig advancedConfig;
try {
advancedConfig = configRepository.getAdvancedConfig(agentRollupId);
} catch (AgentConfigNotFoundException e) {
return ConfigDefaults.ADVANCED_MAX_QUERY_AGGREGATES;
}
if (advancedConfig.hasMaxQueryAggregates()) {
return advancedConfig.getMaxQueryAggregates().getValue();
} else {
return ConfigDefaults.ADVANCED_MAX_QUERY_AGGREGATES;
}
}
private int getMaxServiceCallAggregatesPerTransactionAggregate(String agentRollupId)
throws Exception {
AdvancedConfig advancedConfig;
try {
advancedConfig = configRepository.getAdvancedConfig(agentRollupId);
} catch (AgentConfigNotFoundException e) {
return ConfigDefaults.ADVANCED_MAX_SERVICE_CALL_AGGREGATES;
}
if (advancedConfig.hasMaxServiceCallAggregates()) {
return advancedConfig.getMaxServiceCallAggregates().getValue();
} else {
return ConfigDefaults.ADVANCED_MAX_SERVICE_CALL_AGGREGATES;
}
}
static TTL getAdjustedTTL(TTL ttl, long captureTime, Clock clock) {
return ImmutableTTL.builder()
.generalTTL(Common.getAdjustedTTL(ttl.generalTTL(), captureTime, clock))
.queryTTL(Common.getAdjustedTTL(ttl.queryTTL(), captureTime, clock))
.serviceCallTTL(Common.getAdjustedTTL(ttl.serviceCallTTL(), captureTime, clock))
.profileTTL(Common.getAdjustedTTL(ttl.profileTTL(), captureTime, clock))
.build();
}
@Value.Immutable
interface Table {
String partialName();
List columns();
List clusterKey();
boolean summary();
boolean fromInclusive();
}
@Value.Immutable
@Styles.AllParameters
interface Column {
String name();
String type();
}
@Value.Immutable
interface RollupParams {
String agentRollupId();
int rollupLevel();
TTL adjustedTTL();
int maxQueryAggregatesPerTransactionAggregate();
int maxServiceCallAggregatesPerTransactionAggregate();
}
@Value.Immutable
interface TTL {
int generalTTL();
int queryTTL();
int serviceCallTTL();
int profileTTL();
}
private static class MutableSummary {
private double totalDurationNanos;
private long transactionCount;
}
private static class MutableErrorSummary {
private long errorCount;
private long transactionCount;
}
}
