net.opentsdb.tools.TreeSync Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of opentsdb Show documentation
Show all versions of opentsdb Show documentation
OpenTSDB is a distributed, scalable Time Series Database (TSDB)
written on top of HBase. OpenTSDB was written to address a common need:
store, index and serve metrics collected from computer systems (network
gear, operating systems, applications) at a large scale, and make this
data easily accessible and graphable.
// This file is part of OpenTSDB.
// Copyright (C) 2013 The OpenTSDB Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 2.1 of the License, or (at your
// option) any later version. This program is distributed in the hope that it
// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
// General Public License for more details. You should have received a copy
// of the GNU Lesser General Public License along with this program. If not,
// see uidclass = UniqueId.class;
try {
// Those are all implementation details so they're not part of the
// interface. We access them anyway using reflection. I think this
// is better than marking those public and adding a javadoc comment
// "THIS IS INTERNAL DO NOT USE". If only Java had C++'s "friend" or
// a less stupid notion of a package.
Field f;
f = uidclass.getDeclaredField("CHARSET");
f.setAccessible(true);
CHARSET = (Charset) f.get(null);
} catch (Exception e) {
throw new RuntimeException("static initializer failed", e);
}
}
/** TSDB to use for storage access */
final TSDB tsdb;
/** The ID to start the sync with for this thread */
final long start_id;
/** The end of the ID block to work on */
final long end_id;
/** Diagnostic ID for this thread */
final int thread_id;
/**
* Default constructor, stores the TSDB to use
* @param tsdb The TSDB to use for access
* @param start_id The starting ID of the block we'll work on
* @param quotient The total number of IDs in our block
* @param thread_id The ID of this thread (starts at 0)
*/
public TreeSync(final TSDB tsdb, final long start_id, final double quotient,
final int thread_id) {
this.tsdb = tsdb;
this.start_id = start_id;
this.end_id = start_id + (long) quotient + 1; // teensy bit of overlap
this.thread_id = thread_id;
}
/**
* Performs a tree synchronization using a table scanner across the UID table
* @return 0 if completed successfully, something else if an error occurred
*/
public void run() {
final Scanner scanner = getScanner();
// start the process by loading all of the trees in the system
final List trees;
try {
trees = Tree.fetchAllTrees(tsdb).joinUninterruptibly();
LOG.info("[" + thread_id + "] Complete");
} catch (Exception e) {
LOG.error("[" + thread_id + "] Unexpected Exception", e);
throw new RuntimeException("[" + thread_id + "] Unexpected exception", e);
}
if (trees == null) {
LOG.warn("No tree definitions were found");
return;
} else {
boolean has_enabled_tree = false;
for (Tree tree : trees) {
if (tree.getEnabled()) {
has_enabled_tree = true;
break;
}
}
if (!has_enabled_tree) {
LOG.warn("No enabled trees were found");
return;
}
LOG.info("Found [" + trees.size() + "] trees");
}
// setup an array for storing the tree processing calls so we can block
// until each call has completed
final ArrayList> tree_calls =
new ArrayList>();
final Deferred completed = new Deferred();
/**
* Scanner callback that loops through the UID table recursively until
* the scanner returns a null row set.
*/
final class TsuidScanner implements Callback,
ArrayList>> {
/**
* Fetches the next set of rows from the scanner, adding this class as a
* callback
* @return A meaningless deferred used to wait on until processing has
* completed
*/
public Deferred scan() {
return scanner.nextRows().addCallbackDeferring(this);
}
@Override
public Deferred call(ArrayList> rows)
throws Exception {
if (rows == null) {
completed.callback(true);
return null;
}
for (final ArrayList row : rows) {
// convert to a string one time
final String tsuid = UniqueId.uidToString(row.get(0).key());
/**
* A throttling callback used to wait for the current TSMeta to
* complete processing through the trees before continuing on with
* the next set.
*/
final class TreeBuilderBufferCB implements Callback>> {
@Override
public Boolean call(ArrayList> builder_calls)
throws Exception {
//LOG.debug("Processed [" + builder_calls.size() + "] tree_calls");
return true;
}
}
/**
* Executed after parsing a TSMeta object and loading all of the
* associated UIDMetas. Once the meta has been loaded, this callback
* runs it through each of the configured TreeBuilder objects and
* stores the resulting deferred in an array. Once processing of all
* of the rules has completed, we group the deferreds and call
* BufferCB() to wait for their completion.
*/
final class ParseCB implements Callback, TSMeta> {
final ArrayList>> builder_calls =
new ArrayList>>();
@Override
public Deferred call(TSMeta meta) throws Exception {
if (meta != null) {
LOG.debug("Processing TSMeta: " + meta + " w value: " +
JSON.serializeToString(meta));
// copy the trees into a tree builder object and iterate through
// each builder. We need to do this as a builder is not thread
// safe and cannot be used asynchronously.
final ArrayList tree_builders =
new ArrayList(trees.size());
for (Tree tree : trees) {
if (!tree.getEnabled()) {
continue;
}
final TreeBuilder builder = new TreeBuilder(tsdb, tree);
tree_builders.add(builder);
}
for (TreeBuilder builder : tree_builders) {
builder_calls.add(builder.processTimeseriesMeta(meta));
}
return Deferred.group(builder_calls)
.addCallback(new TreeBuilderBufferCB());
} else {
return Deferred.fromResult(false);
}
}
}
/**
* An error handler used to catch issues when loading the TSMeta such
* as a missing UID name. In these situations we want to log that the
* TSMeta had an issue and continue on.
*/
final class ErrBack implements Callback, Exception> {
@Override
public Deferred call(Exception e) throws Exception {
if (e.getClass().equals(IllegalStateException.class)) {
LOG.error("Invalid data when processing TSUID [" + tsuid + "]", e);
} else if (e.getClass().equals(IllegalArgumentException.class)) {
LOG.error("Invalid data when processing TSUID [" + tsuid + "]", e);
} else if (e.getClass().equals(NoSuchUniqueId.class)) {
LOG.warn("Timeseries [" + tsuid +
"] includes a non-existant UID: " + e.getMessage());
} else {
LOG.error("[" + thread_id + "] Exception while processing TSUID [" +
tsuid + "]", e);
}
return Deferred.fromResult(false);
}
}
// matched a TSMeta column, so request a parsing and loading of
// associated UIDMeta objects, then pass it off to callbacks for
// parsing through the trees.
final Deferred process_tsmeta =
TSMeta.parseFromColumn(tsdb, row.get(0), true)
.addCallbackDeferring(new ParseCB());
process_tsmeta.addErrback(new ErrBack());
tree_calls.add(process_tsmeta);
}
/**
* Another buffer callback that waits for the current set of TSMetas to
* complete their tree calls before we fetch another set of rows from
* the scanner. This necessary to avoid OOM issues.
*/
final class ContinueCB implements Callback,
ArrayList> {
@Override
public Deferred call(ArrayList tsuids)
throws Exception {
LOG.debug("Processed [" + tsuids.size() + "] tree_calls, continuing");
tree_calls.clear();
return scan();
}
}
// request the next set of rows from the scanner, but wait until the
// current set of TSMetas has been processed so we don't slaughter our
// host
Deferred.group(tree_calls).addCallback(new ContinueCB());
return Deferred.fromResult(null);
}
}
/**
* Used to capture unhandled exceptions from the scanner callbacks and
* exit the thread properly
*/
final class ErrBack implements Callback, Exception> {
@Override
public Deferred call(Exception e) throws Exception {
LOG.error("Unexpected exception", e);
completed.callback(false);
return Deferred.fromResult(false);
}
}
final TsuidScanner tree_scanner = new TsuidScanner();
tree_scanner.scan().addErrback(new ErrBack());
try {
completed.joinUninterruptibly();
LOG.info("[" + thread_id + "] Complete");
} catch (Exception e) {
LOG.error("[" + thread_id + "] Scanner Exception", e);
throw new RuntimeException("[" + thread_id + "] Scanner exception", e);
}
return;
}
/**
* Attempts to delete all data generated by the given tree, and optionally,
* the tree definition itself.
* @param tree_id The tree with data to delete
* @param delete_definition Whether or not the tree definition itself should
* be removed from the system
* @return 0 if completed successfully, something else if an error occurred
*/
public int purgeTree(final int tree_id, final boolean delete_definition)
throws Exception {
if (delete_definition) {
LOG.info("Deleting tree branches and definition for: " + tree_id);
} else {
LOG.info("Deleting tree branches for: " + tree_id);
}
Tree.deleteTree(tsdb, tree_id, delete_definition).joinUninterruptibly();
LOG.info("Completed tree deletion for: " + tree_id);
return 0;
}
/**
* Returns a scanner set to scan the range configured for this thread
* @return A scanner on the "name" CF configured for the specified range
* @throws HBaseException if something goes boom
*/
private Scanner getScanner() throws HBaseException {
final short metric_width = TSDB.metrics_width();
final byte[] start_row =
Arrays.copyOfRange(Bytes.fromLong(start_id), 8 - metric_width, 8);
final byte[] end_row =
Arrays.copyOfRange(Bytes.fromLong(end_id), 8 - metric_width, 8);
LOG.debug("[" + thread_id + "] Start row: " + UniqueId.uidToString(start_row));
LOG.debug("[" + thread_id + "] End row: " + UniqueId.uidToString(end_row));
final Scanner scanner = tsdb.getClient().newScanner(tsdb.metaTable());
scanner.setStartKey(start_row);
scanner.setStopKey(end_row);
scanner.setFamily("name".getBytes(CHARSET));
scanner.setQualifier("ts_meta".getBytes(CHARSET));
return scanner;
}
}