Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
co.cask.cdap.metrics.query.MetricsQueryHelper Maven / Gradle / Ivy
/*
* Copyright © 2017 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package co.cask.cdap.metrics.query;
import co.cask.cdap.api.dataset.lib.cube.AggregationFunction;
import co.cask.cdap.api.dataset.lib.cube.Interpolator;
import co.cask.cdap.api.dataset.lib.cube.Interpolators;
import co.cask.cdap.api.dataset.lib.cube.TimeValue;
import co.cask.cdap.api.metrics.MetricDataQuery;
import co.cask.cdap.api.metrics.MetricSearchQuery;
import co.cask.cdap.api.metrics.MetricStore;
import co.cask.cdap.api.metrics.MetricTimeSeries;
import co.cask.cdap.api.metrics.TagValue;
import co.cask.cdap.common.conf.Constants;
import co.cask.cdap.common.utils.TimeMathParser;
import co.cask.cdap.proto.MetricQueryRequest;
import co.cask.cdap.proto.MetricQueryResult;
import co.cask.cdap.proto.MetricTagValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Predicates;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.inject.Inject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
/**
*
*/
public class MetricsQueryHelper {
private static final Logger LOG = LoggerFactory.getLogger(MetricsQueryHelper.class);
public static final String NAMESPACE_STRING = "namespace";
public static final String APP_STRING = "app";
// constants used for request query parsing
private static final String PARAM_COUNT = "count";
private static final String PARAM_START_TIME = "start";
private static final String PARAM_RESOLUTION = "resolution";
private static final String PARAM_END_TIME = "end";
private static final String PARAM_INTERPOLATE = "interpolate";
private static final String PARAM_STEP_INTERPOLATOR = "step";
private static final String PARAM_LINEAR_INTERPOLATOR = "linear";
private static final String PARAM_MAX_INTERPOLATE_GAP = "maxInterpolateGap";
private static final String PARAM_AGGREGATE = "aggregate";
private static final String PARAM_AUTO_RESOLUTION = "auto";
private static final String ANY_TAG_VALUE = "*";
private final MetricStore metricStore;
private static final Map tagNameToHuman;
private static final Map humanToTagName;
static {
ImmutableBiMap mapping = ImmutableBiMap.builder()
.put(Constants.Metrics.Tag.NAMESPACE, NAMESPACE_STRING)
.put(Constants.Metrics.Tag.RUN_ID, "run")
.put(Constants.Metrics.Tag.INSTANCE_ID, "instance")
.put(Constants.Metrics.Tag.COMPONENT, "component")
.put(Constants.Metrics.Tag.HANDLER, "handler")
.put(Constants.Metrics.Tag.METHOD, "method")
.put(Constants.Metrics.Tag.STREAM, "stream")
.put(Constants.Metrics.Tag.DATASET, "dataset")
.put(Constants.Metrics.Tag.APP, APP_STRING)
.put(Constants.Metrics.Tag.SERVICE, "service")
// SERVICE_HANDLER is the same HANDLER
.put(Constants.Metrics.Tag.WORKER, "worker")
.put(Constants.Metrics.Tag.FLOW, "flow")
.put(Constants.Metrics.Tag.FLOWLET, "flowlet")
.put(Constants.Metrics.Tag.FLOWLET_QUEUE, "queue")
.put(Constants.Metrics.Tag.PRODUCER, "producer")
.put(Constants.Metrics.Tag.CONSUMER, "consumer")
.put(Constants.Metrics.Tag.MAPREDUCE, "mapreduce")
.put(Constants.Metrics.Tag.MR_TASK_TYPE, "tasktype")
.put(Constants.Metrics.Tag.WORKFLOW, "workflow")
.put(Constants.Metrics.Tag.SPARK, "spark").build();
tagNameToHuman = mapping;
humanToTagName = mapping.inverse();
}
@Inject
public MetricsQueryHelper(MetricStore metricStore) {
this.metricStore = metricStore;
}
public List searchTags(List tags) throws Exception {
// we want to search the entire range, so startTimestamp is '0' and end Timestamp is Integer.MAX_VALUE and
// limit is -1 , to include the entire search result.
MetricSearchQuery searchQuery = new MetricSearchQuery(0, Integer.MAX_VALUE, -1,
toTagValues(humanToTagNames(parseTagValues(tags))));
return tagValuesToHuman(metricStore.findNextAvailableTags(searchQuery));
}
public Collection searchMetric(List tagValues) throws Exception {
return getMetrics(humanToTagNames(parseTagValues(tagValues)));
}
public Map executeBatchQueries(Map queries) throws Exception {
LOG.trace("Received Queries {}", queries);
Map queryFinalResponse = Maps.newHashMap();
for (Map.Entry query : queries.entrySet()) {
MetricQueryRequest queryRequest = getQueryRequestFromFormat(query.getValue());
queryFinalResponse.put(query.getKey(), executeQuery(queryRequest));
}
return queryFinalResponse;
}
public MetricQueryResult executeTagQuery(List tags, List metrics, List groupByTags,
Map> queryTimeParams) throws Exception {
MetricQueryRequest queryRequest = new MetricQueryRequest(parseTagValuesAsMap(tags), metrics, groupByTags);
setTimeRangeInQueryRequest(queryRequest, queryTimeParams);
return executeQuery(queryRequest);
}
@VisibleForTesting
public MetricStore getMetricStore() {
return metricStore;
}
private Collection getMetrics(List tagValues) throws Exception {
// we want to search the entire range, so startTimestamp is '0' and end Timestamp is Integer.MAX_VALUE and
// limit is -1 , to include the entire search result.
MetricSearchQuery searchQuery =
new MetricSearchQuery(0, Integer.MAX_VALUE, -1, toTagValues(tagValues));
Collection metricNames = metricStore.findMetricNames(searchQuery);
return Lists.newArrayList(Iterables.filter(metricNames, Predicates.notNull()));
}
private List toTagValues(List tagValues) {
return Lists.transform(tagValues, new Function() {
@Nullable
@Override
public TagValue apply(@Nullable MetricTagValue input) {
if (input == null) {
// SHOULD NEVER happen
throw new NullPointerException();
}
return new TagValue(input.getName(), input.getValue());
}
});
}
private List humanToTagNames(List tagValues) {
List result = Lists.newArrayList();
for (MetricTagValue tagValue : tagValues) {
String tagName = humanToTagName(tagValue.getName());
result.add(new MetricTagValue(tagName, tagValue.getValue()));
}
return result;
}
private List parseTagValues(List tags) {
List result = Lists.newArrayList();
for (String tag : tags) {
// split by ':' and add the tagValue to result list
String[] tagSplit = tag.split(":", 2);
if (tagSplit.length == 2) {
String value = tagSplit[1].equals(ANY_TAG_VALUE) ? null : tagSplit[1];
result.add(new MetricTagValue(tagSplit[0], value));
}
}
return result;
}
private String humanToTagName(String humanTagName) {
String replacement = humanToTagName.get(humanTagName);
return replacement != null ? replacement : humanTagName;
}
private List tagValuesToHuman(Collection tagValues) {
List result = Lists.newArrayList();
for (TagValue tagValue : tagValues) {
String human = tagNameToHuman.get(tagValue.getName());
human = human != null ? human : tagValue.getName();
String value = tagValue.getValue() == null ? ANY_TAG_VALUE : tagValue.getValue();
result.add(new MetricTagValue(human, value));
}
return result;
}
private MetricQueryRequest getQueryRequestFromFormat(QueryRequestFormat queryRequestFormat) {
Map> queryParams = Maps.newHashMap();
for (Map.Entry entry : queryRequestFormat.getTimeRange().entrySet()) {
queryParams.put(entry.getKey(), ImmutableList.of(entry.getValue()));
}
MetricQueryRequest queryRequest = new MetricQueryRequest(queryRequestFormat.getTags(),
queryRequestFormat.getMetrics(),
queryRequestFormat.getGroupBy());
setTimeRangeInQueryRequest(queryRequest, queryParams);
return queryRequest;
}
private void setTimeRangeInQueryRequest(MetricQueryRequest request, Map> queryTimeParams) {
Long start =
queryTimeParams.containsKey(PARAM_START_TIME) ?
TimeMathParser.parseTimeInSeconds(queryTimeParams.get(PARAM_START_TIME).get(0)) : null;
Long end =
queryTimeParams.containsKey(PARAM_END_TIME) ?
TimeMathParser.parseTimeInSeconds(queryTimeParams.get(PARAM_END_TIME).get(0)) : null;
Integer count = null;
boolean aggregate =
queryTimeParams.containsKey(PARAM_AGGREGATE) && queryTimeParams.get(PARAM_AGGREGATE).get(0).equals("true") ||
((start == null) && (end == null));
Integer resolution = queryTimeParams.containsKey(PARAM_RESOLUTION) ?
getResolution(queryTimeParams.get(PARAM_RESOLUTION).get(0), start, end) : 1;
Interpolator interpolator = null;
if (queryTimeParams.containsKey(PARAM_INTERPOLATE)) {
long timeLimit = queryTimeParams.containsKey(PARAM_MAX_INTERPOLATE_GAP) ?
Long.parseLong(queryTimeParams.get(PARAM_MAX_INTERPOLATE_GAP).get(0)) : Long.MAX_VALUE;
interpolator = getInterpolator(queryTimeParams.get(PARAM_INTERPOLATE).get(0), timeLimit);
}
if (queryTimeParams.containsKey(PARAM_COUNT)) {
count = Integer.valueOf(queryTimeParams.get(PARAM_COUNT).get(0));
if (start == null && end != null) {
start = end - count * resolution;
} else if (start != null && end == null) {
end = start + count * resolution;
}
} else if (start != null && end != null) {
count = (int) (((end / resolution * resolution) - (start / resolution * resolution)) / resolution + 1);
} else if (!aggregate) {
throw new IllegalArgumentException("At least two of count/start/end parameters " +
"are required for time-range queries ");
}
if (aggregate) {
request.setTimeRange(0L, 0L, 1, Integer.MAX_VALUE, null);
} else {
request.setTimeRange(start, end, count, resolution, interpolator);
}
}
private Integer getResolution(String resolution, Long start, Long end) {
if (resolution.equals(PARAM_AUTO_RESOLUTION)) {
if (start != null && end != null) {
long difference = end - start;
return MetricQueryParser.getResolution(difference).getResolution();
} else {
throw new IllegalArgumentException("if resolution=auto, start and end timestamp " +
"should be provided to determine resolution");
}
} else {
// if not auto, check if the given resolution matches available resolutions that we support.
int resolutionInterval = TimeMathParser.resolutionInSeconds(resolution);
if (!((resolutionInterval == Integer.MAX_VALUE) || (resolutionInterval == 3600) ||
(resolutionInterval == 60) || (resolutionInterval == 1))) {
throw new IllegalArgumentException("Resolution interval not supported, only 1 second, 1 minute and " +
"1 hour resolutions are supported currently");
}
return resolutionInterval;
}
}
private Interpolator getInterpolator(String interpolator, long timeLimit) {
if (PARAM_STEP_INTERPOLATOR.equals(interpolator)) {
return new Interpolators.Step(timeLimit);
} else if (PARAM_LINEAR_INTERPOLATOR.equals(interpolator)) {
return new Interpolators.Linear(timeLimit);
}
return null;
}
private MetricQueryResult executeQuery(MetricQueryRequest queryRequest) throws Exception {
if (queryRequest.getMetrics().size() == 0) {
throw new IllegalArgumentException("Missing metrics parameter in the query");
}
Map tagsSliceBy = humanToTagNames(transformTagMap(queryRequest.getTags()));
MetricQueryRequest.TimeRange timeRange = queryRequest.getTimeRange();
MetricDataQuery query = new MetricDataQuery(timeRange.getStart(), timeRange.getEnd(),
timeRange.getResolutionInSeconds(),
timeRange.getCount(), toMetrics(queryRequest.getMetrics()),
tagsSliceBy, transformGroupByTags(queryRequest.getGroupBy()),
timeRange.getInterpolate());
Collection queryResult = metricStore.query(query);
long endTime = timeRange.getEnd();
if (timeRange.getResolutionInSeconds() == Integer.MAX_VALUE && endTime == 0) {
// for aggregate query, we set the end time to be query time (current time)
endTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
}
return decorate(queryResult, timeRange.getStart(), endTime, timeRange.getResolutionInSeconds());
}
private Map transformTagMap(Map tags) {
return Maps.transformValues(tags, new Function() {
@Override
public String apply(String value) {
if (ANY_TAG_VALUE.equals(value)) {
return null;
} else {
return value;
}
}
});
}
private List transformGroupByTags(List groupBy) {
return Lists.transform(groupBy, new Function() {
@Nullable
@Override
public String apply(@Nullable String input) {
String replacement = humanToTagName.get(input);
return replacement != null ? replacement : input;
}
});
}
private Map parseTagValuesAsMap(List tags) {
List tagValues = parseTagValues(tags);
Map result = Maps.newHashMap();
for (MetricTagValue tagValue : tagValues) {
result.put(tagValue.getName(), tagValue.getValue());
}
return result;
}
private Map humanToTagNames(Map tagValues) {
Map result = Maps.newHashMap();
for (Map.Entry tagValue : tagValues.entrySet()) {
result.put(humanToTagName(tagValue.getKey()), tagValue.getValue());
}
return result;
}
private Map toMetrics(List metrics) {
Map result = Maps.newHashMap();
for (String metric : metrics) {
// todo: figure out metric type
result.put(metric, AggregationFunction.SUM);
}
return result;
}
private MetricQueryResult decorate(Collection series, long startTs, long endTs,
int resolution) {
MetricQueryResult.TimeSeries[] serieses = new MetricQueryResult.TimeSeries[series.size()];
int i = 0;
for (MetricTimeSeries timeSeries : series) {
MetricQueryResult.TimeValue[] timeValues = decorate(timeSeries.getTimeValues());
serieses[i++] = new MetricQueryResult.TimeSeries(timeSeries.getMetricName(),
tagNamesToHuman(timeSeries.getTagValues()), timeValues);
}
return new MetricQueryResult(startTs, endTs, serieses, resolution);
}
private MetricQueryResult.TimeValue[] decorate(List points) {
MetricQueryResult.TimeValue[] timeValues = new MetricQueryResult.TimeValue[points.size()];
int k = 0;
for (TimeValue timeValue : points) {
timeValues[k++] = new MetricQueryResult.TimeValue(timeValue.getTimestamp(), timeValue.getValue());
}
return timeValues;
}
private Map tagNamesToHuman(Map tagValues) {
Map humanTagValues = Maps.newHashMap();
for (Map.Entry tag : tagValues.entrySet()) {
humanTagValues.put(tagNameToHuman.get(tag.getKey()), tag.getValue());
}
return humanTagValues;
}
/**
* Helper class to Deserialize Query requests and based on this
* {@link MetricQueryRequest} will be constructed
*/
public class QueryRequestFormat {
Map tags;
List metrics;
List groupBy;
Map timeRange;
public Map getTags() {
tags = (tags == null) ? Maps.newHashMap() : tags;
return tags;
}
public List getMetrics() {
return metrics;
}
public List getGroupBy() {
groupBy = (groupBy == null) ? Lists.newArrayList() : groupBy;
return groupBy;
}
/**
* time range has aggregate=true or {start, end, count, resolution, interpolate} parameters,
* since start, end can be represented as 'now ('+' or '-')' and not just absolute timestamp,
* we use this format to get those strings and after parsing and determining other parameters, we can construct
* {@link MetricQueryRequest} , similar for resolution.
* @return time range prameters
*/
public Map getTimeRange() {
timeRange = (timeRange == null || timeRange.size() == 0) ? ImmutableMap.of("aggregate", "true") : timeRange;
return timeRange;
}
}
}
