com.wavefront.ingester.AbstractIngesterFormatter Maven / Gradle / Ivy
package com.wavefront.ingester;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.wavefront.common.Clock;
import com.wavefront.data.ParseException;
import org.apache.avro.specific.SpecificRecordBase;
import wavefront.report.Annotation;
import wavefront.report.Histogram;
import javax.annotation.Nullable;
import java.util.*;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import static org.apache.commons.lang.StringUtils.containsAny;
import static org.apache.commons.lang.StringUtils.replace;
/**
* This is the base class for parsing data from plaintext.
*
* @author Suranjan Pramanik ([email protected])
* @author [email protected]
*/
public abstract class AbstractIngesterFormatter {
public static final String SOURCE_TAG_LITERAL = "@SourceTag";
public static final String SOURCE_DESCRIPTION_LITERAL = "@SourceDescription";
public static final String EVENT_LITERAL = "@Event";
private static final String SINGLE_QUOTE_STR = "'";
private static final String ESCAPED_SINGLE_QUOTE_STR = "\\'";
private static final String DOUBLE_QUOTE_STR = "\"";
private static final String ESCAPED_DOUBLE_QUOTE_STR = "\\\"";
private static final List DEFAULT_LOG_MESSAGE_KEYS = Arrays.asList("message", "text");
private static final List DEFAULT_LOG_TIMESTAMP_KEYS = Arrays.asList("timestamp", "log_timestamp");
private static final List DEFAULT_LOG_APPLICATION_KEYS = Collections.singletonList("application");
private static final List DEFAULT_LOG_SERVICE_KEYS = Collections.singletonList("service");
private static final List DEFAULT_LOG_EXCEPTION_KEYS = Arrays.asList("exception", "error_name");
private static final List DEFAULT_LOG_LEVEL_KEYS = Arrays.asList("level", "log_level");
protected final List> elements;
protected AbstractIngesterFormatter(List> elements) {
this.elements = elements;
}
protected interface FormatterElement {
void consume(StringParser parser, T target);
}
/**
* This class can be used to create a parser for content that proxy receives.
*/
public abstract static class IngesterFormatBuilder {
final List> elements = Lists.newArrayList();
public IngesterFormatBuilder caseSensitiveLiterals(List literals) {
elements.add(new Text<>(literals, null, true));
return this;
}
public IngesterFormatBuilder caseSensitiveLiterals(List literals,
BiConsumer textConsumer) {
elements.add(new Text<>(literals, textConsumer, true));
return this;
}
public IngesterFormatBuilder caseInsensitiveLiterals(List literals) {
elements.add(new Text<>(literals, null, false));
return this;
}
public IngesterFormatBuilder text(BiConsumer textConsumer) {
elements.add(new Text<>(textConsumer));
return this;
}
public IngesterFormatBuilder value(BiConsumer valueConsumer) {
elements.add(new Value<>(valueConsumer));
return this;
}
public IngesterFormatBuilder centroids() {
elements.add(new Centroids<>());
return this;
}
public IngesterFormatBuilder timestamp(BiConsumer timestampConsumer) {
elements.add(new Timestamp<>(timestampConsumer, false, false));
return this;
}
public IngesterFormatBuilder optionalTimestamp(BiConsumer timestampConsumer) {
elements.add(new Timestamp<>(timestampConsumer, true, false));
return this;
}
public IngesterFormatBuilder rawTimestamp(BiConsumer timestampConsumer) {
elements.add(new Timestamp<>(timestampConsumer, false, true));
return this;
}
public IngesterFormatBuilder annotationMap(BiConsumer> mapConsumer) {
elements.add(new StringMap<>(mapConsumer));
return this;
}
public IngesterFormatBuilder annotationMap(Function> mapProvider,
BiConsumer> mapConsumer) {
elements.add(new StringMap<>(mapConsumer, mapProvider, null, null));
return this;
}
public IngesterFormatBuilder annotationMap(BiConsumer> mapConsumer,
int limit) {
elements.add(new StringMap<>(mapConsumer, null, limit, null));
return this;
}
public IngesterFormatBuilder annotationList(BiConsumer> listConsumer) {
elements.add(new AnnotationList<>(listConsumer, null));
return this;
}
public IngesterFormatBuilder annotationList(BiConsumer> listConsumer,
Predicate stringPredicate) {
elements.add(new AnnotationList<>(listConsumer, stringPredicate));
return this;
}
public IngesterFormatBuilder annotationList(Function> listProvider,
BiConsumer> listConsumer) {
elements.add(new AnnotationList<>(listConsumer, listProvider, null, null));
return this;
}
public IngesterFormatBuilder annotationList(BiConsumer> listConsumer,
int limit) {
elements.add(new AnnotationList<>(listConsumer, null, limit, null));
return this;
}
public IngesterFormatBuilder annotationMultimap(
BiConsumer>> multimapConsumer) {
elements.add(new StringMultiMap<>(multimapConsumer));
return this;
}
public IngesterFormatBuilder textList(BiConsumer> listConsumer) {
elements.add(new StringList<>(listConsumer));
return this;
}
public abstract AbstractIngesterFormatter build();
}
public static class Text implements FormatterElement {
final List literals;
final BiConsumer textConsumer;
final boolean isCaseSensitive;
Text(@Nullable BiConsumer textConsumer) {
this(null, textConsumer, true);
}
Text(List literals,
@Nullable BiConsumer textConsumer,
boolean isCaseSensitive) {
this.literals = literals;
this.textConsumer = textConsumer;
this.isCaseSensitive = isCaseSensitive;
}
@Override
public void consume(StringParser parser, T target) {
String text = parser.next();
if (!isAllowedLiteral(text)) throw new ParseException("'" + text +
"' is not allowed here!");
if (textConsumer != null) textConsumer.accept(target, text);
}
private boolean isAllowedLiteral(String literal) {
if (literals == null) return true;
for (String allowedLiteral : literals) {
if (isCaseSensitive && literal.equals(allowedLiteral)) return true;
if (!isCaseSensitive && literal.equalsIgnoreCase(allowedLiteral)) return true;
}
return false;
}
}
public static class Value implements FormatterElement {
final BiConsumer valueConsumer;
Value(BiConsumer valueConsumer) {
this.valueConsumer = valueConsumer;
}
@Override
public void consume(StringParser parser, T target) {
String token = parser.next();
if (token == null)
throw new ParseException("Value is missing");
try {
valueConsumer.accept(target, Double.parseDouble(token));
} catch (NumberFormatException nef) {
throw new ParseException("Invalid value: " + token);
}
}
}
public static class Centroids implements FormatterElement {
private static final String WEIGHT = "#";
@Override
public void consume(StringParser parser, T target) {
List counts = new ArrayList<>();
List bins = new ArrayList<>();
while (WEIGHT.equals(parser.peek())) {
parser.next(); // skip the # token
counts.add(parse(parser.next(), "centroid weight", true).intValue());
bins.add(parse(parser.next(), "centroid value", false).doubleValue());
}
if (counts.size() == 0) throw new ParseException("Empty histogram (no centroids)");
Histogram histogram = (Histogram) target.get("value");
histogram.setCounts(counts);
histogram.setBins(bins);
}
private static Number parse(@Nullable String toParse, String name, boolean asInteger) {
if (toParse == null) {
throw new ParseException("Unexpected end of line, expected: " + name);
}
try {
return asInteger ? Integer.parseInt(toParse) : Double.parseDouble(toParse);
} catch (NumberFormatException nef) {
throw new ParseException("Expected: " + name + ", got: " + toParse);
}
}
}
public static class Timestamp implements FormatterElement {
private final BiConsumer timestampConsumer;
private final boolean optional;
private final boolean raw;
Timestamp(BiConsumer timestampConsumer, boolean optional, boolean raw) {
this.timestampConsumer = timestampConsumer;
this.optional = optional;
this.raw = raw;
}
@Override
public void consume(StringParser parser, T target) {
Long timestamp = parseTimestamp(parser, optional, raw);
if (timestamp != null) timestampConsumer.accept(target, timestamp);
}
}
public static class StringList implements FormatterElement {
private final BiConsumer> stringListConsumer;
StringList(BiConsumer> stringListConsumer) {
this.stringListConsumer = stringListConsumer;
}
@Override
public void consume(StringParser parser, T target) {
List list = new ArrayList<>();
while (parser.hasNext()) {
list.add(parser.next());
}
stringListConsumer.accept(target, list);
}
}
public static class StringMap implements FormatterElement {
private final BiConsumer> stringMapConsumer;
private final Function> stringMapProvider;
private final Integer limit;
private final Predicate predicate;
StringMap(BiConsumer> stringMapConsumer) {
this(stringMapConsumer, null, null, null);
}
StringMap(BiConsumer> stringMapConsumer,
@Nullable Function> stringMapProvider,
@Nullable Integer limit,
@Nullable Predicate predicate) {
this.stringMapConsumer = stringMapConsumer;
this.stringMapProvider = stringMapProvider;
this.limit = limit;
this.predicate = predicate;
}
@Override
public void consume(StringParser parser, T target) {
Map stringMap = null;
if (stringMapProvider != null) {
stringMap = stringMapProvider.apply(target);
}
if (stringMap == null) {
stringMap = Maps.newHashMap();
}
int i = 0;
while (parser.hasNext() && (limit == null || i < limit) &&
(predicate == null || predicate.test(parser.peek()))) {
parseKeyValuePair(parser, stringMap::put);
i++;
}
stringMapConsumer.accept(target, stringMap);
}
}
public static class StringMultiMap implements FormatterElement {
private final BiConsumer>> annotationMultimapConsumer;
StringMultiMap(BiConsumer>> annotationMultimapConsumer) {
this.annotationMultimapConsumer = annotationMultimapConsumer;
}
@Override
public void consume(StringParser parser, T target) {
Map> multimap = new HashMap<>();
while (parser.hasNext()) {
parseKeyValuePair(parser, (k, v) -> {
multimap.computeIfAbsent(k, x -> new ArrayList<>()).add(v);
});
}
annotationMultimapConsumer.accept(target, multimap);
}
}
public static class AnnotationList implements FormatterElement {
private final BiConsumer> annotationListConsumer;
private final Function> annotationListProvider;
private final Integer limit;
private final Predicate predicate;
AnnotationList(BiConsumer> annotationListConsumer,
Predicate predicate) {
this(annotationListConsumer, null, null, predicate);
}
AnnotationList(BiConsumer> annotationListConsumer,
@Nullable Function> annotationListProvider,
@Nullable Integer limit,
@Nullable Predicate predicate) {
this.annotationListConsumer = annotationListConsumer;
this.annotationListProvider = annotationListProvider;
this.limit = limit;
this.predicate = predicate;
}
@Override
public void consume(StringParser parser, T target) {
List annotations = null;
if (annotationListProvider != null) {
annotations = annotationListProvider.apply(target);
}
if (annotations == null) {
annotations = new ArrayList<>();
}
List annotationList = annotations;
int i = 0;
while (parser.hasNext() && (limit == null || i < limit) &&
(predicate == null || predicate.test(parser.peek()))) {
parseKeyValuePair(parser, (k, v) -> annotationList.add(new Annotation(k, v)));
i++;
}
annotationListConsumer.accept(target, annotationList);
}
}
/**
* Infers timestamp resolution and normalizes it to milliseconds
* @param timestamp timestamp in seconds, milliseconds, microseconds or nanoseconds
* @return timestamp in milliseconds
*/
public static long timestampInMilliseconds(Double timestamp) {
long timestampLong = timestamp.longValue();
if (timestampLong < 1_000_000_000_000L) {
// less than 13 digits: treat it as seconds
return (long)(1000 * timestamp);
} else if (timestampLong < 10_000_000_000_000L) {
// 13 digits: treat as milliseconds
return timestampLong;
} else if (timestampLong < 10_000_000_000_000_000L) {
// 16 digits: treat as microseconds
return timestampLong / 1000;
} else {
// 19 digits: treat as nanoseconds.
return timestampLong / 1000000;
}
}
private static Long parseTimestamp(StringParser parser, boolean optional, boolean raw) {
String peek = parser.peek();
if (peek == null || !Character.isDigit(peek.charAt(0))) {
if (optional) {
return null;
} else {
throw new ParseException("Expected timestamp, found " +
(peek == null ? "end of line" : peek));
}
}
try {
Double timestamp = Double.parseDouble(peek);
parser.next();
if (raw) {
// as-is
return timestamp.longValue();
}
return timestampInMilliseconds(timestamp);
} catch (NumberFormatException nfe) {
throw new ParseException("Invalid timestamp value: " + peek);
}
}
private static void parseKeyValuePair(StringParser parser,
BiConsumer kvConsumer) {
String annotationKey = parser.next();
String op = parser.next();
if (op == null) {
throw new ParseException("Tag keys and values must be separated by '=', " +
"nothing found after '" + annotationKey + "'");
}
if (!op.equals("=")) {
throw new ParseException("Tag keys and values must be separated by '=', found " + op);
}
String annotationValue = parser.next();
if (annotationValue == null) {
throw new ParseException("Value missing for " + annotationKey);
}
kvConsumer.accept(annotationKey, annotationValue);
}
/**
* @param text Text to unquote.
* @return Extracted value from inside a quoted string.
*/
@SuppressWarnings("WeakerAccess") // Has users.
public static String unquote(String text) {
if (text.startsWith(DOUBLE_QUOTE_STR)) {
String quoteless = text.substring(1, text.length() - 1);
if (containsAny(quoteless, ESCAPED_DOUBLE_QUOTE_STR)) {
return replace(quoteless, ESCAPED_DOUBLE_QUOTE_STR, DOUBLE_QUOTE_STR);
}
return quoteless;
} else if (text.startsWith(SINGLE_QUOTE_STR)) {
String quoteless = text.substring(1, text.length() - 1);
if (containsAny(quoteless, ESCAPED_SINGLE_QUOTE_STR)) {
return replace(quoteless, ESCAPED_SINGLE_QUOTE_STR, SINGLE_QUOTE_STR);
}
return quoteless;
}
return text;
}
@Nullable
public static String getHostAndNormalizeTags(@Nullable List annotations,
@Nullable List customSourceTags,
boolean replaceTag) {
String source = null;
String host = null;
if (annotations != null) {
Iterator iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
if (annotation.getKey().equals("source")) {
iter.remove();
source = annotation.getValue();
} else if (annotation.getKey().equals("host")) {
iter.remove();
host = annotation.getValue();
} else if (annotation.getKey().equals("tag") && replaceTag) {
// TODO: investigate why we add underscore here
annotation.setKey("_tag");
}
}
if (host != null) {
if (source == null) {
source = host;
} else {
annotations.add(new Annotation("_host", host));
}
}
if (source == null && customSourceTags != null) {
// iterate over the set of custom tags, breaking when one is found
for (String tag : customSourceTags) {
// nested loops are not pretty but we need to ensure the order of customSourceTags
for (Annotation annotation : annotations) {
if (annotation.getKey().equals(tag)) {
source = annotation.getValue();
break;
}
}
if (source != null) break;
}
}
}
return source;
}
@Nullable
public static String getLogMessage(@Nullable List annotations,
@Nullable List customLogMessageTags) {
String logMessage = null;
if (annotations != null) {
Iterator iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
for (String defaultLogMessageKey : DEFAULT_LOG_MESSAGE_KEYS) {
if (annotation.getKey().equals(defaultLogMessageKey)) {
iter.remove();
logMessage = annotation.getValue();
break;
}
}
}
if (logMessage == null && customLogMessageTags != null) {
// iterate over the set of custom message tags, breaking when one is found
for (String tag : customLogMessageTags) {
// nested loops are not pretty but we need to ensure the order of customLogMessageTags
iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
if (annotation.getKey().equals(tag)) {
logMessage = annotation.getValue();
iter.remove();
break;
}
}
if (logMessage != null) break;
}
}
}
return (logMessage == null)? "" : logMessage;
}
@Nullable
public static Long getLogTimestamp(@Nullable List annotations,
@Nullable List customLogTimestampTags) {
String timestampStr = null;
if (annotations != null) {
Iterator iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
for (String defaultLogTimestampKey : DEFAULT_LOG_TIMESTAMP_KEYS) {
if (annotation.getKey().equals(defaultLogTimestampKey)) {
iter.remove();
timestampStr = annotation.getValue();
break;
}
}
}
if (timestampStr == null && customLogTimestampTags != null) {
// iterate over the set of custom timestamp tags, breaking when one is found
for (String tag : customLogTimestampTags) {
// nested loops are not pretty but we need to ensure the order of customLogTimestampTags
iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
if (annotation.getKey().equals(tag)) {
timestampStr = annotation.getValue();
iter.remove();
break;
}
}
if (timestampStr != null) break;
}
}
}
if (timestampStr == null) {
return Clock.now();
}
Long timestamp = null;
// We're only supporting timestamp in epoch format with various resolutions (seconds, milliseconds,
// microseconds or nanoseconds) as input. We will normalize to millisecond resolution
try {
timestamp = timestampInMilliseconds(Double.parseDouble(timestampStr));
} catch (NumberFormatException ignore) {
timestamp = Clock.now();
}
return timestamp;
}
@Nullable
public static String getLogApplication(@Nullable List annotations,
@Nullable List customLogApplicationTags) {
String applicationStr = null;
if (annotations != null) {
Iterator iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
for (String defaultLogApplicationKey : DEFAULT_LOG_APPLICATION_KEYS) {
if (annotation.getKey().equals(defaultLogApplicationKey)) {
iter.remove();
applicationStr = annotation.getValue();
break;
}
}
}
if (applicationStr == null && customLogApplicationTags != null) {
// iterate over the set of custom application tags, breaking when one is found
for (String tag : customLogApplicationTags) {
// nested loops are not pretty but we need to ensure the order of customLogMessageTags
iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
if (annotation.getKey().equals(tag)) {
applicationStr = annotation.getValue();
iter.remove();
break;
}
}
if (applicationStr != null) break;
}
}
}
return (applicationStr == null)? "none" : applicationStr;
}
@Nullable
public static String getLogService(@Nullable List annotations,
@Nullable List customLogServiceTags) {
String serviceStr = null;
if (annotations != null) {
Iterator iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
for (String defaultLogServiceKey : DEFAULT_LOG_SERVICE_KEYS) {
if (annotation.getKey().equals(defaultLogServiceKey)) {
iter.remove();
serviceStr = annotation.getValue();
break;
}
}
}
if (serviceStr == null && customLogServiceTags != null) {
// iterate over the set of custom service tags, breaking when one is found
for (String tag : customLogServiceTags) {
// nested loops are not pretty but we need to ensure the order of customLogMessageTags
iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
if (annotation.getKey().equals(tag)) {
serviceStr = annotation.getValue();
iter.remove();
break;
}
}
if (serviceStr != null) break;
}
}
}
return (serviceStr == null)? "none" : serviceStr;
}
public static List getDefaultLogMessageKeys() {
return DEFAULT_LOG_MESSAGE_KEYS;
}
@Nullable
public static String getLogLevel(@Nullable List annotations,
@Nullable List customLogLevelTags) {
String logLvlStr = null;
if (annotations != null) {
Iterator iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
for (String defaultLogLevelKey : DEFAULT_LOG_LEVEL_KEYS) {
if (annotation.getKey().equals(defaultLogLevelKey)) {
iter.remove();
logLvlStr = annotation.getValue();
break;
}
}
}
if (logLvlStr == null && customLogLevelTags != null) {
// iterate over the set of custom level tags, breaking when one is found
for (String tag : customLogLevelTags) {
// nested loops are not pretty but we need to ensure the order of customLogMessageTags
iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
if (annotation.getKey().equals(tag)) {
logLvlStr = annotation.getValue();
iter.remove();
break;
}
}
if (logLvlStr != null) break;
}
}
}
return (logLvlStr == null)? "" : logLvlStr;
}
@Nullable
public static String getLogException(@Nullable List annotations,
@Nullable List customLogExceptionTags) {
String exceptionStr = null;
if (annotations != null) {
Iterator iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
for (String defaultLogExceptionKey : DEFAULT_LOG_EXCEPTION_KEYS) {
if (annotation.getKey().equals(defaultLogExceptionKey)) {
iter.remove();
exceptionStr = annotation.getValue();
break;
}
}
}
if (exceptionStr == null && customLogExceptionTags != null) {
// iterate over the set of custom exception tags, breaking when one is found
for (String tag : customLogExceptionTags) {
// nested loops are not pretty but we need to ensure the order of customLogMessageTags
iter = annotations.iterator();
while (iter.hasNext()) {
Annotation annotation = iter.next();
if (annotation.getKey().equals(tag)) {
exceptionStr = annotation.getValue();
iter.remove();
break;
}
}
if (exceptionStr != null) break;
}
}
}
return (exceptionStr == null)? "" : exceptionStr;
}
public T drive(String input, @Nullable Supplier defaultHostNameSupplier,
String customerId) {
return drive(input, defaultHostNameSupplier, customerId, null, null, null, null, null, null, null, null);
}
public abstract T drive(String input, @Nullable Supplier defaultHostNameSupplier,
String customerId, @Nullable List customSourceTags, @Nullable List customLogTimestampTags,
@Nullable List customLogMessageTags, List customLogApplicationTags, List customLogServiceTags,
@Nullable List customLogLevelTags, @Nullable List customExceptionTags,@Nullable IngesterContext ingesterContext);
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy