com.yahoo.vespa.model.admin.otel.OpenTelemetryConfigGenerator Maven / Gradle / Ivy
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.admin.otel;
import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
import ai.vespa.metricsproxy.metric.model.prometheus.PrometheusUtil;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.yahoo.config.model.ApplicationConfigProducerRoot.StatePortInfo;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterMembership;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Zone;
import com.yahoo.vespa.model.Service;
import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.Set;
import java.util.TreeSet;
import static com.yahoo.vespa.defaults.Defaults.getDefaults;
/**
* @author olaa
*/
public class OpenTelemetryConfigGenerator {
private final boolean useTls;
private final String ca_file;
private final String cert_file;
private final String key_file;
private List statePorts = new ArrayList<>();
private final Zone zone;
private final ApplicationId applicationId;
private final boolean isHostedVespa;
OpenTelemetryConfigGenerator(Zone zone, ApplicationId applicationId, boolean isHostedVespa) {
this.zone = zone;
this.applicationId = applicationId;
this.isHostedVespa = isHostedVespa;
boolean isCd = true;
boolean isPublic = true;
if (zone != null) {
isCd = zone.system().isCd();
isPublic = zone.system().isPublic();
this.useTls = true;
} else {
// for manual testing
this.useTls = false;
}
if (isCd) {
if (isPublic) {
this.ca_file = "/opt/vespa/var/vespa/trust-store.pem";
this.cert_file = "/var/lib/sia/certs/vespa.external.cd.tenant.cert.pem";
this.key_file = "/var/lib/sia/keys/vespa.external.cd.tenant.key.pem";
} else {
this.ca_file = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem";
this.cert_file = "/var/lib/sia/certs/vespa.vespa.cd.tenant.cert.pem";
this.key_file = "/var/lib/sia/keys/vespa.vespa.cd.tenant.key.pem";
}
} else {
if (isPublic) {
this.ca_file = "/opt/vespa/var/vespa/trust-store.pem";
this.cert_file = "/var/lib/sia/certs/vespa.external.tenant.cert.pem";
this.key_file = "/var/lib/sia/keys/vespa.external.tenant.key.pem";
} else {
this.ca_file = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem";
this.cert_file = "/var/lib/sia/certs/vespa.vespa.tenant.cert.pem";
this.key_file = "/var/lib/sia/keys/vespa.vespa.tenant.key.pem";
}
}
}
String receiverName(int index) {
return "prometheus_simple/s" + index;
}
private void addReceivers(JsonGenerator g) throws java.io.IOException {
g.writeFieldName("receivers");
g.writeStartObject();
int counter = 0;
for (var statePort : statePorts) {
addReceiver(g, ++counter, statePort);
}
g.writeEndObject(); // receivers
}
private void addReceiver(JsonGenerator g, int index, StatePortInfo statePort) throws java.io.IOException {
g.writeFieldName(receiverName(index));
g.writeStartObject();
g.writeStringField("collection_interval", "60s");
g.writeStringField("endpoint", statePort.hostName() + ":" + statePort.portNumber());
addUrlInfo(g);
if (useTls) addTls(g);
{
g.writeFieldName("labels");
var dimVals = serviceAttributes(statePort.service());
// these will be tagged as dimension-name/label-value
// attributes on all metrics from this /state/v1 port
g.writeStartObject();
for (var entry : dimVals.entrySet()) {
if (entry.getValue() != null) {
g.writeStringField(entry.getKey(), entry.getValue());
}
}
String ph = findParentHost(statePort.hostName());
if (isHostedVespa && ph != null) {
g.writeStringField("parentHostname", ph);
}
g.writeEndObject();
}
g.writeEndObject();
}
// note: this pattern should match entire node name
static private final Pattern expectedNodeName1 = Pattern.compile("[a-z0-9]+-v6-[0-9]+[.].+");
static private final Pattern expectedNodeName2 = Pattern.compile("[a-z]*[0-9]+[a-z][.].+");
// matches the part we want to replace with just a dot
static private final Pattern replaceNodeName1 = Pattern.compile("-v6-[0-9]+[.]");
static private final Pattern replaceNodeName2 = Pattern.compile("[a-z][.]");
static String findParentHost(String nodeName) {
if (expectedNodeName1.matcher(nodeName).matches()) {
return replaceNodeName1.matcher(nodeName).replaceFirst(".");
}
if (expectedNodeName2.matcher(nodeName).matches()) {
return replaceNodeName2.matcher(nodeName).replaceFirst(".");
}
return null;
}
private void addTls(JsonGenerator g) throws java.io.IOException {
g.writeFieldName("tls");
g.writeStartObject();
g.writeStringField("ca_file", ca_file);
g.writeStringField("cert_file", cert_file);
g.writeBooleanField("insecure_skip_verify", true);
g.writeStringField("key_file", key_file);
g.writeEndObject(); // tls
}
private void addUrlInfo(JsonGenerator g) throws java.io.IOException {
g.writeStringField("metrics_path", "/state/v1/metrics");
g.writeFieldName("params");
g.writeStartObject();
g.writeStringField("format", "prometheus");
g.writeEndObject();
}
private void addExporters(JsonGenerator g) throws java.io.IOException {
g.writeFieldName("exporters");
g.writeStartObject();
addFileExporter(g);
g.writeEndObject(); // exporters
}
private void addFileExporter(JsonGenerator g) throws java.io.IOException {
g.writeFieldName("file");
g.writeStartObject();
g.writeStringField("path", getDefaults().underVespaHome("logs/vespa/otel-test.json"));
{
g.writeFieldName("rotation");
g.writeStartObject();
g.writeNumberField("max_megabytes", 10);
g.writeNumberField("max_days", 3);
g.writeNumberField("max_backups", 1);
g.writeEndObject(); // rotation
}
g.writeEndObject(); // file
}
private void addProcessors(JsonGenerator g) throws java.io.IOException {
g.writeFieldName("processors");
g.writeStartObject();
addResourceProcessor(g);
addRenameProcessor(g);
addFilterProcessor(g);
g.writeEndObject();
}
private void addRenameProcessor(JsonGenerator g) throws java.io.IOException {
g.writeFieldName("metricstransform/rename");
g.writeStartObject();
g.writeFieldName("transforms");
g.writeStartArray();
var metrics = MetricsConsumer.vespa9.metrics();
for (var metric : metrics.values()) {
if (! metric.name.equals(metric.outputName)) {
String oldName = PrometheusUtil.sanitize(metric.name);
String newName = PrometheusUtil.sanitize(metric.outputName);
addRenameAction(g, oldName, newName);
}
}
g.writeEndArray();
g.writeEndObject();
}
private void addRenameAction(JsonGenerator g, String oldName, String newName) throws java.io.IOException {
g.writeStartObject();
g.writeStringField("include", oldName);
g.writeStringField("action", "update");
g.writeStringField("new_name", newName);
g.writeEndObject();
}
private void addResourceProcessor(JsonGenerator g) throws java.io.IOException {
g.writeFieldName("resource");
g.writeStartObject();
g.writeFieldName("attributes");
g.writeStartArray();
// common attributes for all metrics from all services;
// which application and which cloud/system/zone/environment
addAttributeInsert(g, PublicDimensions.ZONE, zoneAttr());
addAttributeInsert(g, PublicDimensions.APPLICATION_ID, appIdAttr());
addAttributeInsert(g, "system", systemAttr());
addAttributeInsert(g, "tenantName", tenantAttr());
addAttributeInsert(g, "applicationName", appNameAttr());
addAttributeInsert(g, "instanceName", appInstanceAttr());
addAttributeInsert(g, "cloud", cloudAttr());
g.writeEndArray();
g.writeEndObject();
}
private void addAttributeInsert(JsonGenerator g, String key, String value) throws java.io.IOException {
if (value == null) return;
g.writeStartObject();
g.writeStringField("key", key);
g.writeStringField("value", value);
g.writeStringField("action", "insert");
g.writeEndObject();
}
static private Set wantedMetrics() {
Set result = new TreeSet<>();
var metrics = MetricsConsumer.vespa9.metrics();
for (var metric : metrics.values()) {
String oldName = PrometheusUtil.sanitize(metric.name);
String newName = PrometheusUtil.sanitize(metric.outputName);
result.add(oldName);
result.add(newName);
}
return result;
}
private void addFilterProcessor(JsonGenerator g) throws java.io.IOException {
g.writeFieldName("filter/metricset");
g.writeStartObject();
g.writeFieldName("metrics");
g.writeStartObject();
g.writeFieldName("include");
g.writeStartObject();
g.writeStringField("match_type", "strict");
g.writeFieldName("metric_names");
g.writeStartArray();
for (String metricName : wantedMetrics()) {
g.writeString(metricName);
}
g.writeEndArray();
g.writeEndObject();
g.writeEndObject();
g.writeEndObject();
}
private void addServiceBlock(JsonGenerator g) throws java.io.IOException {
g.writeFieldName("service");
g.writeStartObject();
{
g.writeFieldName("telemetry");
g.writeStartObject();
{
g.writeFieldName("logs");
g.writeStartObject();
g.writeStringField("level", "debug");
g.writeEndObject();
}
g.writeEndObject();
}
{
g.writeFieldName("pipelines");
g.writeStartObject();
addMetricsPipelines(g);
g.writeEndObject(); // pipelines
}
g.writeEndObject(); // service
}
private void addMetricsPipelines(JsonGenerator g) throws java.io.IOException {
g.writeFieldName("metrics");
g.writeStartObject();
{
g.writeFieldName("receivers");
g.writeStartArray();
int counter = 0;
for (var statePort : statePorts) {
g.writeString(receiverName(++counter));
}
g.writeEndArray();
}
g.writeFieldName("processors");
g.writeStartArray();
g.writeString("metricstransform/rename");
g.writeString("filter/metricset");
g.writeString("resource");
g.writeEndArray();
{
g.writeFieldName("exporters");
g.writeStartArray();
g.writeString("otlphttp/gw");
g.writeEndArray();
}
g.writeEndObject(); // metrics
}
// For now - mostly dummy config
/*
TODO: Create config
1. polling /state/v1 handler of every service (mostly done)
2. Processing with mapping/filtering from metric sets
3. Exporter to correct endpoint (alternatively amended)
*/
public String generate() {
if (statePorts.isEmpty()) {
return "";
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
try {
JsonGenerator g = new JsonFactory().createGenerator(out, JsonEncoding.UTF8);
g.writeStartObject();
addReceivers(g);
addExporters(g);
addProcessors(g);
addServiceBlock(g);
g.writeEndObject(); // root
g.close();
} catch (java.io.IOException e) {
System.err.println("unexpected error: " + e);
return "";
}
return out.toString(StandardCharsets.UTF_8);
}
void addStatePorts(List portList) {
this.statePorts = portList;
}
List referencedPaths() {
return List.of(ca_file, cert_file, key_file);
}
private String zoneAttr() {
if (zone == null) return null;
return zone.environment().value() + "." + zone.region().value();
}
private String appIdAttr() {
if (applicationId == null) return null;
return applicationId.toFullString();
}
private String systemAttr() {
if (zone == null) return null;
return zone.system().value();
}
private String tenantAttr() {
if (applicationId == null) return null;
return applicationId.tenant().value();
}
private String appNameAttr() {
if (applicationId == null) return null;
return applicationId.application().value();
}
private String appInstanceAttr() {
if (applicationId == null) return null;
return applicationId.instance().value();
}
private String cloudAttr() {
if (zone == null) return null;
return zone.cloud().name().value();
}
private String getDeploymentCluster(ClusterSpec cluster) {
if (applicationId == null) return null;
if (zone == null) return null;
String appString = applicationId.toFullString();
return String.join(".", appString,
zone.environment().value(),
zone.region().value(),
cluster.id().value());
}
private Map serviceAttributes(Service svc) {
Map dimvals = new LinkedHashMap<>();
// currently unused - can be added if necessary:
// dimvals.put("local_service_name", svc.getServiceName());
// dimvals.put("local_service_type", svc.getServiceType());
String cName = svc.getServicePropertyString("clustername", null);
if (cName != null) {
// overridden by cluster membership below (if available)
dimvals.put(PublicDimensions.INTERNAL_CLUSTER_ID, cName);
}
String cType = svc.getServicePropertyString("clustertype", null);
if (cType != null) {
// overridden by cluster membership below (if available)
dimvals.put(PublicDimensions.INTERNAL_CLUSTER_TYPE, cType);
}
var hostResource = svc.getHost();
if (hostResource != null) {
hostResource.spec().membership().map(ClusterMembership::cluster).ifPresent(cluster -> {
dimvals.put(PublicDimensions.DEPLOYMENT_CLUSTER, getDeploymentCluster(cluster));
// overrides value above
dimvals.put(PublicDimensions.INTERNAL_CLUSTER_TYPE, cluster.type().name());
// alternative to above
dimvals.put(PublicDimensions.INTERNAL_CLUSTER_ID, cluster.id().value());
cluster.group().ifPresent(group -> dimvals.put(PublicDimensions.GROUP_ID, group.toString()));
});
}
return dimvals;
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy