Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.metrics;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.DelegatingConfiguration;
import org.apache.flink.configuration.MetricOptions;
import org.apache.flink.core.plugin.PluginManager;
import org.apache.flink.metrics.MetricConfig;
import org.apache.flink.metrics.reporter.InstantiateViaFactory;
import org.apache.flink.metrics.reporter.InterceptInstantiationViaReflection;
import org.apache.flink.metrics.reporter.MetricReporter;
import org.apache.flink.metrics.reporter.MetricReporterFactory;
import org.apache.flink.runtime.metrics.scope.ScopeFormat;
import org.apache.flink.shaded.guava18.com.google.common.collect.Iterators;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* Encapsulates everything needed for the instantiation and configuration of a {@link MetricReporter}.
*/
public final class ReporterSetup {
private static final Logger LOG = LoggerFactory.getLogger(ReporterSetup.class);
// regex pattern to split the defined reporters
private static final Pattern reporterListPattern = Pattern.compile("\\s*,\\s*");
// regex pattern to extract the name from reporter configuration keys, e.g. "rep" from "metrics.reporter.rep.class"
private static final Pattern reporterClassPattern = Pattern.compile(
Pattern.quote(ConfigConstants.METRICS_REPORTER_PREFIX) +
// [\S&&[^.]] = intersection of non-whitespace and non-period character classes
"([\\S&&[^.]]*)\\." +
'(' + Pattern.quote(ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX) + '|' + Pattern.quote(ConfigConstants.METRICS_REPORTER_FACTORY_CLASS_SUFFIX) + ')');
private final String name;
private final MetricConfig configuration;
private final MetricReporter reporter;
public ReporterSetup(final String name, final MetricConfig configuration, MetricReporter reporter) {
this.name = name;
this.configuration = configuration;
this.reporter = reporter;
}
public Optional getDelimiter() {
return Optional.ofNullable(configuration.getString(ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, null));
}
public Optional getIntervalSettings() {
return Optional.ofNullable(configuration.getString(ConfigConstants.METRICS_REPORTER_INTERVAL_SUFFIX, null));
}
public Set getExcludedVariables() {
String excludedVariablesList = configuration.getString(ConfigConstants.METRICS_REPORTER_EXCLUDED_VARIABLES, null);
if (excludedVariablesList == null) {
return Collections.emptySet();
} else {
final Set excludedVariables = new HashSet<>();
for (String exclusion : excludedVariablesList.split(";")) {
excludedVariables.add(ScopeFormat.asVariable(exclusion));
}
return Collections.unmodifiableSet(excludedVariables);
}
}
public String getName() {
return name;
}
@VisibleForTesting
MetricConfig getConfiguration() {
return configuration;
}
public MetricReporter getReporter() {
return reporter;
}
@VisibleForTesting
public static ReporterSetup forReporter(String reporterName, MetricReporter reporter) {
return createReporterSetup(reporterName, new MetricConfig(), reporter);
}
@VisibleForTesting
public static ReporterSetup forReporter(String reporterName, MetricConfig metricConfig, MetricReporter reporter) {
return createReporterSetup(reporterName, metricConfig, reporter);
}
private static ReporterSetup createReporterSetup(String reporterName, MetricConfig metricConfig, MetricReporter reporter) {
reporter.open(metricConfig);
return new ReporterSetup(reporterName, metricConfig, reporter);
}
public static List fromConfiguration(final Configuration configuration, @Nullable final PluginManager pluginManager) {
String includedReportersString = configuration.getString(MetricOptions.REPORTERS_LIST, "");
Set namedReporters = findEnabledReportersInConfiguration(configuration, includedReportersString);
if (namedReporters.isEmpty()) {
return Collections.emptyList();
}
final List> reporterConfigurations = loadReporterConfigurations(configuration, namedReporters);
final Map reporterFactories = loadAvailableReporterFactories(pluginManager);
return setupReporters(reporterFactories, reporterConfigurations);
}
private static Set findEnabledReportersInConfiguration(Configuration configuration, String includedReportersString) {
Set includedReporters = reporterListPattern.splitAsStream(includedReportersString)
.filter(r -> !r.isEmpty()) // splitting an empty string results in an empty string on jdk9+
.collect(Collectors.toSet());
// use a TreeSet to make the reporter order deterministic, which is useful for testing
Set namedOrderedReporters = new TreeSet<>(String::compareTo);
// scan entire configuration for keys starting with METRICS_REPORTER_PREFIX and determine the set of enabled reporters
for (String key : configuration.keySet()) {
if (key.startsWith(ConfigConstants.METRICS_REPORTER_PREFIX)) {
Matcher matcher = reporterClassPattern.matcher(key);
if (matcher.matches()) {
String reporterName = matcher.group(1);
if (includedReporters.isEmpty() || includedReporters.contains(reporterName)) {
if (namedOrderedReporters.contains(reporterName)) {
LOG.warn("Duplicate class configuration detected for reporter {}.", reporterName);
} else {
namedOrderedReporters.add(reporterName);
}
} else {
LOG.info("Excluding reporter {}, not configured in reporter list ({}).", reporterName, includedReportersString);
}
}
}
}
return namedOrderedReporters;
}
private static List> loadReporterConfigurations(Configuration configuration, Set namedReporters) {
final List> reporterConfigurations = new ArrayList<>(namedReporters.size());
for (String namedReporter: namedReporters) {
DelegatingConfiguration delegatingConfiguration = new DelegatingConfiguration(
configuration,
ConfigConstants.METRICS_REPORTER_PREFIX + namedReporter + '.');
reporterConfigurations.add(Tuple2.of(namedReporter, delegatingConfiguration));
}
return reporterConfigurations;
}
private static Map loadAvailableReporterFactories(@Nullable PluginManager pluginManager) {
final Map reporterFactories = new HashMap<>(2);
final Iterator factoryIterator = getAllReporterFactories(pluginManager);
// do not use streams or for-each loops here because they do not allow catching individual ServiceConfigurationErrors
// such an error might be caused if the META-INF/services contains an entry to a non-existing factory class
while (factoryIterator.hasNext()) {
try {
MetricReporterFactory factory = factoryIterator.next();
String factoryClassName = factory.getClass().getName();
MetricReporterFactory existingFactory = reporterFactories.get(factoryClassName);
if (existingFactory == null) {
reporterFactories.put(factoryClassName, factory);
LOG.debug("Found reporter factory {} at {} ",
factoryClassName,
new File(factory.getClass().getProtectionDomain().getCodeSource().getLocation().toURI()).getCanonicalPath());
} else {
LOG.warn("Multiple implementations of the same reporter were found in 'lib' and/or 'plugins' directories for {}. It is recommended to remove redundant reporter JARs to resolve used versions' ambiguity.", factoryClassName);
}
} catch (Exception | ServiceConfigurationError e) {
LOG.warn("Error while loading reporter factory.", e);
}
}
return Collections.unmodifiableMap(reporterFactories);
}
private static Iterator getAllReporterFactories(@Nullable PluginManager pluginManager) {
final Iterator factoryIteratorSPI = ServiceLoader.load(MetricReporterFactory.class).iterator();
final Iterator factoryIteratorPlugins = pluginManager != null
? pluginManager.load(MetricReporterFactory.class)
: Collections.emptyIterator();
return Iterators.concat(factoryIteratorPlugins, factoryIteratorSPI);
}
private static List setupReporters(Map reporterFactories, List> reporterConfigurations) {
List reporterSetups = new ArrayList<>(reporterConfigurations.size());
for (Tuple2 reporterConfiguration: reporterConfigurations) {
String reporterName = reporterConfiguration.f0;
Configuration reporterConfig = reporterConfiguration.f1;
try {
Optional metricReporterOptional = loadReporter(reporterName, reporterConfig, reporterFactories);
metricReporterOptional.ifPresent(reporter -> {
MetricConfig metricConfig = new MetricConfig();
reporterConfig.addAllToProperties(metricConfig);
reporterSetups.add(createReporterSetup(reporterName, metricConfig, reporter));
});
}
catch (Throwable t) {
LOG.error("Could not instantiate metrics reporter {}. Metrics might not be exposed/reported.", reporterName, t);
}
}
return reporterSetups;
}
private static Optional loadReporter(
final String reporterName,
final Configuration reporterConfig,
final Map reporterFactories)
throws ClassNotFoundException, IllegalAccessException, InstantiationException {
final String reporterClassName = reporterConfig.getString(ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, null);
final String factoryClassName = reporterConfig.getString(ConfigConstants.METRICS_REPORTER_FACTORY_CLASS_SUFFIX, null);
if (factoryClassName != null) {
return loadViaFactory(factoryClassName, reporterName, reporterConfig, reporterFactories);
}
if (reporterClassName != null) {
final Optional interceptingFactory = reporterFactories.values().stream()
.filter(factory -> {
InterceptInstantiationViaReflection annotation = factory.getClass().getAnnotation(InterceptInstantiationViaReflection.class);
return annotation != null && annotation.reporterClassName().equals(reporterClassName);
})
.findAny();
if (interceptingFactory.isPresent()) {
return loadViaFactory(reporterConfig, interceptingFactory.get());
}
return loadViaReflection(reporterClassName, reporterName, reporterConfig, reporterFactories);
}
LOG.warn("No reporter class nor factory set for reporter {}. Metrics might not be exposed/reported.", reporterName);
return Optional.empty();
}
private static Optional loadViaFactory(
final String factoryClassName,
final String reporterName,
final Configuration reporterConfig,
final Map reporterFactories) {
MetricReporterFactory factory = reporterFactories.get(factoryClassName);
if (factory == null) {
LOG.warn("The reporter factory ({}) could not be found for reporter {}. Available factories: {}.", factoryClassName, reporterName, reporterFactories.keySet());
return Optional.empty();
} else {
return loadViaFactory(reporterConfig, factory);
}
}
private static Optional loadViaFactory(
final Configuration reporterConfig,
final MetricReporterFactory factory) {
final MetricConfig metricConfig = new MetricConfig();
reporterConfig.addAllToProperties(metricConfig);
return Optional.of(factory.createMetricReporter(metricConfig));
}
private static Optional loadViaReflection(
final String reporterClassName,
final String reporterName,
final Configuration reporterConfig,
final Map reporterFactories) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
final Class reporterClass = Class.forName(reporterClassName);
final InstantiateViaFactory alternativeFactoryAnnotation = reporterClass.getAnnotation(InstantiateViaFactory.class);
if (alternativeFactoryAnnotation != null) {
final String alternativeFactoryClassName = alternativeFactoryAnnotation.factoryClassName();
LOG.info("The reporter configuration of {} is out-dated (but still supported)." +
" Please configure a factory class instead: '{}{}.{}: {}' to ensure that the configuration" +
" continues to work with future versions.",
reporterName,
ConfigConstants.METRICS_REPORTER_PREFIX,
reporterName,
ConfigConstants.METRICS_REPORTER_FACTORY_CLASS_SUFFIX,
alternativeFactoryClassName);
return loadViaFactory(alternativeFactoryClassName, reporterName, reporterConfig, reporterFactories);
}
return Optional.of((MetricReporter) reporterClass.newInstance());
}
}
