Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.metrics;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.DelegatingConfiguration;
import org.apache.flink.configuration.MetricOptions;
import org.apache.flink.core.plugin.PluginManager;
import org.apache.flink.metrics.MetricConfig;
import org.apache.flink.metrics.reporter.InstantiateViaFactory;
import org.apache.flink.metrics.reporter.InterceptInstantiationViaReflection;
import org.apache.flink.metrics.reporter.MetricReporter;
import org.apache.flink.metrics.reporter.MetricReporterFactory;
import org.apache.flink.runtime.metrics.scope.ScopeFormat;
import org.apache.flink.shaded.guava18.com.google.common.collect.Iterators;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* Encapsulates everything needed for the instantiation and configuration of a {@link
* MetricReporter}.
*/
public final class ReporterSetup {
private static final Logger LOG = LoggerFactory.getLogger(ReporterSetup.class);
// regex pattern to split the defined reporters
private static final Pattern reporterListPattern = Pattern.compile("\\s*,\\s*");
// regex pattern to extract the name from reporter configuration keys, e.g. "rep" from
// "metrics.reporter.rep.class"
private static final Pattern reporterClassPattern =
Pattern.compile(
Pattern.quote(ConfigConstants.METRICS_REPORTER_PREFIX)
+
// [\S&&[^.]] = intersection of non-whitespace and non-period character
// classes
"([\\S&&[^.]]*)\\."
+ '('
+ Pattern.quote(ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX)
+ '|'
+ Pattern.quote(ConfigConstants.METRICS_REPORTER_FACTORY_CLASS_SUFFIX)
+ ')');
private final String name;
private final MetricConfig configuration;
private final MetricReporter reporter;
public ReporterSetup(
final String name, final MetricConfig configuration, MetricReporter reporter) {
this.name = name;
this.configuration = configuration;
this.reporter = reporter;
}
public Optional getDelimiter() {
return Optional.ofNullable(
configuration.getString(ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, null));
}
public Optional getIntervalSettings() {
return Optional.ofNullable(
configuration.getString(ConfigConstants.METRICS_REPORTER_INTERVAL_SUFFIX, null));
}
public Set getExcludedVariables() {
String excludedVariablesList =
configuration.getString(ConfigConstants.METRICS_REPORTER_EXCLUDED_VARIABLES, null);
if (excludedVariablesList == null) {
return Collections.emptySet();
} else {
final Set excludedVariables = new HashSet<>();
for (String exclusion : excludedVariablesList.split(";")) {
excludedVariables.add(ScopeFormat.asVariable(exclusion));
}
return Collections.unmodifiableSet(excludedVariables);
}
}
public String getName() {
return name;
}
@VisibleForTesting
MetricConfig getConfiguration() {
return configuration;
}
public MetricReporter getReporter() {
return reporter;
}
@VisibleForTesting
public static ReporterSetup forReporter(String reporterName, MetricReporter reporter) {
return createReporterSetup(reporterName, new MetricConfig(), reporter);
}
@VisibleForTesting
public static ReporterSetup forReporter(
String reporterName, MetricConfig metricConfig, MetricReporter reporter) {
return createReporterSetup(reporterName, metricConfig, reporter);
}
private static ReporterSetup createReporterSetup(
String reporterName, MetricConfig metricConfig, MetricReporter reporter) {
reporter.open(metricConfig);
return new ReporterSetup(reporterName, metricConfig, reporter);
}
public static List fromConfiguration(
final Configuration configuration, @Nullable final PluginManager pluginManager) {
String includedReportersString = configuration.getString(MetricOptions.REPORTERS_LIST, "");
Set namedReporters =
findEnabledReportersInConfiguration(configuration, includedReportersString);
if (namedReporters.isEmpty()) {
return Collections.emptyList();
}
final List> reporterConfigurations =
loadReporterConfigurations(configuration, namedReporters);
final Map reporterFactories =
loadAvailableReporterFactories(pluginManager);
return setupReporters(reporterFactories, reporterConfigurations);
}
private static Set findEnabledReportersInConfiguration(
Configuration configuration, String includedReportersString) {
Set includedReporters =
reporterListPattern
.splitAsStream(includedReportersString)
.filter(r -> !r.isEmpty()) // splitting an empty string results in
// an empty string on jdk9+
.collect(Collectors.toSet());
// use a TreeSet to make the reporter order deterministic, which is useful for testing
Set namedOrderedReporters = new TreeSet<>(String::compareTo);
// scan entire configuration for keys starting with METRICS_REPORTER_PREFIX and determine
// the set of enabled reporters
for (String key : configuration.keySet()) {
if (key.startsWith(ConfigConstants.METRICS_REPORTER_PREFIX)) {
Matcher matcher = reporterClassPattern.matcher(key);
if (matcher.matches()) {
String reporterName = matcher.group(1);
if (includedReporters.isEmpty() || includedReporters.contains(reporterName)) {
if (namedOrderedReporters.contains(reporterName)) {
LOG.warn(
"Duplicate class configuration detected for reporter {}.",
reporterName);
} else {
namedOrderedReporters.add(reporterName);
}
} else {
LOG.info(
"Excluding reporter {}, not configured in reporter list ({}).",
reporterName,
includedReportersString);
}
}
}
}
return namedOrderedReporters;
}
private static List> loadReporterConfigurations(
Configuration configuration, Set namedReporters) {
final List> reporterConfigurations =
new ArrayList<>(namedReporters.size());
for (String namedReporter : namedReporters) {
DelegatingConfiguration delegatingConfiguration =
new DelegatingConfiguration(
configuration,
ConfigConstants.METRICS_REPORTER_PREFIX + namedReporter + '.');
reporterConfigurations.add(Tuple2.of(namedReporter, delegatingConfiguration));
}
return reporterConfigurations;
}
private static Map loadAvailableReporterFactories(
@Nullable PluginManager pluginManager) {
final Map reporterFactories = new HashMap<>(2);
final Iterator factoryIterator =
getAllReporterFactories(pluginManager);
// do not use streams or for-each loops here because they do not allow catching individual
// ServiceConfigurationErrors
// such an error might be caused if the META-INF/services contains an entry to a
// non-existing factory class
while (factoryIterator.hasNext()) {
try {
MetricReporterFactory factory = factoryIterator.next();
String factoryClassName = factory.getClass().getName();
MetricReporterFactory existingFactory = reporterFactories.get(factoryClassName);
if (existingFactory == null) {
reporterFactories.put(factoryClassName, factory);
LOG.debug(
"Found reporter factory {} at {} ",
factoryClassName,
new File(
factory.getClass()
.getProtectionDomain()
.getCodeSource()
.getLocation()
.toURI())
.getCanonicalPath());
} else {
LOG.warn(
"Multiple implementations of the same reporter were found in 'lib' and/or 'plugins' directories for {}. It is recommended to remove redundant reporter JARs to resolve used versions' ambiguity.",
factoryClassName);
}
} catch (Exception | ServiceConfigurationError e) {
LOG.warn("Error while loading reporter factory.", e);
}
}
return Collections.unmodifiableMap(reporterFactories);
}
private static Iterator getAllReporterFactories(
@Nullable PluginManager pluginManager) {
final Iterator factoryIteratorSPI =
ServiceLoader.load(MetricReporterFactory.class).iterator();
final Iterator factoryIteratorPlugins =
pluginManager != null
? pluginManager.load(MetricReporterFactory.class)
: Collections.emptyIterator();
return Iterators.concat(factoryIteratorPlugins, factoryIteratorSPI);
}
private static List setupReporters(
Map reporterFactories,
List> reporterConfigurations) {
List reporterSetups = new ArrayList<>(reporterConfigurations.size());
for (Tuple2 reporterConfiguration : reporterConfigurations) {
String reporterName = reporterConfiguration.f0;
Configuration reporterConfig = reporterConfiguration.f1;
try {
Optional metricReporterOptional =
loadReporter(reporterName, reporterConfig, reporterFactories);
metricReporterOptional.ifPresent(
reporter -> {
MetricConfig metricConfig = new MetricConfig();
reporterConfig.addAllToProperties(metricConfig);
reporterSetups.add(
createReporterSetup(reporterName, metricConfig, reporter));
});
} catch (Throwable t) {
LOG.error(
"Could not instantiate metrics reporter {}. Metrics might not be exposed/reported.",
reporterName,
t);
}
}
return reporterSetups;
}
private static Optional loadReporter(
final String reporterName,
final Configuration reporterConfig,
final Map reporterFactories)
throws ClassNotFoundException, IllegalAccessException, InstantiationException {
final String reporterClassName =
reporterConfig.getString(ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, null);
final String factoryClassName =
reporterConfig.getString(
ConfigConstants.METRICS_REPORTER_FACTORY_CLASS_SUFFIX, null);
if (factoryClassName != null) {
return loadViaFactory(
factoryClassName, reporterName, reporterConfig, reporterFactories);
}
if (reporterClassName != null) {
final Optional interceptingFactory =
reporterFactories.values().stream()
.filter(
factory -> {
InterceptInstantiationViaReflection annotation =
factory.getClass()
.getAnnotation(
InterceptInstantiationViaReflection
.class);
return annotation != null
&& annotation
.reporterClassName()
.equals(reporterClassName);
})
.findAny();
if (interceptingFactory.isPresent()) {
return loadViaFactory(reporterConfig, interceptingFactory.get());
}
return loadViaReflection(
reporterClassName, reporterName, reporterConfig, reporterFactories);
}
LOG.warn(
"No reporter class nor factory set for reporter {}. Metrics might not be exposed/reported.",
reporterName);
return Optional.empty();
}
private static Optional loadViaFactory(
final String factoryClassName,
final String reporterName,
final Configuration reporterConfig,
final Map reporterFactories) {
MetricReporterFactory factory = reporterFactories.get(factoryClassName);
if (factory == null) {
LOG.warn(
"The reporter factory ({}) could not be found for reporter {}. Available factories: {}.",
factoryClassName,
reporterName,
reporterFactories.keySet());
return Optional.empty();
} else {
return loadViaFactory(reporterConfig, factory);
}
}
private static Optional loadViaFactory(
final Configuration reporterConfig, final MetricReporterFactory factory) {
final MetricConfig metricConfig = new MetricConfig();
reporterConfig.addAllToProperties(metricConfig);
return Optional.of(factory.createMetricReporter(metricConfig));
}
private static Optional loadViaReflection(
final String reporterClassName,
final String reporterName,
final Configuration reporterConfig,
final Map reporterFactories)
throws ClassNotFoundException, IllegalAccessException, InstantiationException {
final Class reporterClass = Class.forName(reporterClassName);
final InstantiateViaFactory alternativeFactoryAnnotation =
reporterClass.getAnnotation(InstantiateViaFactory.class);
if (alternativeFactoryAnnotation != null) {
final String alternativeFactoryClassName =
alternativeFactoryAnnotation.factoryClassName();
LOG.info(
"The reporter configuration of {} is out-dated (but still supported)."
+ " Please configure a factory class instead: '{}{}.{}: {}' to ensure that the configuration"
+ " continues to work with future versions.",
reporterName,
ConfigConstants.METRICS_REPORTER_PREFIX,
reporterName,
ConfigConstants.METRICS_REPORTER_FACTORY_CLASS_SUFFIX,
alternativeFactoryClassName);
return loadViaFactory(
alternativeFactoryClassName, reporterName, reporterConfig, reporterFactories);
}
return Optional.of((MetricReporter) reporterClass.newInstance());
}
}
