Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.inlong.sort.base.metric;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.Meter;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.SimpleCounter;
import org.apache.inlong.audit.AuditOperator;
import org.apache.inlong.sort.base.Constants;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import static org.apache.inlong.sort.base.Constants.NUM_BYTES_IN;
import static org.apache.inlong.sort.base.Constants.NUM_BYTES_IN_FOR_METER;
import static org.apache.inlong.sort.base.Constants.NUM_BYTES_IN_PER_SECOND;
import static org.apache.inlong.sort.base.Constants.NUM_RECORDS_IN;
import static org.apache.inlong.sort.base.Constants.NUM_RECORDS_IN_FOR_METER;
import static org.apache.inlong.sort.base.Constants.NUM_RECORDS_IN_PER_SECOND;
/**
* A collection class for handling metrics
*/
public class SourceMetricData implements MetricData {
private final MetricGroup metricGroup;
private final Map labels;
private Counter numRecordsIn;
private Counter numBytesIn;
private Counter numRecordsInForMeter;
private Counter numBytesInForMeter;
private Meter numRecordsInPerSecond;
private Meter numBytesInPerSecond;
private AuditOperator auditOperator;
public SourceMetricData(MetricOption option, MetricGroup metricGroup) {
this.metricGroup = metricGroup;
this.labels = option.getLabels();
ThreadSafeCounter recordsInCounter = new ThreadSafeCounter();
ThreadSafeCounter bytesInCounter = new ThreadSafeCounter();
switch (option.getRegisteredMetric()) {
default:
recordsInCounter.inc(option.getInitRecords());
bytesInCounter.inc(option.getInitBytes());
registerMetricsForNumRecordsIn(recordsInCounter);
registerMetricsForNumBytesIn(bytesInCounter);
registerMetricsForNumBytesInForMeter(new ThreadSafeCounter());
registerMetricsForNumRecordsInForMeter(new ThreadSafeCounter());
registerMetricsForNumBytesInPerSecond();
registerMetricsForNumRecordsInPerSecond();
break;
}
if (option.getIpPorts().isPresent()) {
AuditOperator.getInstance().setAuditProxy(option.getIpPortList());
this.auditOperator = AuditOperator.getInstance();
}
}
/**
* Default counter is {@link SimpleCounter}
* groupId and streamId and nodeId are label value, user can use it filter metric data when use metric reporter
* prometheus
*/
public void registerMetricsForNumRecordsInForMeter() {
registerMetricsForNumRecordsInForMeter(new SimpleCounter());
}
/**
* User can use custom counter that extends from {@link Counter}
* groupId and streamId and nodeId are label value, user can use it filter metric data when use metric reporter
* prometheus
*/
public void registerMetricsForNumRecordsInForMeter(Counter counter) {
numRecordsInForMeter = registerCounter(NUM_RECORDS_IN_FOR_METER, counter);
}
/**
* Default counter is {@link SimpleCounter}
* groupId and streamId and nodeId are label value, user can use it filter metric data when use metric reporter
* prometheus
*/
public void registerMetricsForNumBytesInForMeter() {
registerMetricsForNumBytesInForMeter(new SimpleCounter());
}
/**
* User can use custom counter that extends from {@link Counter}
* groupId and streamId and nodeId are label value, user can use it filter metric data when use metric reporter
* prometheus
*/
public void registerMetricsForNumBytesInForMeter(Counter counter) {
numBytesInForMeter = registerCounter(NUM_BYTES_IN_FOR_METER, counter);
}
/**
* Default counter is {@link SimpleCounter}
* groupId and streamId and nodeId are label value, user can use it filter metric data when use metric reporter
* prometheus
*/
public void registerMetricsForNumRecordsIn() {
registerMetricsForNumRecordsIn(new SimpleCounter());
}
/**
* User can use custom counter that extends from {@link Counter}
* groupId and streamId and nodeId are label value, user can use it filter metric data when use metric reporter
* prometheus
*/
public void registerMetricsForNumRecordsIn(Counter counter) {
numRecordsIn = registerCounter(NUM_RECORDS_IN, counter);
}
/**
* Default counter is {@link SimpleCounter}
* groupId and streamId and nodeId are label value, user can use it filter metric data when use metric reporter
* prometheus
*/
public void registerMetricsForNumBytesIn() {
registerMetricsForNumBytesIn(new SimpleCounter());
}
/**
* User can use custom counter that extends from {@link Counter}
* groupId and streamId and nodeId are label value, user can use it filter metric data when use metric reporter
* prometheus
*/
public void registerMetricsForNumBytesIn(Counter counter) {
numBytesIn = registerCounter(NUM_BYTES_IN, counter);
}
public void registerMetricsForNumRecordsInPerSecond() {
numRecordsInPerSecond = registerMeter(NUM_RECORDS_IN_PER_SECOND, this.numRecordsInForMeter);
}
public void registerMetricsForNumBytesInPerSecond() {
numBytesInPerSecond = registerMeter(NUM_BYTES_IN_PER_SECOND, this.numBytesInForMeter);
}
public Counter getNumRecordsIn() {
return numRecordsIn;
}
public Counter getNumBytesIn() {
return numBytesIn;
}
public Meter getNumRecordsInPerSecond() {
return numRecordsInPerSecond;
}
public Meter getNumBytesInPerSecond() {
return numBytesInPerSecond;
}
public Counter getNumRecordsInForMeter() {
return numRecordsInForMeter;
}
public Counter getNumBytesInForMeter() {
return numBytesInForMeter;
}
@Override
public MetricGroup getMetricGroup() {
return metricGroup;
}
@Override
public Map getLabels() {
return labels;
}
public void outputMetricsWithEstimate(Object data) {
long size = data.toString().getBytes(StandardCharsets.UTF_8).length;
outputMetrics(1, size);
}
public void outputMetrics(long rowCountSize, long rowDataSize) {
if (numRecordsIn != null) {
this.numRecordsIn.inc(rowCountSize);
}
if (numBytesIn != null) {
this.numBytesIn.inc(rowDataSize);
}
if (numRecordsInForMeter != null) {
this.numRecordsInForMeter.inc(rowCountSize);
}
if (numBytesInForMeter != null) {
this.numBytesInForMeter.inc(rowDataSize);
}
if (auditOperator != null) {
auditOperator.add(
Constants.AUDIT_SORT_INPUT,
getGroupId(),
getStreamId(),
System.currentTimeMillis(),
rowCountSize,
rowDataSize);
}
}
@Override
public String toString() {
return "SourceMetricData{"
+ "metricGroup=" + metricGroup
+ ", labels=" + labels
+ ", numRecordsIn=" + numRecordsIn.getCount()
+ ", numBytesIn=" + numBytesIn.getCount()
+ ", numRecordsInForMeter=" + numRecordsInForMeter.getCount()
+ ", numBytesInForMeter=" + numBytesInForMeter.getCount()
+ ", numRecordsInPerSecond=" + numRecordsInPerSecond.getRate()
+ ", numBytesInPerSecond=" + numBytesInPerSecond.getRate()
+ ", auditOperator=" + auditOperator
+ '}';
}
}
