• Home
• io.streamnative.pulsar.handlers
• pulsar-protocol-handler-kafka
• 3.3.1.6
• source code
• RequestStats.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

io.streamnative.pulsar.handlers.kop.RequestStats Maven / Gradle / Ivy

/**
 * Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
 */
/**
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.streamnative.pulsar.handlers.kop;

import static io.streamnative.pulsar.handlers.kop.KopServerStats.ACTIVE_CHANNEL_COUNT;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.ALIVE_CHANNEL_COUNT;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.BATCH_COUNT_PER_MEMORYRECORDS;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.CATEGORY_SERVER;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.FETCH_DECODE;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.GROUP_SCOPE;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.MESSAGE_PUBLISH;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.MESSAGE_QUEUED_LATENCY;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.MESSAGE_READ;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.NETWORK_TOTAL_BYTES_IN;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.NETWORK_TOTAL_BYTES_OUT;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.PARTITION_SCOPE;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.PENDING_TOPIC_LATENCY;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.PREPARE_METADATA;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.PRODUCE_ENCODE;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.REQUEST_PARSE_LATENCY;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.REQUEST_QUEUE_SIZE;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.RESPONSE_BLOCKED_LATENCY;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.RESPONSE_BLOCKED_TIMES;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.SERVER_SCOPE;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.TOPIC_SCOPE;
import static io.streamnative.pulsar.handlers.kop.KopServerStats.WAITING_FETCHES_TRIGGERED;

import com.google.common.annotations.VisibleForTesting;
import io.streamnative.pulsar.handlers.kop.schemaregistry.SchemaRegistryStats;
import io.streamnative.pulsar.handlers.kop.stats.NullStatsLogger;
import io.streamnative.pulsar.handlers.kop.stats.StatsLogger;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.stats.Counter;
import org.apache.bookkeeper.stats.Gauge;
import org.apache.bookkeeper.stats.OpStatsLogger;
import org.apache.bookkeeper.stats.annotations.StatsDoc;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.protocol.ApiKeys;

/**
 * Kop request stats metric for prometheus metrics.
 */
@StatsDoc(
    name = SERVER_SCOPE,
    category = CATEGORY_SERVER,
    help = "KOP request stats"
)

@Getter
@Slf4j
public class RequestStats {

    public static final AtomicInteger REQUEST_QUEUE_SIZE_INSTANCE = new AtomicInteger(0);
    public static final AtomicInteger BATCH_COUNT_PER_MEMORY_RECORDS_INSTANCE = new AtomicInteger(0);
    public static final AtomicInteger ALIVE_CHANNEL_COUNT_INSTANCE = new AtomicInteger(0);
    public static final AtomicInteger ACTIVE_CHANNEL_COUNT_INSTANCE = new AtomicInteger(0);


    public static final RequestStats NULL_INSTANCE = new RequestStats(NullStatsLogger.INSTANCE);

    private final StatsLogger statsLogger;

    @StatsDoc(
            name = WAITING_FETCHES_TRIGGERED,
            help = "number of pending fetches that woke up due to some data produced"
    )
    private final Counter waitingFetchesTriggered;

    @StatsDoc(
            name = REQUEST_PARSE_LATENCY,
            help = "parse ByteBuf to request latency"
    )
    private final OpStatsLogger requestParseLatencyStats;

    @StatsDoc(
            name = RESPONSE_BLOCKED_TIMES,
            help = "response blocked times"
    )
    private final Counter responseBlockedTimes;

    @StatsDoc(
            name = RESPONSE_BLOCKED_LATENCY,
            help = "response blocked latency"
    )
    private final OpStatsLogger responseBlockedLatency;

    @StatsDoc(
            name = PENDING_TOPIC_LATENCY,
            help = "pending topic latency stats of Kop"
    )
    private final OpStatsLogger pendingTopicLatencyStats;

    @StatsDoc(
        name = PRODUCE_ENCODE,
        help = "produce encode stats of Kop"
    )
    private final OpStatsLogger produceEncodeStats;

    @StatsDoc(
        name = MESSAGE_PUBLISH,
        help = "message publish stats from kop to pulsar broker"
    )
    private final OpStatsLogger messagePublishStats;

    @StatsDoc(
        name = MESSAGE_QUEUED_LATENCY,
        help = "message queued stats from kop to pulsar broker"
    )
    private final OpStatsLogger messageQueuedLatencyStats;

    @StatsDoc(
            name = PREPARE_METADATA,
            help = "stats of preparing metadata in fetch request"
    )
    private final OpStatsLogger prepareMetadataStats;

    @StatsDoc(
            name = MESSAGE_READ,
            help = "stats of performing a single cursor's async-read within fetch request"
    )
    private final OpStatsLogger messageReadStats;

    @StatsDoc(
            name = FETCH_DECODE,
            help = "stats of decoding entries in fetch request"
    )
    private final OpStatsLogger fetchDecodeStats;

    @StatsDoc(
            name = NETWORK_TOTAL_BYTES_IN,
            help = "total bytes received"
    )
    private final Counter networkTotalBytesIn;

    @StatsDoc(
            name = NETWORK_TOTAL_BYTES_OUT,
            help = "total bytes sent out"
    )
    private final Counter networkTotalBytesOut;

    private final Map apiKeysToStatsLogger = new ConcurrentHashMap<>();

    private final Map cachedLoggersForTopicPartitions = new ConcurrentHashMap<>();
    private final Map, StatsLogger> cachedLoggersForTopicPartitionsAndGroups =
            new ConcurrentHashMap<>();

    private final Map cachedRequestStatsForTenants = new ConcurrentHashMap<>();

    public RequestStats(StatsLogger statsLogger) {
        this.statsLogger = statsLogger;

        this.requestParseLatencyStats = statsLogger.getOpStatsLogger(REQUEST_PARSE_LATENCY);

        this.responseBlockedLatency = statsLogger.getOpStatsLogger(RESPONSE_BLOCKED_LATENCY);
        this.responseBlockedTimes = statsLogger.getCounter(RESPONSE_BLOCKED_TIMES);

        this.pendingTopicLatencyStats = statsLogger.getOpStatsLogger(PENDING_TOPIC_LATENCY);
        this.produceEncodeStats = statsLogger.getOpStatsLogger(PRODUCE_ENCODE);
        this.messagePublishStats = statsLogger.getOpStatsLogger(MESSAGE_PUBLISH);
        this.messageQueuedLatencyStats = statsLogger.getOpStatsLogger(MESSAGE_QUEUED_LATENCY);

        this.prepareMetadataStats = statsLogger.getOpStatsLogger(PREPARE_METADATA);
        this.messageReadStats = statsLogger.getOpStatsLogger(MESSAGE_READ);
        this.fetchDecodeStats  = statsLogger.getOpStatsLogger(FETCH_DECODE);
        this.waitingFetchesTriggered = statsLogger.getCounter(WAITING_FETCHES_TRIGGERED);
        this.networkTotalBytesIn = statsLogger.getCounter(NETWORK_TOTAL_BYTES_IN);
        this.networkTotalBytesOut = statsLogger.getCounter(NETWORK_TOTAL_BYTES_OUT);

        statsLogger.registerGauge(REQUEST_QUEUE_SIZE, new Gauge() {
            @Override
            public Number getDefaultValue() {
                return 0;
            }

            @Override
            public Number getSample() {
                return REQUEST_QUEUE_SIZE_INSTANCE;
            }
        });

        statsLogger.registerGauge(BATCH_COUNT_PER_MEMORYRECORDS, new Gauge() {
            @Override
            public Number getDefaultValue() {
                return 0;
            }

            @Override
            public Number getSample() {
                return BATCH_COUNT_PER_MEMORY_RECORDS_INSTANCE;
            }
        });

        statsLogger.registerGauge(ALIVE_CHANNEL_COUNT, new Gauge() {
            @Override
            public Number getDefaultValue() {
                return 0;
            }

            @Override
            public Number getSample() {
                return ALIVE_CHANNEL_COUNT_INSTANCE;
            }
        });

        statsLogger.registerGauge(ACTIVE_CHANNEL_COUNT, new Gauge() {
            @Override
            public Number getDefaultValue() {
                return 0;
            }

            @Override
            public Number getSample() {
                return ACTIVE_CHANNEL_COUNT_INSTANCE;
            }
        });
    }

    public StatsLogger getStatsLoggerForTopicPartition(TopicPartition topicPartition) {
        return cachedLoggersForTopicPartitions.computeIfAbsent(topicPartition,
                __ -> statsLogger
                        .scopeLabel(TOPIC_SCOPE, topicPartition.topic())
                        .scopeLabel(PARTITION_SCOPE, String.valueOf(topicPartition.partition())));
    }

    public StatsLogger getStatsLoggerForTopicPartitionAndGroup(TopicPartition topicPartition, String groupId) {
        return cachedLoggersForTopicPartitionsAndGroups.computeIfAbsent(
                Pair.of(topicPartition, groupId),
                __ -> getStatsLoggerForTopicPartition(topicPartition)
                        .scopeLabel(GROUP_SCOPE, groupId));
    }

    /**
     * Get the stats logger for Kafka requests.
     *
     * @param apiKey the {@link ApiKeys} object that represents the Kafka request's type
     * @param statsName the stats name
     * @return
     */
    public OpStatsLogger getRequestStatsLogger(final ApiKeys apiKey, final String statsName) {
        return apiKeysToStatsLogger.computeIfAbsent(apiKey,
                __ -> statsLogger.scopeLabel(KopServerStats.REQUEST_SCOPE, apiKey.name)
        ).getOpStatsLogger(statsName);
    }

    @VisibleForTesting
    public Set getApiKeysSet() {
        return new TreeSet<>(apiKeysToStatsLogger.keySet());
    }

    public RequestStats forTenant(String tenant) {
        return cachedRequestStatsForTenants.computeIfAbsent(tenant,
                __ -> new RequestStats(statsLogger.scopeLabel("tenant", tenant)));
    }

    public SchemaRegistryStats generateSchemaRegistryStats() {
        return SchemaRegistryStats.builder()
                    .schemasCreated(statsLogger.getCounter(SchemaRegistryStats.SR_REGISTERED_COUNT))
                    .schemasDeleted(statsLogger.getCounter(SchemaRegistryStats.SR_DELETED_COUNT))
                    .apiCallsSuccess(statsLogger.getCounter(SchemaRegistryStats.SR_API_SUCCESS_COUNT))
                    .apiCallsFailure(statsLogger.getCounter(SchemaRegistryStats.SR_API_FAILURE_COUNT))
                    .avroSchemasCreated(statsLogger.getCounter(SchemaRegistryStats.SR_AVRO_SCHEMAS_CREATED))
                    .avroSchemasDeleted(statsLogger.getCounter(SchemaRegistryStats.SR_AVRO_SCHEMAS_DELETED))
                    .jsonSchemasCreated(statsLogger.getCounter(SchemaRegistryStats.SR_JSON_SCHEMAS_CREATED))
                    .jsonSchemasDeleted(statsLogger.getCounter(SchemaRegistryStats.SR_JSON_SCHEMAS_DELETED))
                    .protobufSchemasCreated(statsLogger.getCounter(SchemaRegistryStats.SR_PB_SCHEMAS_CREATED))
                    .protobufSchemasDeleted(statsLogger.getCounter(SchemaRegistryStats.SR_PB_SCHEMAS_DELETED))
                    .build();
    }

}