com.hazelcast.internal.monitors.OverloadedConnectionsPlugin Maven / Gradle / Ivy

Go to download
/*
 * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.internal.monitors;

import com.hazelcast.instance.GroupProperties;
import com.hazelcast.internal.serialization.SerializationService;
import com.hazelcast.logging.ILogger;
import com.hazelcast.nio.ConnectionManager;
import com.hazelcast.nio.OutboundFrame;
import com.hazelcast.nio.Packet;
import com.hazelcast.nio.tcp.TcpIpConnection;
import com.hazelcast.nio.tcp.TcpIpConnectionManager;
import com.hazelcast.nio.tcp.nonblocking.NonBlockingSocketWriter;
import com.hazelcast.nio.tcp.spinning.SpinningSocketWriter;
import com.hazelcast.spi.impl.NodeEngineImpl;
import com.hazelcast.spi.impl.operationservice.impl.operations.Backup;
import com.hazelcast.util.ItemCounter;

import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.Queue;
import java.util.Random;
import java.util.Set;

import static com.hazelcast.instance.GroupProperty.PERFORMANCE_MONITOR_OVERLOADED_CONNECTIONS_PERIOD_SECONDS;
import static com.hazelcast.instance.GroupProperty.PERFORMANCE_MONITOR_OVERLOADED_CONNECTIONS_SAMPLES;
import static com.hazelcast.instance.GroupProperty.PERFORMANCE_MONITOR_OVERLOADED_CONNECTIONS_THRESHOLD;
import static java.lang.Math.min;
import static java.util.Collections.emptySet;

/**
 * The OverloadedConnectionsPlugin checks all the connections and samples the content of the packet
 * queues if the size is above a certain threshold. This is very useful to figure out when huge
 * amount of memory is consumed due to pending packets.
 *
 * Currently the sampling has a lot of overhead since the value needs to be deserialized. That is why this
 * plugin is disabled by default.
 */
public class OverloadedConnectionsPlugin extends PerformanceMonitorPlugin {
    private static final Queue EMPTY_QUEUE = new LinkedList();

    private final SerializationService serializationService;
    private final ILogger logger;
    private final ItemCounter occurrenceMap = new ItemCounter();
    private final ArrayList packets = new ArrayList();
    private final Random random = new Random();
    private final NumberFormat defaultFormat = NumberFormat.getPercentInstance();
    private final NodeEngineImpl nodeEngine;
    private final long periodMillis;
    private final int threshold;
    private final int samples;

    public OverloadedConnectionsPlugin(NodeEngineImpl nodeEngine) {
        this.nodeEngine = nodeEngine;
        this.serializationService = nodeEngine.getSerializationService();
        this.logger = nodeEngine.getLogger(OverloadedConnectionsPlugin.class);
        this.defaultFormat.setMinimumFractionDigits(3);

        GroupProperties props = nodeEngine.getGroupProperties();
        this.periodMillis = props.getMillis(PERFORMANCE_MONITOR_OVERLOADED_CONNECTIONS_PERIOD_SECONDS);
        this.threshold = props.getInteger(PERFORMANCE_MONITOR_OVERLOADED_CONNECTIONS_THRESHOLD);
        this.samples = props.getInteger(PERFORMANCE_MONITOR_OVERLOADED_CONNECTIONS_SAMPLES);
    }

    @Override
    public long getPeriodMillis() {
        return periodMillis;
    }

    @Override
    public void onStart() {
        logger.info("Plugin:active, period-millis:" + periodMillis + " threshold:" + threshold + " samples:" + samples);
    }

    @Override
    public void run(PerformanceLogWriter writer) {
        writer.startSection("OverloadedConnections");

        Set connections = getTcpIpConnections();
        for (TcpIpConnection connection : connections) {
            clear();
            scan(writer, connection, false);

            clear();
            scan(writer, connection, true);
        }

        writer.endSection();
    }

    private Set getTcpIpConnections() {
        ConnectionManager connectionManager = nodeEngine.getNode().getConnectionManager();
        if (connectionManager instanceof TcpIpConnectionManager) {
            return ((TcpIpConnectionManager) connectionManager).getActiveConnections();
        } else {
            return emptySet();
        }
    }

    private void scan(PerformanceLogWriter writer, TcpIpConnection connection, boolean priority) {
        Queue q = getOutboundQueue(connection, priority);

        int sampleCount = sample(q);
        if (sampleCount < 0) {
            return;
        }

        render(writer, connection, priority, sampleCount);
    }

    private Queue getOutboundQueue(TcpIpConnection connection, boolean priority) {
        if (connection.getSocketWriter() instanceof NonBlockingSocketWriter) {
            NonBlockingSocketWriter writer = (NonBlockingSocketWriter) connection.getSocketWriter();
            return priority ? writer.urgentWriteQueue : writer.writeQueue;
        } else if (connection.getSocketWriter() instanceof SpinningSocketWriter) {
            SpinningSocketWriter writer = (SpinningSocketWriter) connection.getSocketWriter();
            return priority ? writer.urgentWriteQueue : writer.writeQueue;
        } else {
            return EMPTY_QUEUE;
        }
    }

    private void render(PerformanceLogWriter writer, TcpIpConnection connection, boolean priority, int sampleCount) {
        writer.startSection(connection.toString());

        writer.writeKeyValueEntry(priority ? "urgentPacketCount" : "packetCount", packets.size());
        writer.writeKeyValueEntry("sampleCount", sampleCount);
        renderSamples(writer, sampleCount);

        writer.endSection();
    }

    private void renderSamples(PerformanceLogWriter writer, int sampleCount) {
        writer.startSection("samples");

        for (String key : occurrenceMap.keySet()) {
            long value = occurrenceMap.get(key);
            if (value == 0) {
                continue;
            }

            double percentage = (1d * value) / sampleCount;
            writer.writeEntry(key + " sampleCount=" + value + " " + defaultFormat.format(percentage));
        }
        writer.endSection();
    }

    private void clear() {
        occurrenceMap.reset();
        packets.clear();
    }

    /**
     * Samples the queue.
     *
     * @param q the queue to sample.
     * @return the number of samples. If there were not sufficient samples, -1 is returned.
     */
    private int sample(Queue q) {
        for (OutboundFrame frame : q) {
            packets.add(frame);
        }

        if (packets.size() < threshold) {
            return -1;
        }

        int sampleCount = min(samples, packets.size());
        int actualSampleCount = 0;
        for (int k = 0; k < sampleCount; k++) {
            OutboundFrame packet = packets.get(random.nextInt(packets.size()));
            String key = toKey(packet);

            if (key != null) {
                actualSampleCount++;
                occurrenceMap.add(key, 1);
            }
        }

        return actualSampleCount;
    }

    String toKey(OutboundFrame packet) {
        if (packet instanceof Packet) {
            try {
                Object result = serializationService.toObject(packet);
                if (result == null) {
                    return "null";
                } else if (result instanceof Backup) {
                    Backup backup = (Backup) result;
                    return Backup.class.getName() + "#" + backup.getBackupOp().getClass().getName();
                } else {
                    return result.getClass().getName();
                }
            } catch (Exception ignore) {
                logger.severe(ignore);
                return null;
            }
        } else {
            return packet.getClass().getName();
        }
    }
}