org.apache.cassandra.net.OutboundConnections Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of cassandra-all Show documentation
The Apache Cassandra Project develops a highly scalable second-generation distributed database, bringing together Dynamo's fully distributed design and Bigtable's ColumnFamily-based data model.
There is a newer version: 5.0.2
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.cassandra.net;

import java.nio.channels.ClosedChannelException;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;

import org.apache.cassandra.utils.concurrent.FutureCombiner;
import org.apache.cassandra.utils.concurrent.Condition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.carrotsearch.hppc.ObjectObjectHashMap;
import io.netty.util.concurrent.Future; //checkstyle: permit this import
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.InternodeOutboundMetrics;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;

import static java.lang.Math.max;
import static org.apache.cassandra.config.CassandraRelevantProperties.OTCP_LARGE_MESSAGE_THRESHOLD;
import static org.apache.cassandra.gms.Gossiper.instance;
import static org.apache.cassandra.net.FrameEncoderCrc.HEADER_AND_TRAILER_LENGTH;
import static org.apache.cassandra.net.MessagingService.current_version;
import static org.apache.cassandra.net.ConnectionType.URGENT_MESSAGES;
import static org.apache.cassandra.net.ConnectionType.LARGE_MESSAGES;
import static org.apache.cassandra.net.ConnectionType.SMALL_MESSAGES;
import static org.apache.cassandra.net.ResourceLimits.*;
import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition;

/**
 * Groups a set of outbound connections to a given peer, and routes outgoing messages to the appropriate connection
 * (based upon message's type or size). Contains a {@link OutboundConnection} for each of the
 * {@link ConnectionType} types.
 */
public class OutboundConnections
{
    private static final Logger logger = LoggerFactory.getLogger(OutboundConnections.class);

    private static final int HEADER_LENGTH = 8  // magic number
                                            + 1  // token
                                            + 4  // compressed length
                                            + 4  // uncompressed length
                                            + 4; // checksum

    @VisibleForTesting
    public static final int LARGE_MESSAGE_THRESHOLD = OTCP_LARGE_MESSAGE_THRESHOLD.getInt()
    - max(max(HEADER_LENGTH, HEADER_AND_TRAILER_LENGTH), FrameEncoderLZ4.HEADER_AND_TRAILER_LENGTH);

    private final Condition metricsReady = newOneTimeCondition();
    private volatile InternodeOutboundMetrics metrics;
    private final Limit reserveCapacity;

    private OutboundConnectionSettings template;
    public final OutboundConnection small;
    public final OutboundConnection large;
    public final OutboundConnection urgent;

    private OutboundConnections(OutboundConnectionSettings template)
    {
        this.template = template = template.withDefaultReserveLimits();
        reserveCapacity = new Concurrent(template.applicationSendQueueReserveEndpointCapacityInBytes);
        EndpointAndGlobal reserveCapacityInBytes = new EndpointAndGlobal(reserveCapacity, template.applicationSendQueueReserveGlobalCapacityInBytes);
        this.small = new OutboundConnection(SMALL_MESSAGES, template, reserveCapacityInBytes);
        this.large = new OutboundConnection(LARGE_MESSAGES, template, reserveCapacityInBytes);
        this.urgent = new OutboundConnection(URGENT_MESSAGES, template, reserveCapacityInBytes);
    }

    /**
     * Select the appropriate connection for the provided message and use it to send the message.
     */
    public void enqueue(Message msg, ConnectionType type) throws ClosedChannelException
    {
        connectionFor(msg, type).enqueue(msg);
    }

    static  OutboundConnections tryRegister(ConcurrentMap in, K key, OutboundConnectionSettings settings)
    {
        OutboundConnections connections = in.get(key);
        if (connections == null)
        {
            connections = new OutboundConnections(settings);
            OutboundConnections existing = in.putIfAbsent(key, connections);

            if (existing == null)
            {
                connections.metrics = new InternodeOutboundMetrics(settings.to, connections);
                connections.metricsReady.signalAll();
            }
            else
            {
                connections.metricsReady.signalAll();
                connections.close(false);
                connections = existing;
            }
        }
        return connections;
    }

    /**
     * Reconnect to the peer using the given {@code addr}. Outstanding messages in each channel will be sent on the
     * current channel. Typically this function is used for something like EC2 public IP addresses which need to be used
     * for communication between EC2 regions.
     *
     * @param addr IP Address to use (and prefer) going forward for connecting to the peer
     */
    synchronized Future reconnectWithNewIp(InetAddressAndPort addr)
    {
        template = template.withConnectTo(addr);
        return FutureCombiner.nettySuccessListener(
            apply(c -> c.reconnectWith(template))
        );
    }

    /**
     * Close the connections permanently
     *
     * @param flushQueues {@code true} if existing messages in the queue should be sent before closing.
     */
    synchronized Future scheduleClose(long time, TimeUnit unit, boolean flushQueues)
    {
        // immediately release our metrics, so that if we need to re-open immediately we can safely register a new one
        releaseMetrics();
        return FutureCombiner.nettySuccessListener(
            apply(c -> c.scheduleClose(time, unit, flushQueues))
        );
    }

    /**
     * Close the connections permanently
     *
     * @param flushQueues {@code true} if existing messages in the queue should be sent before closing.
     */
    synchronized Future close(boolean flushQueues)
    {
        // immediately release our metrics, so that if we need to re-open immediately we can safely register a new one
        releaseMetrics();
        return FutureCombiner.nettySuccessListener(
            apply(c -> c.close(flushQueues))
        );
    }

    private void releaseMetrics()
    {
        try
        {
            metricsReady.await();
        }
        catch (InterruptedException e)
        {
            throw new UncheckedInterruptedException(e);
        }

        if (metrics != null)
            metrics.release();
    }

    /**
     * Close each netty channel and its socket
     */
    void interrupt()
    {
        // must return a non-null value for ImmutableList.of()
        apply(OutboundConnection::interrupt);
    }

    /**
     * Apply the given function to each of the connections we are pooling, returning the results as a list
     */
    private  List apply(Function f)
    {
        return ImmutableList.of(
            f.apply(urgent), f.apply(small), f.apply(large)
        );
    }

    @VisibleForTesting
    OutboundConnection connectionFor(Message message)
    {
        return connectionFor(message, null);
    }

    private OutboundConnection connectionFor(Message msg, ConnectionType forceConnection)
    {
        return connectionFor(connectionTypeFor(msg, forceConnection));
    }

    private static ConnectionType connectionTypeFor(Message msg, ConnectionType specifyConnection)
    {
        if (specifyConnection != null)
            return specifyConnection;

        if (msg.serializedSize(current_version) > LARGE_MESSAGE_THRESHOLD)
        {
            if (msg.verb().priority == Verb.Priority.P0)
            {
                NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES,
                                 "Enqueued URGENT message which exceeds large message threshold");

                if (logger.isTraceEnabled())
                    logger.trace("{} message with size {} exceeded large message threshold {}",
                                 msg.verb(),
                                 msg.serializedSize(current_version),
                                 LARGE_MESSAGE_THRESHOLD);
            }

            return LARGE_MESSAGES;
        }

        return msg.verb().priority == Verb.Priority.P0
               ? URGENT_MESSAGES
               : SMALL_MESSAGES;
    }

    @VisibleForTesting
    final OutboundConnection connectionFor(ConnectionType type)
    {
        switch (type)
        {
            case SMALL_MESSAGES:
                return small;
            case LARGE_MESSAGES:
                return large;
            case URGENT_MESSAGES:
                return urgent;
            default:
                throw new IllegalArgumentException("unsupported connection type: " + type);
        }
    }

    public long usingReserveBytes()
    {
        return reserveCapacity.using();
    }

    long expiredCallbacks()
    {
        return metrics.expiredCallbacks.getCount();
    }

    void incrementExpiredCallbackCount()
    {
        metrics.expiredCallbacks.mark();
    }

    OutboundConnectionSettings template()
    {
        return template;
    }

    private static class UnusedConnectionMonitor
    {
        UnusedConnectionMonitor(MessagingService messagingService)
        {
            this.messagingService = messagingService;
        }

        static class Counts
        {
            final long small, large, urgent;
            Counts(long small, long large, long urgent)
            {
                this.small = small;
                this.large = large;
                this.urgent = urgent;
            }
        }

        final MessagingService messagingService;
        ObjectObjectHashMap prevEndpointToCounts = new ObjectObjectHashMap<>();

        private void closeUnusedSinceLastRun()
        {
            ObjectObjectHashMap curEndpointToCounts = new ObjectObjectHashMap<>();
            for (OutboundConnections connections : messagingService.channelManagers.values())
            {
                Counts cur = new Counts(
                    connections.small.submittedCount(),
                    connections.large.submittedCount(),
                    connections.urgent.submittedCount()
                );
                curEndpointToCounts.put(connections.template.to, cur);

                Counts prev = prevEndpointToCounts.get(connections.template.to);
                if (prev == null)
                    continue;

                if (cur.small != prev.small && cur.large != prev.large && cur.urgent != prev.urgent)
                    continue;

                if (cur.small == prev.small && cur.large == prev.large && cur.urgent == prev.urgent
                    && !instance.isKnownEndpoint(connections.template.to))
                {
                    logger.info("Closing outbound connections to {}, as inactive and not known by Gossiper",
                                connections.template.to);
                    // close entirely if no traffic and the endpoint is unknown
                    messagingService.closeOutboundNow(connections);
                    continue;
                }

                if (cur.small == prev.small)
                    connections.small.interrupt();

                if (cur.large == prev.large)
                    connections.large.interrupt();

                if (cur.urgent == prev.urgent)
                    connections.urgent.interrupt();
            }

            prevEndpointToCounts = curEndpointToCounts;
        }
    }

    static void scheduleUnusedConnectionMonitoring(MessagingService messagingService, ScheduledExecutorService executor, long delay, TimeUnit units)
    {
        executor.scheduleWithFixedDelay(new UnusedConnectionMonitor(messagingService)::closeUnusedSinceLastRun, 0L, delay, units);
    }

    @VisibleForTesting
    static OutboundConnections unsafeCreate(OutboundConnectionSettings template)
    {
        OutboundConnections connections = new OutboundConnections(template);
        connections.metricsReady.signalAll();
        return connections;
    }

}