org.apache.cassandra.transport.Server Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of cassandra-all Show documentation
Show all versions of cassandra-all Show documentation
Palantir open source project
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.transport;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLParameters;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.channel.*;
import io.netty.channel.epoll.Epoll;
import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.epoll.EpollServerSocketChannel;
import io.netty.channel.group.ChannelGroup;
import io.netty.channel.group.DefaultChannelGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.ByteToMessageDecoder;
import io.netty.handler.ssl.SslHandler;
import io.netty.handler.timeout.IdleState;
import io.netty.handler.timeout.IdleStateEvent;
import io.netty.handler.timeout.IdleStateHandler;
import io.netty.util.Version;
import io.netty.util.concurrent.EventExecutor;
import io.netty.util.concurrent.GlobalEventExecutor;
import io.netty.util.internal.logging.InternalLoggerFactory;
import io.netty.util.internal.logging.Slf4JLoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.EncryptionOptions;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.metrics.ClientMetrics;
import org.apache.cassandra.security.SSLFactory;
import org.apache.cassandra.service.*;
import org.apache.cassandra.transport.messages.EventMessage;
import org.apache.cassandra.utils.FBUtilities;
public class Server implements CassandraDaemon.Server
{
static
{
InternalLoggerFactory.setDefaultFactory(new Slf4JLoggerFactory());
}
private static final Logger logger = LoggerFactory.getLogger(Server.class);
private static final boolean enableEpoll = Boolean.valueOf(System.getProperty("cassandra.native.epoll.enabled", "true"));
public static final int VERSION_1 = 1;
public static final int VERSION_2 = 2;
public static final int VERSION_3 = 3;
public static final int VERSION_4 = 4;
public static final int CURRENT_VERSION = VERSION_4;
private final ConnectionTracker connectionTracker = new ConnectionTracker();
private final Connection.Factory connectionFactory = new Connection.Factory()
{
public Connection newConnection(Channel channel, int version)
{
return new ServerConnection(channel, version, connectionTracker);
}
};
public final InetSocketAddress socket;
private final AtomicBoolean isRunning = new AtomicBoolean(false);
private EventLoopGroup workerGroup;
private EventExecutor eventExecutorGroup;
public Server(InetSocketAddress socket)
{
this.socket = socket;
EventNotifier notifier = new EventNotifier(this);
StorageService.instance.register(notifier);
MigrationManager.instance.register(notifier);
registerMetrics();
}
public Server(String hostname, int port)
{
this(new InetSocketAddress(hostname, port));
}
public Server(InetAddress host, int port)
{
this(new InetSocketAddress(host, port));
}
public Server(int port)
{
this(new InetSocketAddress(port));
}
public void start()
{
if(!isRunning())
{
run();
}
}
public void stop()
{
if (isRunning.compareAndSet(true, false))
close(false);
}
@VisibleForTesting
public void stopAndAwaitTermination()
{
if (isRunning.compareAndSet(true, false))
close(true);
}
public boolean isRunning()
{
return isRunning.get();
}
private void run()
{
// Configure the server.
eventExecutorGroup = new RequestThreadPoolExecutor();
boolean hasEpoll = enableEpoll ? Epoll.isAvailable() : false;
if (hasEpoll)
{
workerGroup = new EpollEventLoopGroup();
logger.info("Netty using native Epoll event loop");
}
else
{
workerGroup = new NioEventLoopGroup();
logger.info("Netty using Java NIO event loop");
}
ServerBootstrap bootstrap = new ServerBootstrap()
.group(workerGroup)
.channel(hasEpoll ? EpollServerSocketChannel.class : NioServerSocketChannel.class)
.childOption(ChannelOption.TCP_NODELAY, true)
.childOption(ChannelOption.SO_LINGER, 0)
.childOption(ChannelOption.SO_KEEPALIVE, DatabaseDescriptor.getRpcKeepAlive())
.childOption(ChannelOption.ALLOCATOR, CBUtil.allocator)
.childOption(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, 32 * 1024)
.childOption(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, 8 * 1024);
final EncryptionOptions.ClientEncryptionOptions clientEnc = DatabaseDescriptor.getClientEncryptionOptions();
if (clientEnc.enabled)
{
if (clientEnc.optional)
{
logger.info("Enabling optionally encrypted CQL connections between client and server");
bootstrap.childHandler(new OptionalSecureInitializer(this, clientEnc));
}
else
{
logger.info("Enabling encrypted CQL connections between client and server");
bootstrap.childHandler(new SecureInitializer(this, clientEnc));
}
}
else
{
bootstrap.childHandler(new Initializer(this));
}
// Bind and start to accept incoming connections.
logger.info("Using Netty Version: {}", Version.identify().entrySet());
logger.info("Starting listening for CQL clients on {}...", socket);
ChannelFuture bindFuture = bootstrap.bind(socket);
if (!bindFuture.awaitUninterruptibly().isSuccess())
throw new IllegalStateException(String.format("Failed to bind port %d on %s.", socket.getPort(), socket.getAddress().getHostAddress()));
connectionTracker.allChannels.add(bindFuture.channel());
isRunning.set(true);
StorageService.instance.setRpcReady(true);
}
private void registerMetrics()
{
ClientMetrics.instance.addCounter("connectedNativeClients", new Callable()
{
@Override
public Integer call() throws Exception
{
return connectionTracker.getConnectedClients();
}
});
}
private void close()
{
close(false);
}
private void closeAndAwait()
{
close(true);
}
private void close(boolean awaitTermination)
{
// Close opened connections
connectionTracker.closeAll();
workerGroup.shutdownGracefully();
eventExecutorGroup.shutdown();
logger.info("Stop listening for CQL clients");
if (awaitTermination)
{
try
{
workerGroup.awaitTermination(1, TimeUnit.MINUTES);
eventExecutorGroup.awaitTermination(1, TimeUnit.MINUTES);
}
catch (InterruptedException e)
{
logger.error(e.getMessage());
}
}
workerGroup = null;
eventExecutorGroup = null;
StorageService.instance.setRpcReady(false);
}
public static class ConnectionTracker implements Connection.Tracker
{
// TODO: should we be using the GlobalEventExecutor or defining our own?
public final ChannelGroup allChannels = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE);
private final EnumMap groups = new EnumMap<>(Event.Type.class);
public ConnectionTracker()
{
for (Event.Type type : Event.Type.values())
groups.put(type, new DefaultChannelGroup(type.toString(), GlobalEventExecutor.INSTANCE));
}
public void addConnection(Channel ch, Connection connection)
{
allChannels.add(ch);
}
public void register(Event.Type type, Channel ch)
{
groups.get(type).add(ch);
}
public void send(Event event)
{
groups.get(event.type).writeAndFlush(new EventMessage(event));
}
public void closeAll()
{
allChannels.close().awaitUninterruptibly();
}
public int getConnectedClients()
{
/*
- When server is running: allChannels contains all clients' connections (channels)
plus one additional channel used for the server's own bootstrap.
- When server is stopped: the size is 0
*/
return allChannels.size() != 0 ? allChannels.size() - 1 : 0;
}
}
private static class Initializer extends ChannelInitializer
{
// Stateless handlers
private static final Message.ProtocolDecoder messageDecoder = new Message.ProtocolDecoder();
private static final Message.ProtocolEncoder messageEncoder = new Message.ProtocolEncoder();
private static final Frame.Decompressor frameDecompressor = new Frame.Decompressor();
private static final Frame.Compressor frameCompressor = new Frame.Compressor();
private static final Frame.Encoder frameEncoder = new Frame.Encoder();
private static final Message.ExceptionHandler exceptionHandler = new Message.ExceptionHandler();
private static final Message.Dispatcher dispatcher = new Message.Dispatcher();
private static final ConnectionLimitHandler connectionLimitHandler = new ConnectionLimitHandler();
private final Server server;
public Initializer(Server server)
{
this.server = server;
}
protected void initChannel(Channel channel) throws Exception
{
ChannelPipeline pipeline = channel.pipeline();
// Add the ConnectionLimitHandler to the pipeline if configured to do so.
if (DatabaseDescriptor.getNativeTransportMaxConcurrentConnections() > 0
|| DatabaseDescriptor.getNativeTransportMaxConcurrentConnectionsPerIp() > 0)
{
// Add as first to the pipeline so the limit is enforced as first action.
pipeline.addFirst("connectionLimitHandler", connectionLimitHandler);
}
long idleTimeout = DatabaseDescriptor.nativeTransportIdleTimeout();
if (idleTimeout > 0)
{
pipeline.addLast("idleStateHandler", new IdleStateHandler(false, 0, 0, idleTimeout, TimeUnit.MILLISECONDS)
{
@Override
protected void channelIdle(ChannelHandlerContext ctx, IdleStateEvent evt)
{
logger.info("Closing client connection {} after timeout of {}ms", channel.remoteAddress(), idleTimeout);
ctx.close();
}
});
}
//pipeline.addLast("debug", new LoggingHandler());
pipeline.addLast("frameDecoder", new Frame.Decoder(server.connectionFactory));
pipeline.addLast("frameEncoder", frameEncoder);
pipeline.addLast("frameDecompressor", frameDecompressor);
pipeline.addLast("frameCompressor", frameCompressor);
pipeline.addLast("messageDecoder", messageDecoder);
pipeline.addLast("messageEncoder", messageEncoder);
// The exceptionHandler will take care of handling exceptionCaught(...) events while still running
// on the same EventLoop as all previous added handlers in the pipeline. This is important as the used
// eventExecutorGroup may not enforce strict ordering for channel events.
// As the exceptionHandler runs in the EventLoop as the previous handlers we are sure all exceptions are
// correctly handled before the handler itself is removed.
// See https://issues.apache.org/jira/browse/CASSANDRA-13649
pipeline.addLast("exceptionHandler", exceptionHandler);
pipeline.addLast(server.eventExecutorGroup, "executor", dispatcher);
}
}
protected abstract static class AbstractSecureIntializer extends Initializer
{
private final SSLContext sslContext;
private final EncryptionOptions encryptionOptions;
protected AbstractSecureIntializer(Server server, EncryptionOptions encryptionOptions)
{
super(server);
this.encryptionOptions = encryptionOptions;
try
{
this.sslContext = SSLFactory.createSSLContext(encryptionOptions, encryptionOptions.require_client_auth);
}
catch (IOException e)
{
throw new RuntimeException("Failed to setup secure pipeline", e);
}
}
protected final SslHandler createSslHandler() {
SSLEngine sslEngine = sslContext.createSSLEngine();
sslEngine.setUseClientMode(false);
String[] suites = SSLFactory.filterCipherSuites(sslEngine.getSupportedCipherSuites(), encryptionOptions.cipher_suites);
if(encryptionOptions.require_endpoint_verification)
{
SSLParameters sslParameters = sslEngine.getSSLParameters();
sslParameters.setEndpointIdentificationAlgorithm("HTTPS");
sslEngine.setSSLParameters(sslParameters);
}
sslEngine.setEnabledCipherSuites(suites);
sslEngine.setNeedClientAuth(encryptionOptions.require_client_auth);
sslEngine.setEnabledProtocols(SSLFactory.ACCEPTED_INCOMING_PROTOCOLS);
return new SslHandler(sslEngine);
}
}
private static class OptionalSecureInitializer extends AbstractSecureIntializer
{
public OptionalSecureInitializer(Server server, EncryptionOptions encryptionOptions)
{
super(server, encryptionOptions);
}
protected void initChannel(final Channel channel) throws Exception
{
super.initChannel(channel);
channel.pipeline().addFirst("sslDetectionHandler", new ByteToMessageDecoder()
{
@Override
protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List