Please wait. This can take some minutes ...
Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
io.hekate.network.netty.NettyServer Maven / Gradle / Ivy
/*
* Copyright 2020 The Hekate Project
*
* The Hekate Project licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package io.hekate.network.netty;
import io.hekate.codec.CodecFactory;
import io.hekate.core.internal.util.ArgAssert;
import io.hekate.core.internal.util.ConfigCheck;
import io.hekate.network.NetworkEndpoint;
import io.hekate.network.NetworkServer;
import io.hekate.network.NetworkServerCallback;
import io.hekate.network.NetworkServerFailure;
import io.hekate.network.NetworkServerFailure.Resolution;
import io.hekate.network.NetworkServerFuture;
import io.hekate.network.NetworkServerHandlerConfig;
import io.hekate.network.internal.NettyChannelSupport;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.epoll.EpollServerSocketChannel;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.ssl.SslContext;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static io.hekate.network.NetworkServer.State.STARTED;
import static io.hekate.network.NetworkServer.State.STARTING;
import static io.hekate.network.NetworkServer.State.STOPPED;
import static io.hekate.network.NetworkServer.State.STOPPING;
import static java.util.Collections.emptyList;
import static java.util.Collections.synchronizedMap;
class NettyServer implements NetworkServer, NettyChannelSupport {
private static final Logger log = LoggerFactory.getLogger(NettyServer.class);
private static final boolean DEBUG = log.isDebugEnabled();
private static final boolean TRACE = log.isTraceEnabled();
private final Object mux = new Object();
private final boolean autoAccept;
private final int hbInterval;
private final int hbLossThreshold;
private final boolean hbDisabled;
private final boolean tcpNoDelay;
private final Integer soReceiveBufferSize;
private final Integer soSendBuffer;
private final Boolean soReuseAddress;
private final Integer soBacklog;
private final SslContext ssl;
private final Map> codecs = synchronizedMap(new HashMap<>());
private final Map handlers = new ConcurrentHashMap<>();
private final Map clients = new IdentityHashMap<>();
private final EventLoopGroup acceptors;
private final EventLoopGroup workers;
private final NettyMetricsFactory metrics;
private Channel server;
private NetworkServerCallback callback;
private NetworkServerFuture startFuture;
private NetworkServerFuture stopFuture;
private boolean failoverInProgress;
// Volatile since address is updated after server start (to reflect a real port)
// and can be accessed in non-synchronized context.
private volatile InetSocketAddress address;
// Volatile since it can be accessed in non-synchronized context.
private volatile State state = STOPPED;
public NettyServer(NettyServerFactory factory) {
ArgAssert.notNull(factory, "Factory");
ConfigCheck check = ConfigCheck.get(NettyServerFactory.class);
check.notNull(factory.getAcceptorEventLoop(), "acceptor event loop");
check.notNull(factory.getWorkerEventLoop(), "worker event loop");
autoAccept = factory.isAutoAccept();
hbInterval = factory.getHeartbeatInterval();
hbLossThreshold = factory.getHeartbeatLossThreshold();
hbDisabled = factory.isDisableHeartbeats();
tcpNoDelay = factory.isTcpNoDelay();
soReceiveBufferSize = factory.getSoReceiveBufferSize();
soSendBuffer = factory.getSoSendBufferSize();
soReuseAddress = factory.getSoReuseAddress();
soBacklog = factory.getSoBacklog();
ssl = factory.getSsl();
metrics = factory.getMetrics();
acceptors = factory.getAcceptorEventLoop();
workers = factory.getWorkerEventLoop();
checkWorkerEventLoopType(check, workers);
if (factory.getHandlers() != null) {
factory.getHandlers().forEach(this::addHandler);
}
}
@Override
public InetSocketAddress address() {
return address;
}
@Override
public State state() {
return state;
}
@Override
public NetworkServerFuture start(InetSocketAddress bindAddress) {
return start(bindAddress, null);
}
@Override
public NetworkServerFuture start(InetSocketAddress address, NetworkServerCallback callback) {
ArgAssert.notNull(address, "Address");
synchronized (mux) {
if (state != STOPPED) {
throw new IllegalStateException("Server is in " + state + " state [address=" + this.address + ']');
}
if (DEBUG) {
log.debug("Starting [address={}]", this.address);
}
this.state = STARTING;
this.address = address;
this.callback = callback;
startFuture = new NetworkServerFuture();
doStart(0);
return startFuture;
}
}
@Override
public void startAccepting() {
synchronized (mux) {
if (server != null && !server.config().isAutoRead()) {
if (DEBUG) {
log.debug("Start accepting [address={}]", address);
}
server.config().setAutoRead(true);
}
}
}
@Override
public NetworkServerFuture stop() {
return doStop(null);
}
@Override
public void addHandler(NetworkServerHandlerConfig> cfg) {
@SuppressWarnings("unchecked")
NetworkServerHandlerConfig objCfg = (NetworkServerHandlerConfig)cfg;
addHandler(copy(objCfg));
}
public void addHandler(NettyServerHandlerConfig> cfg) {
synchronized (mux) {
ConfigCheck check = validate(cfg);
@SuppressWarnings("unchecked")
NettyServerHandlerConfig nettyCfg = (NettyServerHandlerConfig)cfg;
NettyServerHandlerConfig copy = copy(nettyCfg);
copy.setEventLoop(cfg.getEventLoop());
checkWorkerEventLoopType(check, copy.getEventLoop());
if (DEBUG) {
log.debug("Adding handler [protocol={}]", copy);
}
NettyMetricsSink metricsSink = null;
if (metrics != null) {
metricsSink = metrics.createSink(copy.getProtocol());
}
NettyServerHandler registration = new NettyServerHandler(copy, metricsSink);
handlers.put(copy.getProtocol(), registration);
codecs.put(copy.getProtocol(), copy.getCodecFactory());
}
}
@Override
public List> removeHandler(String protocol) {
ArgAssert.notNull(protocol, "Protocol");
if (DEBUG) {
log.debug("Removing handler [protocol={}]", protocol);
}
synchronized (mux) {
handlers.remove(protocol);
codecs.remove(protocol);
List> liveClients = new ArrayList<>();
for (NettyServerClient client : clients.values()) {
String clientProtocol = client.protocol();
if (clientProtocol != null && clientProtocol.equals(protocol)) {
liveClients.add(client);
}
}
return liveClients;
}
}
@Override
public List> clients(String protocol) {
NettyServerHandler handler = handlers.get(protocol);
if (handler != null) {
return handler.clients();
}
return emptyList();
}
@Override
public Optional nettyChannel() {
synchronized (mux) {
return Optional.ofNullable(server);
}
}
private void doStart(int attempt) {
assert Thread.holdsLock(mux) : "Thread must hold lock.";
ServerBootstrap boot = new ServerBootstrap();
if (acceptors instanceof EpollEventLoopGroup) {
if (DEBUG) {
log.debug("Using EPOLL server socket channel.");
}
boot.channel(EpollServerSocketChannel.class);
} else {
if (DEBUG) {
log.debug("Using NIO server socket channel.");
}
boot.channel(NioServerSocketChannel.class);
}
boot.group(acceptors, workers);
setOpts(boot);
setChildOpts(boot);
boot.handler(new ChannelInboundHandlerAdapter() {
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
mayBeRetry(ctx.channel(), attempt, cause);
}
});
boot.childHandler(new ChannelInitializer() {
@Override
protected void initChannel(SocketChannel channel) throws Exception {
InetSocketAddress remoteAddress = channel.remoteAddress();
InetSocketAddress localAddress = channel.localAddress();
synchronized (mux) {
if (state == STOPPING || state == STOPPED) {
if (DEBUG) {
log.debug("Closing connection since server is in {} state [address={}].", state, remoteAddress);
}
channel.close();
return;
}
// Setup pipeline.
ChannelPipeline pipe = channel.pipeline();
// Configure SSL.
if (ssl != null) {
pipe.addLast(ssl.newHandler(channel.alloc()));
}
// Message codecs.
NetworkProtocolCodec codec = new NetworkProtocolCodec(codecs);
pipe.addLast(new NetworkProtocolVersion.Decoder());
pipe.addLast(codec.encoder());
pipe.addLast(codec.decoder());
// Client handler.
NettyServerClient client = new NettyServerClient(
remoteAddress,
localAddress,
ssl != null,
hbInterval,
hbLossThreshold,
hbDisabled,
handlers,
workers
);
pipe.addLast(client);
// Register client to this server.
clients.put(channel, client);
// Unregister client on disconnect.
channel.closeFuture().addListener(close -> {
if (DEBUG) {
log.debug("Removing connection from server registry [address={}]", remoteAddress);
}
synchronized (mux) {
clients.remove(channel);
}
});
}
}
});
ChannelFuture bindFuture = boot.bind(address);
server = bindFuture.channel();
bindFuture.addListener((ChannelFutureListener)bind -> {
if (bind.isSuccess()) {
synchronized (mux) {
failoverInProgress = false;
if (state == STARTING) {
state = STARTED;
// Updated since port can be automatically assigned by the underlying OS.
address = (InetSocketAddress)bind.channel().localAddress();
if (DEBUG) {
log.debug("Started [address={}]", address);
}
if (!startFuture.isDone() && callback != null) {
callback.onStart(this);
}
startFuture.complete(this);
}
}
} else {
mayBeRetry(bind.channel(), attempt, bind.cause());
}
});
}
private void mayBeRetry(Channel channel, int attempt, Throwable cause) {
boolean stopWithError = true;
if (cause instanceof IOException) {
synchronized (mux) {
if (state == STARTED || state == STARTING) {
InetSocketAddress newAddress = null;
long delay = 0;
if (callback != null) {
NetworkServerFailure failure = new NettyServerFailure(cause, attempt, address);
try {
Resolution resolution = callback.onFailure(this, failure);
if (resolution != null && !resolution.isFailure()) {
newAddress = resolution.retryAddress();
if (newAddress == null) {
// Reuse old address.
newAddress = address;
}
delay = resolution.retryDelay();
}
} catch (RuntimeException | Error e) {
if (log.isErrorEnabled()) {
log.error("Got an unexpected runtime error while notifying network server callback on failure.", e);
}
}
}
if (newAddress != null) {
if (DEBUG) {
log.debug("Network server encountered an I/O error ...will try to restart after {} ms "
+ "[old-address={}, new-address={}]", delay, address, newAddress, cause);
}
channel.close();
stopWithError = false;
failoverInProgress = true;
address = newAddress;
Runnable failoverTask = () -> {
try {
synchronized (mux) {
if (failoverInProgress) {
failoverInProgress = false;
doStart(attempt + 1);
}
}
} catch (RuntimeException | Error e) {
if (log.isErrorEnabled()) {
log.error("Got an unexpected runtime error during network server failover.", e);
}
}
};
if (delay > 0) {
acceptors.schedule(failoverTask, delay, TimeUnit.MILLISECONDS);
} else {
acceptors.submit(failoverTask);
}
}
}
}
}
if (stopWithError) {
if (DEBUG) {
log.debug("Network server encountered an error and will be stopped [address={}]", address, cause);
}
doStop(cause);
}
}
private NetworkServerFuture doStop(Throwable cause) {
synchronized (mux) {
if (state == STOPPING) {
return stopFuture;
} else if (state == STARTING || state == STARTED) {
State oldState = state;
state = STOPPING;
failoverInProgress = false;
if (DEBUG) {
log.debug("Stopping [address={}]", address);
}
NetworkServerCallback localCallback = this.callback;
NetworkServerFuture localStartFuture = this.startFuture;
NetworkServerFuture localStopFuture = this.stopFuture = new NetworkServerFuture();
server.close().addListener(serverClose -> {
CompletableFuture allClientsClosed = new CompletableFuture<>();
synchronized (mux) {
// Close client connections.
if (clients.isEmpty()) {
allClientsClosed.complete(null);
} else {
List clientsCopy = new ArrayList<>(clients.keySet());
clients.clear();
AtomicInteger remaining = new AtomicInteger(clientsCopy.size());
clientsCopy.forEach(channel -> {
if (DEBUG) {
log.debug("Closing connection due to server shutdown [address={}]", channel.remoteAddress());
}
// Close channel by scheduling an asynchronous task on its event loop.
// Need to do in order to make sure that there is no disruption in case
// if this channel is currently migrating onto a different event loop
// (see handshake logic in NettyServerClient).
channel.eventLoop().execute(() ->
channel.close().addListener(closed -> {
if (remaining.decrementAndGet() == 0) {
allClientsClosed.complete(null);
}
})
);
});
}
}
allClientsClosed.thenRun(() -> {
synchronized (mux) {
state = STOPPED;
server = null;
if (oldState == STARTED && localCallback != null && !localStopFuture.isDone()) {
localCallback.onStop(this);
}
if (oldState == STARTING && cause != null) {
localStartFuture.completeExceptionally(cause);
} else {
localStartFuture.complete(this);
}
if (DEBUG) {
log.debug("Stopped [address={}]", address);
}
localStopFuture.complete(this);
// Cleanup fields only if they were not changed by a concurrent start.
// JVM identity check to make sure that objects are exactly the same.
if (startFuture == localStartFuture) {
startFuture = null;
}
// JVM identity check to make sure that objects are exactly the same.
if (stopFuture == localStopFuture) {
stopFuture = null;
}
// JVM identity check to make sure that objects are exactly the same.
if (callback == localCallback) {
callback = null;
}
}
});
});
return localStopFuture;
} else {
if (TRACE) {
log.trace("Skipped stop request since server is in {} state [address={}]", state, address);
}
}
}
return NetworkServerFuture.completed(this);
}
private NettyServerHandlerConfig copy(NetworkServerHandlerConfig source) {
NettyServerHandlerConfig copy = new NettyServerHandlerConfig<>();
copy.setProtocol(source.getProtocol());
copy.setHandler(source.getHandler());
copy.setCodecFactory(source.getCodecFactory());
copy.setLoggerCategory(source.getLoggerCategory());
return copy;
}
private ConfigCheck validate(NetworkServerHandlerConfig> handler) {
assert Thread.holdsLock(mux) : "Thread must hold lock.";
ConfigCheck check = ConfigCheck.get(NetworkServerHandlerConfig.class);
check.notEmpty(handler.getProtocol(), "protocol");
check.validSysName(handler.getProtocol(), "protocol");
check.unique(handler.getProtocol(), handlers.keySet(), "protocol");
check.notNull(handler.getHandler(), "handler");
check.notNull(handler.getCodecFactory() != null, "codec factory");
return check;
}
private void setChildOpts(ServerBootstrap boot) {
boot.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
setChildUserOpt(boot, ChannelOption.TCP_NODELAY, tcpNoDelay);
setChildUserOpt(boot, ChannelOption.SO_RCVBUF, soReceiveBufferSize);
setChildUserOpt(boot, ChannelOption.SO_SNDBUF, soSendBuffer);
}
private void setOpts(ServerBootstrap boot) {
setUserOpt(boot, ChannelOption.SO_BACKLOG, soBacklog);
setUserOpt(boot, ChannelOption.SO_RCVBUF, soReceiveBufferSize);
setUserOpt(boot, ChannelOption.SO_REUSEADDR, soReuseAddress);
if (!autoAccept) {
setUserOpt(boot, ChannelOption.AUTO_READ, false);
}
}
private void setChildUserOpt(ServerBootstrap boot, ChannelOption opt, O value) {
if (value != null) {
if (DEBUG) {
log.debug("Setting option {} = {} [address={}]", opt, value, address);
}
boot.childOption(opt, value);
}
}
private void setUserOpt(ServerBootstrap boot, ChannelOption opt, O value) {
if (value != null) {
if (DEBUG) {
log.debug("Setting option {} = {} [address={}]", opt, value, address);
}
boot.option(opt, value);
}
}
private void checkWorkerEventLoopType(ConfigCheck check, EventLoopGroup group) {
if (group != null) {
check.isTrue(acceptors.getClass().isAssignableFrom(group.getClass()), "Can't mix different types of event loop groups "
+ "[acceptors=" + acceptors.getClass().getName() + ", workers=" + group.getClass().getName() + ']');
}
}
@Override
public String toString() {
return getClass().getSimpleName()
+ "[address=" + address
+ ", state=" + state
+ ", handlers=" + handlers.keySet()
+ ']';
}
}