All Downloads are FREE. Search and download functionalities are using the official Maven repository.
Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.envisioniot.sub.client.internal.netty.SubClient Maven / Gradle / Ivy
package com.envisioniot.sub.client.internal.netty;
import com.envisioniot.sub.client.internal.MessageListener;
import com.envisioniot.sub.client.internal.RequestFuture;
import com.envisioniot.sub.client.internal.SubThread;
import com.envisioniot.sub.client.internal.netty.processor.AuthRspProcessor;
import com.envisioniot.sub.client.internal.netty.processor.PullRspProcessor;
import com.envisioniot.sub.client.internal.netty.processor.SubRspProcessor;
import com.envisioniot.sub.common.constants.MessageConstant;
import com.envisioniot.sub.common.generated.SubProto;
import com.envisioniot.sub.common.model.SubCategory;
import com.envisioniot.sub.common.model.TPartition;
import com.envisioniot.sub.common.netty.ChannelWriter;
import com.envisioniot.sub.common.netty.RegManager;
import com.envisioniot.sub.common.utils.CommonUtils;
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelOption;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioSocketChannel;
import org.apache.log4j.Logger;
import java.util.Map;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
/**
* created by jie.jin on 2018/12/24.
*/
public class SubClient {
private String host = "localhost";
private int port = 9003;
private String accessKey;
private String secret;
private SubCategory subCategory;
private String subId;
private String consumerGroup;
private boolean autoCommit = true;
private int commitInterval = 3000;
private MessageListener msgListener;
private int requestTimeout = 10000;
private int requestQueueCap = 2;
private BlockingQueue firedRequests;
private long lastCommit = 0L;
private Map consumedOffsets;
private AtomicLong idGen = new AtomicLong(0L);
private AtomicBoolean threadStarted = new AtomicBoolean(false);
private SendThread sendThread;
private UserProcessThread userProcessThread;
private Bootstrap bootstrap;
private static SubClient client;
static NioEventLoopGroup clientEventLoopGroup = new NioEventLoopGroup();
private static final Logger LOG = Logger.getLogger(SubClient.class);
public static SubClient instance() {
if (null == client) {
synchronized (SubClient.class) {
if (null == client) {
client = new SubClient();
}
}
}
return client;
}
private static void register() {
RegManager.register(SubProto.CmdId.idle_req_VALUE, SubProto.IdleReq.class);
RegManager.register(SubProto.CmdId.auth_req_VALUE, SubProto.AuthReq.class);
RegManager.register(SubProto.CmdId.auth_rsp_VALUE, SubProto.AuthRsp.class, new AuthRspProcessor());
RegManager.register(SubProto.CmdId.sub_req_VALUE, SubProto.SubReq.class);
RegManager.register(SubProto.CmdId.sub_rsp_VALUE, SubProto.SubRsp.class, new SubRspProcessor());
RegManager.register(SubProto.CmdId.pull_req_VALUE, SubProto.PullReq.class);
RegManager.register(SubProto.CmdId.pull_rsp_VALUE, SubProto.PullRsp.class, new PullRspProcessor());
RegManager.register(SubProto.CmdId.commit_req_VALUE, SubProto.CommitDTO.class);
}
private SubClient() {
register();
this.consumedOffsets = new ConcurrentHashMap<>();
this.bootstrap = new Bootstrap();
this.bootstrap.group(clientEventLoopGroup);
this.bootstrap.channel(NioSocketChannel.class);
this.bootstrap.remoteAddress(this.host, this.port);
this.bootstrap.option(ChannelOption.SO_KEEPALIVE, true);
this.bootstrap.option(ChannelOption.TCP_NODELAY, true);
this.bootstrap.handler(new SubClientChannelInitializer(false));
}
public void connect() {
if (null == this.firedRequests) {
this.firedRequests = new LinkedBlockingQueue<>(requestQueueCap);
}
this.firedRequests.clear();
this.bootstrap.connect().addListener(
new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
if (!future.isSuccess()) {
future.channel().eventLoop().schedule(new Runnable()
{
@Override
public void run()
{
SubClient.this.connect();
}
}, 5, TimeUnit.SECONDS);
}
LOG.info(future.isSuccess() ? "connect success" : "connect failed");
}
}
);
}
public void startPull(ChannelHandlerContext ctx) {
if (threadStarted.compareAndSet(false, true)) {
this.sendThread = new SendThread(ctx);
this.sendThread.start();
this.userProcessThread = new UserProcessThread(ctx);
this.userProcessThread.start();
} else {
this.sendThread.resetCtx(ctx);
this.userProcessThread.resetCtx(ctx);
}
}
//TODO: support stop
public String getHost() {
return host;
}
public int getPort() {
return port;
}
public String getAccessKey() {
return accessKey;
}
public String getSecret() {
return secret;
}
public SubCategory getSubCategory() {
return subCategory;
}
public String getSubId() {
return subId;
}
public String getConsumerGroup() {
return consumerGroup;
}
public BlockingQueue getFiredRequests() {
return firedRequests;
}
public SubClient setHost(String host) {
this.host = host;
this.bootstrap.remoteAddress(this.host, this.port);
return this;
}
public SubClient setPort(int port) {
this.port = port;
this.bootstrap.remoteAddress(this.host, this.port);
return this;
}
public SubClient setAccessKey(String accessKey) {
this.accessKey = accessKey;
return this;
}
public SubClient setSecret(String secret) {
this.secret = secret;
return this;
}
public SubClient setSubCategory(SubCategory subCategory) {
this.subCategory = subCategory;
return this;
}
public SubClient setSubId(String subId) {
this.subId = subId;
return this;
}
public SubClient setConsumerGroup(String consumerGroup) {
this.consumerGroup = consumerGroup;
return this;
}
public SubClient setMessageListener(MessageListener listener) {
this.msgListener = listener;
return this;
}
public SubClient enableAutoCommit() {
this.autoCommit = true;
return this;
}
public SubClient disableAutoCommit() {
this.autoCommit = false;
return this;
}
public SubClient setAutoCommitInterval(int interval) {
this.commitInterval = interval;
return this;
}
public SubClient enablePreFetch( ) {
this.requestQueueCap = 2;
return this;
}
public SubClient disablePreFetch( ) {
this.requestQueueCap = 1;
return this;
}
public SubClient setRequestTimeout(int timeout) {
this.requestTimeout = timeout;
return this;
}
private SubProto.CommitDTO takeOutCommits() {
SubProto.CommitDTO.Builder builder = SubProto.CommitDTO.newBuilder();
for (Map.Entry entry: this.consumedOffsets.entrySet()) {
TPartition tp = entry.getKey();
Long offset = entry.getValue() + 1;
builder.addCommits(SubProto.Commit.newBuilder().setTopic(tp.topic()).setPartition(tp.partition())
.setOffset(offset).build());
}
this.consumedOffsets.clear();
return builder.build();
}
private String getParameters(String ... args) {
if (args.length % 2 != 0) {
LOG.error("illegal parameters...");
return null;
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < args.length; ++i) {
if (i % 2 == 0) {
if (sb.length() > 0) {
sb.append("&");
}
sb.append(args[i]);
sb.append("=");
} else {
sb.append(args[i]);
}
}
return sb.toString();
}
/**
* simple flow control, i.e., we have limited requests on flight
*/
private class SendThread extends SubThread {
private long idleCnt = 0L;
private AtomicReference ctx = new AtomicReference<>();
private SendThread(ChannelHandlerContext ctx) {
super("send-thread", false);
this.ctx.set(ctx);
}
@Override
public void run() {
while (!closed.get()) {
if (firePullRequest()) {
idleCnt = 0;
} else {
++idleCnt;
}
CommonUtils.doBackoff(idleCnt);
}
}
public void resetCtx(ChannelHandlerContext ctx) {
this.ctx.set(ctx);
}
private boolean firePullRequest() {
if (null == this.ctx.get() || !this.ctx.get().channel().isWritable()) {
return false;
}
SubProto.PullReq pull = SubProto.PullReq.newBuilder().setId(idGen.getAndIncrement()).build();
RequestFuture future = new RequestFuture(pull.getId());
try {
firedRequests.put(future);
} catch (InterruptedException e) {
LOG.error("put firedRequests queue exception. ", e);
return false;
}
ChannelWriter.writeToChannel(this.ctx.get(), pull);
return true;
}
}
private class UserProcessThread extends SubThread {
private long idleCnt = 0L;
private AtomicReference ctx = new AtomicReference<>();
private UserProcessThread(ChannelHandlerContext ctx) {
super("user-process-thread", false);
this.ctx.set(ctx);
}
@Override
public void run() {
while (!closed.get()) {
RequestFuture rf = firedRequests.peek();
if (null == rf) {
++idleCnt;
CommonUtils.doBackoff(idleCnt);
continue;
} else {
idleCnt = 0L;
}
SubProto.PullRsp response = null;
try {
if (rf.awaitDone(requestTimeout, TimeUnit.MILLISECONDS)) {
response = rf.value();
}
} catch (InterruptedException e) {
LOG.error("future await interrupted.", e);
}
synchronized (firedRequests) {
if (rf == firedRequests.peek()) {
firedRequests.poll();
}
}
if (null == response) {
LOG.warn(String.format("wait for response of future (%d) timeout.", rf.getId()));
continue;
}
//TODO: check response code to decide if we need to force close channel
if (response.getCode() != 0) {
LOG.warn("receive rsp code: " + response.getCode());
}
if (LOG.isDebugEnabled()) {
LOG.debug("receive message batch size: " + response.getMsgDTO().getMessagesCount());
}
for (SubProto.Message message: response.getMsgDTO().getMessagesList()) {
if (null == msgListener ) {
LOG.error("msgListener is null");
break;
}
if (!MessageConstant.NOT_MATCHED_MESSAGE.equals(message.getValue())) {
msgListener.onMessage(message);
}
consumedOffsets.put(new TPartition(message.getTopic(), message.getPartition()), message.getOffset());
}
doAutoCommit();
}
}
public void resetCtx(ChannelHandlerContext ctx) {
this.ctx.set(ctx);
}
private void doAutoCommit() {
if (!autoCommit) {
return;
}
long now = System.currentTimeMillis();
if (now - lastCommit <= commitInterval) {
return;
}
SubProto.CommitDTO commitDTO = takeOutCommits();
if (commitDTO.getCommitsCount() > 0) {
ChannelWriter.writeToChannel(this.ctx.get(), commitDTO, 3000L);
lastCommit = now;
if (LOG.isDebugEnabled()) {
LOG.debug("Auto commit: " + commitDTO.toString());
}
}
}
}
public static void main(String[] args) {
final AtomicInteger counter = new AtomicInteger();
final long startTime = System.currentTimeMillis();
MessageListener listener = new MessageListener() {
@Override
public void onMessage(SubProto.Message msg) {
if (counter.get() % 10000 == 0) {
LOG.info("sample message: " + msg.getValue());
}
counter.incrementAndGet();
}
};
ExecutorService executor = Executors.newSingleThreadScheduledExecutor();
executor.submit(new Runnable() {
@Override
public void run() {
while(true) {
LOG.info("QPS: " + counter.get() / ((System.currentTimeMillis() - startTime)/1000.0));
try {
Thread.sleep(1000L);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
});
SubClient.instance()
.setSubId("othersub")
.setSubCategory(SubCategory.EVENT)
.setHost("10.27.21.254")
.setPort(9003)
.setAccessKey("accesskey001")
.setSecret("secret001")
.setConsumerGroup("xgroup")
.enableAutoCommit()
.setMessageListener(listener)
.connect();
// customer/model,
}
}
