com.github.lontime.extredisson.container.RealConsumeHandler Maven / Gradle / Ivy
The newest version!
package com.github.lontime.extredisson.container;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import com.github.lontime.base.commonj.components.ComponentInterfaceHelper;
import com.github.lontime.base.commonj.utils.CollectionHelper;
import com.github.lontime.base.commonj.utils.CommonManager;
import com.github.lontime.base.commonj.utils.StringHelper;
import com.github.lontime.base.logging.GLogger;
import com.github.lontime.base.serial.model.Message;
import com.github.lontime.extredisson.RedissonInstance;
import com.github.lontime.extredisson.codec.StringBytesMapCodec;
import com.github.lontime.extredisson.common.EndpointMessage;
import com.github.lontime.extredisson.common.EndpointMessageContext;
import com.github.lontime.extredisson.common.ProjectRedisKeyGenerator;
import com.github.lontime.extredisson.common.Topic;
import com.github.lontime.extredisson.configuration.ConsumerOption;
import com.github.lontime.extredisson.configuration.OptionResolver;
import com.github.lontime.extredisson.listener.RedisMessageListener;
import com.github.lontime.shaded.org.redisson.api.RScript;
import com.github.lontime.shaded.org.redisson.api.RStream;
import com.github.lontime.shaded.org.redisson.api.RedissonClient;
import com.github.lontime.shaded.org.redisson.api.StreamMessageId;
import com.github.lontime.shaded.org.redisson.api.stream.StreamReadArgs;
import com.github.lontime.shaded.org.redisson.api.stream.StreamReadGroupArgs;
import com.github.lontime.shaded.org.redisson.client.RedisException;
import com.github.lontime.shaded.org.redisson.client.codec.StringCodec;
import static com.github.lontime.extredisson.common.Constants.SERVICE_DATA_KEY;
/**
* RealConsumeHandler.
*
* @author lontime
* @since 1.0
*/
public class RealConsumeHandler {
private static final String INIT_KEY = "__INIT__";
protected final ConsumerOption consumerOption;
private volatile StreamMessageId currentMessageId;
private volatile long lastAccessTime = 0;
private volatile long lastCleanTime = 0;
private String realConnectionName;
private final Topic topic;
public RealConsumeHandler(ConsumerOption option) {
this.consumerOption = option;
this.topic = Topic.create(option);
}
private List findRegisterListeners(String listenerName) {
return ComponentInterfaceHelper.get(RedisMessageListener.class, listenerName)
.stream().filter(s -> s.kind() == consumerOption.getKind())
.collect(Collectors.toList());
}
public void initialize() {
this.currentMessageId = consumerOption.getMessageId();
this.realConnectionName = topic.resolveConnectionName();
GLogger.defaults().info(getClass(), "Initialize consume options {0}, realConnectionName {1}", consumerOption, realConnectionName);
initStream();
}
public boolean doLoop() {
keepAlive();
boolean result = doLoopInternal();
if (consumerOption.getEnableClean()) {
cleanAsync();
}
return result;
}
private boolean doLoopInternal() {
final String listenerName = consumerOption.getListener();
final List listeners = findRegisterListeners(listenerName);
if (CollectionHelper.isEmpty(listeners)) {
GLogger.defaults().debugv("doLoop Stream<{0}> listeners({1}) lazyListener:{2} broadcast:{3} is null!",
topic.redisKey(), listenerName, consumerOption.getLazyListener(),
consumerOption.getBroadcast());
return consumerOption.getLazyListener();
}
final String groupName = consumerOption.getGroupName();
GLogger.defaults().debugv("【{3}】Starting consume of name: {0}, Stream name: {1}, Connection name: {2} listener name: {4}, group name:{5}, broadcast:{6}",
consumerOption.getName(), topic.redisKey(), realConnectionName,
Thread.currentThread().getName(), listenerName, groupName, consumerOption.getBroadcast());
final Map> map;
if (consumerOption.getBroadcast()) {
map = latest();
} else {
if (consumerOption.getDisableGroup()) {
map = read();
} else {
map = readGroup();
}
}
if (CollectionHelper.isEmpty(map)) {
GLogger.defaults().debugv("RedissonClient Stream<{0}> map is null, group {1} !",
topic.redisKey(), (StringHelper.isEmpty(groupName) ? "" : groupName));
return true;
}
saveCurrentMessageId(map);
for (RedisMessageListener listener : listeners) {
listener.onMessage(buildMessageContext(), buildMessages(map));
}
GLogger.defaults().debugv("【{3}】Ending consume of name: {0}, Stream name: {1}, Connection name: {2}",
consumerOption.getName(), topic.redisKey(), realConnectionName, Thread.currentThread().getName());
return true;
}
private void cleanAsync() {
final long currentTime = System.currentTimeMillis();
if ((currentTime - lastCleanTime) < consumerOption.getLocalCleanInterval().toMillis()) {
return;
}
lastCleanTime = currentTime;
final long interval = consumerOption.getCleanInterval().toMillis();
RedissonInstance.get().getBucket(topic.redisKeyOfClean())
.trySetAsync(String.valueOf(currentTime + interval), interval, TimeUnit.MILLISECONDS)
.thenAcceptAsync(this::cleanInternal, CommonManager.getInstance().getExecutorService());
}
private void cleanInternal(boolean result) {
if (!result) {
return;
}
GLogger.defaults().info(getClass(), "start executeClean for topic {0}, {1}, {2}", topic.getOrig(),
topic.redisKey(), consumerOption.getRef());
final RScript script = client().getScript();
final String limitSize = String.valueOf((int) (consumerOption.getMaxLen() * consumerOption.getMaxLenRate()));
if (consumerOption.getRef()) {
script.eval(RScript.Mode.READ_ONLY,
"local len1 = redis.call('XLEN', KEYS[1]); "
+ "if len1 > tonumber(KEYS[3]) then "
+ " redis.call('XTRIM', KEYS[1], 'MAXLEN', '~', tonumber(KEYS[4])); "
+ "end; "
+ "local len2 = redis.call('XLEN', KEYS[2]); "
+ "if len2 > tonumber(KEYS[3]) then "
+ " redis.call('XTRIM', KEYS[2], 'MAXLEN', '~', tonumber(KEYS[4])); "
+ "end; "
+ "return 1;"
, RScript.ReturnType.VALUE, Arrays.asList(topic.redisKeyRef(), topic.redisKey(), limitSize, consumerOption.getMaxLen()));
return;
}
script.eval(RScript.Mode.READ_ONLY,
"local len1 = redis.call('XLEN', KEYS[1]); "
+ "if len1 > tonumber(KEYS[2]) then "
+ " redis.call('XTRIM', KEYS[1], 'MAXLEN', '~', tonumber(KEYS[3])); "
+ "end; "
+ "return 1;"
, RScript.ReturnType.VALUE, Arrays.asList(topic.redisKey(), limitSize, consumerOption.getMaxLen()));
}
private void saveCurrentMessageId(final Map> map) {
final Set streamMessageIds = map.keySet();
GLogger.defaults().info(getClass(), "RedissonClient Stream<{0}> pull size {1}, ids:{3}, currentMessageId:{2}!",
consumerOption.getTopic(), streamMessageIds.size(), currentMessageId,
streamMessageIds.stream().map(StreamMessageId::toString).collect(Collectors.joining(",")));
final StreamMessageId maxMessageId = streamMessageIds.stream()
.max(Comparator.comparing(StreamMessageId::toString)).orElse(null);
if (maxMessageId != null) {
currentMessageId = maxMessageId;
}
}
private void keepAlive() {
final long currentTime = System.currentTimeMillis();
if ((currentTime - lastAccessTime) > consumerOption.getHeartbeatInterval().toMillis()) {
keepAliveInternal();
lastAccessTime = currentTime;
}
}
//
// private void serviceKeepAlive() {
// final String serviceName = consumerOption.getListener();
// final RegistryOption registryOption = OptionResolver.getInstance().registry();
// final String ip = OptionResolver.getInstance().ip();
// final RScript script = RedissonInstance.get().getScript(registryOption.getName(),StringCodec.INSTANCE);
// script.evalAsync(RScript.Mode.READ_ONLY,
// "redis.call('zadd', KEYS[1], ARGV[1], ARGV[2]); "
// + "redis.call('zadd', KEYS[2], ARGV[1], ARGV[3]); "
// + "redis.call('zadd', KEYS[3], ARGV[1], KEYS[4]); "
// + "redis.call('hset', KEYS[4], 'service', ARGV[2], 'ip', ARGV[4], 'topic', ARGV[3], 'redis', ARGV[5], 'consumer', ARGV[6], 'listener', ARGV[7], 'kind', ARGV[8], 'tag', ARGV[9]); "
// + "return 1;"
// , RScript.ReturnType.VALUE, Arrays.asList(ProjectRedisKeyGenerator.allService().getFullName(),
// ProjectRedisKeyGenerator.allTopic().getFullName(),
// ProjectRedisKeyGenerator.aliveService(serviceName).getFullName(),
// ProjectRedisKeyGenerator.ipService(serviceName).getFullName()),
// System.currentTimeMillis(), serviceName, realTopicName,
// ip, realConnectionName, consumerOption.getName(),
// consumerOption.getListener(), consumerOption.getKind().name(),
// Optional.ofNullable(consumerOption.getTag()).orElse(""));
// }
private void keepAliveInternal() {
final String ip = OptionResolver.getInstance().ip();
final RScript script = client().getScript(StringCodec.INSTANCE);
script.evalAsync(RScript.Mode.READ_ONLY,
"redis.call('zadd', KEYS[1], ARGV[1], ARGV[2]); "
+ "redis.call('zadd', KEYS[2], ARGV[1], KEYS[3]); "
+ "redis.call('hset', KEYS[3], 'listener', ARGV[3], 'ip', ARGV[4], 'actualTopic', ARGV[9], 'topic', ARGV[2], 'redis', ARGV[5], 'consumer', ARGV[6], 'kind', ARGV[7], 'ref', KEYS[4], 'tag', ARGV[8]); "
+ "return 1;"
, RScript.ReturnType.VALUE, Arrays.asList(ProjectRedisKeyGenerator.watchAllTopic().getFullName(),
topic.redisKeyOfAlive(),
topic.redisKeyOfIp(),
consumerOption.getRef().toString()),
System.currentTimeMillis(), topic.getName(),
consumerOption.getListener(), ip, realConnectionName, consumerOption.getName(),
consumerOption.getKind().name(), Optional.ofNullable(consumerOption.getTag()).orElse(""),
topic.redisKey());
}
private EndpointMessageContext buildMessageContext() {
final EndpointMessageContext context = new EndpointMessageContext();
context.setConnectionName(realConnectionName);
context.setTopic(topic);
context.setConsumerName(consumerOption.getName());
context.setConsumerGroupName(consumerOption.getGroupName());
context.setNextStreamName(consumerOption.getNext());
context.setListener(consumerOption.getListener());
context.setGroupNoAck(consumerOption.getGroupNoAck());
context.setReference(consumerOption.getRef());
return context;
}
private List buildMessages(Map> map) {
final List messages = new ArrayList<>(map.size());
for (Map.Entry> entry : map.entrySet()) {
final Map values = entry.getValue();
if (CollectionHelper.isEmpty(values)) {
continue;
}
if (values.containsKey(SERVICE_DATA_KEY)) {
final String id = entry.getKey().toString();
final Message message = Message.fromBytes(values.get(SERVICE_DATA_KEY));
messages.add(EndpointMessage.create(id, message));
}
}
return messages;
}
private Map> latest() {
return getStream().rangeReversed(consumerOption.getInitLatestSize(), StreamMessageId.MAX, StreamMessageId.MIN);
}
private Map> read() {
GLogger.defaults().debug(getClass(), "Consumer using group currentMessageId:{0}!!!", currentMessageId);
if (currentMessageId == null) {
return latest();
}
final StreamReadArgs args = StreamReadArgs.greaterThan(currentMessageId);
if (consumerOption.getBatchSize() != null) {
args.count(consumerOption.getBatchSize());
}
if (consumerOption.getTimeout() != null) {
args.timeout(consumerOption.getTimeout());
}
try {
return getStream().read(args);
} catch (Exception e) {
GLogger.defaults().warnv(e, "read is error");
return Collections.emptyMap();
}
}
private void initStream() {
if (consumerOption.getBroadcast() || consumerOption.getDisableGroup()) {
return;
}
final Long result = client().getScript().eval(RScript.Mode.READ_ONLY,
"local ss = redis.call('exists', KEYS[1]); "
+ "if ss == 0 then "
+ " local msgid = redis.call('xadd', KEYS[1], '*', KEYS[2], '1'); "
+ " redis.call('xdel', KEYS[1], msgid); "
+ "end; "
+ "local values = redis.call('XINFO', 'GROUPS', KEYS[1]); "
+ "for index, item in ipairs(values) do "
+ " local founded = false; "
+ " for i, v in ipairs(item) do "
+ " if founded then"
+ " if item[i] == KEYS[3] then "
+ " return redis.call('exists', KEYS[1]); "
+ " end; "
+ " found = false; "
+ " end; "
+ " if i % 2 ~= 0 then "
+ " if item[i] == 'name' then "
+ " founded = true; "
+ " end; "
+ " end; "
+ " end; "
+ "end; "
+ "redis.call('xgroup', 'CREATE', KEYS[1], KEYS[3], '$'); "
+ "return redis.call('exists', KEYS[1]);"
, RScript.ReturnType.VALUE, Arrays.asList(topic.redisKey(), INIT_KEY,
consumerOption.getGroupName()));
GLogger.defaults().info(getClass(), "checkStream result {0}", result);
}
private Map> readGroup() {
final String groupName = consumerOption.getGroupName();
final StreamReadGroupArgs args = StreamReadGroupArgs.neverDelivered();
if (consumerOption.getBatchSize() != null) {
args.count(consumerOption.getBatchSize());
}
if (consumerOption.getTimeout() != null && consumerOption.getTimeout().getSeconds() > 0) {
args.timeout(consumerOption.getTimeout());
} /*else if (consumerOption.getInterval() != null && consumerOption.getInterval().getSeconds() > 0) {
args.timeout(consumerOption.getInterval());
}*/
if (consumerOption.getGroupNoAck() != null && consumerOption.getGroupNoAck()) {
args.noAck();
}
try {
return getStream().readGroup(groupName, consumerOption.getName(), args);
} catch (Exception e) {
if (e instanceof RedisException) {
initStream();
}
GLogger.defaults().warn(getClass(), e, "read group is error");
return Collections.emptyMap();
}
}
private RStream getStream() {
return client().getStream(topic.redisKey(), StringBytesMapCodec.INSTANCE);
}
private RedissonClient client() {
return RedissonInstance.get().client(topic.getOrig());
}
}