All Downloads are FREE. Search and download functionalities are using the official Maven repository.

de.otto.synapse.messagestore.redis.RedisRingBufferMessageStore Maven / Gradle / Ivy

Go to download

Support for Redis as a MessageStore.

There is a newer version: 0.33.0
Show newest version
package de.otto.synapse.messagestore.redis;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.annotations.Beta;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import de.otto.synapse.channel.ChannelPosition;
import de.otto.synapse.channel.ShardPosition;
import de.otto.synapse.messagestore.Index;
import de.otto.synapse.messagestore.MessageStore;
import de.otto.synapse.messagestore.MessageStoreEntry;
import de.otto.synapse.translator.*;
import org.slf4j.Logger;
import org.springframework.dao.DataAccessException;
import org.springframework.data.redis.core.*;

import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import static de.otto.synapse.channel.ChannelPosition.channelPosition;
import static de.otto.synapse.channel.ShardPosition.fromPosition;
import static de.otto.synapse.translator.ObjectMappers.currentObjectMapper;
import static java.util.Arrays.asList;
import static java.util.Spliterators.spliteratorUnknownSize;
import static org.slf4j.LoggerFactory.getLogger;

/**
 * Redis-based implementation of a WritableMessageStore.
 *
 * 

 *     The store can be configured like a ring-buffer to only store the latest N messages.
 * 

 */
@Beta
public class RedisRingBufferMessageStore implements MessageStore {

    private static final Logger LOG = getLogger(RedisRingBufferMessageStore.class);

    private static final int CHARACTERISTICS = Spliterator.ORDERED | Spliterator.NONNULL | Spliterator.IMMUTABLE;

    private final String name;
    private final RedisTemplate redisTemplate;
    private final int batchSize;
    private final int maxSize;
    private final Encoder encoder;
    private final Decoder decoder;

    /**
     * @param name the name of the message store
     * @param batchSize the size of the batches used to fetch messages from Redis
     * @param ringBufferSize the maximum number of messages stored in the ring-buffer
     * @param stringRedisTemplate the RedisTemplate used to access Redis
     */
    public RedisRingBufferMessageStore(final String name,
                                       final int batchSize,
                                       final int ringBufferSize,
                                       final RedisTemplate stringRedisTemplate) {
        this(name, batchSize, ringBufferSize, stringRedisTemplate, new TextEncoder(MessageFormat.V2), new TextDecoder());
    }

    /**
     * @param name the name of the message store
     * @param batchSize the size of the batches used to fetch messages from Redis
     * @param ringBufferSize the maximum number of messages stored in the ring-buffer
     * @param stringRedisTemplate the RedisTemplate used to access Redis
     * @param messageEncoder the encoder used to encode messages into the string-representation stored in Redis
     * @param messageDecoder the decoder used to decode messages from the string-representation stored in Redis
     */
    public RedisRingBufferMessageStore(final String name,
                                       final int batchSize,
                                       final int ringBufferSize,
                                       final RedisTemplate stringRedisTemplate,
                                       final Encoder messageEncoder,
                                       final Decoder messageDecoder) {
        this.name = name;
        this.redisTemplate = stringRedisTemplate;
        this.batchSize = batchSize;
        this.maxSize = ringBufferSize;
        this.encoder = messageEncoder;
        this.decoder = messageDecoder;
    }

    @Override
    @SuppressWarnings("unchecked")
    public void add(final MessageStoreEntry entry) {
        // This will contain the results of all ops in the transaction
        final List txResults = redisTemplate.execute(new SessionCallback>() {
            public List execute(final RedisOperations operations) throws DataAccessException {
                operations.multi();

                // Store shard position per channel in Redis Hash:
                entry.getTextMessage().getHeader().getShardPosition().ifPresent(shardPosition -> {
                    final String channelPosHashKey = name + "-" + entry.getChannelName() + "-channelPos";
                    final BoundHashOperations channelPosHash = operations.boundHashOps(channelPosHashKey);
                    channelPosHash.put(shardPosition.shardName(), shardPosition.position());
                });

                // Store channelName in Redis Set
                final String channelNamesSetKey = name + "-channels";
                final BoundSetOperations channelNamesSet = operations.boundSetOps(channelNamesSetKey);
                channelNamesSet.add(entry.getChannelName());

                // Encode entry into a string and store it in a Redis list:
                final String messagesListKey = name + "-messages";
                final BoundListOperations messagesList = operations.boundListOps(messagesListKey);
                messagesList.rightPush(encode(entry));
                // Trim the list to  elements
                messagesList.trim(-maxSize, -1);

                return operations.exec();
            }
        });
        LOG.debug("Redis returned with " + txResults);
    }

    @Override
    public Set getChannelNames() {
        Set members = redisTemplate
                .boundSetOps(name + "-channels")
                .members();
        return members;
    }

    @Override
    public ImmutableSet getIndexes() {
        return ImmutableSet.of();
    }

    @Override
    public ChannelPosition getLatestChannelPosition(final String channelName) {
        final Set shardPositions = redisTemplate
                .boundHashOps(name + "-" + channelName + "-channelPos")
                .entries()
                .entrySet()
                .stream()
                .map(entry -> fromPosition(entry.getKey().toString(), entry.getValue().toString()))
                .collect(Collectors.toSet());
        return channelPosition(shardPositions);
    }

    @Override
    public Stream stream() {
        final Iterator messageIterator = new BatchedRedisListIterator<>(
                redisTemplate,
                this::decode,
                name + "-messages",
                batchSize
        );
        return StreamSupport.stream(
                spliteratorUnknownSize(messageIterator, CHARACTERISTICS),
                false
        );
    }

    /**
     * Guaranteed to throw an exception and leave the message store unmodified.
     *
     * @throws UnsupportedOperationException always
     * @deprecated Unsupported operation.
     */
    @Override
    public Stream stream(Index index, String value) {
        throw new UnsupportedOperationException();
    }

    @Override
    public long size() {
        return redisTemplate.boundListOps(name + "-messages").size();
    }

    @Override
    public void close() {
    }

    public void clear() {
        final List keys = new ArrayList<>(asList(name + "-channels", name + "-messages"));
        getChannelNames().forEach(channel -> keys.add(name + "-" + channel + "-channelPos"));
        redisTemplate.delete(keys);
    }

    private String encode(final MessageStoreEntry entry) {
        try {
            return currentObjectMapper().writeValueAsString(ImmutableMap.of(
                    "channelName", entry.getChannelName(),
                    "message", encoder.apply(entry.getTextMessage())));
        } catch (final JsonProcessingException e) {
            throw new IllegalStateException("Failed to encode MessageStoreEntry " + entry + ": " + e.getMessage(), e);
        }
    }

    private MessageStoreEntry decode(final String value) {
        try {
            final Map map = currentObjectMapper().readValue(value, Map.class);
            return MessageStoreEntry.of(
                    map.get("channelName").toString(),
                    decoder.apply(map.get("message").toString()));
        } catch (final IOException e) {
            throw new IllegalStateException("Failed to decode MessageStoreEntry from " + value + ": " + e.getMessage(), e);
        }
    }
}
    

    

    
            
    
            







    
© 2015 - 2024 Weber Informatics LLC | Privacy Policy