All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.hazelcast.jet.impl.connector.StreamEventJournalP Maven / Gradle / Ivy

Go to download

There is a newer version: 4.5.4
Show newest version
/*
 * Copyright (c) 2008-2018, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.jet.impl.connector;

import com.hazelcast.cache.journal.EventJournalCacheEvent;
import com.hazelcast.client.config.ClientConfig;
import com.hazelcast.client.impl.HazelcastClientProxy;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.HazelcastInstanceNotActiveException;
import com.hazelcast.core.ICompletableFuture;
import com.hazelcast.core.Partition;
import com.hazelcast.internal.journal.EventJournalInitialSubscriberState;
import com.hazelcast.internal.journal.EventJournalReader;
import com.hazelcast.jet.Traverser;
import com.hazelcast.jet.Traversers;
import com.hazelcast.jet.core.AbstractProcessor;
import com.hazelcast.jet.core.BroadcastKey;
import com.hazelcast.jet.core.Processor;
import com.hazelcast.jet.core.ProcessorMetaSupplier;
import com.hazelcast.jet.core.ProcessorSupplier;
import com.hazelcast.jet.core.WatermarkGenerationParams;
import com.hazelcast.jet.core.WatermarkSourceUtil;
import com.hazelcast.jet.core.processor.Processors;
import com.hazelcast.jet.core.processor.SourceProcessors;
import com.hazelcast.jet.function.DistributedFunction;
import com.hazelcast.jet.function.DistributedPredicate;
import com.hazelcast.jet.pipeline.JournalInitialPosition;
import com.hazelcast.map.journal.EventJournalMapEvent;
import com.hazelcast.nio.Address;
import com.hazelcast.projection.Projection;
import com.hazelcast.ringbuffer.ReadResultSet;
import com.hazelcast.util.function.Predicate;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import java.util.stream.IntStream;

import static com.hazelcast.client.HazelcastClient.newHazelcastClient;
import static com.hazelcast.jet.Traversers.traverseStream;
import static com.hazelcast.jet.Util.entry;
import static com.hazelcast.jet.core.BroadcastKey.broadcastKey;
import static com.hazelcast.jet.impl.util.ExceptionUtil.peel;
import static com.hazelcast.jet.impl.util.ExceptionUtil.rethrow;
import static com.hazelcast.jet.impl.util.LoggingUtil.logFinest;
import static com.hazelcast.jet.impl.util.Util.arrayIndexOf;
import static com.hazelcast.jet.impl.util.Util.processorToPartitions;
import static com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_CURRENT;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.mapping;
import static java.util.stream.Collectors.toList;
import static java.util.stream.IntStream.range;

/**
 * @see SourceProcessors#streamMapP
 */
public final class StreamEventJournalP extends AbstractProcessor {

    private static final int MAX_FETCH_SIZE = 128;

    @Nonnull
    private final EventJournalReader eventJournalReader;
    @Nonnull
    private final Predicate predicate;
    @Nonnull
    private final Projection projection;
    @Nonnull
    private final JournalInitialPosition initialPos;
    @Nonnull
    private final int[] partitionIds;
    @Nonnull
    private final WatermarkSourceUtil watermarkSourceUtil;

    private final boolean isRemoteReader;

    // keep track of next offset to emit and read separately, as even when the
    // outbox is full we can still poll for new items.
    @Nonnull
    private final long[] emitOffsets;

    @Nonnull
    private final long[] readOffsets;

    private ICompletableFuture>[] readFutures;

    // currently processed resultSet, it's partitionId and iterating position
    @Nullable
    private ReadResultSet resultSet;
    private int currentPartitionIndex = -1;
    private int resultSetPosition;

    private Traverser, long[]>> snapshotTraverser;
    private Traverser traverser = Traversers.empty();

    StreamEventJournalP(
            @Nonnull EventJournalReader eventJournalReader,
            @Nonnull List assignedPartitions,
            @Nonnull DistributedPredicate predicateFn,
            @Nonnull DistributedFunction projectionFn,
            @Nonnull JournalInitialPosition initialPos,
            boolean isRemoteReader,
            @Nonnull WatermarkGenerationParams wmGenParams
    ) {
        this.eventJournalReader = eventJournalReader;
        this.predicate = (Serializable & Predicate) predicateFn::test;
        this.projection = toProjection(projectionFn);
        this.initialPos = initialPos;
        this.isRemoteReader = isRemoteReader;

        partitionIds = assignedPartitions.stream().mapToInt(Integer::intValue).toArray();
        emitOffsets = new long[partitionIds.length];
        readOffsets = new long[partitionIds.length];

        watermarkSourceUtil = new WatermarkSourceUtil<>(wmGenParams);

        // Do not coalesce partition WMs because the number of partitions
        // is far larger than the number of consumers by default and it is
        // not configurable on a per journal basis. This creates
        // excessive latency when the number of events are relatively low
        // where we have to wait for all partitions to advance before
        // advancing the watermark.
        // The side effect of not coalescing is that when the job is
        // restarted and catching up, there might be dropped late events due to
        // several events being read from one partition before the rest and
        // the partition advancing ahead of others.
        // This might be changed in the future and/or made optional.
        watermarkSourceUtil.increasePartitionCount(1);
    }

    @Override
    protected void init(@Nonnull Context context) throws Exception {
        ICompletableFuture[] futures = new ICompletableFuture[partitionIds.length];
        Arrays.setAll(futures, i -> eventJournalReader.subscribeToEventJournal(partitionIds[i]));
        for (int i = 0; i < futures.length; i++) {
            emitOffsets[i] = readOffsets[i] = getSequence(futures[i].get());
        }
    }

    @Override
    public boolean complete() {
        if (readFutures == null) {
            initialRead();
        }
        if (!emitFromTraverser(traverser)) {
            return false;
        }
        do {
            tryGetNextResultSet();
            if (resultSet == null) {
                break;
            }
            emitResultSet();
        } while (resultSet == null);
        return false;
    }

    private void emitResultSet() {
        assert resultSet != null : "null resultSet";
        while (resultSetPosition < resultSet.size()) {
            T event = resultSet.get(resultSetPosition);
            emitOffsets[currentPartitionIndex] = resultSet.getSequence(resultSetPosition) + 1;
            resultSetPosition++;
            if (event != null) {
                // Always use partition index of 0, treating all the partitions the
                // same for coalescing purposes.
                traverser = watermarkSourceUtil.handleEvent(event, 0);
                if (!emitFromTraverser(traverser)) {
                    return;
                }
            }
        }
        // we're done with current resultSet
        resultSetPosition = 0;
        resultSet = null;
    }

    @Override
    public boolean saveToSnapshot() {
        if (!emitFromTraverser(traverser)) {
            return false;
        }

        if (snapshotTraverser == null) {
            snapshotTraverser = traverseStream(IntStream.range(0, partitionIds.length)
                    .mapToObj(pIdx -> entry(
                            broadcastKey(partitionIds[pIdx]),
                            // Always use partition index of 0, treating all the partitions the
                            // same for coalescing purposes.
                            new long[] {emitOffsets[pIdx], watermarkSourceUtil.getWatermark(0)})));
        }
        boolean done = emitFromTraverserToSnapshot(snapshotTraverser);
        if (done) {
            logFinest(getLogger(), "Saved snapshot. partitions=%s, offsets=%s",
                    Arrays.toString(partitionIds), Arrays.toString(emitOffsets));
            snapshotTraverser = null;
        }
        return done;
    }

    @Override
    protected void restoreFromSnapshot(@Nonnull Object key, @Nonnull Object value) {
        int partitionId = ((BroadcastKey) key).key();
        int partitionIndex = arrayIndexOf(partitionId, partitionIds);
        long offset = ((long[]) value)[0];
        long wm = ((long[]) value)[1];
        if (partitionIndex >= 0) {
            readOffsets[partitionIndex] = offset;
            emitOffsets[partitionIndex] = offset;
            // Always use partition index of 0, treating all the partitions the
            // same for coalescing purposes.
            watermarkSourceUtil.restoreWatermark(0, wm);
        }
    }

    @Override
    public boolean finishSnapshotRestore() {
        logFinest(getLogger(), "Restored snapshot. partitions=%s, offsets=%s",
                Arrays.toString(partitionIds), Arrays.toString(readOffsets));
        return true;
    }

    @Override
    public boolean isCooperative() {
        return true;
    }

    private void initialRead() {
        readFutures = new ICompletableFuture[partitionIds.length];
        for (int i = 0; i < readFutures.length; i++) {
            readFutures[i] = readFromJournal(partitionIds[i], readOffsets[i]);
        }
    }

    private long getSequence(EventJournalInitialSubscriberState state) {
        return initialPos == START_FROM_CURRENT ? state.getNewestSequence() + 1 : state.getOldestSequence();
    }

    private void tryGetNextResultSet() {
        while (resultSet == null && ++currentPartitionIndex < partitionIds.length) {
            ICompletableFuture> future = readFutures[currentPartitionIndex];
            if (!future.isDone()) {
                continue;
            }
            resultSet = toResultSet(future);
            int partitionId = partitionIds[currentPartitionIndex];
            if (resultSet != null) {
                assert resultSet.size() > 0 : "empty resultSet";
                long prevSequence = readOffsets[currentPartitionIndex];
                long lostCount = resultSet.getNextSequenceToReadFrom() - resultSet.readCount() - prevSequence;
                if (lostCount > 0) {
                    getLogger().warning(lostCount +  " events lost for partition "
                            + partitionId + " due to journal overflow when reading from event journal."
                            + " Increase journal size to avoid this error. nextSequenceToReadFrom="
                            + resultSet.getNextSequenceToReadFrom() + ", readCount=" + resultSet.readCount()
                            + ", prevSeq=" + prevSequence);
                }
                readOffsets[currentPartitionIndex] = resultSet.getNextSequenceToReadFrom();
            }
            // make another read on the same partition
            readFutures[currentPartitionIndex] = readFromJournal(partitionId, readOffsets[currentPartitionIndex]);
        }

        if (currentPartitionIndex == partitionIds.length) {
            currentPartitionIndex = -1;
            traverser = watermarkSourceUtil.handleNoEvent();
        }
    }

    private ReadResultSet toResultSet(ICompletableFuture> future) {
        try {
            return future.get();
        } catch (ExecutionException e) {
            Throwable ex = peel(e);
            if (ex instanceof HazelcastInstanceNotActiveException && !isRemoteReader) {
                // This exception can be safely ignored - it means the instance was shutting down,
                // so we shouldn't unnecessarily throw an exception here.
                return null;
            } else {
                throw rethrow(ex);
            }
        } catch (InterruptedException e) {
            throw rethrow(e);
        }
    }

    private ICompletableFuture> readFromJournal(int partition, long offset) {
        logFinest(getLogger(), "Reading from partition %d and offset %d", partition, offset);
        return eventJournalReader.readFromEventJournal(offset,
                1, MAX_FETCH_SIZE, partition, predicate, projection);
    }

    private static  Projection toProjection(Function projectionFn) {
        return new Projection() {
            @Override
            public T transform(E input) {
                return projectionFn.apply(input);
            }
        };
    }

    private static class ClusterMetaSupplier implements ProcessorMetaSupplier {

        static final long serialVersionUID = 1L;

        private final SerializableClientConfig serializableConfig;
        private final DistributedFunction> eventJournalReaderSupplier;
        private final DistributedPredicate predicate;
        private final DistributedFunction projection;
        private final JournalInitialPosition initialPos;
        private final WatermarkGenerationParams wmGenParams;

        private transient int remotePartitionCount;
        private transient Map> addrToPartitions;

        ClusterMetaSupplier(
                @Nullable ClientConfig clientConfig,
                @Nonnull DistributedFunction> eventJournalReaderSupplier,
                @Nonnull DistributedPredicate predicate,
                @Nonnull DistributedFunction projection,
                @Nonnull JournalInitialPosition initialPos,
                @Nonnull WatermarkGenerationParams wmGenParams
        ) {
            this.serializableConfig = clientConfig == null ? null : new SerializableClientConfig(clientConfig);
            this.eventJournalReaderSupplier = eventJournalReaderSupplier;
            this.predicate = predicate;
            this.projection = projection;
            this.initialPos = initialPos;
            this.wmGenParams = wmGenParams;
        }

        @Override
        public int preferredLocalParallelism() {
            return serializableConfig != null ? 1 : 2;
        }

        @Override
        public void init(@Nonnull Context context) {
            if (serializableConfig != null) {
                initRemote();
            } else {
                initLocal(context.jetInstance().getHazelcastInstance().getPartitionService().getPartitions());
            }
        }

        private void initRemote() {
            HazelcastInstance client = newHazelcastClient(serializableConfig.asClientConfig());
            try {
                HazelcastClientProxy clientProxy = (HazelcastClientProxy) client;
                remotePartitionCount = clientProxy.client.getClientPartitionService().getPartitionCount();
            } finally {
                client.shutdown();
            }
        }

        private void initLocal(Set partitions) {
            addrToPartitions = partitions.stream()
                                         .collect(groupingBy(p -> p.getOwner().getAddress(),
                                                 mapping(Partition::getPartitionId, toList())));
        }

        @Override @Nonnull
        public Function get(@Nonnull List
 addresses) {
            if (addrToPartitions == null) {
                // assign each remote partition to a member
                addrToPartitions = range(0, remotePartitionCount)
                        .boxed()
                        .collect(groupingBy(partition -> addresses.get(partition % addresses.size())));
            }

            return address -> new ClusterProcessorSupplier<>(addrToPartitions.get(address),
                    serializableConfig, eventJournalReaderSupplier, predicate, projection, initialPos,
                    wmGenParams);
        }

    }

    private static class ClusterProcessorSupplier implements ProcessorSupplier {

        static final long serialVersionUID = 1L;

        @Nonnull
        private final List ownedPartitions;
        @Nullable
        private final SerializableClientConfig serializableClientConfig;
        @Nonnull
        private final DistributedFunction> eventJournalReaderSupplier;
        @Nonnull
        private final DistributedPredicate predicate;
        @Nonnull
        private final DistributedFunction projection;
        @Nonnull
        private final JournalInitialPosition initialPos;
        @Nonnull
        private final WatermarkGenerationParams wmGenParams;

        private transient HazelcastInstance client;
        private transient EventJournalReader eventJournalReader;

        ClusterProcessorSupplier(
                @Nonnull List ownedPartitions,
                @Nullable SerializableClientConfig serializableClientConfig,
                @Nonnull DistributedFunction> eventJournalReaderSupplier,
                @Nonnull DistributedPredicate predicate,
                @Nonnull DistributedFunction projection,
                @Nonnull JournalInitialPosition initialPos,
                @Nonnull WatermarkGenerationParams wmGenParams
        ) {
            this.ownedPartitions = ownedPartitions;
            this.serializableClientConfig = serializableClientConfig;
            this.eventJournalReaderSupplier = eventJournalReaderSupplier;
            this.predicate = predicate;
            this.projection = projection;
            this.initialPos = initialPos;
            this.wmGenParams = wmGenParams;
        }

        @Override
        public void init(@Nonnull Context context) {
            HazelcastInstance instance = context.jetInstance().getHazelcastInstance();
            if (serializableClientConfig != null) {
                client = newHazelcastClient(serializableClientConfig.asClientConfig());
                instance = client;
            }
            eventJournalReader = eventJournalReaderSupplier.apply(instance);
        }

        @Override
        public void close(Throwable error) {
            if (client != null) {
                client.shutdown();
            }
        }

        @Override @Nonnull
        public List get(int count) {
            return processorToPartitions(count, ownedPartitions)
                    .values().stream()
                    .map(this::processorForPartitions)
                    .collect(toList());
        }

        private Processor processorForPartitions(List partitions) {
            return partitions.isEmpty()
                    ? Processors.noopP().get()
                    : new StreamEventJournalP<>(eventJournalReader, partitions, predicate, projection,
                    initialPos, client != null, wmGenParams);
        }
    }

    @SuppressWarnings("unchecked")
    public static  ProcessorMetaSupplier streamMapP(
            @Nonnull String mapName,
            @Nonnull DistributedPredicate> predicate,
            @Nonnull DistributedFunction, T> projection,
            @Nonnull JournalInitialPosition initialPos,
            WatermarkGenerationParams wmGenParams
    ) {
        return new ClusterMetaSupplier<>(null,
                instance -> (EventJournalReader>) instance.getMap(mapName),
                predicate, projection, initialPos, wmGenParams);
    }

    @SuppressWarnings("unchecked")
    public static  ProcessorMetaSupplier streamRemoteMapP(
            @Nonnull String mapName,
            @Nonnull ClientConfig clientConfig,
            @Nonnull DistributedPredicate> predicate,
            @Nonnull DistributedFunction, T> projection,
            @Nonnull JournalInitialPosition initialPos,
            @Nonnull WatermarkGenerationParams wmGenParams) {
        return new ClusterMetaSupplier<>(clientConfig,
                instance -> (EventJournalReader>) instance.getMap(mapName),
                predicate, projection, initialPos, wmGenParams);
    }

    @SuppressWarnings("unchecked")
    public static  ProcessorMetaSupplier streamCacheP(
            @Nonnull String cacheName,
            @Nonnull DistributedPredicate> predicate,
            @Nonnull DistributedFunction, T> projection,
            @Nonnull JournalInitialPosition initialPos,
            @Nonnull WatermarkGenerationParams wmGenParams) {
        return new ClusterMetaSupplier<>(null,
                inst -> (EventJournalReader>) inst.getCacheManager().getCache(cacheName),
                predicate, projection, initialPos, wmGenParams);
    }

    @SuppressWarnings("unchecked")
    public static  ProcessorMetaSupplier streamRemoteCacheP(
            @Nonnull String cacheName,
            @Nonnull ClientConfig clientConfig,
            @Nonnull DistributedPredicate> predicate,
            @Nonnull DistributedFunction, T> projection,
            @Nonnull JournalInitialPosition initialPos,
            @Nonnull WatermarkGenerationParams wmGenParams) {
        return new ClusterMetaSupplier<>(clientConfig,
                inst -> (EventJournalReader>) inst.getCacheManager().getCache(cacheName),
                predicate, projection, initialPos, wmGenParams);
    }
}
    

    

    
            
    
            







    
© 2015 - 2024 Weber Informatics LLC | Privacy Policy