All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.hazelcast.jet.pipeline.test.ParallelStreamP Maven / Gradle / Ivy

Go to download

Core Hazelcast Module

There is a newer version: 5.5.0
Show newest version
/*
 * Copyright (c) 2008-2021, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.jet.pipeline.test;

import com.hazelcast.jet.Traverser;
import com.hazelcast.jet.core.AbstractProcessor;
import com.hazelcast.jet.core.AppendableTraverser;
import com.hazelcast.jet.core.EventTimeMapper;
import com.hazelcast.jet.core.EventTimePolicy;
import com.hazelcast.logging.ILogger;

import javax.annotation.Nonnull;

import java.util.List;
import java.util.stream.IntStream;

import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;


/**
 * Implements a mock stream source with parallelism. It emits the items
 * from the generator functions you supply, preserving the order of each
 * generator.
 *
 * @param  type of emitted items
 */
public class ParallelStreamP extends AbstractProcessor {

    private static final long NANOS_PER_SECOND = SECONDS.toNanos(1);

    private final long nanoTimeMillisToCurrentTimeMillis = determineTimeOffset();
    private final long periodNanos;
    private final EventTimeMapper eventTimeMapper;
    private int globalProcessorIndex;
    private long startNanoTime;
    private long totalParallelism;

    private long nowNanoTime;

    private long sequence;
    private Traverser traverser = new AppendableTraverser<>(2);

    private final List> generators;
    private List> assignedGenerators;

    private ILogger logger;

    /**
     * Creates a processor that generates items using its assigned
     * generator functions. This processor picks its generator functions
     * according to its global processor index.
     *
     * @param eventsPerSecondPerGenerator the desired event rate for each generator
     * @param eventTimePolicy parameters for watermark generation
     * @param generators list of generator functions used in source
     */
    public ParallelStreamP(long eventsPerSecondPerGenerator, EventTimePolicy eventTimePolicy,
                    List> generators) {
        this.startNanoTime = System.currentTimeMillis(); // temporarily holds the parameter value until init
        this.periodNanos = NANOS_PER_SECOND / eventsPerSecondPerGenerator;
        this.eventTimeMapper = new EventTimeMapper<>(eventTimePolicy);
        eventTimeMapper.addPartitions(generators.size());
        this.generators = generators;
    }

    @Override
    protected void init(@Nonnull Context context) {
        totalParallelism = context.totalParallelism();
        globalProcessorIndex = context.globalProcessorIndex();
        startNanoTime = MILLISECONDS.toNanos(startNanoTime + nanoTimeMillisToCurrentTimeMillis) +
                globalProcessorIndex * periodNanos;
        // Assigned generator
        assignedGenerators = IntStream.range(0, generators.size())
                .filter(i -> i % totalParallelism == globalProcessorIndex)
                .mapToObj(generators::get)
                .collect(toList());

        logger = context.logger();
    }

    @Override
    public boolean complete() {
        nowNanoTime = System.nanoTime();
        try {
            emitEvents();
        } catch (Exception e) {
            logger.severe(e);
        }
        return false;
    }

    private void emitEvents() throws Exception {
        long emitUpTo = (nowNanoTime - startNanoTime) / periodNanos;
        while (emitFromTraverser(traverser) && sequence < emitUpTo) {
            long timestampNanoTime = startNanoTime + sequence * periodNanos;
            long timestamp = NANOSECONDS.toMillis(timestampNanoTime) - nanoTimeMillisToCurrentTimeMillis;
            for (GeneratorFunction generator : assignedGenerators) {
                T item = generator.generate(timestamp, sequence);
                traverser = eventTimeMapper.flatMapEvent(nowNanoTime, item, globalProcessorIndex, timestamp);
            }
            sequence++;
        }
    }

    private static long determineTimeOffset() {
        return NANOSECONDS.toMillis(System.nanoTime()) - System.currentTimeMillis();
    }

}
    

    

    
            
    
            







    
© 2015 - 2024 Weber Informatics LLC | Privacy Policy