All Downloads are FREE. Search and download functionalities are using the official Maven repository.

io.github.shanqiang.sp.StreamProcessing Maven / Gradle / Ivy

The newest version!
package io.github.shanqiang.sp;

import io.github.shanqiang.JStream;
import io.github.shanqiang.sp.input.StreamTable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.Duration;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import static java.util.Objects.requireNonNull;

public class StreamProcessing {
    private static final Logger logger = LoggerFactory.getLogger(StreamProcessing.class);

    protected final String name;
    protected final int thread;
    private final List computeThreads;
    protected final StreamTable[] streamTables;
    //考虑到watermark和noDataDelay和网络延迟等情况在流表已经全部fetch完之后增加一个delay时间再返回true
    private final long finishDelay;
    private volatile long finishTime;
    private static boolean failFast = true;
    private static Throwable globalException;
    private static final Set allSP = new HashSet<>();

    public synchronized static void setFailFast(boolean failFast) {
        StreamProcessing.failFast = failFast;
    }

    public synchronized static void handleException(Throwable t) {
        logger.error("", t);
        if (failFast) {
            System.exit(-1);
        }

        globalException = t;
        for (StreamProcessing sp : allSP) {
            sp.stop();
        }
        allSP.clear();
    }

    public StreamProcessing(StreamTable... streamingTables) {
        this(Runtime.getRuntime().availableProcessors() * 2, streamingTables);
    }

    public StreamProcessing(int thread, StreamTable... streamTables) {
        this(thread, Duration.ofSeconds(33), streamTables);
    }

    public StreamProcessing(String name, int thread, StreamTable... streamTables) {
        this(name, thread, Duration.ofSeconds(33), streamTables);
    }

    public StreamProcessing(int thread, Duration finishDelay, StreamTable... streamTables) {
        this("", thread, finishDelay, streamTables);
    }

    public StreamProcessing(String name, int thread, Duration finishDelay, StreamTable... streamTables) {
        this.name = requireNonNull(name);
        this.thread = thread;
        computeThreads = new ArrayList<>(thread);
        this.streamTables = requireNonNull(streamTables);
        this.finishDelay = finishDelay.toMillis();
        allSP.add(this);

        JStream.startServer();
    }

    public int getThread() {
        return thread;
    }

    private boolean isFinished() {
        if (streamTables.length <= 0) {
            return false;
        }

        for (int i = 0; i < streamTables.length; i++) {
            if (!streamTables[i].isFinished()) {
                return false;
            }
        }

        long now = System.currentTimeMillis();
        if (0 == finishTime) {
            finishTime = now;
        }
        if (now - finishTime >= finishDelay) {
            return true;
        }
        return false;
    }

    public void compute(Compute compute) {
        computeNoWait(compute);
        join();
    }

    public void computeNoWait(Compute compute) {
        for (int i = 0; i < streamTables.length; i++) {
            streamTables[i].start();
        }

        for (int i = 0; i < thread; i++) {
            int finalI = i;
            Thread t = new Thread(new Runnable() {
                @Override
                public void run() {
                    try {
                        while (!isFinished() && !Thread.interrupted()) {
                            compute.compute(finalI);
                        }
                    } catch (InterruptedException e) {
                        logger.info("interrupted");
                    } catch (Throwable t) {
                        handleException(t);
                    }
                }
            }, name + "-" + i);
            computeThreads.add(t);
        }

        for (int i = 0; i < thread; i++) {
            computeThreads.get(i).start();
        }
    }

    public void stop() {
        for (int i = 0; i < thread; i++) {
            computeThreads.get(i).interrupt();
        }
    }

    private synchronized static void throwIf() {
        if (null != globalException) {
            throw new RuntimeException(globalException);
        }
    }

    public void join() {
        throwIf();
        for (int i = 0; i < thread; i++) {
            try {
                computeThreads.get(i).join();
            } catch (InterruptedException e) {
                //等待过程中被interrupt直接退出即可
                logger.info("interrupted");
            }
        }
        computeThreads.clear();
        throwIf();

        finishTime = 0;
    }
}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy