io.github.shanqiang.sp.StreamProcessing Maven / Gradle / Ivy
The newest version!
package io.github.shanqiang.sp;
import io.github.shanqiang.JStream;
import io.github.shanqiang.sp.input.StreamTable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import static java.util.Objects.requireNonNull;
public class StreamProcessing {
private static final Logger logger = LoggerFactory.getLogger(StreamProcessing.class);
protected final String name;
protected final int thread;
private final List computeThreads;
protected final StreamTable[] streamTables;
//考虑到watermark和noDataDelay和网络延迟等情况在流表已经全部fetch完之后增加一个delay时间再返回true
private final long finishDelay;
private volatile long finishTime;
private static boolean failFast = true;
private static Throwable globalException;
private static final Set allSP = new HashSet<>();
public synchronized static void setFailFast(boolean failFast) {
StreamProcessing.failFast = failFast;
}
public synchronized static void handleException(Throwable t) {
logger.error("", t);
if (failFast) {
System.exit(-1);
}
globalException = t;
for (StreamProcessing sp : allSP) {
sp.stop();
}
allSP.clear();
}
public StreamProcessing(StreamTable... streamingTables) {
this(Runtime.getRuntime().availableProcessors() * 2, streamingTables);
}
public StreamProcessing(int thread, StreamTable... streamTables) {
this(thread, Duration.ofSeconds(33), streamTables);
}
public StreamProcessing(String name, int thread, StreamTable... streamTables) {
this(name, thread, Duration.ofSeconds(33), streamTables);
}
public StreamProcessing(int thread, Duration finishDelay, StreamTable... streamTables) {
this("", thread, finishDelay, streamTables);
}
public StreamProcessing(String name, int thread, Duration finishDelay, StreamTable... streamTables) {
this.name = requireNonNull(name);
this.thread = thread;
computeThreads = new ArrayList<>(thread);
this.streamTables = requireNonNull(streamTables);
this.finishDelay = finishDelay.toMillis();
allSP.add(this);
JStream.startServer();
}
public int getThread() {
return thread;
}
private boolean isFinished() {
if (streamTables.length <= 0) {
return false;
}
for (int i = 0; i < streamTables.length; i++) {
if (!streamTables[i].isFinished()) {
return false;
}
}
long now = System.currentTimeMillis();
if (0 == finishTime) {
finishTime = now;
}
if (now - finishTime >= finishDelay) {
return true;
}
return false;
}
public void compute(Compute compute) {
computeNoWait(compute);
join();
}
public void computeNoWait(Compute compute) {
for (int i = 0; i < streamTables.length; i++) {
streamTables[i].start();
}
for (int i = 0; i < thread; i++) {
int finalI = i;
Thread t = new Thread(new Runnable() {
@Override
public void run() {
try {
while (!isFinished() && !Thread.interrupted()) {
compute.compute(finalI);
}
} catch (InterruptedException e) {
logger.info("interrupted");
} catch (Throwable t) {
handleException(t);
}
}
}, name + "-" + i);
computeThreads.add(t);
}
for (int i = 0; i < thread; i++) {
computeThreads.get(i).start();
}
}
public void stop() {
for (int i = 0; i < thread; i++) {
computeThreads.get(i).interrupt();
}
}
private synchronized static void throwIf() {
if (null != globalException) {
throw new RuntimeException(globalException);
}
}
public void join() {
throwIf();
for (int i = 0; i < thread; i++) {
try {
computeThreads.get(i).join();
} catch (InterruptedException e) {
//等待过程中被interrupt直接退出即可
logger.info("interrupted");
}
}
computeThreads.clear();
throwIf();
finishTime = 0;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy