com.huaweicloud.dws.client.worker.ExecutionPool Maven / Gradle / Ivy
package com.huaweicloud.dws.client.worker;
import com.huaweicloud.dws.client.DwsClient;
import com.huaweicloud.dws.client.DwsConfig;
import com.huaweicloud.dws.client.action.AbstractAction;
import com.huaweicloud.dws.client.action.GetAction;
import com.huaweicloud.dws.client.action.ScanAction;
import com.huaweicloud.dws.client.action.WriteAction;
import com.huaweicloud.dws.client.collector.ActionCollector;
import com.huaweicloud.dws.client.exception.DwsClientException;
import com.huaweicloud.dws.client.exception.ExceptionCode;
import com.huaweicloud.dws.client.op.Get;
import lombok.extern.slf4j.Slf4j;
import java.io.Closeable;
import java.io.IOException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @ProjectName: dws-connector
* @ClassName: ExecutionPool
* @Description: 资源池
* @Date: 2023/1/9 19:06
* @Version: 1.0
*/
@Slf4j
public class ExecutionPool implements Closeable {
public static final long AWAIT_TIME = 500L;
private final String name;
/**
* 所有的执行者（按照并发，每个初始化一个执行者）
*/
private final ActionExecutor[] executors;
/**
* 后台任务执行
*/
private final BackgroundExecutor backgroundJob;
private final WaitActionTaskExecutor waitActionTask;
private final ReadActionExecutor readActionJob;
/**
* 配置
*/
private final DwsConfig config;
/**
* 是否在运行
*/
private final AtomicBoolean started;
private final AtomicBoolean flushing;
/**
* 执行任务线程池
*/
private ExecutorService workerExecutorService;
private final ThreadFactory workerThreadFactory;
/**
* 后台任务线程池
*/
private ExecutorService backgroundExecutorService;
private final ThreadFactory backgroundThreadFactory;
private final ActionCollector collector;
private final ArrayBlockingQueue queue;
public ActionCollector getCollector() {
return collector;
}
public ArrayBlockingQueue getQueue() {
return queue;
}
/**
* 检查状态是否能够正常使用资源池，如果不允许使用则阻塞业务线程
*/
public void checkStatus() throws DwsClientException {
if (!flushing.get()) {
return;
}
// 在强制执行刷库任务期间 阻塞写入
log.info("wait flush end.");
try {
synchronized (flushing) {
while (flushing.get()) {
flushing.wait();
}
}
} catch (Exception e) {
throw DwsClientException.fromException(e);
}
log.info("end wait flush end.");
}
/**
* 强制刷资源池中的任
*/
public void flush() throws DwsClientException {
synchronized (flushing) {
try {
flushing.set(true);
// 所有收集器中的缓存提交资源池执行
collector.flush();
// 等待资源池中任务全部执行完成
waitActionTask.flush();
} finally {
flushing.set(false);
flushing.notifyAll();
}
}
}
public ExecutionPool(String name, DwsConfig config, DwsClient client) {
this.name = name;
this.config = config;
this.queue = new ArrayBlockingQueue<>(config.getReadBatchQueueSize());
this.collector = new ActionCollector(config, this);
workerThreadFactory = new ThreadFactory() {
private final AtomicInteger threadNumber = new AtomicInteger(1);
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r);
t.setName(name + "-worker-" + threadNumber.getAndIncrement());
t.setDaemon(true);
return t;
}
};
backgroundThreadFactory = new ThreadFactory() {
private final AtomicInteger threadNumber = new AtomicInteger(1);
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r);
t.setName(name + "-background-" + threadNumber.getAndIncrement());
t.setDaemon(true);
return t;
}
};
started = new AtomicBoolean(false);
flushing = new AtomicBoolean(false);
waitActionTask = new WaitActionTaskExecutor(config, started, client);
backgroundJob = new BackgroundExecutor(config, started, collector, client);
readActionJob = new ReadActionExecutor(config, started, collector, client, config.getReadBatchSize());
int workerSize = Math.max(config.getThreadSize(), config.getReadThreadSize());
executors = new ActionExecutor[workerSize];
for (int i = 0; i < workerSize; i++) {
executors[i] = new ActionExecutor(started, config);
}
start();
}
private synchronized void start() {
if (started.compareAndSet(false, true)) {
log.info("Execution Pool start name = {}", name);
// 初始化线程池
workerExecutorService = new ThreadPoolExecutor(executors.length, executors.length, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1), workerThreadFactory, new ThreadPoolExecutor.AbortPolicy());
backgroundExecutorService = new ThreadPoolExecutor(3, 3, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1), backgroundThreadFactory, new ThreadPoolExecutor.AbortPolicy());
for (int i = 0; i < executors.length; i++) {
// 把每个干活的都拉起来
workerExecutorService.execute(executors[i]);
}
// 后台任务开始跑起来
backgroundExecutorService.execute(backgroundJob);
backgroundExecutorService.execute(readActionJob);
backgroundExecutorService.execute(waitActionTask);
// 系统停止时执行关闭
/**
* 程序关闭时回调任务
*/
if (config.isAutoCloseDwsClient()) {
Runtime.getRuntime().addShutdownHook(new Thread(this::close));
}
}
}
public boolean isRunning() {
return started.get();
}
public boolean submit(AbstractAction action) throws DwsClientException {
return submit(action, false);
}
/**
* @param isSync 是否异步，异步直接会放入后台监听、获取执行结果
* @return
* @throws DwsClientException
*/
public boolean submit(AbstractAction action, boolean isSync) throws DwsClientException {
// 已经close的不允许提交
if (!isRunning()) {
throw new DwsClientException(ExceptionCode.ALREADY_CLOSE, "this pool is closed.");
}
int start;
int end;
if (action instanceof WriteAction) {
start = 0;
end = Math.min(config.getThreadSize(), executors.length);
} else if (action instanceof GetAction || action instanceof ScanAction) {
start = Math.max(0, executors.length - config.getReadThreadSize());
end = executors.length;
} else {
start = 0;
end = executors.length;
}
// 尝试在每个任务执行器上提交执行
for (int i = start; i < end; ++i) {
ActionExecutor executor = executors[i];
// 寻找一个空闲的executor 执行该任务
if (executor.submit(action)) {
if (isSync) {
waitActionTask.submitAction(action);
}
return true;
}
}
return false;
}
/**
* 检查后台任务是否有异常
*/
public void tryException() throws DwsClientException {
backgroundJob.tryException();
readActionJob.tryException();
waitActionTask.tryException();
}
public void waitAction(AbstractAction action) {
waitActionTask.submitAction(action);
}
@Override
public void close() {
try {
// 先停止后台任务后再关闭资源
backgroundJob.close();
readActionJob.close();
waitActionTask.close();
} catch (IOException e) {
log.error("", e);
}
if (started.compareAndSet(true, false)) {
for (ActionExecutor executor : executors) {
try {
executor.close();
} catch (IOException e) {
log.error("", e);
}
}
try {
backgroundExecutorService.shutdown();
while (!backgroundExecutorService.awaitTermination(AWAIT_TIME, TimeUnit.MILLISECONDS)) {
log.info("wait background executorService termination[{}]", name);
}
} catch (Exception e) {
log.error("shutdown background error.", e);
}
backgroundExecutorService = null;
try {
workerExecutorService.shutdown();
while (!workerExecutorService.awaitTermination(AWAIT_TIME, TimeUnit.MILLISECONDS)) {
log.info("wait worker executorService termination[{}]", name);
}
workerExecutorService = null;
} catch (Exception e) {
log.error("shutdown worker error.", e);
}
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy