Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.alibaba.schedulerx.worker.container.ThreadContainerPool Maven / Gradle / Ivy
package com.alibaba.schedulerx.worker.container;
import com.alibaba.schedulerx.common.domain.JobType;
import com.alibaba.schedulerx.common.util.ConfigUtil;
import com.alibaba.schedulerx.common.util.StringUtils;
import com.alibaba.schedulerx.worker.discovery.GroupManager;
import com.alibaba.schedulerx.worker.domain.JobContext;
import com.alibaba.schedulerx.worker.domain.WorkerConstants;
import com.alibaba.schedulerx.worker.util.JobProcessorUtil;
import com.alibaba.schedulerx.worker.util.WorkerConfigUtil;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
/**
*
* @author xiaomeng.hxm
*/
public class ThreadContainerPool extends ContainerPool {
private static ThreadContainerPool instance = new ThreadContainerPool();
private ThreadContainerPool() {}
private Map threadPoolMap = Maps.newConcurrentMap();
private ThreadLocal contextThreadLocal = new ThreadLocal();
private static DualConcurrencyThreadPoolExecutor SHARED_THREAD_POOL;
private static final Map jobInstanceLockMap = Maps.newConcurrentMap();
private Cache jobThreadPoolMap = CacheBuilder.newBuilder().expireAfterAccess(48, TimeUnit.HOURS).build();
public static ThreadContainerPool getInstance() {
return instance;
}
public ThreadPoolExecutor getSharedThreadPool() {
return SHARED_THREAD_POOL;
}
/**
* 基于Processor独立线程池执行
* @return
*/
private boolean executeInProcessorThreadPool(final long jobInstanceId, Container container){
if (container instanceof ThreadContainer) {
ThreadContainer threadContainer = (ThreadContainer) container;
Map processorThreadPoolSize = (Map)ConfigUtil.getWorkerConfig().getProperty(WorkerConstants.PROCESSOR_THREAD_POOL_SIZE);
if (processorThreadPoolSize == null || processorThreadPoolSize.size() == 0) {
return false;
}
JobContext context = threadContainer.getContext();
if (JobType.JAVA.getKey().equalsIgnoreCase(context.getJobType())) {
String processorName = JobProcessorUtil.getSimpleClassName(context.getContent());
if (StringUtils.isNotEmpty(processorName)) {
Integer size = processorThreadPoolSize.get(processorName);
if (size != null && size.intValue() > 0) {
executeInJobThreadPool(jobInstanceId, processorName, size, container);
return true;
}
}
}
}
return false;
}
/**
* 在任务级别线程池中运行任务
* @param key
* @param poolSize
* @param container
*/
private void executeInJobThreadPool(final long jobInstanceId, final String key, int poolSize, Container container){
// 任务级别线程池
if (jobThreadPoolMap.getIfPresent(key) == null) {
synchronized (this) {
if (jobThreadPoolMap.getIfPresent(key) == null) {
poolSize = poolSize > 0? poolSize:5;
ThreadPoolExecutor jobThreadPool = new ThreadPoolExecutor(poolSize, poolSize,
5, TimeUnit.MINUTES, new LinkedBlockingQueue(),
new ThreadFactory() {
int index = 0;
@Override
public Thread newThread(Runnable runnable) {
return new Thread(runnable, "Schedulerx-Container-Thread-"
+ key + "-" + (index++));
}
});
jobThreadPool.allowCoreThreadTimeOut(true);
jobThreadPoolMap.put(key, jobThreadPool);
}
}
}
Future future = jobThreadPoolMap.getIfPresent(key).submit((Runnable) container);
addFuture(container, jobInstanceId, future);
}
@Override
public void submit(final long jobId, final long jobInstanceId, long taskId, Container container, int consumerSize) {
// Processor独立线程池中执行
if (executeInProcessorThreadPool(jobInstanceId, container)) {
return;
}
boolean enableShareContainerPool = WorkerConfigUtil.isEnableShareContainerPool();
if (!enableShareContainerPool) {
String mode = WorkerConfigUtil.getThreadPoolMode();
if (WorkerConstants.THREAD_POOL_MODE_JOB.equals(mode)) {
executeInJobThreadPool(jobInstanceId, Long.toString(jobId), consumerSize, container);
} else {
if (!threadPoolMap.containsKey(jobInstanceId)) {
synchronized (this) {
if (!threadPoolMap.containsKey(jobInstanceId)) {
ThreadPoolExecutor threadPool = new ThreadPoolExecutor(consumerSize, consumerSize,
30, TimeUnit.SECONDS,
new LinkedBlockingQueue(),
new ThreadFactory() {
int index = 0;
@Override
public Thread newThread(Runnable runnable) {
return new Thread(runnable, "Schedulerx-Container-Thread-"
+ jobId + "_" + jobInstanceId + "-" + (index++));
}
});
threadPoolMap.put(jobInstanceId, threadPool);
threadPool.allowCoreThreadTimeOut(true);
}
}
}
// threadPoolMap.get(jobInstanceId).execute((Runnable) container);
Future future = threadPoolMap.get(jobInstanceId).submit((Runnable) container);
addFuture(container, jobInstanceId, future);
}
} else {
if (SHARED_THREAD_POOL == null) {
synchronized (this) {
if (SHARED_THREAD_POOL == null) {
int poolSize = ConfigUtil.getWorkerConfig().getInt(WorkerConstants.SHARE_POOL_SIZE, WorkerConstants.SHARE_POOL_SIZE_DEFAULT);
int queueSize = ConfigUtil.getWorkerConfig().getInt(WorkerConstants.SHARE_POOL_QUEUE_SIZE, Integer.MAX_VALUE);
SHARED_THREAD_POOL = new DualConcurrencyThreadPoolExecutor(poolSize, queueSize,
new ThreadFactory() {
int index = 0;
@Override
public Thread newThread(Runnable runnable) {
return new Thread(runnable, "Schedulerx-Shared-Container-Thread-" + (index++));
}
});
// SHARED_THREAD_POOL = new ThreadPoolExecutor(poolSize, poolSize,
// 30, TimeUnit.SECONDS,
// new LinkedBlockingQueue(queueSize),
// new ThreadFactory() {
// int index = 0;
// @Override
// public Thread newThread(Runnable runnable) {
// return new Thread(runnable, "Schedulerx-Shared-Container-Thread-" + (index++));
// }
// });
}
}
}
// SHARED_THREAD_POOL.execute((Runnable) container);
SHARED_THREAD_POOL.registerSemaphore(jobInstanceId, consumerSize);
Future future = SHARED_THREAD_POOL.submit((Runnable) container);
addFuture(container, jobInstanceId, future);
}
}
private void addFuture(Container container, final long jobInstanceId, Future future) {
Set futureSet = futureMap.get(jobInstanceId);
if (futureSet == null) {
synchronized (futureMap) {
futureSet = futureMap.get(jobInstanceId);
if (futureSet == null) {
futureSet = Sets.newConcurrentHashSet();
futureMap.put(jobInstanceId, futureSet);
}
}
}
if (container instanceof ThreadContainer) {
((ThreadContainer) container).setFuture(future);
}
futureSet.add(future);
}
@Override
public synchronized boolean destroyByInstance(long jobInstanceId, boolean mayInterruptIfRunning) {
if (threadPoolMap.containsKey(jobInstanceId)) {
ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor)threadPoolMap.get(jobInstanceId);
if (mayInterruptIfRunning) {
List runnableList = threadPoolExecutor.shutdownNow();
if (CollectionUtils.isNotEmpty(runnableList)) {
for (Runnable runnable : runnableList) {
if (runnable instanceof ThreadContainer) {
ContainerPool containerPool = ContainerFactory.getContainerPool();
String uniqueId = ((ThreadContainer) runnable).getContext().getUniqueId();
containerPool.remove(uniqueId);
}
}
}
} else {
threadPoolExecutor.getQueue().clear();
threadPoolExecutor.shutdown();
}
threadPoolMap.remove(jobInstanceId);
futureMap.remove(jobInstanceId);
} else if (futureMap.containsKey(jobInstanceId)){
for (Future future:futureMap.get(jobInstanceId)) {
future.cancel(mayInterruptIfRunning);
}
futureMap.remove(jobInstanceId);
}
if (SHARED_THREAD_POOL != null) {
SHARED_THREAD_POOL.clear(jobInstanceId);
}
return super.destroyByInstance(jobInstanceId, mayInterruptIfRunning);
}
@Override
public void setContext(JobContext jobContext) {
contextThreadLocal.set(jobContext);
// 对专业版记录worker执行次数
boolean isAdvanced = GroupManager.INSTANCE.isAdvancedVersion(jobContext.getGroupId());
if (isAdvanced) {
this.record(jobContext.getJobId(), jobContext.getGroupId());
}
}
@Override
public JobContext getContext() {
return contextThreadLocal.get();
}
@Override
public void removeContext() {
contextThreadLocal.remove();
}
@Override
public AtomicLong getInstanceLock(long jobInstanceId, Long serialNum) {
AtomicLong lock = jobInstanceLockMap.get(Long.valueOf(jobInstanceId));
if(lock == null){
synchronized (this){
lock = jobInstanceLockMap.get(Long.valueOf(jobInstanceId));
if(lock == null){
lock = new AtomicLong(serialNum==null?0:serialNum);
jobInstanceLockMap.put(Long.valueOf(jobInstanceId), lock);
}
}
}
if (serialNum != null){
lock.set(serialNum);
}
return lock;
}
@Override
public void releaseInstanceLock(long jobInstanceId) {
jobInstanceLockMap.remove(Long.valueOf(jobInstanceId));
}
@Override
public void shutdown(ShutdownMode mode) throws InterruptedException {
Set services = Sets.newHashSet();
if (MapUtils.isNotEmpty(jobThreadPoolMap.asMap())) {
services.addAll(jobThreadPoolMap.asMap().values());
}
if (MapUtils.isNotEmpty(threadPoolMap)) {
services.addAll(threadPoolMap.values());
}
if (SHARED_THREAD_POOL != null) {
services.add(SHARED_THREAD_POOL);
}
// 清理队列中的任务
if (ShutdownMode.WAIT_RUNNING.equals(mode)){
for (ExecutorService service:services){
if (service instanceof ThreadPoolExecutor) {
((ThreadPoolExecutor) service).getQueue().clear();
}
}
}
// 停止所有运行线程池
for (ExecutorService service:services){
doShutdown(service, mode);
}
// 等待所有线程池运行结束
if(!ShutdownMode.IMMEDIATE.equals(mode)) {
for (ExecutorService service : services) {
service.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
}
}
}
private void doShutdown(ExecutorService executorService, ShutdownMode mode) {
if (ShutdownMode.IMMEDIATE.equals(mode)) {
//立即停止所有
executorService.shutdownNow();
} else {
executorService.shutdown();
}
}
@Override
public String genThreadName(Long jobId, Long jobInstanceId, Long taskId) {
String name = "Schedulerx-Container-Thread-" + jobId + "_" + jobInstanceId;
if (taskId != null) {
name += "_" + taskId;
}
return name;
}
public static void main(String[] args) {
Set services = Sets.newHashSet();
services.add(null);
for (ExecutorService service :services) {
service.shutdown();
}
System.out.println(services.size());
}
}
