All Downloads are FREE. Search and download functionalities are using the official Maven repository.
Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.alibaba.schedulerx.worker.batch.ContainerStatusReqHandler Maven / Gradle / Ivy
package com.alibaba.schedulerx.worker.batch;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ThreadPoolExecutor;
import com.alibaba.schedulerx.common.domain.Metrics;
import com.alibaba.schedulerx.common.domain.Pair;
import com.alibaba.schedulerx.common.domain.TaskStatus;
import com.alibaba.schedulerx.common.monitor.MetricsCollector;
import com.alibaba.schedulerx.common.util.JsonUtil;
import com.alibaba.schedulerx.protocol.Worker.ContainerBatchReportTaskStatuesRequest;
import com.alibaba.schedulerx.protocol.Worker.ContainerReportTaskStatusRequest;
import com.alibaba.schedulerx.protocol.Worker.TaskStatusInfo;
import com.alibaba.schedulerx.worker.SchedulerxWorker;
import com.alibaba.schedulerx.worker.container.ThreadContainerPool;
import com.alibaba.schedulerx.worker.log.LogFactory;
import com.alibaba.schedulerx.worker.log.Logger;
import com.alibaba.schedulerx.worker.util.WorkerConfigUtil;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
/**
* batch report container task status to task master
* @author yanxun on 2019/1/6.
*/
public class ContainerStatusReqHandler extends BaseReqHandler {
private String taskMasterAkkaPath;
// private boolean enableShareContainerPool = WorkerConfigUtil.isEnableShareContainerPool();
private static final Logger LOGGER = LogFactory.getLogger(ContainerStatusReqHandler.class);
@Override
public void process(long jobInstanceId, List reqs, String workerAddr) {
batchProcessSvc.submit(new BatchStatuesReportRunnable(jobInstanceId, (List)reqs));
}
public ContainerStatusReqHandler(long jobInstanceId, int coreBatchThreadNum, int maxBatchThreadNum,
int batchSize, ReqQueue queue, String taskMasterAkkaPath) {
super(jobInstanceId, coreBatchThreadNum, maxBatchThreadNum, batchSize, queue,
"Schedulerx-Container-Batch-Statuses-Process-Thread-", "Schedulerx-Container-Batch-Statues-Retrieve-Thread-");
this.taskMasterAkkaPath = taskMasterAkkaPath;
// defaultSleepMs = 10;
}
private class BatchStatuesReportRunnable implements Runnable {
private long jobInstanceId;
private List statues;
BatchStatuesReportRunnable(long jobInstanceId, List reqs) {
this.jobInstanceId = jobInstanceId;
this.statues = reqs;
}
@Override
public void run() {
try {
boolean enableShareContainerPool = WorkerConfigUtil.isEnableShareContainerPool();
if (enableShareContainerPool) {
//如果开启共享线程池，statues可能会有多个jobInstanceId，需要先split成不同的list
Map, List> taskStatusRequestMap = Maps.newHashMap();
for (ContainerReportTaskStatusRequest req : statues) {
long jobInstanceId = req.getJobInstanceId();
long serialNum = req.getSerialNum();
Pair key = new Pair<>(jobInstanceId, serialNum);
if (taskStatusRequestMap.containsKey(key)) {
taskStatusRequestMap.get(key).add(req);
} else {
List reqsByInstance = Lists.newArrayList(req);
taskStatusRequestMap.put(key, reqsByInstance);
}
}
//针对不同的jobInstanceId，构造batchStatusRequests
for (Entry, List> entry : taskStatusRequestMap.entrySet()) {
Pair entryKey = entry.getKey();
String instanceMasterActorPath = null;
List taskStatuses = Lists.newArrayList();
int finishCount = 0;
for (ContainerReportTaskStatusRequest req : entry.getValue()) {
instanceMasterActorPath = req.getInstanceMasterActorPath();
TaskStatusInfo.Builder builder = TaskStatusInfo.newBuilder()
.setTaskId(req.getTaskId())
.setStatus(req.getStatus());
if (req.hasTaskName()) {
builder.setTaskName(req.getTaskName());
}
if (req.hasResult()) {
builder.setResult(req.getResult());
}
if(req.hasProgress()) {
builder.setProgress(req.getProgress());
}
if (req.hasTraceId()) {
builder.setTraceId(req.getTraceId());
}
if (TaskStatus.parseValue(req.getStatus()).isFinish()) {
finishCount++;
}
taskStatuses.add(builder.build());
}
if (instanceMasterActorPath != null) {
ContainerReportTaskStatusRequest taskStatusRequest = entry.getValue().get(0);
Metrics metrics = null;
ThreadPoolExecutor sharedThreadPool = ThreadContainerPool.getInstance().getSharedThreadPool();
if (finishCount > 0 && sharedThreadPool != null) {
// 可用大小可用线程数 + 线程数等量缓冲区
metrics = MetricsCollector.getMetrics();
if (metrics != null) {
Integer availableSize = (sharedThreadPool.getCorePoolSize() - sharedThreadPool.getActiveCount()) +
((int) Math.sqrt(sharedThreadPool.getCorePoolSize()) - sharedThreadPool.getQueue().size()) + finishCount;
metrics.setSharePoolAvailableSize(availableSize);
}
}
ContainerBatchReportTaskStatuesRequest request = ContainerBatchReportTaskStatuesRequest.newBuilder()
.setJobId(taskStatusRequest.getJobId())
.setJobInstanceId(entryKey.getFirst())
.addAllTaskStatues(taskStatuses)
.setTaskMasterAkkaPath(instanceMasterActorPath)
.setWorkerAddr(taskStatusRequest.getWorkerAddr())
.setWorkerId(taskStatusRequest.getWorkerId())
.setSerialNum(entryKey.getSecond())
.setMetricsJson(metrics!=null? JsonUtil.toJson(metrics):"")
.build();
SchedulerxWorker.AtLeastDeliveryRoutingActor.tell(request, null);
LOGGER.info("jobInstanceId={}, serialNum={}, batch report status={} to task master, size:{}", entryKey.getFirst(), entryKey.getSecond(),
taskStatusRequest.getStatus(), taskStatuses.size());
} else {
LOGGER.error("instanceMasterActorPath is null, jobInstanceId={}", jobInstanceId);
}
}
} else {
// some attrs are duplicated in all reqs, for example: workAddr, workerId, jobId, jobInstanceId, taskMasterPath
// get first one used for all reqs.
ContainerReportTaskStatusRequest taskStatusRequest = statues.get(0);
TaskStatusInfo.Builder builder;
Map> serialTaskStatusMap = Maps.newHashMap();
for (ContainerReportTaskStatusRequest req : statues) {
builder = TaskStatusInfo.newBuilder().setTaskId(req.getTaskId()).setStatus(req.getStatus());
if (req.hasTaskName()) {
builder.setTaskName(req.getTaskName());
}
if (req.hasResult()) {
builder.setResult(req.getResult());
}
if(req.hasProgress()) {
builder.setProgress(req.getProgress());
}
if (req.hasTraceId()) {
builder.setTraceId(req.getTraceId());
}
List taskStatuses = serialTaskStatusMap.get(req.getSerialNum());
if (taskStatuses == null) {
taskStatuses = Lists.newArrayList();
serialTaskStatusMap.put(req.getSerialNum(), taskStatuses);
}
taskStatuses.add(builder.build());
}
for (Map.Entry> entry:serialTaskStatusMap.entrySet()) {
ContainerBatchReportTaskStatuesRequest request = ContainerBatchReportTaskStatuesRequest.newBuilder()
.setJobId(taskStatusRequest.getJobId())
.setJobInstanceId(taskStatusRequest.getJobInstanceId())
.addAllTaskStatues(entry.getValue())
.setTaskMasterAkkaPath(taskMasterAkkaPath)
.setWorkerAddr(taskStatusRequest.getWorkerAddr())
.setWorkerId(taskStatusRequest.getWorkerId())
.setSerialNum(entry.getKey())
.build();
SchedulerxWorker.AtLeastDeliveryRoutingActor.tell(request, null);
LOGGER.info("jobInstanceId={}, serialNum={}, batch report status={} to task master, size:{}", jobInstanceId, entry.getKey(), taskStatusRequest.getStatus(), entry.getValue().size());
}
}
} catch (Throwable e) {
LOGGER.error(e);
} finally {
activeRunnableNum.decrementAndGet();
}
}
}
public String getTaskMasterAkkaPath() {
return taskMasterAkkaPath;
}
}
