Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.alibaba.schedulerx.worker.master.persistence.ServerTaskPersistence Maven / Gradle / Ivy
package com.alibaba.schedulerx.worker.master.persistence;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.collections.CollectionUtils;
import com.alibaba.schedulerx.common.domain.InstanceStatus;
import com.alibaba.schedulerx.common.domain.TaskStatus;
import com.alibaba.schedulerx.common.util.HessianUtil;
import com.alibaba.schedulerx.common.util.JsonUtil;
import com.alibaba.schedulerx.protocol.Worker;
import com.alibaba.schedulerx.protocol.Worker.BatchTaskStatues;
import com.alibaba.schedulerx.protocol.Worker.ContainerReportTaskStatusRequest;
import com.alibaba.schedulerx.protocol.Worker.MasterStartContainerRequest;
import com.alibaba.schedulerx.protocol.Worker.TaskMessage;
import com.alibaba.schedulerx.protocol.Worker.WorkerBatchCreateTasksRequest;
import com.alibaba.schedulerx.protocol.Worker.WorkerBatchCreateTasksResponse;
import com.alibaba.schedulerx.protocol.Worker.WorkerBatchReportTaskStatuesRequest;
import com.alibaba.schedulerx.protocol.Worker.WorkerBatchReportTaskStatuesResponse;
import com.alibaba.schedulerx.protocol.Worker.WorkerBatchUpdateTaskStatusRequest;
import com.alibaba.schedulerx.protocol.Worker.WorkerBatchUpdateTaskStatusResponse;
import com.alibaba.schedulerx.protocol.Worker.WorkerClearTasksRequest;
import com.alibaba.schedulerx.protocol.Worker.WorkerClearTasksResponse;
import com.alibaba.schedulerx.protocol.Worker.WorkerCreateTaskRequest;
import com.alibaba.schedulerx.protocol.Worker.WorkerPullTasksRequest;
import com.alibaba.schedulerx.protocol.Worker.WorkerPullTasksResponse;
import com.alibaba.schedulerx.protocol.Worker.WorkerQueryJobInstanceStatusRequest;
import com.alibaba.schedulerx.protocol.Worker.WorkerQueryJobInstanceStatusResponse;
import com.alibaba.schedulerx.protocol.utils.FutureUtils;
import com.alibaba.schedulerx.worker.SchedulerxWorker;
import com.alibaba.schedulerx.worker.discovery.GroupManager;
import com.alibaba.schedulerx.worker.discovery.ServerDiscovery;
import com.alibaba.schedulerx.worker.discovery.ServerDiscoveryFactory;
import com.alibaba.schedulerx.worker.domain.TaskInfo;
import com.alibaba.schedulerx.worker.log.LogFactory;
import com.alibaba.schedulerx.worker.log.Logger;
import com.alibaba.schedulerx.worker.processor.BizSubTask;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.protobuf.ByteString;
/**
*
* @author xiaomeng.hxm
*/
public class ServerTaskPersistence implements TaskPersistence {
private static final Logger LOGGER = LogFactory.getLogger(ServerTaskPersistence.class);
private ServerDiscovery serverDiscovery;
private final String groupId;
public ServerTaskPersistence(String groupId) {
this.groupId = groupId;
this.serverDiscovery = ServerDiscoveryFactory.getDiscovery(groupId);
}
//@Override
//public void updateTaskStatus(long jobId, long jobInstanceId, long taskId, TaskStatus status, String workerAddr,
// String workerId) throws Exception {
// String uniqueId = IdUtil.getUniqueId(jobId, jobInstanceId, taskId);
// ReportTaskStatusEvent reportTaskStatusEvent = new ReportTaskStatusEvent();
// reportTaskStatusEvent.setUniqueId(uniqueId);
// reportTaskStatusEvent.setStatus(status.getValue());
// reportTaskStatusEvent.setJobInstanceId(jobInstanceId);
// reportTaskStatusEvent.setJobId(jobId);
// reportTaskStatusEvent.setTaskId(taskId);
// if (workerAddr != null && !workerAddr.isEmpty()) {
// reportTaskStatusEvent.setWorkerAddr(workerAddr);
// }
// if (workerId != null) {
// reportTaskStatusEvent.setWorkerId(workerId);
// }
// SchedulerxWorker.AtLeastDeliveryRoutingActor.tell(reportTaskStatusEvent, null);
//}
@Override
public int updateTaskStatus(long jobInstanceId, List taskIds, TaskStatus status, String workerId,
String workerAddr) throws Exception {
int res = -1;
if (CollectionUtils.isEmpty(taskIds)) {
return res;
}
WorkerBatchReportTaskStatuesRequest.Builder builder = WorkerBatchReportTaskStatuesRequest.newBuilder();
BatchTaskStatues batchTaskStatues = BatchTaskStatues.newBuilder()
.setStatus(status.getValue())
.setWorkerId(workerId)
.setWorkerAddr(workerAddr)
.addAllTaskIds(taskIds)
.build();
builder.setJobInstanceId(jobInstanceId);
builder.setGroupId(groupId);
builder.addTaskStatues(batchTaskStatues);
try {
WorkerBatchReportTaskStatuesResponse response = (WorkerBatchReportTaskStatuesResponse) FutureUtils.awaitResult(
serverDiscovery.getTaskStatusRouter(), builder.build(), 15);
if (!response.getSuccess()) {
LOGGER.error("batch update task status of jobInstance={} error, {}", jobInstanceId, response.getMessage());
throw new IOException(response.getMessage());
} else {
// get affect cnt
res = response.getAffectCnt();
}
} catch (Throwable e) {
String errMsg = "batch update task status of jobInstanceId=" + jobInstanceId + " timeout.";
LOGGER.error(errMsg, e);
throw e;
}
return res;
}
@Override
public void updateTaskStatues(List taskStatusInfos) throws Exception{
if (CollectionUtils.isEmpty(taskStatusInfos)) {
return;
}
ContainerReportTaskStatusRequest info = taskStatusInfos.get(0);
Map>> status2WorkIdAddr2TaskIds = PersistenceUtil.getTaskStatusMap(taskStatusInfos);
WorkerBatchReportTaskStatuesRequest.Builder builder = WorkerBatchReportTaskStatuesRequest.newBuilder();
for (Entry>> entry : status2WorkIdAddr2TaskIds.entrySet()) {
int status = entry.getKey();
Map> workerAddr2TaskIds = entry.getValue();
for (Entry> entry1 : workerAddr2TaskIds.entrySet()) {
String[] workerIdAddr = entry1.getKey().split("@");
List statusInfos = entry1.getValue();
List taskIds = Lists.transform(statusInfos, new Function() {
@Override
public Long apply(Worker.TaskStatusInfo task) {
return task.getTaskId();
}
});
BatchTaskStatues batchTaskStatues = BatchTaskStatues.newBuilder()
.setStatus(status)
.setWorkerId(workerIdAddr[0])
.setWorkerAddr(workerIdAddr[1])
.addAllTaskIds(taskIds)
.addAllTaskStatues(statusInfos)
.build();
builder.addTaskStatues(batchTaskStatues);
builder.setGroupId(groupId);
}
}
builder.setJobInstanceId(info.getJobInstanceId());
SchedulerxWorker.AtLeastDeliveryRoutingActor.tell(builder.build(), null);
}
@Override
public void clearTasks(long jobInstanceId) throws Exception {
WorkerClearTasksRequest request = WorkerClearTasksRequest.newBuilder().setJobInstanceId(jobInstanceId).build();
try {
WorkerClearTasksResponse response = (WorkerClearTasksResponse)FutureUtils.awaitResult(
serverDiscovery.getMapMasterRouter(), request, 5);
if (response.getSuccess()) {
LOGGER.info("clear tasks of jobInstance[{}] sucessfully", jobInstanceId);
} else {
LOGGER.error("clear tasks of jobInstance[{}] error, {}", jobInstanceId, response.getMessage());
throw new IOException(response.getMessage());
}
} catch (Throwable e) {
String errMsg = "clear tasks of jobInstanceId=" + jobInstanceId + " timeout.";
LOGGER.error(errMsg, e);
throw new IOException(errMsg);
}
}
/**
* do nothing, already create by server.
* @param jobId
* @param jobInstanceId
* @param taskId
* @param taskName
* @param taskBody
* @throws Exception
*/
@Override
public void createTask(long jobId, long jobInstanceId, long taskId, String taskName, ByteString taskBody) throws Exception {
}
@Override
public void createTasks(List containers, String workerId, String workerAddr) throws Exception {
if (CollectionUtils.isEmpty(containers)) {
return;
}
long jobInstanceId = containers.get(0).getJobInstanceId();
WorkerBatchCreateTasksRequest.Builder builder = WorkerBatchCreateTasksRequest.newBuilder();
boolean isAdvancedVersion = GroupManager.INSTANCE.isAdvancedVersion(this.groupId);
for (MasterStartContainerRequest taskInfo : containers) {
WorkerCreateTaskRequest.Builder taskRequestBuilder = WorkerCreateTaskRequest.newBuilder()
.setJobId(taskInfo.getJobId())
.setJobInstanceId(taskInfo.getJobInstanceId())
.setTaskId(taskInfo.getTaskId())
.setTaskName(taskInfo.getTaskName())
.setTaskBody(taskInfo.getTask());
Object taskBodyObj = HessianUtil.toObject(taskInfo.getTask().toByteArray());
if(isAdvancedVersion && taskBodyObj instanceof BizSubTask){
BizSubTask bizSubTask = (BizSubTask)taskBodyObj;
taskRequestBuilder.setLabelMap(JsonUtil.toJson(bizSubTask.labelMap()));
}
builder.addTask(taskRequestBuilder.build());
}
builder.setJobInstanceId(jobInstanceId);
builder.setWorkerId(workerId);
builder.setWorkerAddr(workerAddr);
try {
WorkerBatchCreateTasksResponse response = (WorkerBatchCreateTasksResponse)FutureUtils.awaitResult(
serverDiscovery.getMapMasterRouter(), builder.build(), 90);
if (response.getSuccess()) {
LOGGER.info("batch create tasks to Server successfully, jobInstanceId={}, size={}", jobInstanceId, containers.size());
} else {
LOGGER.error("batch create tasks error, jobInstanceId={}, reason={}.", jobInstanceId, response.getMessage());
throw new IOException(response.getMessage());
}
} catch (Throwable e) {
String errMsg = "batch create tasks timeout, size="+containers.size()+", jobInstanceId=" + jobInstanceId;
LOGGER.error(errMsg, e);
throw new IOException(errMsg);
}
}
@Override
public List pull(long jobInstanceId, int pageSize) throws Exception {
List taskInfos = Lists.newArrayList();
WorkerPullTasksRequest request = WorkerPullTasksRequest.newBuilder()
.setJobInstanceId(jobInstanceId)
.setPageSize(pageSize)
.build();
try {
WorkerPullTasksResponse response = (WorkerPullTasksResponse) FutureUtils.awaitResult(
serverDiscovery.getMapMasterRouter(), request, 30);
if (response.getSuccess()) {
List taskMessages = response.getTaskMessageList();
for (TaskMessage taskMessage : taskMessages) {
taskInfos.add(convert2TaskInfo(taskMessage));
}
} else {
LOGGER.error("pull tasks of jobInstance[{}] failed, {}", jobInstanceId, response.getMessage());
}
} catch (Throwable e) {
String errMsg = "pull task[jobInstanceId=" + jobInstanceId + ", pageSize=" + pageSize + "] timeout.";
LOGGER.error(errMsg, e);
throw e;
}
return taskInfos;
}
@Override
public InstanceStatus checkInstanceStatus(Long jobInstanceId) throws Exception {
InstanceStatus status = InstanceStatus.UNKNOWN;
WorkerQueryJobInstanceStatusRequest request = WorkerQueryJobInstanceStatusRequest.newBuilder()
.setJobInstanceId(jobInstanceId).build();
try {
WorkerQueryJobInstanceStatusResponse response = (WorkerQueryJobInstanceStatusResponse) FutureUtils.awaitResult(
serverDiscovery.getMapMasterRouter(), request, 30);
if (response.getSuccess()) {
status = InstanceStatus.parseValue(response.getStatus());
} else {
LOGGER.error("query job instance status failed, " + response.getMessage());
}
} catch (Throwable e) {
String errMsg = "checkInstanceStatus of jobInstanceId=" + jobInstanceId + " timeout.";
LOGGER.error(errMsg+ " server={}", serverDiscovery.getActiveServerAddr(), e);
throw new IOException(errMsg);
}
return status;
}
@Override
public InstanceStatus checkInstanceStatus(Long jobInstanceId, Long batchNo) throws Exception {
return null;
}
private TaskInfo convert2TaskInfo(TaskMessage taskMessage) {
return TaskInfo.newBuilder()
.setTaskId(taskMessage.getTaskId())
.setTaskName(taskMessage.getTaskName())
.setTaskBody(taskMessage.getTaskBody().toByteArray())
.build();
}
@Override
public void initTable() throws Exception {
}
@Override
public int batchUpdateTaskStatus(long jobInstanceId, TaskStatus status, String workerId, String workerAddr) {
int res = -1;
WorkerBatchUpdateTaskStatusRequest.Builder builder = WorkerBatchUpdateTaskStatusRequest.newBuilder();
builder.setJobInstanceId(jobInstanceId);
builder.setStatus(status.getValue());
if (workerAddr != null) {
builder.setWorkerAddr(workerAddr);
builder.setWorkerId(workerId);
}
WorkerBatchUpdateTaskStatusRequest request = builder.build();
try {
WorkerBatchUpdateTaskStatusResponse response = (WorkerBatchUpdateTaskStatusResponse)FutureUtils.awaitResult(
serverDiscovery.getMapMasterRouter(), request, 5);
if (response.getSuccess()) {
res = response.getAffectCnt();
LOGGER.debug("batch update status=>{} to Server sucessfully, jobInstanceId={}, workerAddr={}", status, jobInstanceId, workerAddr);
} else {
LOGGER.error("batch update status failed, " + response.getMessage());
}
} catch (Throwable e) {
LOGGER.error("batchUpdateTaskStatus of jobInstanceId={} timeout", jobInstanceId, e);
}
return res;
}
}
