com.alibaba.schedulerx.worker.master.GridTaskMaster Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of schedulerx2-worker Show documentation
There is a newer version: 1.12.2
package com.alibaba.schedulerx.worker.master;

import java.io.IOException;
import java.util.List;

import org.apache.commons.collections.CollectionUtils;

import com.alibaba.schedulerx.common.domain.JobInstanceInfo;
import com.alibaba.schedulerx.common.domain.MapTaskXAttrs;
import com.alibaba.schedulerx.common.domain.Metrics;
import com.alibaba.schedulerx.common.domain.TaskDispatchMode;
import com.alibaba.schedulerx.common.monitor.MetricsCollector;
import com.alibaba.schedulerx.common.util.JsonUtil;
import com.alibaba.schedulerx.worker.batch.ReqQueue;
import com.alibaba.schedulerx.worker.batch.TMStatusReqHandler;
import com.alibaba.schedulerx.worker.batch.TaskPushReqHandler;
import com.alibaba.schedulerx.worker.domain.WorkerConstants;
import com.alibaba.schedulerx.worker.log.LogFactory;
import com.alibaba.schedulerx.worker.log.Logger;
import com.alibaba.schedulerx.worker.master.persistence.H2MemoryPersistence;
import com.alibaba.schedulerx.worker.processor.ProcessResult;
import com.alibaba.schedulerx.worker.pull.TaskPullReqHandler;
import com.google.protobuf.ByteString;

import akka.actor.ActorContext;

/**
 * GridTaskMaster, using MemoryPersistence
 *
 * @author xiaomeng.hxm
 */
public class GridTaskMaster extends MapTaskMaster {
    private static final Logger LOGGER = LogFactory.getLogger(GridTaskMaster.class);

    public GridTaskMaster(JobInstanceInfo jobInstanceInfo, ActorContext actorContext) throws Exception {
        super(jobInstanceInfo, actorContext);
        this.taskPersistence = H2MemoryPersistence.getInstance();
        this.taskPersistence.initTable();
        long jobInstanceId = jobInstanceInfo.getJobInstanceId();
        taskStatusReqQueue = new ReqQueue<>(jobInstanceId, 10 * 10000);
        taskStatusReqBatchHandler = new TMStatusReqHandler<>(jobInstanceId, 1, 1, 3000, taskStatusReqQueue);

        if (jobInstanceInfo.getXattrs() != null) {
            this.xAttrs = JsonUtil.fromJson(jobInstanceInfo.getXattrs(), MapTaskXAttrs.class);
        }
        if (xAttrs != null && xAttrs.getTaskDispatchMode().equals(TaskDispatchMode.PULL.getValue())) {
            taskBlockingQueue = new ReqQueue<>(jobInstanceId, 10 * 10000);
            taskDispatchReqHandler = new TaskPullReqHandler<>(jobInstanceId, 1, 1, pageSize * jobInstanceInfo.getAllWorkers().size(),
                    taskBlockingQueue);
        } else {
            int batchSize = pageSize * jobInstanceInfo.getAllWorkers().size();
            if(isWorkerLoadRouter()) {
                batchSize = 2 * jobInstanceInfo.getAllWorkers().size();
            }

            Long dispatchDelay = parseDispatchSpeed();
            if (dispatchDelay != null) {
                // 开启了限速分发
                batchSize = 1;
                taskBlockingQueue = new ReqQueue<>(jobInstanceId, Integer.MAX_VALUE);
            } else {
                taskBlockingQueue = new ReqQueue<>(jobInstanceId, 10 * 10000);
            }
            taskDispatchReqHandler = new TaskPushReqHandler<>(jobInstanceId, 1, 1, batchSize,
                    taskBlockingQueue, 3000, dispatchDelay);
        }
    }

    @Override
    public boolean map(List taskList, String taskName) throws Exception {
        if (CollectionUtils.isEmpty(taskList)) {
            LOGGER.warn("map taskList is empty, taskName:{}", taskName);
            return false;
        }
        LOGGER.info("map taskList, jobInstanceId={}, taskName:{}, taskList size:{}", jobInstanceInfo.getJobInstanceId(),
                taskName, taskList.size());
        int counter = taskCounter.addAndGet(taskList.size());
        if (xAttrs.getTaskDispatchMode().equals(TaskDispatchMode.PULL.getValue()) && counter > WorkerConstants.PULL_MODEL_TASK_SIZE_MAX) {
            LOGGER.error("jobInstanceId={}, pullModel, task counter={}, beyond {} !",
                    jobInstanceInfo.getJobInstanceId(), counter, WorkerConstants.PULL_MODEL_TASK_SIZE_MAX);
            throw new IOException("task size of pullModel can't beyond " + WorkerConstants.PULL_MODEL_TASK_SIZE_MAX);
        }
        this.doMetricsCheck();
        return super.map(taskList, taskName);
    }

    /**
     * Master各类指标信息检查
     */
    protected void doMetricsCheck()throws IOException{
        Metrics vmDetail =  MetricsCollector.getMetrics();
        if (vmDetail != null) {
            double usedMemoryPercent = vmDetail.getHeap5Usage();
            if (usedMemoryPercent > WorkerConstants.USER_MEMORY_PERCENT_DEFAULT) {
                throw new IOException("jvm memory usage:" + usedMemoryPercent * 100  
                    + ",beyond " + WorkerConstants.USER_MEMORY_PERCENT_DEFAULT * 100 + "%!");
            }
        }
    }

    @Override
    public ProcessResult postFinish(long jobInstanceId) {
        ProcessResult postResult = super.postFinish(jobInstanceId);
        try {
            taskPersistence.clearTasks(jobInstanceId);
        } catch (Throwable e) {
            LOGGER.error("", e);
        }
        return postResult;
    }
}