com.github.ltsopensource.queue.AbstractPreLoader Maven / Gradle / Ivy

Go to download
package com.github.ltsopensource.queue;

import com.github.ltsopensource.core.AppContext;
import com.github.ltsopensource.core.commons.concurrent.ConcurrentHashSet;
import com.github.ltsopensource.core.commons.utils.Callable;
import com.github.ltsopensource.core.commons.utils.*;
import com.github.ltsopensource.core.constant.Constants;
import com.github.ltsopensource.core.constant.ExtConfig;
import com.github.ltsopensource.core.factory.NamedThreadFactory;
import com.github.ltsopensource.core.support.NodeShutdownHook;
import com.github.ltsopensource.core.support.SystemClock;
import com.github.ltsopensource.queue.domain.JobPo;

import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * @author Robert HG ([email protected]) on 8/14/15.
 */
public abstract class AbstractPreLoader implements PreLoader {

    private int loadSize;
    // 预取阀值
    private double factor;

    private ConcurrentHashMap JOB_MAP = new ConcurrentHashMap();

    // 加载的信号
    private ConcurrentHashSet LOAD_SIGNAL = new ConcurrentHashSet();
    private ScheduledExecutorService LOAD_EXECUTOR_SERVICE = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("LTS-PreLoader", true));
    @SuppressWarnings("unused")
    private ScheduledFuture scheduledFuture;
    private AtomicBoolean start = new AtomicBoolean(false);
    private String FORCE_PREFIX = "force_"; // 强制加载的信号
    private ConcurrentHashMap LOADING = new ConcurrentHashMap();

    public AbstractPreLoader(final AppContext appContext) {
        if (start.compareAndSet(false, true)) {

            loadSize = appContext.getConfig().getParameter(ExtConfig.JOB_TRACKER_PRELOADER_SIZE, 300);
            factor = appContext.getConfig().getParameter(ExtConfig.JOB_TRACKER_PRELOADER_FACTOR, 0.2);
            long interval = appContext.getConfig().getParameter(ExtConfig.JOB_TRACKER_PRELOADER_SIGNAL_CHECK_INTERVAL, 100);

            scheduledFuture = LOAD_EXECUTOR_SERVICE.scheduleWithFixedDelay(new Runnable() {
                @Override
                public void run() {
                    doLoad();
                }
            }, interval, interval, TimeUnit.MILLISECONDS);

            NodeShutdownHook.registerHook(appContext, this.getClass().getName(), new Callable() {
                @Override
                public void call() throws Exception {
                    scheduledFuture.cancel(true);
                    LOAD_EXECUTOR_SERVICE.shutdown();
                    start.set(false);
                }
            });
        }
    }

    private void doLoad() {
        for (final String loadTaskTrackerNodeGroup : LOAD_SIGNAL) {
            new Thread(new Runnable() {
                @Override
                public void run() {
                    AtomicBoolean loading = LOADING.get(loadTaskTrackerNodeGroup);
                    if (loading == null) {
                        loading = new AtomicBoolean(false);
                        AtomicBoolean _loading = LOADING.putIfAbsent(loadTaskTrackerNodeGroup, loading);
                        if (_loading != null) {
                            loading = _loading;
                        }
                    }
                    if (loading.compareAndSet(false, true)) {
                        try {
                            handleSignal(loadTaskTrackerNodeGroup);
                        } finally {
                            loading.compareAndSet(true, false);
                        }
                    }
                }
            }).start();
        }
    }

    private void handleSignal(String loadTaskTrackerNodeGroup) {
        // 是否是强制加载
        boolean force = false;
        if (loadTaskTrackerNodeGroup.startsWith(FORCE_PREFIX)) {
            loadTaskTrackerNodeGroup = loadTaskTrackerNodeGroup.replaceFirst(FORCE_PREFIX, "");
            force = true;
        }

        JobPriorityBlockingDeque queue = JOB_MAP.get(loadTaskTrackerNodeGroup);
        if (queue == null) {
            return;
        }
        int size = queue.size();
        if (force || (isInFactor(size))) {

            int needLoadSize = loadSize + size;
            if (force) {
                // 强制加载全量加载吧
                needLoadSize = loadSize;
            }
            // load
            PeriodUtils.start();
            List loads = null;
            try {
                loads = load(loadTaskTrackerNodeGroup, needLoadSize);
            } finally {
                PeriodUtils.end("AbstractPreLoader.load loadTaskTrackerNodeGroup:{},loadSide={}", loadTaskTrackerNodeGroup, needLoadSize);
            }
            // 加入到内存中
            if (CollectionUtils.isNotEmpty(loads)) {
                PeriodUtils.start();
                try {
                    for (JobPo load : loads) {
                        if (!queue.offer(load)) {
                            // 没有成功说明已经满了
                            if (force) {
                                // force场景，移除队列尾部的，插入新的
                                queue.pollLast();
                                queue.offer(load);
                            } else {
                                break;
                            }
                        }
                    }
                } finally {
                    PeriodUtils.end("AbstractPreLoader.offer loadTaskTrackerNodeGroup:{},loadSide={}", loadTaskTrackerNodeGroup, needLoadSize);
                }
            }
        }
        LOAD_SIGNAL.remove(loadTaskTrackerNodeGroup);
    }

    public JobPo take(String taskTrackerNodeGroup, String taskTrackerIdentity) {
        while (true) {
            JobPo jobPo = get(taskTrackerNodeGroup);
            if (jobPo == null) {
                DotLogUtils.dot("Empty JobQueue, taskTrackerNodeGroup:{}, taskTrackerIdentity:{}", taskTrackerNodeGroup, taskTrackerIdentity);
                return null;
            }
            // update jobPo
            PeriodUtils.start();
            try {
                if (lockJob(taskTrackerNodeGroup, jobPo.getJobId(),
                        taskTrackerIdentity, jobPo.getTriggerTime(),
                        jobPo.getGmtModified())) {
                    jobPo.setTaskTrackerIdentity(taskTrackerIdentity);
                    jobPo.setIsRunning(true);
                    jobPo.setGmtModified(SystemClock.now());
                    return jobPo;
                }
            } finally {
                PeriodUtils.end("AbstractPreLoader.lockJob taskId:{}", jobPo.getTaskId());
            }
        }
    }

    @Override
    public void load(String taskTrackerNodeGroup) {
        if (StringUtils.isEmpty(taskTrackerNodeGroup)) {
            for (String key : JOB_MAP.keySet()) {
                LOAD_SIGNAL.add(FORCE_PREFIX + key);
            }
            return;
        }
        LOAD_SIGNAL.add(FORCE_PREFIX + taskTrackerNodeGroup);
    }

    @Override
    public void loadOne2First(String taskTrackerNodeGroup, String jobId) {
        JobPo jobPo = getJob(taskTrackerNodeGroup, jobId);
        if (jobPo == null) {
            return;
        }
        JobPriorityBlockingDeque queue = getQueue(taskTrackerNodeGroup);
        jobPo.setInternalExtParam(Constants.OLD_PRIORITY, String.valueOf(jobPo.getPriority()));

        jobPo.setPriority(Integer.MIN_VALUE);

        if (!queue.offer(jobPo)) {
            queue.pollLast(); // 移除优先级最低的一个
            queue.offer(jobPo);
        }
    }

    protected abstract JobPo getJob(String taskTrackerNodeGroup, String jobId);

    /**
     * 锁定任务
     */
    protected abstract boolean lockJob(String taskTrackerNodeGroup,
                                       String jobId,
                                       String taskTrackerIdentity,
                                       Long triggerTime,
                                       Long gmtModified);

    /**
     * 加载任务
     */
    protected abstract List load(String loadTaskTrackerNodeGroup, int loadSize);

    private JobPo get(String taskTrackerNodeGroup) {

        JobPriorityBlockingDeque queue = getQueue(taskTrackerNodeGroup);

        int size = queue.size();
        DotLogUtils.dot("AbstractPreLoader.queue size:{},taskTrackerNodeGroup:{}", size, taskTrackerNodeGroup);
        if (isInFactor(size)) {
            // 触发加载的请求
            if (!LOAD_SIGNAL.contains(taskTrackerNodeGroup)) {
                LOAD_SIGNAL.add(taskTrackerNodeGroup);
                doLoad();
            }
        }
        JobPo jobPo = queue.poll();
        if (jobPo != null && jobPo.getPriority() == Integer.MIN_VALUE) {
            if (CollectionUtils.isNotEmpty(jobPo.getInternalExtParams())) {
                if (jobPo.getInternalExtParams().containsKey(Constants.OLD_PRIORITY)) {
                    try {
                        int priority = Integer.parseInt(jobPo.getInternalExtParam(Constants.OLD_PRIORITY));
                        jobPo.getInternalExtParams().remove(Constants.OLD_PRIORITY);
                        jobPo.setPriority(priority);
                    } catch (NumberFormatException ignored) {
                    }
                }
            }
        }
        return jobPo;
    }

    private boolean isInFactor(int size) {
        return size / (loadSize * 1.0) < factor;
    }

    private JobPriorityBlockingDeque getQueue(String taskTrackerNodeGroup) {
        JobPriorityBlockingDeque queue = JOB_MAP.get(taskTrackerNodeGroup);
        if (queue == null) {
            queue = new JobPriorityBlockingDeque(loadSize);
            JobPriorityBlockingDeque oldQueue = JOB_MAP.putIfAbsent(taskTrackerNodeGroup, queue);
            if (oldQueue != null) {
                queue = oldQueue;
            }
        }
        return queue;
    }
}