uw.task.conf.TaskServerConfig Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of uw-task Show documentation
uw-task包是一个分布式任务框架，通过uw-task可以快速构建基于docker的分布式任务体系，同时支持基于influx的任务运维监控。
The newest version!
package uw.task.conf;

import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;

import javax.annotation.PreDestroy;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.core.AcknowledgeMode;
import org.springframework.amqp.core.Binding;
import org.springframework.amqp.core.Binding.DestinationType;
import org.springframework.amqp.core.DirectExchange;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.rabbit.connection.ConnectionFactory;
import org.springframework.amqp.rabbit.core.RabbitAdmin;
import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer;
import org.springframework.amqp.rabbit.listener.adapter.MessageListenerAdapter;
import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter;
import org.springframework.amqp.support.postprocessor.GUnzipPostProcessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;

import uw.task.TaskCroner;
import uw.task.TaskData;
import uw.task.TaskRunner;
import uw.task.api.TaskAPI;
import uw.task.entity.TaskContact;
import uw.task.entity.TaskCronerConfig;
import uw.task.entity.TaskRunnerConfig;
import uw.task.mq.TaskConsumer;

/**
 * 根据从服务器端加载的信息来配置服务和任务项。
 * 
 * @author axeon
 */
@Component
public class TaskServerConfig {

	private static final Logger log = LoggerFactory.getLogger(TaskServerConfig.class);

	@Autowired
	ApplicationContext context;

	@Autowired
	ConnectionFactory connectionFactory;

	@Autowired
	TaskConsumer taskConsumer;

	@Autowired
	TaskProperties taskProperties;

	@Autowired
	TaskAPI taskAPI;

	@Autowired
	RabbitAdmin rabbitAdmin;

	@Autowired
	ObjectMapper objectMapper;

	/**
	 * 是否已经跑过一次
	 */
	boolean isRan = false;

	/**
	 * container缓存。key=主机配置,key=任务名,value=container。
	 */
	private ConcurrentHashMap> runnerContainerMap = new ConcurrentHashMap<>();

	/**
	 * Runner任务实例缓存。
	 */
	private ConcurrentHashMap> runnerMap = new ConcurrentHashMap<>();

	/**
	 * Cron任务实例缓存。
	 */
	private ConcurrentHashMap cronerMap = new ConcurrentHashMap<>();

	/**
	 * 泛型类型缓存
	 */
	private ConcurrentHashMap dataParamMap = new ConcurrentHashMap<>();

	/**
	 * 运行主机配置
	 */
	private List hostConfig = null;

	/**
	 * 上次更新配置时间
	 */
	private long lastUpdateTime = 0;

	/**
	 * 获得任务
	 * 
	 * @param taskClass
	 * @return
	 */
	public TaskRunner getRunner(String taskClass) {
		return runnerMap.get(taskClass);
	}

	/**
	 * 转换taskdata。
	 * 
	 * @param taskData
	 * @return
	 */
	public TaskData convertTaskData(TaskData taskData) {
		TaskData convertData = null;
		JavaType type = dataParamMap.get(taskData.getTaskClass());
		if (type != null) {
			try {
				convertData = objectMapper.convertValue(taskData, type);
			} catch (Exception e) {
				log.error(e.getMessage(), e);
			}
		}
		return convertData == null ? taskData : convertData;
	}

	/**
	 * 配置RabbitAdmin
	 * 
	 * @param connectionFactory
	 * @return
	 */
	@Bean
	RabbitAdmin rabbitAdmin(ConnectionFactory connectionFactory) {
		return new RabbitAdmin(connectionFactory);
	}

	/**
	 * 更新主机状态。30秒更新一次。
	 */
	@Scheduled(fixedRate = 30000)
	public void updateStatus() {
		if (log.isDebugEnabled()) {
			log.debug("正在提交主机状态报告...");
		}
		taskAPI.updateHostStatus();
	}

	/**
	 * 注册当前所有的服务。 每隔5分钟刷新一次。
	 */
	@Scheduled(fixedRate = 300000)
	public void updateConfig() {
		long startUpdateTimeMilles = System.currentTimeMillis();

		if (log.isDebugEnabled()) {
			log.debug("正在拉取主机配置...");
		}
		// 拉取主机配置。
		List newHostConfig = taskAPI.getHostConfig();

		// 检查有没有需要删除的
		if (hostConfig != null) {
			for (String host : hostConfig) {
				if (!newHostConfig.contains(host)) {
					// 这个需要删除掉队列监听了。
					if (log.isDebugEnabled()) {
						log.debug("正在删除主机配置[{}]对应的队列监听...", host);
					}
					ConcurrentHashMap map = runnerContainerMap.get(host);
					if (map != null) {
						for (SimpleMessageListenerContainer container : map.values()) {
							container.stop();
						}
					}
				}
			}
		}
		// 检查是否有新增
		for (String host : newHostConfig) {
			if (hostConfig != null && !hostConfig.contains(host)) {
				if (log.isDebugEnabled()) {
					log.debug("正在新建主机配置[{}]对应的队列监听...", host);
				}
				for (TaskRunner runner : runnerMap.values()) {
					registerTask(host, runner.getConfig());
				}
			}
		}
		// 设置状态
		hostConfig = newHostConfig;

		// 取得有变化的croner配置列表
		LinkedHashMap cronerConfigMap = updateCronerConfig();
		if (log.isDebugEnabled() && cronerConfigMap.size() > 0) {
			log.debug("发现服务器端拉取的[{}]个更新的TaskCroner配置...", cronerConfigMap.size());
		}
		// 取得有变化的runner配置列表
		LinkedHashMap runnerConfigMap = updateRunnerConfig();
		if (log.isDebugEnabled() && runnerConfigMap.size() > 0) {
			log.debug("发现服务器端拉取的[{}]个更新的TaskRunner配置...", runnerConfigMap.size());
		}

		if (isRan) {
			// 此时执行更新操作
			// 改Cron配置
			if (cronerConfigMap != null) {
				for (TaskCronerConfig tcc : cronerConfigMap.values()) {
					TaskCroner tc = cronerMap.get(tcc.getTaskClass());
					if (tc != null) {
						try {
							if (log.isDebugEnabled()) {
								log.debug("TaskCroner:[{}]正在设定新配置...", tcc.getTaskClass());
							}
							// 设置cron表达式可能会抛错
							tc.setConfig(tcc);
						} catch (Exception e) {
							log.error(e.getMessage(), e);
						}
					} else {
						log.warn("TaskCroner:[{}]更新配置时，未找到匹配的任务类!", tcc.getTaskClass());
					}
				}
			}
			// 改Runner配置，逐一循环并配置。
			if (runnerConfigMap != null) {
				for (TaskRunnerConfig trc : runnerConfigMap.values()) {
					// 设置runner参数
					TaskRunner tr = runnerMap.get(trc.getTaskClass());
					if (tr != null) {
						try {
							if (log.isDebugEnabled()) {
								log.debug("TaskRunner:[{}]正在设定新配置...", trc.getTaskClass());
							}
							tr.setConfig(trc);
						} catch (Exception e) {
							log.error(e.getMessage(), e);
						}
					} else {
						log.warn("TaskRunner:[{}]更新配置时，未找到匹配的任务类!", trc.getTaskClass());
					}
					// 分hostConfig逐个重设队列参数
					Collection> list = runnerContainerMap
							.values();
					for (ConcurrentHashMap map : list) {
						SimpleMessageListenerContainer container = map.get(trc.getTaskClass());
						if (container != null) {
							try {
								container.setConcurrentConsumers(trc.getConsumerNum());
								container.setPrefetchCount(trc.getConsumerNum());
							} catch (Exception e) {
								log.error(e.getMessage(), e);
							}
						} else {
							log.warn("TaskRunner:[{}]更新配置时，未找到匹配的队列监听！", trc.getTaskClass());
						}
					}
				}
			}

		} else {
			// 第一次执行初始化操作
			isRan = true;
			// 设置当前主机上所有的TaskCroner
			Map cronerInstanceMap = context.getBeansOfType(TaskCroner.class);
			for (Entry kv : cronerInstanceMap.entrySet()) {
				try {
					// 拿到任务类名
					String taskClass = kv.getValue().getClass().getName();
					TaskCronerConfig config = cronerConfigMap.get(taskClass);
					TaskCroner tc = kv.getValue();
					// 配置croner任务
					if (config == null) {
						config = tc.initConfig();
						TaskContact contact = tc.initContact();
						config.setTaskClass(taskClass);
						contact.setTaskClass(taskClass);
						if (config != null && contact != null) {
							tc.setConfig(config);
							// 上传配置
							uploadCronerInfo(config, contact);
							if (log.isDebugEnabled()) {
								log.debug("TaskCroner:[{}]未找到服务器端配置，上传默认配置...", taskClass);
								log.debug("TaskCroner:[{}]正在设定配置...", taskClass);
							}
						} else {
							log.error("TaskCroner:[{}]默认配置或联系人信息为空，无法启动！", taskClass);
							continue;
						}
					} else {
						tc.setConfig(config);
						if (log.isDebugEnabled()) {
							log.debug("TaskCroner:[{}]正在设定配置...", taskClass);
						}
					}
					cronerMap.put(taskClass, kv.getValue());
					// 启动croner任务
					tc.start();
				} catch (Exception e) {
					log.error(e.getMessage(), e);
				}
			}

			// 获得当前主机上所有的TaskRunner
			@SuppressWarnings("rawtypes")
			Map runnerInstanceMap = context.getBeansOfType(TaskRunner.class);
			for (@SuppressWarnings("rawtypes") Entry kv : runnerInstanceMap.entrySet()) {
				try {
					// 拿到任务类名
					String taskClass = kv.getValue().getClass().getName();
					TaskRunnerConfig config = runnerConfigMap.get(taskClass);
					TaskRunner tr = kv.getValue();
					// 配置croner任务
					if (config == null) {
						config = tr.initConfig();
						TaskContact contact = tr.initContact();
						config.setTaskClass(taskClass);
						contact.setTaskClass(taskClass);
						if (config != null && contact != null) {
							if (log.isDebugEnabled()) {
								log.debug("TaskRunner:[{}]正在设定配置...", taskClass);
								log.debug("TaskRunner:[{}]未找到服务器端配置，上传默认配置...", taskClass);
							}
							tr.setConfig(config);
							// 上传配置
							uploadRunnerInfo(config, contact);
						} else {
							log.error("TaskRunner:[{}]默认配置或联系人信息为空，无法启动！", taskClass);
							continue;
						}
					} else {
						if (log.isDebugEnabled()) {
							log.debug("TaskRunner:[{}]正在设定配置...", taskClass);
						}
						tr.setConfig(config);
					}
					runnerMap.put(taskClass, kv.getValue());
					// 放在最后，防止出幺蛾子
					constructTaskDataType(taskClass, kv.getValue());
				} catch (Exception e) {
					log.error(e.getMessage(), e);
				}
			}

			// 循环创建监听队列
			for (String host : hostConfig) {
				for (TaskRunner runner : runnerMap.values()) {
					registerTask(host, runner.getConfig());
				}
			}
		}
		lastUpdateTime = startUpdateTimeMilles;

	}

	/**
	 * 更新所有Croner的配置
	 */
	private LinkedHashMap updateCronerConfig() {
		LinkedHashMap map = new LinkedHashMap<>();
		List list = taskAPI.getTaskCronerConfigList(taskProperties.getProject(), lastUpdateTime);
		if (list != null) {
			for (TaskCronerConfig tcc : list) {
				map.put(tcc.getTaskClass(), tcc);
			}
		}
		return map;
	}

	/**
	 * 更新所有Runner的配置
	 */
	private LinkedHashMap updateRunnerConfig() {
		LinkedHashMap map = new LinkedHashMap<>();
		List list = taskAPI.getTaskRunnerConfigList(taskProperties.getProject(), lastUpdateTime);
		if (list != null) {
			for (TaskRunnerConfig trc : list) {
				map.put(trc.getTaskClass(), trc);
			}
		}
		return map;
	}

	/**
	 * 上传Runner信息。
	 * 
	 * @param runner
	 */
	private void uploadRunnerInfo(TaskRunnerConfig config, TaskContact contact) {
		taskAPI.initTaskRunnerConfig(config);
		taskAPI.initTaskContact(contact);
	}

	/**
	 * 上传Runner信息。
	 * 
	 * @param runner
	 */
	private void uploadCronerInfo(TaskCronerConfig config, TaskContact contact) {
		taskAPI.initTaskCronerConfig(config);
		taskAPI.initTaskContact(contact);
	}

	/**
	 * 构建任务数据类型，并缓存
	 * 
	 * @param taskClass
	 * @param taskRunner
	 */
	private void constructTaskDataType(String taskClass, TaskRunner taskRunner) {
		Type interfaceType = taskRunner.getClass().getGenericSuperclass();
		if (interfaceType instanceof ParameterizedType) {
			Type[] runnerTypes = ((ParameterizedType) interfaceType).getActualTypeArguments();
			JavaType[] paramTypes = new JavaType[runnerTypes.length];
			for (int i = 0; i < runnerTypes.length; i++) {
				if (runnerTypes[i] instanceof ParameterizedType) {
					ParameterizedType pt = ((ParameterizedType) runnerTypes[i]);
					System.out.println(pt.getActualTypeArguments());
					Type[] ts = pt.getActualTypeArguments();
					Class[] cs = new Class[ts.length];
					for (int x = 0; x < ts.length; x++) {
						cs[x] = (Class) ts[x];
					}
					paramTypes[i] = objectMapper.getTypeFactory().constructParametricType((Class) pt.getRawType(),
							cs);
				} else {
					paramTypes[i] = objectMapper.constructType(runnerTypes[i]);
				}
			}
			JavaType taskDataType = objectMapper.getTypeFactory().constructParametricType(TaskData.class, paramTypes);
			dataParamMap.put(taskClass, taskDataType);
		}
	}

	/**
	 * 注册单个任务
	 * 
	 * @param taskClass
	 */
	private void registerTask(String hostConfig, TaskRunnerConfig runnerConfig) {
		if (log.isDebugEnabled()) {
			log.debug("TaskRunner:[{}]正在注册并已启动监听...", runnerConfig.getTaskClass());
		}
		String queueName = runnerConfig.getTaskClass() + "$" + hostConfig;
		// 定义队列
		rabbitAdmin.declareQueue(new Queue(queueName, true));
		// 定义交换机
		rabbitAdmin.declareExchange(new DirectExchange(queueName + "@exchange", true, false));
		// 绑定
		rabbitAdmin.declareBinding(
				new Binding(queueName, DestinationType.QUEUE, queueName + "@exchange", queueName + "@route", null));
		// 启动任务监听器
		SimpleMessageListenerContainer factory = new SimpleMessageListenerContainer();
		factory.setConnectionFactory(connectionFactory);
		factory.setMaxConcurrentConsumers(runnerConfig.getConsumerNum());
		factory.setPrefetchCount(runnerConfig.getConsumerNum());
		factory.setAcknowledgeMode(AcknowledgeMode.AUTO);
		factory.setQueueNames(queueName);
		MessageListenerAdapter listenerAdapter = new MessageListenerAdapter(taskConsumer, "process");
		listenerAdapter.setMessageConverter(new Jackson2JsonMessageConverter());
		// listenerAdapter.setReplyPostProcessor(new GZipPostProcessor());
		factory.setMessageListener(listenerAdapter);
		factory.setMessageConverter(new Jackson2JsonMessageConverter());
		factory.setAfterReceivePostProcessors(new GUnzipPostProcessor());
		factory.setAutoStartup(true);
		factory.afterPropertiesSet();
		factory.start();
		getContainerMap(hostConfig).put(runnerConfig.getTaskClass(), factory);

	}

	/**
	 * 根据配置名获得ContainerMap，如果没有则新建。
	 * 
	 * @param hostConfig
	 * @return
	 */
	private ConcurrentHashMap getContainerMap(String hostConfig) {
		ConcurrentHashMap map = runnerContainerMap.get(hostConfig);
		if (map == null) {
			map = new ConcurrentHashMap<>();
			runnerContainerMap.put(hostConfig, map);
		}
		return map;
	}

	/**
	 * 停止所有的任务。
	 */
	@PreDestroy
	private void stopAllTaskRunner() {
		for (ConcurrentHashMap map : runnerContainerMap.values()) {
			for (SimpleMessageListenerContainer container : map.values()) {
				container.stop();
			}
		}
	}

}