Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.alibaba.schedulerx.worker.timer.TaskMasterUpdateWorkersTimer Maven / Gradle / Ivy
package com.alibaba.schedulerx.worker.timer;
import com.alibaba.schedulerx.common.domain.JSONResult;
import com.alibaba.schedulerx.common.domain.JobInstanceInfo;
import com.alibaba.schedulerx.common.domain.ResponseCode;
import com.alibaba.schedulerx.common.domain.RouteStrategyInfoForSync;
import com.alibaba.schedulerx.common.domain.TriggerType;
import com.alibaba.schedulerx.common.util.ConfigUtil;
import com.alibaba.schedulerx.common.util.JsonUtil;
import com.alibaba.schedulerx.common.util.UnirestUtil;
import com.alibaba.schedulerx.worker.discovery.GroupManager;
import com.alibaba.schedulerx.worker.domain.WorkerConstants;
import com.alibaba.schedulerx.worker.log.LogFactory;
import com.alibaba.schedulerx.worker.log.Logger;
import com.alibaba.schedulerx.worker.master.TaskMaster;
import com.alibaba.schedulerx.worker.master.TaskMasterPool;
import com.alibaba.schedulerx.worker.route.Router;
import com.alibaba.schedulerx.worker.route.RouterFactory;
import com.alibaba.schedulerx.worker.route.RouterManager;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.ListUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.net.URLEncoder;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
/**
* update running job instance workers
* @author zhaibian
* @version $Id: TaskMasterUpdateWorkersTimer.java, v 0.1 2019年03月06日 11:44 zhaibian Exp $
*/
public class TaskMasterUpdateWorkersTimer extends AbstractTimerTask {
private TaskMasterPool masterPool = TaskMasterPool.INSTANCE;
private RouterManager routerManager = RouterManager.INSTANCE;
protected static final Logger LOGGER = LogFactory.getLogger(TaskMasterUpdateWorkersTimer.class);
private static int errorCount = 0;
@Override
public String getName() {
return "TaskMasterUpdateWorkersTimer";
}
@Override
public long getInitialDelay() {
return 60;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public void run() {
try {
Collection allTaskMaster = masterPool.getAllTaskMaster();
if (CollectionUtils.isEmpty(allTaskMaster)) {
return;
}
// jobId -> jobInstance list
Map> jobInstanceMap = Maps.newHashMap();
// appGroupId -> jobId list
Map> appGroupJob = Maps.newHashMap();
// appGroupId -> group_id
Map appGroupIdMap = Maps.newHashMap();
for (TaskMaster taskMaster : allTaskMaster) {
Long jobId = taskMaster.getJobInstanceInfo().getJobId();
if (TriggerType.MANUAL.getValue() == taskMaster.getJobInstanceInfo().getTriggerType()) {
// 手动运行时不更新worker列表，防止指定的机器列表发生变化
continue;
}
List list = jobInstanceMap.get(jobId);
if (list == null) {
list = Lists.newArrayList();
jobInstanceMap.put(jobId, list);
}
list.add(taskMaster.getJobInstanceInfo());
Long appGroupId = taskMaster.getJobInstanceInfo().getAppGroupId();
String groupId = taskMaster.getJobInstanceInfo().getGroupId();
appGroupIdMap.put(appGroupId, groupId);
List agList = appGroupJob.get(appGroupId);
if (agList == null) {
agList = Lists.newArrayList();
appGroupJob.put(appGroupId, agList);
}
agList.add(jobId);
}
// grep has designate jobIds
// 通过rpc调用获取一个应用下被打标的jobId和未被打标的jobId
Map> hasNotDesignatedAppGroupJob = Maps.newHashMap();
List hasDesignatedJobIds = Lists.newArrayList();
List hasRouteStrategyJobIds = Lists.newArrayList();
for (Entry> entry : appGroupJob.entrySet()) {
// value maybe int type
String groupId = appGroupIdMap.get(entry.getKey());
JSONResult result = grepHasDesignateJobIds(entry.getKey(), entry.getValue(), GroupManager.INSTANCE.getAppKeyByGroupId(groupId));
Set idsWithDesignate = Sets.newHashSet();
if (result != null && result.getData() != null) {
if (result.getCode() == ResponseCode.JOB_HAS_ROUTE_STRATEGY) {
Map> idsMap = JsonUtil.fromJson(
JsonUtil.toJson(result.getData()), new TypeToken>>() {}.getType());
idsWithDesignate = idsMap.get("designate");
Set idsWithRouteStrategy = idsMap.get("routeStrategy");
if (CollectionUtils.isNotEmpty(idsWithRouteStrategy)) {
Set idSet = new HashSet<>(idsWithRouteStrategy.size());
for (Number n : idsWithRouteStrategy) {
idSet.add(n.longValue());
}
hasRouteStrategyJobIds.addAll(idSet);
}
} else {
if (result.getData() instanceof Collection) {
idsWithDesignate = new HashSet<>((Collection)result.getData());
}
}
}
if (CollectionUtils.isNotEmpty(idsWithDesignate)) {
Set idSet = new HashSet<>(idsWithDesignate.size());
for (Number n : idsWithDesignate) {
idSet.add(n.longValue());
}
hasDesignatedJobIds.addAll(idSet);
hasNotDesignatedAppGroupJob.put(entry.getKey(), ListUtils.removeAll(entry.getValue(), idSet));
} else {
hasNotDesignatedAppGroupJob.put(entry.getKey(), entry.getValue());
}
}
// update has not designated jobs workers
if (MapUtils.isNotEmpty(hasNotDesignatedAppGroupJob)) {
for (Entry> entry : hasNotDesignatedAppGroupJob.entrySet()) {
String groupId = appGroupIdMap.get(entry.getKey());
Set allWorkers = getAllWorkers(entry.getKey(), -1L, GroupManager.INSTANCE.getAppKeyByGroupId(groupId));
for (Long jobId : entry.getValue()) {
if (hasRouteStrategyJobIds.contains(jobId)) {
RouteStrategyInfoForSync strategyInfoForSync = getRouteStrategyInfoForSync(entry.getKey(), jobId, GroupManager.INSTANCE.getAppKeyByGroupId(groupId));
updateWorkers(allWorkers, jobInstanceMap.get(jobId), strategyInfoForSync);
} else {
updateWorkers(allWorkers, jobInstanceMap.get(jobId));
}
}
}
}
// update has designated job workers
for (Long jobId : hasDesignatedJobIds) {
Long appGroupId = jobInstanceMap.get(jobId).get(0).getAppGroupId();
String groupId = appGroupIdMap.get(appGroupId);
Set allWorkers = getAllWorkers(appGroupId, jobId, GroupManager.INSTANCE.getAppKeyByGroupId(groupId));
updateWorkers(allWorkers, jobInstanceMap.get(jobId));
}
errorCount = 0;
} catch (Exception ex) {
if(errorCount++ < 10) {
LOGGER.warn("can not update master workers. {}", ex.getMessage());
} else {
LOGGER.error("update master workers error.", ex);
}
}
}
private JSONResult grepHasDesignateJobIds(Long appGroupId, List jobIds, String appKey) throws Exception{
String url = "http://{0}/app/grepHasDesignateJobIds.json?appGroupId={1}&jobIds={2}&appKey={3}"
+ "&grepRouteStrategyJobIds=true";
return UnirestUtil.getResult(url, null,
ConfigUtil.getWorkerConfig().getString(WorkerConstants.WORKER_DOMAIN_NAME),
appGroupId, StringUtils.join(jobIds, ","), URLEncoder.encode((appKey == null) ? "" : appKey, "UTF-8"));
}
private void updateWorkers(Set allWorkers, List instanceInfos) {
updateWorkers(allWorkers, instanceInfos, null);
}
private void updateWorkers(Set allWorkers, List instanceInfos,
RouteStrategyInfoForSync strategyInfoForSync) {
if (CollectionUtils.isEmpty(allWorkers)) {
return;
}
for (JobInstanceInfo instanceInfo : instanceInfos) {
if (instanceInfo.getAllWorkers().size() != allWorkers.size()
|| !instanceInfo.getAllWorkers().containsAll(allWorkers)) {
instanceInfo.setAllWorkers(Lists.newCopyOnWriteArrayList(allWorkers));
LOGGER.info("update appGroupId={} instanceId={} workers.", instanceInfo.getAppGroupId(),
instanceInfo.getJobInstanceId());
}
if (isRouteStrategyInfoNeedSync(instanceInfo, strategyInfoForSync)) {
// type和content更新时由于会影响strategyList的值，需要重新生成一个router
instanceInfo.setRouteStrategyType(strategyInfoForSync.getType());
instanceInfo.setRouteStrategyContent(strategyInfoForSync.getStrategyContent());
Router router = RouterFactory.buildRouter(strategyInfoForSync.getType(), strategyInfoForSync.getStrategyContent());
routerManager.updateRouter(instanceInfo.getAppGroupId(), instanceInfo.getJobId(), router);
}
if (strategyInfoForSync != null && strategyInfoForSync.getTargetWorkerAddrsMap() != null) {
instanceInfo.setTargetWorkerAddrsMap(strategyInfoForSync.getTargetWorkerAddrsMap());
}
}
}
private Set getAllWorkers(Long appGroupId, Long jobId, String appKey) throws Exception {
String url = "http://{0}/app/getAllUsefulWorkerList.json?appGroupId={1}&jobId={2}&appKey={3}";
try {
return UnirestUtil.getSetData(url,
ConfigUtil.getWorkerConfig().getString(WorkerConstants.WORKER_DOMAIN_NAME),
appGroupId, jobId, URLEncoder.encode((appKey == null) ? "" : appKey, "UTF-8"));
} catch (Exception ex) {
if(errorCount++ < 10) {
LOGGER.warn("getAllWorkers failed. {}", ex.getMessage());
} else {
LOGGER.error("getAllWorkers failed.", ex);
}
}
return null;
}
private Map> getTargetWorkerAddrsMap(Long appGroupId, Long jobId, String appKey) throws Exception {
String url = "http://{0}/worker/v1/appgroup/getTaregtWorkerAddrsMap.json?appGroupId={1}&jobId={2}&appKey={3}";
return UnirestUtil.getMapData(url,
ConfigUtil.getWorkerConfig().getString(WorkerConstants.WORKER_DOMAIN_NAME),
appGroupId, jobId, URLEncoder.encode((appKey == null) ? "" : appKey, "UTF-8"));
}
private RouteStrategyInfoForSync getRouteStrategyInfoForSync(Long appGroupId, Long jobId, String appKey) throws Exception {
String url = "http://{0}/worker/v1/appgroup/getRouteStrategyInfoForSync.json?appGroupId={1}&jobId={2}&appKey={3}";
JSONResult result = UnirestUtil.getResult(url, null,
ConfigUtil.getWorkerConfig().getString(WorkerConstants.WORKER_DOMAIN_NAME),
appGroupId, jobId, URLEncoder.encode((appKey == null) ? "" : appKey, "UTF-8"));
if (result.isSuccess()) {
String jsonStr = JsonUtil.toJson(result.getData());
RouteStrategyInfoForSync routeStrategyInfoForSync = JsonUtil.fromJson(jsonStr, RouteStrategyInfoForSync.class);
return routeStrategyInfoForSync;
} else {
return null;
}
}
private boolean isRouteStrategyInfoNeedSync(JobInstanceInfo instanceInfo, RouteStrategyInfoForSync routeStrategyInfoForSync) {
if (routeStrategyInfoForSync == null) {
return false;
}
if (!instanceInfo.getRouteStrategyType().equals(routeStrategyInfoForSync.getType()) ||
!instanceInfo.getRouteStrategyContent().equals(routeStrategyInfoForSync.getStrategyContent())) {
return true;
}
return false;
}
}
