Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.weibo.rill.flow.service.facade.DAGRuntimeFacade Maven / Gradle / Ivy
/*
* Copyright 2021-2023 Weibo, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.weibo.rill.flow.service.facade;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.googlecode.aviator.Expression;
import com.weibo.rill.flow.common.exception.TaskException;
import com.weibo.rill.flow.common.function.ResourceStatus;
import com.weibo.rill.flow.common.model.BizError;
import com.weibo.rill.flow.common.model.DAGRecord;
import com.weibo.rill.flow.interfaces.model.http.HttpParameter;
import com.weibo.rill.flow.interfaces.model.mapping.Mapping;
import com.weibo.rill.flow.interfaces.model.resource.Resource;
import com.weibo.rill.flow.interfaces.model.strategy.Progress;
import com.weibo.rill.flow.interfaces.model.task.*;
import com.weibo.rill.flow.olympicene.core.helper.DAGInfoMaker;
import com.weibo.rill.flow.olympicene.core.helper.DAGWalkHelper;
import com.weibo.rill.flow.olympicene.core.model.dag.DAG;
import com.weibo.rill.flow.olympicene.core.model.dag.DAGInfo;
import com.weibo.rill.flow.olympicene.core.model.dag.DAGInvokeMsg;
import com.weibo.rill.flow.olympicene.core.model.dag.DAGStatus;
import com.weibo.rill.flow.olympicene.core.model.task.TaskCategory;
import com.weibo.rill.flow.olympicene.ddl.parser.DAGStringParser;
import com.weibo.rill.flow.olympicene.ddl.serialize.YAMLMapper;
import com.weibo.rill.flow.olympicene.traversal.helper.ContextHelper;
import com.weibo.rill.flow.olympicene.traversal.mappings.JSONPathInputOutputMapping;
import com.weibo.rill.flow.service.component.DAGToolConverter;
import com.weibo.rill.flow.service.invoke.HttpInvokeHelper;
import com.weibo.rill.flow.service.manager.AviatorCache;
import com.weibo.rill.flow.service.service.DAGDescriptorService;
import com.weibo.rill.flow.service.statistic.DAGResourceStatistic;
import com.weibo.rill.flow.service.statistic.TenantTaskStatistic;
import com.weibo.rill.flow.service.storage.LongTermStorage;
import com.weibo.rill.flow.service.storage.RuntimeStorage;
import com.weibo.rill.flow.service.storage.dao.DAGBusinessDAO;
import com.weibo.rill.flow.service.storage.dao.DAGFeatureDAO;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
@Slf4j
@Service
public class DAGRuntimeFacade {
private static final String TASKS = "tasks";
private static final String CONTEXT = "context";
private final JSONPathInputOutputMapping mapping = new JSONPathInputOutputMapping();
private final DAGWalkHelper dagWalkHelper = DAGWalkHelper.getInstance();
@Autowired
private DAGStringParser dagStringParser;
@Autowired
private RuntimeStorage runtimeStorage;
@Autowired
private LongTermStorage longTermStorage;
@Autowired
private DAGDescriptorService dagDescriptorService;
@Autowired
private DAGResourceStatistic dagResourceStatistic;
@Autowired
private HttpInvokeHelper httpInvokeHelper;
@Autowired
private TenantTaskStatistic tenantTaskStatistic;
@Autowired
private AviatorCache aviatorCache;
@Autowired
private OlympiceneFacade olympiceneFacade;
@Autowired
private DAGBusinessDAO dagBusinessDAO;
@Autowired
private DAGFeatureDAO dagFeatureDAO;
public boolean updateDagStatus(String executionId, DAGStatus status) {
if (StringUtils.isBlank(executionId) || status == null) {
throw new IllegalArgumentException("executionId or status is blank");
}
DAGInfo dagInfo = runtimeStorage.getBasicDAGInfo(executionId);
if (dagInfo == null) {
throw new IllegalArgumentException("executionId not found " + executionId);
}
if (dagInfo.getDagStatus().ordinal() >= status.ordinal()) {
throw new IllegalArgumentException("status is " + dagInfo.getDagStatus() + " now, and cannot be set to " + status);
}
// 任务执行完成后，将执行时间信息写入到 invokeMsg 中便于查询
if (status.isCompleted()) {
DAGInvokeMsg dagInvokeMsg = dagInfo.getDagInvokeMsg();
if (dagInvokeMsg != null && CollectionUtils.isNotEmpty(dagInvokeMsg.getInvokeTimeInfos())) {
List invokeTimeInfos = dagInvokeMsg.getInvokeTimeInfos();
invokeTimeInfos.get(invokeTimeInfos.size() - 1).setEndTimeInMillisecond(System.currentTimeMillis());
dagInvokeMsg.setInvokeTimeInfos(invokeTimeInfos);
}
dagInfo.setDagInvokeMsg(dagInvokeMsg);
}
dagInfo.setDagStatus(status);
runtimeStorage.saveDAGInfo(executionId, dagInfo);
// clear dag info and context after expire time
if (status.isCompleted()) {
runtimeStorage.clearDAGInfo(executionId);
runtimeStorage.clearContext(executionId);
}
return true;
}
public Map convertDAGInfo(String dagDescriptor) {
try {
DAG dag = dagStringParser.parse(dagDescriptor);
DAGInfo dagInfo = new DAGInfoMaker().dag(dag).dagStatus(DAGStatus.NOT_STARTED).make();
return makeDAGInfoMap(dagInfo, false);
} catch (Exception e) {
throw new TaskException(BizError.ERROR_DATA_RESTRICTION, e.getMessage());
}
}
private Map makeDAGInfoMap(DAGInfo dagInfo, boolean brief) {
Map ret = Maps.newHashMap();
if (dagInfo == null) {
ret.put(TASKS, "{}");
return ret;
}
ret.put("workspace", dagInfo.getDag().getWorkspace());
ret.put("dag_name", dagInfo.getDag().getDagName());
ret.put("version", dagInfo.getDag().getVersion());
ret.put("type", dagInfo.getDag().getType().getValue());
ret.put("input_schema", dagInfo.getDag().getInputSchema());
ret.put("dag_invoke_msg", dagInfo.getDagInvokeMsg());
ret.put("dag_status", dagInfo.getDagStatus().name());
ret.put("execution_id", dagInfo.getExecutionId());
ret.put("process", dagProgressCalculate(dagInfo));
if (!brief) {
ret.put(TASKS, dagInfo.getTasks().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, it -> DAGToolConverter.convertTaskInfo(it.getValue()))));
}
return ret;
}
private int dagProgressCalculate(DAGInfo dagInfo) {
if (dagInfo.getDagStatus().isCompleted()) {
return 100;
}
Collection tasks = Optional.ofNullable(dagInfo.getTasks()).map(Map::values).orElse(Collections.emptyList());
if (CollectionUtils.isEmpty(tasks)) {
log.info("dagProcessCalculate cannot get task map");
return 0;
}
try {
return doCalculateProgress(tasks);
} catch (Exception e) {
log.warn("calculate progress by default method, errorMsg:{}", e.getMessage());
int allWeight = tasks.size();
double completeWeight = tasks.stream().filter(task -> task.getTaskStatus().isCompleted()).count();
return (int) (completeWeight * 100 / allWeight);
}
}
private int doCalculateProgress(Collection tasks) {
int allWeight = 0;
double completeWeight = 0D;
for (TaskInfo taskInfo : tasks) {
Progress progress = taskInfo.getTask().getProgress();
int weight = Optional.ofNullable(progress).map(Progress::getWeight).orElse(1);
String calculation = Optional.ofNullable(progress).map(Progress::getCalculation).orElse(null);
allWeight += weight;
if (taskInfo.getTaskStatus().isCompleted()) {
completeWeight += weight;
} else if (StringUtils.isNotBlank(calculation)) {
Map params = Optional.ofNullable(taskInfo.getTaskInvokeMsg())
.map(TaskInvokeMsg::getProgressArgs)
.orElse(Maps.newHashMap());
long taskRunningTimeInMillis = Optional.ofNullable(taskInfo.getTaskInvokeMsg())
.map(TaskInvokeMsg::getInvokeTimeInfos)
.filter(CollectionUtils::isNotEmpty)
.map(it -> it.get(it.size() - 1))
.map(it -> System.currentTimeMillis() - it.getStartTimeInMillisecond())
.orElse(0L);
Map env = Maps.newHashMap();
env.put("params", params);
env.put("taskRunningTimeInMillis", taskRunningTimeInMillis);
Expression expression = aviatorCache.getAviatorExpression(calculation);
String value = String.valueOf(expression.execute(env));
if (NumberUtils.isParsable(value)) {
completeWeight += Double.parseDouble(value) * weight;
}
} else if (MapUtils.isNotEmpty(taskInfo.getSubGroupIndexToStatus())) {
double allGroup = taskInfo.getSubGroupIndexToStatus().size();
long completeGroup = taskInfo.getSubGroupIndexToStatus().values().stream().filter(TaskStatus::isCompleted).count();
completeWeight += completeGroup * weight / allGroup;
}
}
return (int) (completeWeight * 100 / allWeight);
}
public Map getBasicDAGInfo(String executionId, boolean brief) {
DAGInfo dagInfo = runtimeStorage.getBasicDAGInfo(executionId);
if (dagInfo == null) {
dagInfo = longTermStorage.getBasicDAGInfo(executionId);
}
Map result = makeDAGInfoMap(dagInfo, brief);
result.put("context", getContext(executionId, null));
result.put("invoke_summary", tenantTaskStatistic.getFlowAggregate(executionId));
return result;
}
@SuppressWarnings("unchecked")
public Map getContext(String executionId, String taskName) {
if (StringUtils.isBlank(taskName) || dagWalkHelper.isAncestorTask(taskName)) {
Map context = runtimeStorage.getContext(executionId);
if (MapUtils.isEmpty(context)) {
context = longTermStorage.getContext(executionId);
}
return Optional.ofNullable(context).orElse(Maps.newHashMap());
}
String subContextField = dagWalkHelper.buildSubTaskContextFieldName(dagWalkHelper.getRootName(taskName));
Collection fields = ImmutableSet.of(subContextField);
Map groupedContext = runtimeStorage.getContext(executionId, fields);
if (MapUtils.isEmpty(groupedContext)) {
groupedContext = longTermStorage.getContext(executionId, fields);
}
return Optional.ofNullable(groupedContext).map(it -> (Map) it.get(subContextField)).orElse(Maps.newHashMap());
}
public Map getSubContext(String executionId, String parentTaskName, Integer groupIndex) {
String routeName = dagWalkHelper.buildTaskInfoRouteName(parentTaskName, String.valueOf(groupIndex));
String taskName = dagWalkHelper.buildTaskInfoName(routeName, "x");
return getContext(executionId, taskName);
}
public Map getDAGInfoByParentName(String executionId, String parentTaskName, String groupIndex) {
String routeName = dagWalkHelper.buildTaskInfoRouteName(parentTaskName, groupIndex);
String taskName = dagWalkHelper.buildTaskInfoName(routeName, "x");
TaskInfo taskInfo = null;
try {
taskInfo = runtimeStorage.getParentTaskInfoWithSibling(executionId, taskName);
} catch (Exception e) {
// do nothing
}
if (taskInfo == null) {
taskInfo = longTermStorage.getTaskInfo(executionId, parentTaskName, groupIndex);
}
Map ret = Maps.newHashMap();
if (taskInfo == null) {
ret.put(TASKS, "{}");
return ret;
}
ret.put(TASKS, taskInfo.getChildren().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, it -> DAGToolConverter.convertTaskInfo(it.getValue()))));
return ret;
}
public Map mappingEvaluation(String type, String mappingRules, String executionId, String taskName, JSONObject data) {
try {
List rules = Lists.newArrayList();
if (StringUtils.isNotBlank(mappingRules)) {
Mapping[] mappings = YAMLMapper.parseObject(mappingRules, Mapping[].class);
rules.addAll(Lists.newArrayList(mappings));
}
Map context = Optional.ofNullable((Map) data.getJSONObject(CONTEXT))
.orElse(Maps.newHashMap());
Map input = Optional.ofNullable((Map) data.getJSONObject("input"))
.orElse(Maps.newHashMap());
Map output = Optional.ofNullable((Map) data.getJSONObject("output"))
.orElse(Maps.newHashMap());
updateValue(type, executionId, taskName, rules, context, output);
mapping.mapping(context, input, output, rules);
return ImmutableMap.of(CONTEXT, context, "input", input, "output", output);
} catch (Exception e) {
throw new TaskException(BizError.ERROR_DATA_RESTRICTION, ExceptionUtils.getStackTrace(e));
}
}
private void updateValue(String type, String executionId, String taskName, List rules, Map context, Map output) {
if (StringUtils.isBlank(type) || (!type.equals("input_eva") && !type.equals("output_eva"))
|| StringUtils.isBlank(executionId) || StringUtils.isBlank(taskName)) {
return;
}
Optional.ofNullable(context).filter(MapUtils::isEmpty).ifPresent(it -> it.putAll(getContext(executionId, taskName)));
TaskInfo taskInfo = getBasicTaskInfo(executionId, taskName);
if (type.equals("input_eva")) {
Optional.ofNullable(rules).filter(CollectionUtils::isEmpty).ifPresent(it -> it.addAll(taskInfo.getTask().getInputMappings()));
}
if (type.equals("output_eva")) {
Optional.ofNullable(rules).filter(CollectionUtils::isEmpty).ifPresent(it -> it.addAll(taskInfo.getTask().getOutputMappings()));
if (MapUtils.isEmpty(output) && !Objects.equals(taskInfo.getTask().getCategory(), TaskCategory.CHOICE.getValue())
&& !Objects.equals(taskInfo.getTask().getCategory(), TaskCategory.FOREACH.getValue())) {
List> subContextList = ContextHelper.getInstance().getSubContextList(runtimeStorage, executionId, taskInfo);
output.put("sub_context", subContextList);
}
}
}
private TaskInfo getBasicTaskInfo(String executionId, String taskName) {
TaskInfo taskInfo = null;
try {
taskInfo = runtimeStorage.getBasicTaskInfo(executionId, taskName);
} catch (Exception e) {
// do nothing
}
if (taskInfo == null) {
taskInfo = longTermStorage.getBasicTaskInfo(executionId, taskName);
}
if (taskInfo == null) {
throw new TaskException(BizError.ERROR_PROCESS_FAIL.getCode(), String.format("can not get %s taskInfo execuitonId:%s", taskName, executionId));
}
return taskInfo;
}
public Map functionDispatchParams(String executionId, String taskName, JSONObject data) {
try {
AtomicReference task = new AtomicReference<>();
String resourceName = Optional.ofNullable(data.getString("resource_name"))
.orElseGet(() -> {
BaseTask baseTask = getBasicTaskInfo(executionId, taskName).getTask();
task.set(baseTask);
return baseTask instanceof FunctionTask ? ((FunctionTask) baseTask).getResourceName() : "http://mock.function.resource";
});
List inputMappings = Optional.ofNullable(data.getString("input_mapping_rules"))
.map(it -> {
try {
Mapping[] mappings = YAMLMapper.parseObject(it, Mapping[].class);
return Lists.newArrayList(mappings);
} catch (IOException e) {
throw new TaskException(BizError.ERROR_DATA_FORMAT, e.getCause());
}
})
.orElseGet(() -> {
BaseTask baseTask = task.get() != null ? task.get() : getBasicTaskInfo(executionId, taskName).getTask();
return Lists.newArrayList(baseTask.getInputMappings());
});
Map context = Optional.ofNullable((Map) data.getJSONObject(CONTEXT))
.orElseGet(() -> getContext(executionId, taskName));
Resource resource = new Resource(resourceName);
String reqExecutionId = Optional.ofNullable(executionId).orElse("mockExecutionId");
String reqTaskName = Optional.ofNullable(taskName).orElse("mockTaskName");
Map input = Maps.newHashMap();
mapping.mapping(context, input, Maps.newHashMap(), inputMappings);
HttpParameter requestParams = httpInvokeHelper.functionRequestParams(reqExecutionId, reqTaskName, resource, input);
Map queryParams = requestParams.getQueryParams();
Map body = requestParams.getBody();
String url = httpInvokeHelper.buildUrl(resource, queryParams);
return ImmutableMap.of("url", url, "body", body);
} catch (Exception e) {
throw new TaskException(BizError.ERROR_DATA_RESTRICTION, ExceptionUtils.getStackTrace(e));
}
}
public Map dependencyCheck(String descriptorId, String descriptor) {
DAG dag = StringUtils.isNotBlank(descriptorId) ?
dagDescriptorService.getDAG(0L, Collections.emptyMap(), descriptorId) : dagStringParser.parse(descriptor);
Map> dependencies = dagWalkHelper.getDependedResources(dag);
List> resourceToNames = dependencies.entrySet().stream()
.map(entry -> ImmutableMap.of("resource_name", entry.getKey(), "names", entry.getValue()))
.collect(Collectors.toList());
return ImmutableMap.of("dependencies", resourceToNames);
}
public Map runtimeDependentResources(List serviceIds) {
Map ret = Maps.newHashMap();
serviceIds.forEach(serviceId -> {
Map> resourceOrder =
dagResourceStatistic.orderDependentResources(serviceId);
ret.put(serviceId, resourceOrder);
});
ret.put("current_time", System.currentTimeMillis());
return ret;
}
public Map clearRuntimeResources(String serviceId, boolean clearAll, List resourceNames) {
return ImmutableMap.of("ret", dagResourceStatistic.clearRuntimeResources(serviceId, clearAll, resourceNames));
}
public Map businessInvokeSummary(String businessKey) {
return ImmutableMap.of("ret", tenantTaskStatistic.getBusinessAggregate(businessKey));
}
public Map getExecutionIds(String executionId, String business, String feature, String status, String code, Long startTime, Long endTime, Integer current, Integer pageSize) {
if (StringUtils.isNotEmpty(executionId)) {
DAGInfo dagInfo = runtimeStorage.getBasicDAGInfo(executionId);
if (Objects.isNull(dagInfo)) {
dagInfo = longTermStorage.getBasicDAGInfo(executionId);
}
if (Objects.isNull(dagInfo)){
return Map.of("total", 0, "items", Lists.newArrayList());
}
JSONObject executionItem = new JSONObject();
executionItem.put("id", 1);
executionItem.put("execution_id", dagInfo.getExecutionId());
executionItem.put("submit_time", dagInfo.getDagInvokeMsg().getInvokeTimeInfos().get(0).getStartTimeInMillisecond());
executionItem.put("business_id", dagInfo.getDag().getWorkspace());
executionItem.put("feature_id", dagInfo.getDag().getDagName());
executionItem.put("status", dagInfo.getDagStatus());
return Map.of("total", 1, "items", List.of(executionItem));
}
List dagRecordList = new ArrayList<>();
if (StringUtils.isNotEmpty(business) && StringUtils.isNotEmpty(feature)) {
dagRecordList.add(DAGRecord.builder()
.businessId(business)
.featureId(feature)
.build());
} else if (StringUtils.isNotEmpty(business) && StringUtils.isEmpty(feature)) {
dagFeatureDAO.getFeature(business).forEach(featureId -> {
DAGRecord record = DAGRecord.builder()
.businessId(business)
.featureId(featureId)
.build();
dagRecordList.add(record);
});
} else {
dagBusinessDAO.getBusiness().forEach(businessId -> dagFeatureDAO.getFeature(businessId).forEach(featureId -> {
DAGRecord record = DAGRecord.builder()
.businessId(businessId)
.featureId(featureId)
.build();
dagRecordList.add(record);
}));
}
List dagStatuses = Lists.newArrayList();
Optional.ofNullable(DAGStatus.parse(status))
.ifPresentOrElse(
dagStatuses::add,
() -> dagStatuses.addAll(Arrays.asList(DAGStatus.values()))
);
long time = Optional.ofNullable(endTime).orElse(System.currentTimeMillis());
List executionCountList = new ArrayList<>();
dagRecordList.forEach(record -> {
String serviceId = record.getBusinessId() + ":" + record.getFeatureId();
dagStatuses.forEach(dagStatus -> {
Map executionCount = olympiceneFacade.getExecutionIdsForBg(serviceId, dagStatus, code, time, current, pageSize);
JSONArray executionIds = Optional.ofNullable(executionCount)
.map(JSONObject::new)
.map(it -> it.getJSONArray("execution_ids"))
.orElse(new JSONArray());
for (int i = 0; i < executionIds.size(); i++) {
JSONObject executionInfo = executionIds.getJSONObject(i);
JSONObject executionItem = new JSONObject();
executionItem.put("id", i + 1);
executionItem.put("execution_id", executionInfo.getString("execution_id"));
executionItem.put("submit_time", executionInfo.get("submit_time"));
executionItem.put("business_id", record.getBusinessId());
executionItem.put("feature_id", record.getFeatureId());
executionItem.put("status", dagStatus.getValue());
executionCountList.add(executionItem);
}
});
}
);
List executionCountResult = executionCountList.stream()
.filter(it -> (startTime.compareTo(it.getLong("submit_time"))) < 0)
.sorted(Comparator.comparing(item -> item.getLong("submit_time"), (s, t) -> Long.compare(t, s)))
.skip((long) (current - 1) * pageSize)
.limit(pageSize)
.toList();
return Map.of("items", executionCountResult, "total", executionCountList.size());
}
}
