net.wicp.tams.common.http.flink.FlinkRestApi Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of common-http Show documentation
The newest version!
package net.wicp.tams.common.http.flink;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.tuple.Pair;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;

import lombok.extern.slf4j.Slf4j;
import net.wicp.tams.common.Result;
import net.wicp.tams.common.apiext.IOUtil;
import net.wicp.tams.common.apiext.StringUtil;
import net.wicp.tams.common.apiext.json.JSONUtil;
import net.wicp.tams.common.exception.ExceptAll;
import net.wicp.tams.common.exception.ProjectExceptionRuntime;
import net.wicp.tams.common.http.HttpClient;
import net.wicp.tams.common.http.HttpResult;

//https://nightlies.apache.org/flink/flink-docs-release-1.17/docs/ops/rest_api/#jars-jarid-run
@Slf4j
public class FlinkRestApi {
	private final String jmUrl;

	public FlinkRestApi(String jmUrl) {
		this.jmUrl = jmUrl;
	}

	/**
	 * 提交任务
	 * 
	 * @param taskName
	 * @param parallelism
	 * @param params
	 * @return
	 */
	public Result submitTask(String taskName, String jarId, int parallelism, String savepointPath,
			Map params) {
		ObjectNode inputParam = JsonNodeFactory.instance.objectNode();
		inputParam.put("entryClass", "net.wicp.tams.common.flink.sqlrun.SqlRunner");
		StringBuffer buff = new StringBuffer(String.format("--taskId %s", taskName));
		if (MapUtils.isNotEmpty(params)) {
			params.remove("taskId");// 以taskName为主
			for (String key : params.keySet()) {
				buff.append(" " + String.format("--%s %s", key, params.get(key)));
			}
		}
		inputParam.put("programArgs", buff.toString());
		inputParam.put("parallelism", parallelism);
		if (StringUtil.isNotNull(savepointPath)) {
			inputParam.put("savepointPath", savepointPath);
		}
		String url = IOUtil.mergeFolderAndFilePath(this.jmUrl, String.format("/jars/%s/run", jarId));
		HttpResult doPost = HttpClient.doPost(url, inputParam);
		ObjectNode parseObject = (ObjectNode) JSONUtil.parserStr(new ObjectMapper(), doPost.getBodyStr());// 成功后会返回jobId

		if (parseObject.has("errors")) {
			if (parseObject.get("errors").asText().contains("DetachedApplicationRunner.tryExecuteJobs")) {// 不处理只报错误，其实是已经提交成功的，
				log.error("调用submitTask失败:", parseObject.get("errors").asText());
				return Result.getSuc();
			} else {
				return Result.getError(parseObject.get("errors").asText());
			}
		}
		return Result.getSuc();
	}

	public Result submitTask(String taskName, int parallelism, String savepointPath, Map params) {
		return submitTask(taskName, "tams_flink-sql-client-1.17.1.jar", parallelism, savepointPath, params);
	}

	public Result submitTask(String taskName) {
		return submitTask(taskName, 1, null, new HashMap<>());
	}

	public List findJobIds(String status) {
		String url = IOUtil.mergeFolderAndFilePath(this.jmUrl, "/jobs");
		HttpResult doGet = HttpClient.doGet(url);
		// "enum" : [ "INITIALIZING", "CREATED", "RUNNING", "FAILING", "FAILED",
		// "CANCELLING", "CANCELED", "FINISHED", "RESTARTING", "SUSPENDED",
		// "RECONCILING" ]
		ObjectNode retObj = null;
		ObjectMapper objectMapper = new ObjectMapper();
		try {
			retObj = (ObjectNode) JSONUtil.parserStr(objectMapper, doGet.getBodyStr());
		} catch (Exception e) {
			throw new ProjectExceptionRuntime(ExceptAll.Project_default, "找job失败", e);
		}
		ArrayNode res = (ArrayNode) retObj.get("jobs");
		List retlist = new ArrayList<>();
		for (int i = 0; i < res.size(); i++) {
			ObjectNode jsonObject = (ObjectNode) res.get(i);
			if (StringUtil.isNull(status) || status.equalsIgnoreCase(jsonObject.get("status").asText())) {
				retlist.add(jsonObject.get("id").asText());
			}
		}
		return retlist;
	}

	public Map findJobNames(String... status) {
		String url = IOUtil.mergeFolderAndFilePath(this.jmUrl, "/jobs/overview");
		HttpResult doGet = HttpClient.doGet(url);
		// "enum" : [ "INITIALIZING", "CREATED", "RUNNING", "FAILING", "FAILED",
		// "CANCELLING", "CANCELED", "FINISHED", "RESTARTING", "SUSPENDED",
		// "RECONCILING" ]
		ObjectNode retObj = null;
		try {
			retObj = (ObjectNode)JSONUtil.parserStr(new ObjectMapper(), doGet.getBodyStr());
		} catch (Exception e) {
			throw new ProjectExceptionRuntime(ExceptAll.Project_default, "找job失败", e);
		}
		ArrayNode res =(ArrayNode) retObj.get("jobs");
		Map retmap = new HashMap<>();
		for (int i = 0; i < res.size(); i++) {
			ObjectNode jsonObject = (ObjectNode)res.get(i);
			if (StringUtil.isNull(status) || ArrayUtils.contains(status, jsonObject.get("state").asText())) {
				retmap.put(jsonObject.get("name").asText(), jsonObject.get("state").asText());
			}
		}
		return retmap;
	}

	// 找到运行中或重启中（已部署）的任务
	public Map findDeployJobNames() {
		return findJobNames(new String[] { "RUNNING", "RESTARTING" });
	}

	// L:总槽数 R:可用槽数
	public Pair flindSlotInfo() {
		String url = IOUtil.mergeFolderAndFilePath(this.jmUrl, "/overview");
		HttpResult doGet = HttpClient.doGet(url);
		ObjectNode retObj = null;
		try {
			retObj = JSONUtil.parserStr(new ObjectMapper(), doGet.getBodyStr()) ;
		} catch (Exception e) {
			throw new ProjectExceptionRuntime(ExceptAll.Project_default, "找job失败", e);
		}
		return Pair.of(retObj.get("slots-total").asInt(), retObj.get("slots-available").asInt());
	}

	public Result cancelTask(String jobId) {
		HttpResult doPatch = HttpClient
				.doPatch(IOUtil.mergeFolderAndFilePath(this.jmUrl, String.format("/jobs/%s?mode=cancel", jobId)));
		ObjectNode parseObject=JSONUtil.parserStr(new ObjectMapper(), doPatch.getBodyStr());
		if (parseObject.isEmpty()) {
			return Result.getSuc();
		} else {
			return Result.getError(parseObject.get("errors").asText());
		}
	}

	// 已运行的job数
	public int hasRunningJobs() {
		List runjobs = findJobIds("RUNNING");
		return runjobs.size();
	}

	/***
	 * sucess retObjs[0] 为取消的jobid
	 */
	public Result cancelTaskFrist() {
		List runjobs = findJobIds("RUNNING");
		if (CollectionUtils.isEmpty(runjobs)) {
			return Result.getError("没有任务任务");
		}
		Result cancelTask = cancelTask(runjobs.get(0));
		if (cancelTask.isSuc()) {
			cancelTask.setRetObjs(runjobs.get(0));
		}
		return cancelTask;
	}

}