net.wicp.tams.common.binlog.alone.PluginAssit Maven / Gradle / Ivy
The newest version!
/*
* **********************************************************************
* Copyright (c) 2022 .
* All rights reserved.
* 项目名称：common
* 项目描述：公共的工具集
* 版权说明：本软件属andy.zhou([email protected])所有。
* ***********************************************************************
*/
package net.wicp.tams.common.binlog.alone;
import java.io.Serializable;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.tuple.Pair;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.protobuf.ExtensionRegistry;
import com.google.protobuf.Message;
import com.googlecode.protobuf.format.JsonFormat;
import lombok.extern.slf4j.Slf4j;
import net.wicp.tams.common.Conf;
import net.wicp.tams.common.apiext.CollectionUtil;
import net.wicp.tams.common.apiext.ReflectAssist;
import net.wicp.tams.common.apiext.StringUtil;
import net.wicp.tams.common.apiext.jdbc.JdbcAssit;
import net.wicp.tams.common.apiext.jdbc.MySqlAssit;
import net.wicp.tams.common.apiext.jdbc.MySqlColBean;
import net.wicp.tams.common.binlog.alone.ListenerConf.ColHis;
import net.wicp.tams.common.binlog.alone.ListenerConf.DuckulaEvent;
import net.wicp.tams.common.binlog.alone.ListenerConf.DuckulaEventItem;
import net.wicp.tams.common.binlog.alone.ListenerConf.OptType;
import net.wicp.tams.common.binlog.alone.ListenerConf.Position;
import net.wicp.tams.common.binlog.alone.ListenerConf.Position.Builder;
import net.wicp.tams.common.binlog.alone.binlog.bean.Rule;
import net.wicp.tams.common.binlog.alone.binlog.bean.RuleItem;
import net.wicp.tams.common.callback.IDbTbConvert;
import net.wicp.tams.common.constant.DateFormatCase;
import net.wicp.tams.common.constant.FieldFormart;
import net.wicp.tams.common.constant.dbType.BinlogType;
import net.wicp.tams.common.constant.dic.YesOrNo;
@Slf4j
public class PluginAssit {
// 工具用方法
public static ColHis convertCosHis(String host, String db, String tb, long time, List colList,
YesOrNo isRds) {
net.wicp.tams.common.binlog.alone.ListenerConf.ColHis.Builder ColHisBuilder = ColHis.newBuilder();
ColHisBuilder.setTime(time);
ColHisBuilder.setTimeStr(DateFormatCase.YYYY_MM_DD_hhmmss.getInstanc()
.format(time == -1 ? System.currentTimeMillis() : time * 1000));
ColHisBuilder.setServerIp(host);
ColHisBuilder.setDb(db);
ColHisBuilder.setTb(tb);
List namelist = new ArrayList();
List datatypelist = new ArrayList();
List coltypelist = new ArrayList();
List keyindexList = new ArrayList();
for (int i = 0; i < colList.size(); i++) {
MySqlColBean ele = colList.get(i);
namelist.add(ele.getColumnNameStandard(FieldFormart.ori));
datatypelist.add(ele.getDataType());
coltypelist.add(ele.getColumnType());
if (ele.isPri()) {
keyindexList.add(i);
}
}
if (YesOrNo.yes == isRds && CollectionUtils.isEmpty(keyindexList)) {// 是rds,最后一个字段是rowkey
namelist.add("_rowkey_");
datatypelist.add("varchar");
coltypelist.add("varchar(255)");
keyindexList.add(namelist.size() - 1);
}
ColHisBuilder.addAllCols(namelist);
ColHisBuilder.addAllColTypes(datatypelist);
ColHisBuilder.addAllColTypes2(coltypelist);
ColHisBuilder.addAllKeyIndex(keyindexList);
ColHis retobj = ColHisBuilder.build();
return retobj;
}
public static String toJsonStr(final Message message) {
String json = new JsonFormat().printToString(message);
return json;
}
// 不分隔数据时的key
public static String getKeySplitNo(String db, String tb) {
final String kafkaKey = String.format("%s|%s|%s", db, tb,
DateFormatCase.yyyyMMddHHmmss.getInstanc().format(System.currentTimeMillis()));
return kafkaKey;
}
//分隔数据时的key
public static ObjectNode getKeySplit(List> keyValues) {
ObjectNode retjson = JsonNodeFactory.instance.objectNode();
for (Pair keyValue : keyValues) {
retjson.put(keyValue.getLeft(), String.valueOf(keyValue.getRight()));
}
return retjson;
}
/***
* 把jsonl转为message
*
* @param jsonstr
* @param clazz
* @return
*/
@SuppressWarnings("unchecked")
public static T toMessage(String jsonstr, Class clazz) {
if (StringUtil.isNull(jsonstr) || clazz == null) {
return null;
}
if (jsonstr.startsWith("\"")) {
jsonstr = jsonstr.substring(1);
}
if (jsonstr.endsWith("\"")) {
jsonstr = jsonstr.substring(0, jsonstr.length() - 1);
}
jsonstr = jsonstr.replace("\\\"", "\"");
try {
Message.Builder builder = (Message.Builder) ReflectAssist.invokeStaticMothed(clazz.getName(), "newBuilder");
new JsonFormat().merge(jsonstr, ExtensionRegistry.newInstance(), builder);
return (T) builder.build();
} catch (Exception e) {
log.error("转换异常", e);
return null;
}
}
/// addProp用于drds,得到插件用的jdbc配置信息，用于生成连接池，L:池的key R:池用的属性
public static List getJdbcPluginDataSourceConf() {
Map> preGroup = Conf.getPreGroup("common.binlog.alone.plugin.jdbc.");
List retlist = new ArrayList();
for (String dbinstanceid : preGroup.keySet()) {
Properties prop = new Properties();
for (String key : preGroup.get(dbinstanceid).keySet()) {
prop.put(String.format("common.jdbc.datasource.%s.%s", dbinstanceid, key),
preGroup.get(dbinstanceid).get(key));
}
Conf.overProp(prop);
retlist.add(dbinstanceid);
}
return retlist;
}
private static Map baseBuilder = new HashMap();
// 得到级联的数据
public static Map proCascadeDatas(Connection connection, DuckulaEvent duckulaEvent, int index,
Rule rule) {
if (CollectionUtils.isNotEmpty(rule.getSubRules())&&(duckulaEvent.getOptType()==OptType.update||duckulaEvent.getOptType()==OptType.delete)) {//级联只处理修改的数据
String[] relakeyAry=null;
if(rule.getItems().containsKey(RuleItem.relakey)) {
relakeyAry=rule.getItems().get(RuleItem.relakey).split(",");
}else {//没有配置就是找主键
relakeyAry=DuckulaAssit.getKeyColname(duckulaEvent);
}
Object[] relakeyValues=new Object[relakeyAry.length];
for (int i = 0; i < relakeyValues.length; i++) {
relakeyValues[i]=DuckulaAssit.getValue(duckulaEvent, index, relakeyAry[i]);
}
Map retmap = new HashMap();
for (Rule subRule : rule.getSubRules()) {// 子规则
// 找级联数据
String packCascadeSql = subRule.packCascadeSql();
List> ress = JdbcAssit.querySqlMapPre(connection, packCascadeSql, false,
relakeyValues);
if (CollectionUtils.isEmpty(ress)) {
return null;
}
// 组装 base Builder
net.wicp.tams.common.binlog.alone.ListenerConf.DuckulaEvent.Builder builder = null;
if (baseBuilder.containsKey(subRule)) {// 加缓存
builder = baseBuilder.get(subRule).clone();
} else {
String db=subRule.getDbOri();
String tb=subRule.getTbOri();
builder = DuckulaEvent.newBuilder();
builder.setDb(db);
builder.setTb(tb);
builder.setDbInstanceId(duckulaEvent.getDbInstanceId());//级联的DbInstanceId与父一致
builder.setOptType(duckulaEvent.getOptType());//级联操作类型与主流保持一致
String[] primarys = MySqlAssit.getPrimary(connection, db, tb);
// 列类型组装
Pair needColAndType = MySqlAssit.needColAndTypeMysql(connection,
db, tb,null,rule.getItems().get(RuleItem.routeColName),
primarys);
builder.addAllCols(Arrays.asList(needColAndType.getLeft()));
for (BinlogType binlogType : needColAndType.getRight()) {
builder.addColsType(
net.wicp.tams.common.binlog.alone.ListenerConf.ColumnType.valueOf(binlogType.name()));
}
builder.setColNum(needColAndType.getLeft().length);
for (String primary : primarys) {
int indexOf = ArrayUtils.indexOf(needColAndType.getLeft(), primary);
builder.addKeyindexs(indexOf);
}
builder.setIsError(false);
builder.setIsCascade(true);
builder.setCommitTime(System.currentTimeMillis());
baseBuilder.put(subRule, builder.clone());
}
// 组装详细的Builder
for (Map res : ress) {
net.wicp.tams.common.binlog.alone.ListenerConf.DuckulaEventItem.Builder itemBuilder = DuckulaEventItem
.newBuilder();
CollectionUtil.filterNull(res, 1);
itemBuilder.putAllAfter(res);
builder.addItems(itemBuilder);
}
retmap.put(subRule, builder.build());
}
return retmap;
} else {
return null;
}
}
/***
* 得到新的dbtb名。
*
* @param rule
* @param duckulaEvent
* @return
*/
public static Pair getNewDbTb(Rule rule, DuckulaEvent duckulaEvent) {
String newDb = rule.getItems().get(RuleItem.db);
String newTb = rule.getItems().get(RuleItem.tb);
IDbTbConvert dbTbConvert = rule.getDbTbConvert();
if (dbTbConvert != null) {
if (StringUtil.isNull(newTb) || "*".equals(newTb)) {
Pair dbAndTb = dbTbConvert.getDbAndTb(rule.getItems().get(RuleItem.db),
Pair.of(duckulaEvent.getDb(), duckulaEvent.getTb()));
newDb = dbAndTb.getLeft();
newTb = dbAndTb.getRight();
}
} else {
if (StringUtil.isNull(newTb)) {
newTb = duckulaEvent.getTb();
}
if (StringUtil.isNull(newDb)) {
newDb = duckulaEvent.getDb();
}
}
return Pair.of(newDb, newTb);
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
public static void setCheckPoint(Position.Builder lastGtidOver) {
Conf.overProp("lastGtidOver.enable", "true", "lastGtidOver.gtid", lastGtidOver.getGtids(), "lastGtidOver.time",
String.valueOf(lastGtidOver.getTime()), "lastGtidOver.timeStr", lastGtidOver.getTimeStr());
}
public static void cleanCheckPoint() {
Conf.overProp("lastGtidOver.enable", "false", "lastGtidOver.gtid", "", "lastGtidOver.time", "",
"lastGtidOver.timeStr", "");
}
public static Position.Builder getCheckPoint() {
boolean enable = Boolean.parseBoolean(Conf.get("lastGtidOver.enable", "false"));
if (enable) {
Builder newBuilder = Position.newBuilder();
newBuilder.setGtids(Conf.get("lastGtidOver.gtid"));
newBuilder.setTime(Conf.getLong("lastGtidOver.time"));
newBuilder.setTimeStr(Conf.get("lastGtidOver.timeStr"));
return newBuilder;
} else {
return null;
}
}
/*
* public static Connection getMainConn(ConnConf.Builder connConf, String
* defaultdb) { String url = null; if (StringUtil.isNotNull(defaultdb)) { url =
* String.format(
* "jdbc:mysql://%s:%s/%s?autoReconnect=true&useUnicode=true&characterEncoding=utf-8",
* connConf.getHost(), connConf.getPort(), defaultdb); } else { url =
* String.format(
* "jdbc:mysql://%s:%s?autoReconnect=true&useUnicode=true&characterEncoding=utf-8",
* connConf.getHost(), connConf.getPort()); } java.sql.Connection conn =
* JdbcConnection.getConnection("com.mysql.jdbc.Driver", url,
* connConf.getUsername(), connConf.getPassword()); return conn; }
*/
}