All Downloads are FREE. Search and download functionalities are using the official Maven repository.
Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.yanyun.log.service.MQDealService Maven / Gradle / Ivy
package com.yanyun.log.service;
import com.alibaba.fastjson.JSON;
import com.yanyun.log.configuration.properties.ElasticSearchProperties;
import com.yanyun.log.configuration.properties.HbaseProperties;
import com.yanyun.log.configuration.properties.KubeMqProperties;
import com.yanyun.log.model.LogModel;
import io.kubemq.sdk.basic.ServerAddressNotSuppliedException;
import io.kubemq.sdk.queue.Message;
import io.kubemq.sdk.queue.Queue;
import io.kubemq.sdk.queue.ReceiveMessagesResponse;
import io.kubemq.sdk.tools.Converter;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLException;
import java.io.IOException;
import java.util.concurrent.ExecutorService;
/**
* 处理MQ消息的服务
*/
public class MQDealService {
private static Logger logger = LoggerFactory.getLogger(MQDealService.class);
private KubeMqProperties kubeMqProperties;
private ElasticSearchProperties elasticSearchProperties;
private RestHighLevelClient restHighLevelClient;
private HbaseProperties hbaseProperties;
private Connection hbaseConnection;
private IndexRequest request;
private TableName tableName;
private Queue queue;
//ES的作业线程池
private ExecutorService writeESThreadPool;
//hbase的作业线程池
private ExecutorService writeHbaseThreadPool;
public MQDealService(KubeMqProperties kubeMqProperties,
ElasticSearchProperties elasticSearchProperties,
RestHighLevelClient restHighLevelClient,
Connection hbaseConnection,
HbaseProperties hbaseProperties,
ExecutorService writeESThreadPool,
ExecutorService writeHbaseThreadPool) {
this.kubeMqProperties = kubeMqProperties;
this.restHighLevelClient = restHighLevelClient;
this.elasticSearchProperties = elasticSearchProperties;
this.hbaseConnection = hbaseConnection;
this.hbaseProperties = hbaseProperties;
this.writeESThreadPool = writeESThreadPool;
this.writeHbaseThreadPool = writeHbaseThreadPool;
}
/**
* 初始化ES的索引
*/
public void initES() {
//看是否存在该Index索引
GetIndexRequest existsIndex = new GetIndexRequest(elasticSearchProperties.getIndexName());
boolean exists = false;
try {
exists = restHighLevelClient.indices().exists(existsIndex, RequestOptions.DEFAULT);
if (!exists) {
//如果不存在该索引,那么创建索引
CreateIndexRequest createIndexRequest = new CreateIndexRequest(elasticSearchProperties.getIndexName());
createIndexRequest.settings(Settings.builder()
.put("index.number_of_shards", elasticSearchProperties.getShards()) //分片数量
.put("index.number_of_replicas", elasticSearchProperties.getReplicas()) //副本数量
);
//创建索引
restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
logger.info("ES索引[{}]初始化完毕...", elasticSearchProperties.getIndexName());
} else {
logger.info("ES索引[{}]存在,可以直接使用...", elasticSearchProperties.getIndexName());
}
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 初始化Hbase,如果没有表用来初始化表使用
*
* @return
*/
public TableName initHbase() {
//首先查看hbase的表是否存在：表的命名方式为: clientId+自定义的表名称
TableName tableName = TableName.valueOf(hbaseProperties.getClientId() + "_" + hbaseProperties.getTableName());
//获得到数据库
Admin admin = null;
try {
admin = hbaseConnection.getAdmin();
boolean flag = admin.tableExists(tableName);
if (!flag) {
//如果没有此表则进行创建
TableDescriptor tableDescriptor = TableDescriptorBuilder
//指定表名称
.newBuilder(tableName)
//指定列族
.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(hbaseProperties.getColumnFamily())).build())
.build();
//同步构建表
admin.createTable(tableDescriptor);
logger.info("Hbase表[{}]初始化完毕...", tableName.getNameAsString());
admin.close();
}
} catch (IOException e) {
logger.error("Hbase创建表[{}]异常,异常原因：{}", tableName.getNameAsString(), e.getMessage());
e.printStackTrace();
}
return tableName;
}
/**
* Bean初始化的方法
*/
public void initBean() throws ServerAddressNotSuppliedException, SSLException {
//初始化ES
initES();
//初始化Hbase
tableName = initHbase();
//初始化索引
request = new IndexRequest(elasticSearchProperties.getIndexName(), "_doc");
//初始化队列
queue = new Queue(kubeMqProperties.getQueueName(),
kubeMqProperties.getClientId(),
kubeMqProperties.getUrl());
//初始化ES处理线程池
//初始化Hbase处理线程池
}
/**
* Bean初始化后调用的方法
* 进行消息处理
*/
public void dealMsg() throws ServerAddressNotSuppliedException, IOException {
//从队列中取出消息
ReceiveMessagesResponse response = queue.ReceiveQueueMessages(kubeMqProperties.getOnceReceiveMaxMessage(), kubeMqProperties.getWaitTimeReceiveMessage());
if (response.getIsError()) {
//如果接收发生异常
logger.error("MQ消息接收失败,失败原因:{}", response.getError());
//通知管理者
} else {
//消息正确接收
Iterable messages = response.getMessages();
messages.forEach(msg -> {
try {
LogModel logModel = (LogModel) Converter.FromByteArray(msg.getBody());
writeESThreadPool.submit(() -> {
//1.写到ES中
WriteToES(logModel);
});
writeHbaseThreadPool.submit(() -> {
//2.写道Hbase中
WriteToHbase(logModel);
});
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
});
}
}
/**
* 写入ES
*
* @param logModel
*/
private void WriteToES(LogModel logModel) {
try {
/**
* 向ES中索引数据，异步发送到ES
*/
request.source(JSON.toJSONString(logModel), XContentType.JSON);
//此处暂时使用异步，以后需要自定义线程池控制,或者改为同步
restHighLevelClient.index(request, RequestOptions.DEFAULT);
logger.info("日志信息写入ES成功...");
/*restHighLevelClient.indexAsync(request, RequestOptions.DEFAULT, new ActionListener() {
@Override
public void onResponse(IndexResponse indexResponse) {
logger.info("日志信息写入ES成功...");
}
@Override
public void onFailure(Exception e) {
logger.error("索引创建失败消息" + e.getMessage());
logger.error("索引创建失败原因" + e.getCause());
}
});*/
} catch (Exception e) {
logger.error("索引过程出错：{}", e.getMessage());
}
}
/**
* 日志消息写入Hbase
*
* @param logModel
*/
private void WriteToHbase(LogModel logModel) {
try {
/**
* 向hbase中写入数据
*/
Table table = hbaseConnection.getTable(tableName);
//rowKey的初步设计为 clientId + tableName + Long.max-timestamps
Put put = new Put(Bytes
.add(Bytes.toBytes(hbaseProperties.getClientId())
, Bytes.toBytes(hbaseProperties.getTableName())
, Bytes.toBytes(Long.MAX_VALUE - System.currentTimeMillis())));
//保存用户账号
put.addColumn(Bytes.toBytes(hbaseProperties.getColumnFamily()),
Bytes.toBytes("username"),
Bytes.toBytes(logModel.getUsername()));
//保存用户操作
put.addColumn(Bytes.toBytes(hbaseProperties.getColumnFamily()),
Bytes.toBytes("optionType"),
Bytes.toBytes(logModel.getOptionType()));
//保存用户操作明细
put.addColumn(Bytes.toBytes(hbaseProperties.getColumnFamily()),
Bytes.toBytes("option"),
Bytes.toBytes(logModel.getOption() != null ? logModel.getOption() : ""));
//保存用户操作时间
put.addColumn(Bytes.toBytes(hbaseProperties.getColumnFamily()),
Bytes.toBytes("optionTime"),
Bytes.toBytes(logModel.getOptionTime()));
//保存用户的IP
put.addColumn(Bytes.toBytes(hbaseProperties.getColumnFamily()),
Bytes.toBytes("ip"),
Bytes.toBytes(logModel.getIp()));
//保存用户的操作结果
put.addColumn(Bytes.toBytes(hbaseProperties.getColumnFamily()),
Bytes.toBytes("optionResult"),
Bytes.toBytes(logModel.getOptionResult()));
//保存用户的操作路径
put.addColumn(Bytes.toBytes(hbaseProperties.getColumnFamily()),
Bytes.toBytes("classPath"),
Bytes.toBytes(logModel.getClassPath()!=null?logModel.getClassPath():""));
//保存用户的区域信息
put.addColumn(Bytes.toBytes(hbaseProperties.getColumnFamily()),
Bytes.toBytes("area"),
Bytes.toBytes(logModel.getArea()!=null?logModel.getArea():"未知"));
//保存用户的浏览器信息
put.addColumn(Bytes.toBytes(hbaseProperties.getColumnFamily()),
Bytes.toBytes("browser"),
Bytes.toBytes(logModel.getBrowser()!=null?logModel.getBrowser():"未知"));
table.put(put);
//关闭table
table.close();
logger.info("日志信息写入hbase成功...");
} catch (Exception e) {
logger.error("日志写入hbase过程出错：{}", e.getMessage());
}
}
}
