com.efficient.elasticsearch.service.ElasticSearchService Maven / Gradle / Ivy
package com.efficient.elasticsearch.service;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.efficient.common.page.Page;
import com.efficient.common.util.JackSonUtil;
import com.efficient.elasticsearch.entity.ElasticSearchRangeEntity;
import com.efficient.elasticsearch.entity.ElasticSearchRequestEntity;
import com.efficient.elasticsearch.entity.RangeEntity;
import com.efficient.elasticsearch.entity.ResponseEntity;
import com.efficient.elasticsearch.parser.SqlParser;
import com.efficient.elasticsearch.parser.TableNameParser;
import com.efficient.elasticsearch.properties.ElasticSearchProperties;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.*;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.client.indices.*;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.UpdateByQueryRequest;
import org.elasticsearch.script.Script;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentType;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* @author TMW
* @since 2024/5/15 10:06
*/
@Slf4j
public class ElasticSearchService implements Serializable {
private static final long serialVersionUID = -7686867207799800071L;
public RestHighLevelClient restHighLevelClient = null;
private String pkFieldName;
private Long maxBuckets;
private boolean dateToTimestamp = false;
private RestClient restClient = null;
private SqlParser sqlParser = null;
private ElasticSearchProperties properties;
/**
* 初始化
*/
public void init(ElasticSearchProperties elasticSearchProperties) {
this.properties = elasticSearchProperties;
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(properties.getUsername(), properties.getPassword()));
RestClientBuilder builder = RestClient.builder(new HttpHost(properties.getIp(), properties.getPort()))
.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder
.setDefaultCredentialsProvider(credentialsProvider)
.setMaxConnTotal(properties.getMaxConnectNum())
.setMaxConnPerRoute(properties.getMaxConnectPerRoute())
)
.setRequestConfigCallback(restClientBuilder -> {
restClientBuilder.setConnectTimeout(properties.getConnectTimeout());
restClientBuilder.setSocketTimeout(properties.getSocketTimeout());
restClientBuilder.setConnectionRequestTimeout(properties.getConnectionRequestTimeout());
return restClientBuilder;
});
restClient = builder.build();
restHighLevelClient = new RestHighLevelClient(builder);
sqlParser = new SqlParser(properties);
dateToTimestamp = properties.isDateToTimestamp();
pkFieldName = properties.getPkFieldName();
maxBuckets = properties.getMaxBuckets();
// 启用打印
if (elasticSearchProperties.isPrintDist()) {
Long printDistInterval = elasticSearchProperties.getPrintDistInterval();
ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
scheduler.scheduleAtFixedRate(() -> {
try {
this.printDisk();
} catch (IOException e) {
log.error("监控ES服务器磁盘异常", e);
}
}, 0, printDistInterval, TimeUnit.SECONDS);
}
}
public void printDisk() throws IOException {
// 执行 Cat API 请求
Request request = new Request("GET", "/_cat/allocation?v=true&pretty");
Response response = restClient.performRequest(request);
if (response.getStatusLine().getStatusCode() != 200) {
log.warn("监控ES服务器磁盘请求失败");
}
// 解析响应并存储为 Map
Map resultMap = new HashMap<>();
try (Scanner scanner = new Scanner(response.getEntity().getContent())) {
String title = scanner.nextLine();
String value = scanner.nextLine();
if (!title.trim().isEmpty() && !title.startsWith("id")) { // 过滤掉标题行和空行
String[] parts = title.split("\\s+", 9); // 使用空格分割，限制为2个元素，以避免索引和值中可能存在的空格干扰
String[] values = value.split("\\s+", 10); // 使用空格分割，限制为2个元素，以避免索引和值中可能存在的空格干扰
for (int i = 0; i < parts.length; i++) {
resultMap.put(parts[i], values[i + 1]);
}
}
}
log.info("resultMap:{}", resultMap);
String diskUsed = resultMap.get("disk.used");
String diskAvail = resultMap.get("disk.avail");
String diskTotal = resultMap.get("disk.total");
double diskPercent = Double.parseDouble(resultMap.get("disk.percent"));
if (diskPercent < 80) {
log.info("磁盘空间已使用：{},占比：{}%", diskUsed, diskPercent);
} else if (diskPercent >= 95) {
log.error("磁盘空间已使用：{},占比：{}%,磁盘空间已超过95%，ElasticSearch 将不能写入数据，只能读取！", diskUsed, diskPercent);
} else {
log.warn("磁盘空间已使用：{},占比：{}%,磁盘空间一旦超过95%，ElasticSearch 将不能写入数据，只能读取！", diskUsed, diskPercent);
}
}
/**
* 注销
*/
public boolean destroy() {
try {
if (Objects.nonNull(restClient)) {
restClient.close();
}
if (Objects.nonNull(restHighLevelClient)) {
restHighLevelClient.close();
}
} catch (Exception e) {
log.error("关闭 restHighLevelClient 失败 ", e);
return false;
}
return true;
}
/**
* 创建索引
*
* 指标解释
*
* @param index 索引名称
* @return 是否成功
*/
public boolean createIndex(String index) throws IOException {
if (isIndexExist(index)) {
log.error("Index is exits!");
return false;
}
CreateIndexRequest request = new CreateIndexRequest(index);
request.settings(Settings.builder()
.put("number_of_shards", 1)
.put("number_of_replicas", 0)
// 关闭自动刷新,实时刷新会降低索引速度
// .put("refresh_interval ", 10)
.put("max_result_window", 20000000)
.build());
CreateIndexResponse response = restHighLevelClient.indices().create(request, RequestOptions.DEFAULT);
log.info("创建索引{}成功", index);
return response.isAcknowledged();
}
/**
* 判断索引是否存在
*
* @param index 索引名称
* @return 是否成功
*/
public boolean isIndexExist(String index) throws IOException {
GetIndexRequest request = new GetIndexRequest(index);
return restHighLevelClient.indices().exists(request, RequestOptions.DEFAULT);
}
/**
* 删除索引
*
* @param index 索引名称
* @return 是否成功
*/
public boolean deleteIndex(String index) throws IOException {
if (!isIndexExist(index)) {
log.error("Index is not exits!");
return false;
}
// 删除索引请求
DeleteIndexRequest request = new DeleteIndexRequest(index);
// 执行客户端请求
AcknowledgedResponse delete = restHighLevelClient.indices().delete(request, RequestOptions.DEFAULT);
log.info("删除索引{}成功", index);
return delete.isAcknowledged();
}
/**
* 保存
*
* @param index 索引名称
* @param data 数据集合
* @return 是否成功
*/
public boolean save(String index, Map data) {
if (Objects.isNull(data)) {
return false;
}
IndexRequest request = new IndexRequest(index);
// 刷新策略
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
formatMap(data);
request.source(data).opType(DocWriteRequest.OpType.INDEX);
final IndexResponse response;
try {
response = restHighLevelClient.index(request, RequestOptions.DEFAULT);
final DocWriteResponse.Result result = response.getResult();
if (Objects.equals(result, DocWriteResponse.Result.UPDATED)) {
log.warn("新增文档已存在，默认执行了修改操作：" + JackSonUtil.toJson(data));
return true;
}
log.info("索引为: {}, 新增数据成功", index);
return Objects.equals(result, DocWriteResponse.Result.CREATED);
} catch (IOException e) {
log.error(index + " 批量保存失败", e);
return false;
}
}
/**
* 格式化 数据
*
* @param map 数据
* @return 格式化后数据
*/
private Map formatMap(Map map) {
map.keySet().forEach(key -> map.compute(key, (k, value) -> formatValue(value)));
return map;
}
private Object formatValue(Object value) {
if (Objects.isNull(value)) {
return null;
} else if (value instanceof java.sql.Date) {
final long dateLong = ((java.sql.Date) value).getTime();
if (dateToTimestamp) {
return processDateLong(dateLong);
}
return new Date(dateLong);
} else if (value instanceof java.sql.Timestamp) {
final long dateLong = ((java.sql.Timestamp) value).getTime();
if (dateToTimestamp) {
return processDateLong(dateLong);
}
return new Date(dateLong);
} else if (value instanceof Date) {
final Date date = (Date) value;
if (dateToTimestamp) {
return DateUtil.beginOfDay(date).getTime();
}
return date;
}
return value;
}
private Long processDateLong(long dateLong) {
return DateUtil.beginOfDay(new Date(dateLong)).getTime();
}
/**
* 保存
*
* @param index 索引名称
* @param data 数据集合
* @return 是否成功
*/
public boolean saveByPkField(String index, Map data) {
return saveByPkField(index, pkFieldName, data);
}
/**
* 保存
*
* @param index 索引名称
* @param data 数据集合
* @param pkFieldName 主键字段名称
* @return 是否成功
*/
public boolean saveByPkField(String index, String pkFieldName, Map data) {
if (Objects.isNull(data)) {
return false;
}
IndexRequest request = new IndexRequest(index);
// 刷新策略
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
formatMap(data);
request.source(data).opType(DocWriteRequest.OpType.INDEX);
if (StrUtil.isNotBlank(pkFieldName)) {
request.id(String.valueOf(data.get(pkFieldName)));
}
final String id = request.id();
final IndexResponse response;
try {
response = restHighLevelClient.index(request, RequestOptions.DEFAULT);
final DocWriteResponse.Result result = response.getResult();
if (Objects.equals(result, DocWriteResponse.Result.UPDATED)) {
log.warn("新增文档已存在，默认执行了修改操作：" + JackSonUtil.toJson(data));
return true;
}
log.info("索引为: {}, id为: {} 新增数据成功", index, id);
return Objects.equals(result, DocWriteResponse.Result.CREATED);
} catch (IOException e) {
log.error(index + " 批量保存失败", e);
return false;
}
}
/**
* 批量保存
*
* @param index 索引名称
* @param recordList 数据集合
* @return 是否成功
*/
public boolean batchSaveWithRecord(String index, List> recordList) {
if (CollUtil.isEmpty(recordList)) {
return false;
}
BulkRequest request = new BulkRequest();
// 刷新策略
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
recordList.forEach(record -> request.add(new IndexRequest(index).source(record)));
final BulkResponse bulk;
try {
bulk = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
return !bulk.hasFailures();
} catch (IOException e) {
log.error(index + " 批量保存失败", e);
return false;
}
}
/**
* 批量保存
*
* @param index 索引名称
* @param mapList 数据集合
* @return 是否成功
*/
public boolean batchSave(String index, List> mapList) {
if (CollUtil.isEmpty(mapList)) {
return false;
}
BulkRequest request = new BulkRequest();
mapList.forEach(record -> request.add(new IndexRequest(index).source(formatMap(record))));
final BulkResponse bulk;
try {
bulk = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
return !bulk.hasFailures();
} catch (IOException e) {
log.error(index + " 批量保存失败", e);
return false;
}
}
/**
* 批量保存
*
* @param index 索引名称
* @param mapList 数据集合
* @return 是否成功
*/
public boolean batchSaveByPkField(String index, List> mapList) {
return batchSaveByPkField(index, pkFieldName, mapList);
}
/**
* 批量保存
*
* @param index 索引名称
* @param mapList 数据集合
* @param pkFieldName 主键字段名称
* @return 是否成功
*/
public boolean batchSaveByPkField(String index, String pkFieldName, List> mapList) {
if (CollUtil.isEmpty(mapList)) {
return false;
}
BulkRequest request = new BulkRequest();
// 刷新策略
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
mapList.forEach(map -> {
final IndexRequest indexRequest = new IndexRequest(index)
.source(formatMap(map));
if (StrUtil.isNotBlank(pkFieldName)) {
indexRequest.id(String.valueOf(map.get(pkFieldName)));
}
request.add(indexRequest);
});
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
final BulkResponse bulk;
try {
bulk = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
log.info("索引为: {}, 批量新增数据条数: {}, {}", index, mapList.size(), !bulk.hasFailures());
return !bulk.hasFailures();
} catch (IOException e) {
log.error(index + " 批量保存失败", e);
return false;
}
}
/**
* 根据条件修改数据
*
* @param index 索引
* @param pkFieldName 主键字段
* @param documentList 更新文档
* @return 更新条数
*/
public boolean batchUpdateByPkField(String index, String pkFieldName, List> documentList) {
if (CollUtil.isEmpty(documentList)) {
return false;
}
BulkRequest bulkRequest = new BulkRequest(index);
// 刷新策略
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
for (Map map : documentList) {
formatMap(map);
bulkRequest.add(new UpdateRequest(index, String.valueOf(map.get(pkFieldName))).doc(map));
}
BulkResponse response;
try {
response = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
log.info("索引为: {}, 批量修改数据数据量: {}, {}", index, documentList.size(), !response.hasFailures());
return !response.hasFailures();
} catch (IOException e) {
log.error(index + " 根据主键修改数据", e);
return false;
}
}
/**
* 根据条件修改数据
*
* @param index 索引
* @param query 查询条件
* @param document 更新文档
* @return 更新条数
* @throws Exception
*/
public long updateWithQuery(String index, QueryBuilder query, Map document) throws Exception {
if (Objects.isNull(document)) {
return 0L;
}
UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(index);
updateByQueryRequest.setQuery(query);
formatMap(document);
StringBuilder script = new StringBuilder();
Set keys = document.keySet();
for (String key : keys) {
String appendValue = "";
Object value = formatValue(document.get(key));
if (value instanceof Number) {
appendValue = value.toString();
} else if (value instanceof String) {
appendValue = "'" + value + "'";
} else if (value instanceof List) {
appendValue = JackSonUtil.toJson(value);
} else {
appendValue = value.toString();
}
script.append("ctx._source.").append(key).append("=").append(appendValue).append(";");
}
updateByQueryRequest.setScript(new Script(script.toString()));
final BulkByScrollResponse bulkByScrollResponse = restHighLevelClient.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT);
return bulkByScrollResponse.getTotal();
}
/**
* 根据主键修改数据
*
* @param index 索引
* @param document 更新文档
* @return 更新条数
*/
public boolean updateByPkField(String index, Map document) {
return updateByPkField(index, pkFieldName, document);
}
/**
* 根据主键修改数据
*
* @param index 索引
* @param pkFieldName 文档主键名称
* @param document 更新文档
* @return 更新条数
*/
public boolean updateByPkField(String index, String pkFieldName, Map document) {
if (Objects.isNull(document)) {
return false;
}
final String id = String.valueOf(document.get(pkFieldName));
UpdateRequest updateRequest = new UpdateRequest(index, id);
// 刷新策略
updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
formatMap(document);
updateRequest.doc(document);
final ActionListener listener = new ActionListener() {
@Override
public void onResponse(UpdateResponse updateResponse) {
log.info("索引为: {}, id为: {} 修改数据成功", index, id);
}
@Override
public void onFailure(Exception e) {
log.error("索引为: {}, id为: {} 修改数据失败, 异常信息: {}", index, id, e.getMessage());
}
};
restHighLevelClient.updateAsync(updateRequest, RequestOptions.DEFAULT, listener);
return true;
}
/**
* 根据主键修改数据
*
* @param index 索引
* @param id 文档主键
* @param document 更新文档
* @return 更新条数
*/
public boolean updateById(String index, String id, Map document) {
if (Objects.isNull(document)) {
return false;
}
UpdateRequest updateRequest = new UpdateRequest(index, id);
// 刷新策略
updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
formatMap(document);
updateRequest.doc(document);
final UpdateResponse response;
try {
response = restHighLevelClient.update(updateRequest, RequestOptions.DEFAULT);
log.info("索引为: {}, id为: {} 修改数据成功", index, id);
return Objects.equals(response.getResult(), DocWriteResponse.Result.UPDATED);
} catch (IOException e) {
log.error(index + " 根据主键修改数据", e);
return false;
}
}
/**
* 通过ID删除数据
*
* @param index 索引，类似数据库
* @param id 数据ID
*/
public boolean deleteDataById(String index, String id) {
try {
DeleteRequest request = new DeleteRequest(index, id);
// 刷新策略
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
restHighLevelClient.delete(request, RequestOptions.DEFAULT);
log.info("索引为: {}, id为: {} 删除数据成功", index, id);
return true;
} catch (IOException e) {
log.error("索引为: " + index + ", id为: " + id + " 删除数据失败", e);
return false;
}
}
/**
* 根据条件修改数据
*
* @param index 索引
* @param ids 主键字段集合
* @return 更新条数
*/
public boolean batchDelete(String index, List ids) {
BulkRequest bulkRequest = new BulkRequest(index);
// 刷新策略
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
if (CollUtil.isEmpty(ids)) {
return true;
}
ids.forEach(id -> bulkRequest.add(new DeleteRequest(index, id)));
BulkResponse response;
try {
response = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
log.info("索引为: {}, 批量删除数据条数: {} 成功", index, ids.size());
return !response.hasFailures();
} catch (IOException e) {
log.error(index + " 根据主键删除数据异常", e);
return false;
}
}
public ResponseEntity initTest() throws Exception {
log.info("EsPlugin Init Test ...");
Request request = new Request("POST", "/_sql?format=json");
final JSONObject jsonObject = new JSONObject();
jsonObject.put("query", "show tables");
jsonObject.put("fetch_size", 100);
final String json = jsonObject.toString();
request.setJsonEntity(json);
final Response response = restClient.performRequest(request);
final String resultStr = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
return JackSonUtil.toObject(resultStr, ResponseEntity.class);
}
/**
* 查找当前最大桶数量
*
* @return 最大桶数量
* @throws Exception
*/
public Long findMaxBuckets() throws Exception {
Request request = new Request("GET", "/_cluster/settings");
final Response response = restClient.performRequest(request);
final String resultStr = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
return JSONUtil.parseObj(resultStr).getByPath("persistent.search.max_buckets", Long.class);
}
/**
* 设置最大桶数量
*
* @param maxBuckets 最大桶数量
* @return 是否成功
* @throws Exception
*/
public boolean putMaxBuckets(Integer maxBuckets) throws Exception {
Request request = new Request("PUT", "/_cluster/settings");
final JSONObject json = new JSONObject();
final JSONObject jsonObject = new JSONObject();
jsonObject.set("search.max_buckets", maxBuckets);
json.set("persistent", jsonObject);
final String jsonStr = json.toString();
request.setJsonEntity(jsonStr);
final Response response = restClient.performRequest(request);
final String resultStr = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
return JSONUtil.parseObj(resultStr).getByPath("acknowledged", Boolean.class);
}
/**
* 根据 sql 分页查询
*
* @param tableName 表名
* @param pageNum 当前页
* @param pageSize 每页数量
* @param sql sql语句
* @param includeFieldList 包含字段
* @return Page