com.sghd.logging.query.GrepUtil Maven / Gradle / Ivy
The newest version!
package com.sghd.logging.query;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.sghd.common.utils.time.DateUtils;
import com.sghd.logging.csv.CsvUtils;
/**
* GREP + AWK 命令生成工具
* @author solq
*/
public class GrepUtil {
private static final Logger logger = LoggerFactory.getLogger(GrepUtil.class);
public static boolean isLinuxOS = false;
static {
String osName = System.getProperty("os.name");
if (!StringUtils.isBlank(osName) && osName.toLowerCase().indexOf("linux") > -1) {
isLinuxOS = true;
} else if ("\\".equals(File.separator)) {
isLinuxOS = false;
} else if ("/".equals(File.separator)) {
isLinuxOS = true;
}
}
private static final String T = "\\t";
// private static final String $ = "$";
private static final String S = "^";
private static final String A = "*";
private static final String PIPE_LINE = " | ";
private static final String NOT_QUERY_COMMAND = ".*";
private static final String DOUBLE_QUOTATION = "\"";
private static final String GREP_COMMAND_START = "grep -P -h ";
private static final String FIND_COMMAND_START = "find ";
private static final String FIND_COMMAND_END = " -type f ";
private static final String FIND_COMMAND_PATH = " -path ";
// private static final String FIND_COMMAND_XARGS = " xargs ";
// 0 0 123 name account 35 rKHJKE@#%%^$&*^)(*&)+_:" \{}|><" -5 1410013414296
// null 11
/***
* @param clz 返回数据类型
* @param awkQuery awk查询
* @param matcher JAVA 层过滤处理
* @param rootDir 查询主目录
* @param fileNames 程序处理完的多个文件
*/
public static List query(Class clz, AwkQuery awkQuery, RecordMatcher matcher, String rootDir,
String... fileNames) {
String command = buildCommand(clz, awkQuery, rootDir, fileNames);
String[] commands = null;
if (isLinuxOS) {
String[] newCommand = { "sh", "-c", command };
commands = newCommand;
} else {
String[] newCommand = { "cmd", "/c", command };
commands = newCommand;
}
List result = new LinkedList<>();
try {
Process process = Runtime.getRuntime().exec(commands);
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream(), "UTF-8"));
for (String line = bufferedReader.readLine(); line != null; line = bufferedReader.readLine()) {
if (line.isEmpty()) {
continue;
}
T obj = CsvUtils.string2Object(line, clz);
if (matcher != null && !matcher.match(obj)) {
continue;
}
result.add(obj);
}
bufferedReader.close();
} catch (IOException e) {
logger.error("读取日志文件错误 : {}", command, e);
throw new RuntimeException(e);
}
return result;
}
public static List query(Class clz, AwkQuery awkQuery, String rootDir, String... fileNames) {
return query(clz, awkQuery, null, rootDir, fileNames);
}
// //////////////////////////////////////命令生成//////////////////////////////////////////////////
/**
* 完整 命令生成
* grep + awk
*/
public static String buildCommand(Class clz, AwkQuery awkQuery, String rootDir, String... fileNames) {
String files = buildFileName(rootDir, fileNames);
final String grepCommand = buildGrepCommand(clz, awkQuery, files);
if (logger.isDebugEnabled()) {
logger.debug("grep command : {}", grepCommand);
}
return grepCommand;
}
/***
* grep + awk 命令生成
* 优先级用 grep 快速匹配数据 然后再用 awk 细过滤 查询效果达到秒级
*/
public static String buildGrepCommand(Object clz, AwkQuery awkQuery, String fileName) {
final StringBuffer stringBuffer = new StringBuffer();
List indes = awkQuery.getIndexs();
boolean canAddFile = StringUtils.isNotBlank(fileName);
if (indes == null || indes.isEmpty()) {
stringBuffer.append(GREP_COMMAND_START);
stringBuffer.append(DOUBLE_QUOTATION);
stringBuffer.append(NOT_QUERY_COMMAND);
stringBuffer.append(DOUBLE_QUOTATION);
if (canAddFile) {
stringBuffer.append(" ");
stringBuffer.append(fileName);
}
} else {
Map fieldIndex = CsvUtils.getFieldIndexs(clz);
int max = valueMax(fieldIndex);
boolean isFirst = true;
boolean isAdd = false;
for (AwkParam awkParam : indes) {
if (!AwkOperateType.EQ.equals(awkParam.getOperate())) {
continue;
}
isAdd = true;
if (!isFirst) {
stringBuffer.append(PIPE_LINE);
}
stringBuffer.append(GREP_COMMAND_START);
stringBuffer.append(DOUBLE_QUOTATION);
int index = fieldIndex.get(awkParam.getName());
if (index == 0) {
stringBuffer.append(S);
} else {
stringBuffer.append(T);
}
stringBuffer.append(awkParam.getValue());
if (index != max) {
stringBuffer.append(T);
}
stringBuffer.append(DOUBLE_QUOTATION);
// 首次追加文件
if (isFirst) {
if (canAddFile) {
stringBuffer.append(" ");
stringBuffer.append(fileName);
}
isFirst = false;
}
}
if (!isAdd) {
stringBuffer.append(GREP_COMMAND_START);
stringBuffer.append(DOUBLE_QUOTATION);
stringBuffer.append(NOT_QUERY_COMMAND);
stringBuffer.append(DOUBLE_QUOTATION);
if (canAddFile) {
stringBuffer.append(" ");
stringBuffer.append(fileName);
}
}
stringBuffer.append(PIPE_LINE);
stringBuffer.append(awkQuery.buildCsvCommand(null));
}
return stringBuffer.toString();
}
// /////////////////////////////////目录处理////////////////////////////////////////
/**
* 生成完整文件
*/
public static String buildFileName(String rootDir, String... fileNames) {
final StringBuffer listFiles = new StringBuffer();
for (String file : fileNames) {
listFiles.append(" ");
listFiles.append(rootDir);
listFiles.append(file);
}
return listFiles.toString();
}
/**
* 生成单个日志文件名
* @param template 配置模板 如 /{date}/{type}/{server}.log
* @param context 注入数据上下文 匹配范围 {1...3} {start...end}
*/
public static String buildFileName(String template, Map context) {
String result = String.valueOf(template);
for (Entry entry : context.entrySet()) {
final String key = entry.getKey();
final String value = entry.getValue();
result = result.replaceAll("\\{" + key + "\\}", value);
}
return result;
}
/**
* 生成默认日志文件名
* @param template 配置模板 如 /{date}/{type}/{server}.log
* @param domain 游戏地址 域名
* @param port 游戏地址 端口
* @param operatorId 查询平台
* @param serverId 查询服
* @param logName 查询日志文件
* @param date 日期
*/
public static String defaultBuildFileName(String template, String domain, int port, String operatorId,
String serverId, String logName, Date date) {
String date_hour = DateUtils.date2String(date, "yyyy-MM-dd-HH");
String date_1 = DateUtils.date2String(date, DateUtils.PATTERN_DATE);
String date_2 = DateUtils.date2String(date, "yyyyMMdd");
final Map context = new HashMap();
context.put("domain", domain);
context.put("port", String.valueOf(port));
context.put("operatorId", operatorId);
if (!isLinuxOS) {
if (serverId.matches("\\{.*\\}")) {
serverId = "*";
}
}
context.put("serverId", serverId);
context.put("type", logName);
context.put("date1", date_1);
context.put("date2", date_2);
context.put("hour", date_hour);
return buildFileName(template, context);
}
/**
* find 命令生成
*/
@Deprecated
public static String buildFindCommand(String rootDir, String fileName) {
// return FIND_COMMAND_START + rootDir + FIND_COMMAND_PATH+ fileName;
final StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append(FIND_COMMAND_START);
stringBuffer.append(rootDir);
stringBuffer.append(FIND_COMMAND_PATH);
stringBuffer.append(DOUBLE_QUOTATION);
stringBuffer.append(A);
stringBuffer.append(fileName);
stringBuffer.append(DOUBLE_QUOTATION);
stringBuffer.append(FIND_COMMAND_END);
return stringBuffer.toString();
}
// /////////////////内部处理/////////////
private static int valueMax(Map values) {
int max = 0;
for (int v : values.values()) {
max = Math.max(max, v);
}
return max;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy