Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.turbospaces.logging.ElasticSearchAppender Maven / Gradle / Ivy
package com.turbospaces.logging;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.Base64;
import java.util.Collection;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Random;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.StringSubstitutor;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.io.SerializedString;
import com.google.common.collect.Lists;
import ch.qos.logback.classic.spi.ILoggingEvent;
import io.sentry.event.Event.Level;
import io.sentry.event.EventBuilder;
import io.sentry.event.interfaces.MessageInterface;
import lombok.Setter;
import okhttp3.ConnectionPool;
import okhttp3.HttpUrl;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okio.BufferedSink;
public class ElasticSearchAppender extends AbstractAppender {
public static final MediaType X_JSON = MediaType.get("application/json");
public static final MediaType X_NDJSON = MediaType.get("application/x-ndjson");
public static final String TIMESTAMP = "@timestamp";
public static final String SEQUENCE = "@sequence";
// ~ we re-use JSON parser as much as we can to lower GC impact
public static final SerializedString ROOT_VALUE_SEPARATOR = new SerializedString(StringUtils.EMPTY);
private final JsonFactory jf = new JsonFactory(); // ~ JSON low-level factory
private final Timer timer = new Timer(true); // ~ remove old indexes in background
private OkHttpClient httpClient;
// ~ options
@Setter
private String template; // ~ for automatic template registration
@Setter
private boolean hourly; // ~ create hourly index or daily instead
@Setter
private int retention = 7; // ~ for how long to keep logs (3 days by default)
@Setter
private String designator = "logs"; // ~ main logging prefix (designator)
@Setter
private int keepAliveTimeout = 30; // ~ how long to keep-alive HTTP connections
@Setter
private boolean retryOnConnectionFailure = true; // ~ should we automatically retry on client side
@Override
public void start() {
super.start();
addInfo(String.format("starting with settings hourly: %s, template: %s, retention: %s, designator: %s", hourly, template, retention, designator));
OkHttpClient.Builder ok = new OkHttpClient.Builder();
ok.connectTimeout(socketTimeout, TimeUnit.SECONDS);
ok.connectionPool(new ConnectionPool(threads, keepAliveTimeout, TimeUnit.SECONDS));
ok.retryOnConnectionFailure(retryOnConnectionFailure);
httpClient = ok.build();
if (Objects.nonNull(getElasticEndpoint())) {
//
// ~ try to register template automatically
//
if (StringUtils.isNotEmpty(template)) {
registerTemplate();
}
// ~ mark started
started = true;
//
// ~ effectively start
//
for (int i = 0; i < threads; i++) {
WorkerThread worker = workers[i];
worker.start();
}
//
// ~ schedule cleanup job randomly (not to bombard all at once)
// ~ auto house-keeping
//
Random rnd = new SecureRandom();
int intervalRandom = 1 + rnd.nextInt((int) TimeUnit.DAYS.toHours(1));
int interval = (int) TimeUnit.HOURS.toMillis(intervalRandom);
int delayRandom = 1 + rnd.nextInt((int) TimeUnit.HOURS.toMinutes(1));
int delay = (int) TimeUnit.MINUTES.toMillis(delayRandom);
addInfo(String.format("log removal task with run hourly at rate: %s hour(s) and initial delay: %s min(s)", intervalRandom, delayRandom));
timer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
if (retention > 0) {
String op = "delete_old_indexes_by_retention";
LocalDate from = LocalDate.now(ZoneOffset.UTC).minusMonths(1);
LocalDate to = LocalDate.now(ZoneOffset.UTC).minusDays(retention);
addInfo(String.format("running log retention task from: %s, to: %s", from, to));
for (int d = 0; d < ChronoUnit.DAYS.between(from, to); d++) {
LocalDate date = from.plusDays(d);
Collection indexes = Lists.newArrayList();
if (hourly) {
for (int i = 0; i < TimeUnit.DAYS.toHours(1); i++) {
LocalDateTime hours = date.atStartOfDay().plusHours(i);
String pattern = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH").format(hours);
String index = String.format("%s-%s-%s", designator, getService(), pattern);
indexes.add(index);
}
} else {
String pattern = DateTimeFormatter.ofPattern("yyyy-MM-dd").format(date);
String index = String.format("%s-%s-%s", designator, getService(), pattern);
indexes.add(index);
}
List success = new LinkedList<>();
for (String index : indexes) {
Request.Builder reqb = new Request.Builder();
reqb.url(baseUrl().addPathSegment(index).build());
reqb.delete();
addAuthInfo(reqb);
Request req = reqb.build();
try (Response response = httpClient.newCall(req).execute()) {
if (response.isSuccessful()) {
addInfo(String.format("deleted index at: %s", index));
success.add(index);
}
} catch (Exception err) {
boolean applied = sendSentryAlert(op, err);
if (applied) {} else {
addError(op, err);
}
}
}
if (success.isEmpty()) {
// ~ maybe add warning
} else {
addInfo("successfully deleted: " + success);
}
}
}
}
}, delay, interval);
}
}
@Override
public void stop() {
super.stop();
//
// ~ cancel timer (remove auto-housekeeping thread gracefully)
//
timer.cancel();
//
// ~ close OK client (not necessary, but nice to have)
//
if (Objects.nonNull(httpClient)) {
httpClient.dispatcher().executorService().shutdown();
httpClient.connectionPool().evictAll();
if (httpClient.cache() != null) {
try {
httpClient.cache().close();
} catch (IOException err) {
addError("unable to close OK cache", err);
}
}
}
}
@Override
protected boolean dryRun() {
return loggingDryRun();
}
@Override
protected void sendBulk(List list) {
LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC);
String pattern = DateTimeFormatter.ofPattern(hourly ? "yyyy-MM-dd-HH" : "yyyy-MM-dd").format(now);
String op = "send_bulk_logs";
try {
Request.Builder reqb = new Request.Builder();
reqb.url(baseUrl().addPathSegment(String.format("%s-%s-%s", designator, getService(), pattern)).addPathSegment("_bulk").build());
reqb.post(new RequestBody() {
@Override
public MediaType contentType() {
return X_NDJSON;
}
@Override
public void writeTo(BufferedSink sink) throws IOException {
OutputStream outputStream = sink.outputStream();
//
// ~ substitute with temporary output stream so that we can put debug info to logs
//
if (trace) {
outputStream = new ByteArrayOutputStream();
}
//
// ~ write directly to sink's output stream (non-trace)
//
try (OutputStreamWriter out = new OutputStreamWriter(outputStream)) {
try (BufferedWriter writer = new BufferedWriter(out)) {
try (JsonGenerator json = jf.createGenerator(writer)) {
json.setRootValueSeparator(ROOT_VALUE_SEPARATOR);
for (SequencedDeferredEvent next : list) {
// ~ write id
try {
json.writeStartObject();
json.writeFieldName("index");
// _id
String hex = Long.toHexString(Double.doubleToLongBits(Math.random()));
int hash = Math.abs(next.event().hashCode());
json.writeStartObject();
json.writeStringField("_id", getSlot() + hex + hash + next.seq());
json.writeEndObject();
json.writeEndObject();
json.flush();
} finally {
writer.newLine();
}
//
// ~ and actual body
//
try {
writeBody(json, next);
} finally {
writer.newLine();
}
}
}
}
} catch (IOException err) {
addError(err.getMessage(), err);
throw err;
} catch (Exception err) {
addError(err.getMessage(), err);
throw new IOException(err);
}
//
// ~ write to info everything we send to ElasticSearch before
//
if (trace) {
String data = outputStream.toString();
addInfo(data);
//
// ~ copy everything to sink's output stream
//
try (ByteArrayInputStream in = new ByteArrayInputStream(data.getBytes())) {
IOUtils.copy(in, sink.outputStream());
}
}
//
// ~ GC
//
list.clear();
}
});
addAuthInfo(reqb);
Request req = reqb.build();
try (Response response = httpClient.newCall(req).execute()) {
if (response.isSuccessful()) {
} else {
boolean applied = sendUnexpectedHttpStatus(op, req, response);
if (BooleanUtils.isFalse(applied)) {
// ~ just to make sure we send log to somewhere
addError(String.format("%s: http_status=%d", op, response.code()));
}
}
}
} catch (InterruptedIOException err) {
addInfo("got interrupted while sending data", err);
} catch (Exception err) {
boolean applied = sendSentryAlert(op, err);
if (BooleanUtils.isFalse(applied)) {
addError(op, err);
}
}
}
private void registerTemplate() {
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
String op = "register_logging_template";
URL resource = classLoader.getResource(template);
if (Objects.isNull(resource)) {
resource = getClass().getClassLoader().getResource(template);
}
try (InputStream io = resource.openStream()) {
String raw = IOUtils.toString(io, StandardCharsets.UTF_8);
StringSubstitutor str = new StringSubstitutor();
str.setEnableUndefinedVariableException(true); // ~ early prevention of undefined variables
String body = str.replace(raw);
Request.Builder reqb = new Request.Builder();
reqb.url(baseUrl().addPathSegment("_template").addPathSegment("logging-" + getService()).build());
reqb.put(RequestBody.create(body, X_JSON));
addAuthInfo(reqb);
Request req = reqb.build();
try (Response response = httpClient.newCall(req).execute()) {
if (response.isSuccessful()) {
addInfo(String.format("successfully registered logging template from %s", resource.toExternalForm()));
} else {
boolean applied = sendUnexpectedHttpStatus(op, req, response);
if (BooleanUtils.isFalse(applied)) {
addError(String.format("%s: http_status: %d", op, response.code()));
}
}
}
} catch (Exception err) {
boolean applied = sendSentryAlert(op, err);
if (BooleanUtils.isFalse(applied)) {
addError(op, err);
}
}
}
private void writeBody(JsonGenerator json, SequencedDeferredEvent data) throws IOException {
ILoggingEvent event = data.event();
String timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(event.getTimeStamp()).atZone(ZoneOffset.UTC));
try {
json.writeStartObject();
// ~ system
json.writeStringField(TIMESTAMP, timestamp); // actual event time stamp
json.writeNumberField(SEQUENCE, data.seq()); // for strict ordered search
// ~ actual
json.writeStringField(Logback.HOST, getHost());
json.writeStringField(Logback.SLOT, getSlot());
json.writeStringField(Logback.RELEASE, getRelease());
// ~ write all formatted values
for (DocumentProperty field : properties.getProperties()) {
String formatted = field.format(event);
if (StringUtils.isNotEmpty(formatted)) {
json.writeStringField(field.getName(), formatted);
}
}
// ~ write MDC values
Map mdc = event.getMDCPropertyMap();
if (mdc != null) {
for (Entry entry : mdc.entrySet()) {
boolean toInclude = true;
if (BooleanUtils.isFalse(mdcNames.isEmpty())) {
toInclude = mdcNames.contains(entry.getKey());
}
if (toInclude) {
if (StringUtils.isNotEmpty(entry.getValue())) {
json.writeStringField(entry.getKey(), entry.getValue());
}
}
}
}
json.writeEndObject();
} finally {
json.flush();
}
}
private boolean sendUnexpectedHttpStatus(String operation, Request req, Response response) {
long now = System.currentTimeMillis();
long prev = lastSentryAlert.get();
long period = now - prev;
if (BooleanUtils.isFalse(alertsDryRun())) {
if (Objects.nonNull(getSentry())) {
if (period >= TimeUnit.MINUTES.toMillis(sentryRateLimit)) {
EventBuilder builder = new EventBuilder();
builder.withTimestamp(new Date());
builder.withMessage(operation);
builder.withLogger(getClass().getName());
builder.withLevel(Level.ERROR);
try {
String payload = response.body().string();
MessageInterface intf = new MessageInterface(payload);
builder.withSentryInterface(intf);
builder.withTag("contentType", req.body().contentType().type());
builder.withTag("contentLenght", Long.toString(req.body().contentLength()));
builder.withTag("status", Integer.toString(response.code()));
} catch (IOException err) {
addError(err.getMessage(), err);
} finally {
lastSentryAlert.compareAndSet(prev, now);
}
getSentry().sendEvent(builder);
return true;
}
}
}
return false;
}
private void addAuthInfo(Request.Builder req) {
String userInfo = getElasticEndpoint().getUserInfo();
if (userInfo != null && !userInfo.isEmpty()) {
String auth = Base64.getEncoder().encodeToString(userInfo.getBytes());
req.addHeader("Authorization", "Basic " + auth);
}
}
private HttpUrl.Builder baseUrl() {
HttpUrl.Builder url = new HttpUrl.Builder();
url.scheme(getElasticEndpoint().getProtocol());
url.host(getElasticEndpoint().getHost());
url.port(getElasticEndpoint().getPort());
if (StringUtils.isNotEmpty(getElasticEndpoint().getPath())) {
url.addPathSegment(getElasticEndpoint().getPath());
}
return url;
}
}
