fi.evolver.basics.spring.log.MessageLogService Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of evolver-basics-spring
There is a newer version: 6.7.0
package fi.evolver.basics.spring.log;

import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.regex.Pattern;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;

import fi.evolver.basics.spring.job.JobStatusService;
import fi.evolver.basics.spring.job.JobStatusService.Job;
import fi.evolver.basics.spring.job.ResultState;
import fi.evolver.basics.spring.job.entity.TaskStatus.TaskState;
import fi.evolver.basics.spring.log.entity.MessageLog;
import fi.evolver.basics.spring.log.entity.MessageLog.Direction;
import fi.evolver.basics.spring.log.entity.MessageLogMetadata;
import fi.evolver.utils.GzipUtils;
import fi.evolver.utils.format.FormatUtils;
import jakarta.persistence.EntityManager;


@Service
public class MessageLogService {
	private static final Logger LOG = LoggerFactory.getLogger(MessageLogService.class);

	private static final int PERSIST_BATCH_SIZE = 500;

	private static final LinkedBlockingQueue messageLogQueue = new LinkedBlockingQueue<>(50000);


	private final EntityManager em;
	private final JobStatusService jobStatusService;
	private final MessageLogStatisticsService messageLogStatisticsService;


	@Value("${git.commit.id.describe:?}")
	private String gitDescription;

	@Value("${application.name.pretty:?}")
	private String applicationName;

	private Pattern redactedHeadersRegex;


	@Autowired
	public MessageLogService(
			EntityManager em,
			JobStatusService jobStatusService,
			MessageLogStatisticsService messageLogStatisticsService) {

		this.em = em;
		this.jobStatusService = jobStatusService;
		this.messageLogStatisticsService = messageLogStatisticsService;
	}


	@Value("${message.log.redactedHeadersRegex:Authorization|.*Api\\W*(Key|Token).*|.*Secret.*}")
	public void setRedactedHeadersRegex(String regex) {
		this.redactedHeadersRegex = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
	}


	public void logMessage(
			LocalDateTime startTime, String messageType, String protocol, String address,
			String requestingSystem, String respondingSystem, Direction direction,
			String requestMessage, Map requestHeaders, String responseMessage, Map responseHeaders,
			String statusCode, String statusMessage, MessageLogMetadata... metadata) {

		logMessage(startTime, messageType, protocol, address, requestingSystem, respondingSystem, direction, requestMessage, requestHeaders, responseMessage, responseHeaders, statusCode, statusMessage, Arrays.asList(metadata));
	}


	public void logMessage(
			LocalDateTime startTime, String messageType, String protocol, String address,
			String requestingSystem, String respondingSystem, Direction direction,
			String requestMessage, Map requestHeaders, String responseMessage, Map responseHeaders,
			String statusCode, String statusMessage, List metadata) {

		MessageLog entry = createLogMessage(
				startTime,
				messageType,
				protocol,
				address,
				requestingSystem,
				respondingSystem,
				direction,
				requestMessage,
				requestHeaders,
				responseMessage,
				responseHeaders,
				metadata);

		logMessage(entry, statusCode, statusMessage);
	}


	public void logZippedMessage(
			LocalDateTime startTime, String messageType, String protocol, String address,
			String requestingSystem, String respondingSystem, Direction direction,
			int requestSize, byte[] requestMessage, Map requestHeaders,
			int responseSize, byte[] responseMessage, Map responseHeaders,
			String statusCode, String statusMessage, MessageLogMetadata... metadata) {

		logZippedMessage(startTime, messageType, protocol, address, requestingSystem, respondingSystem, direction, requestSize, requestMessage, requestHeaders, responseSize, responseMessage, responseHeaders, statusCode, statusMessage, Arrays.asList(metadata));
	}


	public void logZippedMessage(
			LocalDateTime startTime, String messageType, String protocol, String address,
			String requestingSystem, String respondingSystem, Direction direction,
			int requestSize, byte[] requestMessage, Map requestHeaders,
			int responseSize, byte[] responseMessage, Map responseHeaders,
			String statusCode, String statusMessage, List metadata) {

		MessageLog entry = createLogMessageFromZippedData(
				startTime,
				messageType,
				protocol,
				address,
				requestingSystem,
				respondingSystem,
				direction,
				requestSize,
				requestMessage,
				requestHeaders,
				responseSize,
				responseMessage,
				responseHeaders,
				metadata);
		logMessage(entry, statusCode, statusMessage);
	}


	public MessageLog createLogMessage(
			LocalDateTime startTime, String messageType, String protocol, String address,
			String requestingSystem, String respondingSystem, Direction direction,
			String requestMessage, Map requestHeaders, String responseMessage, Map responseHeaders,
			List metadata) {

		return createLogMessageFromZippedData(
				startTime,
				messageType,
				protocol,
				address,
				requestingSystem,
				respondingSystem,
				direction,
				requestMessage == null ? 0: requestMessage.length(),
				createBlobData(requestMessage),
				requestHeaders,
				responseMessage == null ? 0: responseMessage.length(),
				createBlobData(responseMessage),
				responseHeaders,
				metadata);
	}


	public MessageLog createLogMessageFromZippedData(
			LocalDateTime startTime, String messageType, String protocol, String address,
			String requestingSystem, String respondingSystem, Direction direction,
			int requestSize, byte[] requestMessage, Map requestHeaders,
			int responseSize, byte[] responseMessage, Map responseHeaders,
			MessageLogMetadata... metadata) {

		return createLogMessageFromZippedData(startTime, messageType, protocol, address, requestingSystem, respondingSystem, direction, requestSize, requestMessage, requestHeaders, responseSize, responseMessage, responseHeaders, Arrays.asList(metadata));
	}


	public MessageLog createLogMessageFromZippedData(
			LocalDateTime startTime, String messageType, String protocol, String address,
			String requestingSystem, String respondingSystem, Direction direction,
			int requestSize, byte[] requestMessage, Map requestHeaders,
			int responseSize, byte[] responseMessage, Map responseHeaders,
			List metadata) {

		List meta = prepareFullMetadata(requestMessage, responseMessage, metadata);
		return new MessageLog(
				startTime,
				gitDescription,
				messageType,
				protocol,
				address,
				requestingSystem,
				respondingSystem,
				direction,
				requestSize,
				requestMessage,
				prepareHeaders(requestHeaders),
				responseSize,
				responseMessage,
				prepareHeaders(responseHeaders),
				meta);
	}


	private static byte[] createBlobData(String message) {
		return GzipUtils.zip(message, StandardCharsets.UTF_8);
	}


	private String prepareHeaders(Map properties) {
		if (properties == null)
			return null;

		StringBuilder builder = new StringBuilder();
		for (Map.Entry entry : properties.entrySet()) {
			String key = entry.getKey();
			String value = entry.getValue() == null ? null : entry.getValue().toString();
			if (key == null || value == null || key.length() == 0 || value.length() == 0)
				continue;

			if (redactedHeadersRegex.matcher(key).matches())
				value = "*".repeat(value.length());
			builder.append(key).append('=').append(value).append('\n');
		}
		return builder.toString().trim();
	}


	private static List prepareFullMetadata(byte[] request, byte[] response, List metadata) {
		Map metadataByKey = new LinkedHashMap<>();
		MessageLogMetadataAttribute.listMetadata().forEach(m -> metadataByKey.put(m.getKey(), m));
		if (metadata != null)
			metadata.forEach(m -> metadataByKey.put(m.getKey(), m));
		if (request != null)
			metadataByKey.put("RequestMessageDigest", new MessageLogMetadata("RequestMessageDigest", digest(request)));
		if (response != null)
			metadataByKey.put("ResponseMessageDigest", new MessageLogMetadata("ResponseMessageDigest", digest(response)));
		return new ArrayList<>(metadataByKey.values());
	}


	private static String digest(byte[] data) {
		if (data == null)
			return null;

		try {
			MessageDigest messageDigest = MessageDigest.getInstance("MD5");
			String digest = FormatUtils.toHex(messageDigest.digest(data));
			return FormatUtils.padLeft(digest, 32, '0');
		}
		catch (Exception e) {
			LOG.error("Failed generating digest for message", e);
			return null;
		}
	}



	public void logMessage(MessageLog entry, String statusCode, String statusMessage) {
		if (entry == null) {
			LOG.error("Trying to log NULL entry to MESSAGE LOG");
			return;
		}

		entry.setEndState(
				(int)ChronoUnit.MILLIS.between(entry.getStartTime(), LocalDateTime.now()),
				statusCode,
				statusMessage);

		try {
			if (!messageLogQueue.offer(entry))
				LOG.error("MESSAGE log message buffer OVERFLOW");
		} catch (RuntimeException e) {
			LOG.error("ADDING message log entry FAILED", e);
		}
	}


	@Transactional(propagation = Propagation.REQUIRES_NEW)
	public boolean persistMessageLog() {
		int count = messageLogQueue.size();
		int persisted = 0;
		TaskState state = TaskState.FAILED;

		try (Job job = jobStatusService.start(MessageLogService.class)) {
			try {
				List messages = new ArrayList<>(PERSIST_BATCH_SIZE);
				int drained = messageLogQueue.drainTo(messages, PERSIST_BATCH_SIZE);
				for (MessageLog message : messages) {
					em.persist(message);
					updateMessageStatistics(message);
				}

				persisted = drained;
				state = persisted > 0 ? TaskState.DONE : TaskState.NOTHING_TO_DO;
			}
			catch (RuntimeException e) {
				LOG.error("PERSISTING {} log messages FAILED, {} left in queue", persisted, count - persisted, e);
			}
			finally {
				job.setResultState(new ResultState(state, "%s %s / %s", state.isSuccess() ? "Persisted" : "Failed persisting", persisted, count));
			}
		}

		return count > persisted;
	}


	private void updateMessageStatistics(MessageLog messageLog) {
		try {
			messageLogStatisticsService.updateMessageState(messageLog);
		}
		catch (Exception e) {
			LOG.error("Could not update message log statistics", e);
		}
	}


	public String getApplicationName() {
		return applicationName;
	}

}