fi.evolver.basics.spring.messaging.sender.AwsSender Maven / Gradle / Ivy

Go to download
package fi.evolver.basics.spring.messaging.sender;

import static fi.evolver.basics.spring.messaging.util.SendUtils.PROPERTY_CONNECT_TIMEOUT_MS;
import static fi.evolver.basics.spring.messaging.util.SendUtils.PROPERTY_READ_TIMEOUT_MS;
import static fi.evolver.basics.spring.messaging.util.SendUtils.mapMetadata;

import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URI;
import java.net.URL;
import java.nio.CharBuffer;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.util.Collections;
import java.util.Date;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;

import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.CountingInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.amazonaws.ClientConfiguration;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.BasicSessionCredentials;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.services.kinesis.AmazonKinesis;
import com.amazonaws.services.kinesis.AmazonKinesisClientBuilder;
import com.amazonaws.services.kinesis.model.PutRecordRequest;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
import com.amazonaws.services.securitytoken.model.AssumeRoleRequest;
import com.amazonaws.services.securitytoken.model.Credentials;
import com.amazonaws.services.sns.AmazonSNS;
import com.amazonaws.services.sns.AmazonSNSClientBuilder;
import com.amazonaws.services.sns.model.PublishRequest;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
import com.amazonaws.services.sqs.model.SendMessageRequest;

import fi.evolver.basics.spring.log.MessageLogService;
import fi.evolver.basics.spring.log.entity.MessageLog.Direction;
import fi.evolver.basics.spring.messaging.SendResult;
import fi.evolver.basics.spring.messaging.entity.Message;
import fi.evolver.basics.spring.messaging.entity.MessageTargetConfig;
import fi.evolver.basics.spring.messaging.util.SendUtils;
import fi.evolver.utils.CommunicationException;
import fi.evolver.utils.TagUtils;
import fi.evolver.utils.stream.FinishingInputStream;


@Component
public class AwsSender implements Sender {
	private static final Logger LOG = LoggerFactory.getLogger(AwsSender.class);

	private static final String PROTOCOL_KINESIS = "kinesis";
	private static final String PROTOCOL_S3 = "s3";
	private static final String PROTOCOL_S3_SNS = "s3+sns";
	private static final String PROTOCOL_S3_SQS = "s3+sqs";
	private static final String PROTOCOL_SNS = "sns";
	private static final String PROTOCOL_SQS = "sqs";

	private static final int DEFAULT_TTL_MS = 1000 * 60 * 60;

	private static final String PROPERTY_DELAY_SECONDS = "DelayS";
	private static final String PROPERTY_ENCRYPTION = "Encryption";
	private static final String PROPERTY_GROUP_ID = "GroupId";
	private static final String PROPERTY_KMS_KEY_ARN = "KmsKeyArn";
	private static final String PROPERTY_QUEUE = "Queue";
	private static final String PROPERTY_SNS_TOPIC_ARN = "TopicArn";
	private static final String PROPERTY_KINESIS_STREAM_NAME = "KinesisStreamName";
	private static final String PROPERTY_KINESIS_PARTITION_KEY = "KinesisPartitionKey";
	private static final String PROPERTY_STS_ASSUME_ROLE_ARN = "StsRoleArn";
	private static final String PROPERTY_STS_ROLE_DURATION_SECONDS = "StsRoleDurationSeconds";
	private static final String PROPERTY_STS_SESSION_NAME = "StsSessionName";
	private static final String PROPERTY_TIME_TO_LIVE_MS = "TimeToLiveMs";
	private static final String PROPERTY_ACCESS_KEY = "AccessKey";
	private static final String PROPERTY_SECRET_ACCESS_KEY = "SecretAccessKey";

	private static final String AWS_V4_SIGNER_TYPE = "AWSS3V4SignerType";

	private static final String STATUS_FAILED = "FAILED";
	private static final String STATUS_OK = "OK";

	private static final Set PROTOCOLS = Set.of(PROTOCOL_S3, PROTOCOL_SNS, PROTOCOL_S3_SNS, PROTOCOL_SQS, PROTOCOL_S3_SQS, PROTOCOL_KINESIS);


	private final MessageLogService messageLogService;


	@Autowired
	public AwsSender(MessageLogService messageLogService) {
		this.messageLogService = messageLogService;
	}


	@Override
	public SendResult send(Message message, URI uri) {
		switch (uri.getScheme().toLowerCase()) {
			case PROTOCOL_S3:
				return uploadToS3(message, uri);
			case PROTOCOL_SNS:
				return publishMessageToSns(message, uri);
			case PROTOCOL_S3_SNS:
				return uploadToS3AndPublishUrlToSns(message, uri);
			case PROTOCOL_SQS:
				return sendMessageToSqs(message, uri);
			case PROTOCOL_S3_SQS:
				return uploadToS3AndSendUrlToSqs(message, uri);
			case PROTOCOL_KINESIS:
				return publishMessageToKinesis(message, uri);
			default:
				throw new IllegalArgumentException("Unsupported protocol: " + uri.getScheme());
		}
	}


	private SendResult uploadToS3(Message message, URI uri) {
		LocalDateTime start = LocalDateTime.now();
		String statusCode = STATUS_FAILED;
		long requestLength = -1L;

		try {
			AWSCredentialsProvider credentialsProvider = getCredentialsProvider(message, uri);
			ClientConfiguration config = buildClientConfiguration(message.getMessageTargetConfig());
			if (credentialsProvider instanceof AWSStaticCredentialsProvider)
				config.setSignerOverride(AWS_V4_SIGNER_TYPE);

			requestLength = getInputLength(message);
			upload(message, requestLength, uri, config, credentialsProvider, false);
			statusCode = STATUS_OK;
			return SendResult.success();
		}
		catch (IOException e) {
			LOG.warn("S3 upload failed", e);
			return SendResult.error("S3 upload failed: %s", e.getMessage());
		}
		finally {
			messageLogService.logZippedMessage(
					start,
					message.getMessageType(),
					PROTOCOL_S3,
					uri.toString(),
					messageLogService.getApplicationName(),
					message.getTargetSystem(),
					Direction.OUTBOUND,
					(int) requestLength,
					message.getCompressedData(),
					Collections.emptyMap(),
					0,
					null,
					null,
					statusCode,
					null,
					mapMetadata(message.getMetadata()));
		}
	}


	private SendResult publishMessageToSns(Message message, URI uri) {
		LocalDateTime start = LocalDateTime.now();
		String statusCode = STATUS_FAILED;
		String body = null;

		try (Reader reader = new InputStreamReader(message.getDataStream(), StandardCharsets.UTF_8)) {
			AWSCredentialsProvider credentialsProvider = getCredentialsProvider(message, uri);
			ClientConfiguration config = buildClientConfiguration(message.getMessageTargetConfig());

			body = IOUtils.toString(reader);
			if (body != null && !body.isEmpty())
				publishToSns(message, body, uri, config, credentialsProvider);
			statusCode = STATUS_OK;
			return SendResult.success();
		}
		catch (IOException e) {
			LOG.warn("SNS publish failed", e);
			return SendResult.error("SNS publish failed: %s", e.getMessage());
		}
		finally {
			messageLogService.logMessage(
					start,
					message.getMessageType(),
					PROTOCOL_SNS,
					uri.toString(),
					messageLogService.getApplicationName(),
					message.getTargetSystem(),
					Direction.OUTBOUND,
					body,
					Collections.emptyMap(),
					null,
					null,
					statusCode,
					null,
					mapMetadata(message.getMetadata()));
		}
	}


	private SendResult publishMessageToKinesis(Message message, URI uri) {
		LocalDateTime start = LocalDateTime.now();
		String statusCode = STATUS_FAILED;
		String body = null;

		try (Reader reader = new InputStreamReader(message.getDataStream(), StandardCharsets.UTF_8)) {
			body = IOUtils.toString(reader);
			AWSCredentialsProvider credentialsProvider = getCredentialsProvider(message, uri);
			ClientConfiguration config = buildClientConfiguration(message.getMessageTargetConfig());
			publishToKinesis(message, body, uri, config, credentialsProvider);
			statusCode = STATUS_OK;
			return SendResult.success();
		}
		catch (IOException e) {
			LOG.warn("Kinesis publish failed", e);
			return SendResult.error("Kinesis publish failed: %s", e.getMessage());
		}
		finally {
			messageLogService.logMessage(
					start,
					message.getMessageType(),
					PROTOCOL_KINESIS,
					uri.toString(),
					messageLogService.getApplicationName(),
					message.getTargetSystem(),
					Direction.OUTBOUND,
					body,
					Collections.emptyMap(),
					null,
					null,
					statusCode,
					null,
					mapMetadata(message.getMetadata())
			);
		}
	}


	private SendResult sendMessageToSqs(Message message, URI uri) {
		LocalDateTime start = LocalDateTime.now();
		String statusCode = STATUS_FAILED;
		String body = null;

		try (Reader reader = new InputStreamReader(message.getDataStream(), StandardCharsets.UTF_8)) {
			body = IOUtils.toString(reader);
			AWSCredentialsProvider credentialsProvider = getCredentialsProvider(message, uri);
			ClientConfiguration config = buildClientConfiguration(message.getMessageTargetConfig());

			sendToSqs(message, body, uri, config, credentialsProvider);
			statusCode = STATUS_OK;
			return SendResult.success();
		}
		catch (IOException e) {
			LOG.warn("SQS send failed", e);
			return SendResult.error("SQS send failed: %s", e.getMessage());
		}
		finally {
			messageLogService.logMessage(
					start,
					message.getMessageType(),
					PROTOCOL_SQS,
					uri.toString(),
					messageLogService.getApplicationName(),
					message.getTargetSystem(),
					Direction.OUTBOUND,
					body,
					Collections.emptyMap(),
					null,
					null,
					statusCode,
					null,
					mapMetadata(message.getMetadata()));
		}
	}


	private SendResult uploadToS3AndPublishUrlToSns(Message message, URI uri) {
		LocalDateTime start = LocalDateTime.now();
		String statusCode = STATUS_FAILED;
		long requestLength = -1L;

		try {
			AWSCredentialsProvider credentialsProvider = getCredentialsProvider(message, uri);
			ClientConfiguration config = buildClientConfiguration(message.getMessageTargetConfig());
			if (credentialsProvider instanceof AWSStaticCredentialsProvider)
				config.setSignerOverride(AWS_V4_SIGNER_TYPE);

			requestLength = getInputLength(message);
			URL preSignedUrl = upload(message, requestLength, uri, config, credentialsProvider, true);
			if (preSignedUrl == null)
				throw new IllegalStateException("Did not get presigned S3 uri from upload");

			config.setSignerOverride(null);
			publishToSns(message, preSignedUrl.toString(), uri, config, credentialsProvider);
			statusCode = STATUS_OK;
			return SendResult.success();
		}
		catch (IOException e) {
			LOG.warn("S3 upload + SNS publish failed", e);
			return SendResult.error("S3 upload + SNS publish failed: %s", e.getMessage());
		}
		finally {
			messageLogService.logZippedMessage(
					start,
					message.getMessageType(),
					PROTOCOL_S3_SNS,
					uri.toString(),
					messageLogService.getApplicationName(),
					message.getTargetSystem(),
					Direction.OUTBOUND,
					(int) requestLength,
					message.getCompressedData(),
					Collections.emptyMap(),
					0,
					null,
					null,
					statusCode,
					null,
					mapMetadata(message.getMetadata()));
		}
	}


	private SendResult uploadToS3AndSendUrlToSqs(Message message, URI uri) {
		LocalDateTime start = LocalDateTime.now();
		String statusCode = STATUS_FAILED;
		long requestLength = -1L;

		try {
			AWSCredentialsProvider credentialsProvider = getCredentialsProvider(message, uri);
			ClientConfiguration config = buildClientConfiguration(message.getMessageTargetConfig());

			requestLength = getInputLength(message);
			URL preSignedUrl = upload(message, requestLength, uri, config, credentialsProvider, true);
			if (preSignedUrl == null)
				throw new IllegalStateException("Did not get presigned S3 uri from upload");

			config.setSignerOverride(null);
			sendToSqs(message, preSignedUrl.toString(), uri, config, credentialsProvider);
			statusCode = STATUS_OK;
			return SendResult.success();
		}
		catch (IOException e) {
			LOG.warn("S3 upload + SQS send failed", e);
			return SendResult.error("S3 upload + SQS send failed: %s", e.getMessage());
		}
		finally {
			messageLogService.logZippedMessage(
					start,
					message.getMessageType(),
					PROTOCOL_S3_SQS,
					uri.toString(),
					messageLogService.getApplicationName(),
					message.getTargetSystem(),
					Direction.OUTBOUND,
					(int) requestLength,
					message.getCompressedData(),
					Collections.emptyMap(),
					0,
					null,
					null,
					statusCode,
					null,
					mapMetadata(message.getMetadata()));
		}
	}


	private static AWSCredentialsProvider getCredentialsProvider(Message message, URI uri) throws CommunicationException {
		AWSCredentialsProvider baseCredentialsProvider = createBaseCredentialsProvider(message);

		Optional stsRoleArn = SendUtils.getTagReplacedTargetProperty(message, PROPERTY_STS_ASSUME_ROLE_ARN);
		if (stsRoleArn.isPresent()) {
			AWSSecurityTokenService stsClient = AWSSecurityTokenServiceClientBuilder.standard()
					.withCredentials(baseCredentialsProvider)
					.withRegion(parseHost(uri))
					.build();

			AssumeRoleRequest assumeRequest = new AssumeRoleRequest()
					.withRoleArn(stsRoleArn.get())
					.withExternalId(UUID.randomUUID().toString())
					.withDurationSeconds(message.getMessageTargetConfig().getProperty(PROPERTY_STS_ROLE_DURATION_SECONDS).map(Integer::parseInt).orElse(3600))
					.withRoleSessionName(message.getMessageTargetConfig().getProperty(PROPERTY_STS_SESSION_NAME).orElse("DefaultSessionName"));

			Credentials credentials = stsClient.assumeRole(assumeRequest).getCredentials();

			BasicSessionCredentials temporaryCredentials = new BasicSessionCredentials(
					credentials.getAccessKeyId(),
					credentials.getSecretAccessKey(),
					credentials.getSessionToken());

			return new AWSStaticCredentialsProvider(temporaryCredentials);
		}

		return baseCredentialsProvider;
	}


	private static AWSCredentialsProvider createBaseCredentialsProvider(Message message) {
		Optional accessKey = message.getMessageTargetConfig().getProperty(PROPERTY_ACCESS_KEY);
		Optional secretAccessKey = message.getMessageTargetConfig().getProperty(PROPERTY_SECRET_ACCESS_KEY);
		if (accessKey.isPresent() && secretAccessKey.isPresent())
			return new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey.get(), secretAccessKey.get()));
		return new InstanceProfileCredentialsProvider(false);
	}


	private static URL upload(Message message, long contentLength, URI uri, ClientConfiguration config, AWSCredentialsProvider credentialsProvider, boolean generatePreSignedUrl) throws CommunicationException {
		try {
			AmazonS3 s3Client = buildS3Client(uri, config, credentialsProvider);

			String[] pathParts = uri.getPath().split("/", 3);
			if (pathParts.length < 3)
				throw new IllegalArgumentException(String.format("Failed parsing S3 bucket name and filename from URI path: %s", uri.getPath()));

			String bucketName = pathParts[1];
			String fileName = pathParts[2];
			fileName = TagUtils.replaceTags(fileName, TagUtils.TIMESTAMP);

			ObjectMetadata metadata = new ObjectMetadata();
			metadata.setContentLength(contentLength);
			message.getMessageTargetConfig().getProperty(PROPERTY_ENCRYPTION)
					.filter(a -> !"false".equalsIgnoreCase(a))
					.map(a -> "true".equalsIgnoreCase(a) ? ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION : a)
					.ifPresent(metadata::setSSEAlgorithm);

			PutObjectRequest request = new PutObjectRequest(bucketName, fileName, SendUtils.createDataStream(message), metadata);

			message.getMessageTargetConfig().getProperty(PROPERTY_KMS_KEY_ARN)
					.ifPresent(key -> request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(key)));

			s3Client.putObject(request);

			if (generatePreSignedUrl)
				return generatePreSignedUrl(s3Client, fileName, bucketName, message);

			return null;
		}
		catch (CommunicationException e) {
			throw e;
		}
		catch (IOException e) {
			throw new CommunicationException(e, "Failed reading message to input stream");
		}
		catch (SdkClientException e) {
			throw new CommunicationException(e, "Failed uploading to AWS S3");
		}
	}


	private static URL generatePreSignedUrl(AmazonS3 s3Client, String key, String bucketName, Message message) throws CommunicationException {
		try {
			int ttl = message.getMessageTargetConfig().getIntProperty(PROPERTY_TIME_TO_LIVE_MS).orElse(DEFAULT_TTL_MS);
			GeneratePresignedUrlRequest generatePresignedUrlRequest =
					new GeneratePresignedUrlRequest(bucketName, key)
							.withExpiration(new Date(System.currentTimeMillis() + ttl));

			return s3Client.generatePresignedUrl(generatePresignedUrlRequest);
		}
		catch (SdkClientException e) {
			throw new CommunicationException(e, "Failed generating S3 pre signed URL for %s", key);
		}
	}


	private static void publishToSns(Message message, String text, URI uri, ClientConfiguration config, AWSCredentialsProvider credentialsProvider) throws CommunicationException {
		try {
			AmazonSNS snsClient = buildSnsClient(uri, config, credentialsProvider);
			String snsTopicArn = SendUtils.getTagReplacedTargetProperty(message, PROPERTY_SNS_TOPIC_ARN)
					.orElseThrow(() -> new CommunicationException("Missing required property: %s", PROPERTY_SNS_TOPIC_ARN));

			PublishRequest request = new PublishRequest()
					.withTopicArn(snsTopicArn)
					.withMessage(text);
			snsClient.publish(request);
		}
		catch (Exception e) {
			throw new CommunicationException(e, "Failed publishing message (%s) to SNS", text);
		}
	}

	private static void publishToKinesis(Message message, String text, URI uri, ClientConfiguration config, AWSCredentialsProvider credentialsProvider) throws CommunicationException {
		try {
			AmazonKinesis kinesisClient = buildKinesisClient(uri, config, credentialsProvider);
			String kinesisStreamName = SendUtils.getTagReplacedTargetProperty(message, PROPERTY_KINESIS_STREAM_NAME)
					.orElseThrow(() -> new CommunicationException("Missing required property %s", PROPERTY_KINESIS_STREAM_NAME));

			String partition = SendUtils.getTagReplacedTargetProperty(message, PROPERTY_KINESIS_PARTITION_KEY)
					.orElseThrow(() -> new CommunicationException("Missing required property %s", PROPERTY_KINESIS_PARTITION_KEY));

			PutRecordRequest request = new PutRecordRequest()
					.withStreamName(kinesisStreamName)
					.withPartitionKey(partition)
					.withData(
							StandardCharsets.UTF_8.newEncoder().encode(CharBuffer.wrap(text))
					);

			kinesisClient.putRecord(request);
		}
		catch (Exception e) {
			throw new CommunicationException(e, "Failed publishing message (%s) to Kinesis", text);
		}
	}


	private static void sendToSqs(Message message, String text, URI uri, ClientConfiguration config, AWSCredentialsProvider credentialsProvider) throws CommunicationException {
		try {
			AmazonSQS sqsClient = buildSqsClient(uri, config, credentialsProvider);
			String queue = SendUtils.getTagReplacedTargetProperty(message, PROPERTY_QUEUE)
					.orElseThrow(() -> new CommunicationException("Missing required property: %s", PROPERTY_QUEUE));

			SendMessageRequest request = new SendMessageRequest()
					.withQueueUrl(sqsClient.getQueueUrl(queue).getQueueUrl())
					.withMessageBody(text);

			if (queue.endsWith(".fifo")) {
				String groupId = SendUtils.getTagReplacedTargetProperty(message, PROPERTY_GROUP_ID).orElse("default");
				request = request.withMessageGroupId(groupId);
				request.setMessageDeduplicationId(String.format("%s-%s", message.getMessageChainId(), message.getId()));
			} else {
				int delaySeconds = message.getMessageTargetConfig().getIntProperty(PROPERTY_DELAY_SECONDS).orElse(5);
				request = request.withDelaySeconds(delaySeconds);
			}

			sqsClient.sendMessage(request);
		}
		catch (Exception e) {
			throw new CommunicationException(e, "Failed sending message (%s) to SQS", text);
		}
	}


	private static AmazonKinesis buildKinesisClient(URI uri, ClientConfiguration config, AWSCredentialsProvider credentialsProvider) throws CommunicationException {
		return AmazonKinesisClientBuilder.standard()
				.withRegion(parseHost(uri))
				.withClientConfiguration(config)
				.withCredentials(credentialsProvider)
				.build();
	}


	private static AmazonSNS buildSnsClient(URI uri, ClientConfiguration config, AWSCredentialsProvider credentialsProvider) throws CommunicationException {
		return AmazonSNSClientBuilder.standard()
				.withRegion(parseHost(uri))
				.withClientConfiguration(config)
				.withCredentials(credentialsProvider)
				.build();
	}


	private static AmazonSQS buildSqsClient(URI uri, ClientConfiguration config, AWSCredentialsProvider credentialsProvider) throws CommunicationException {
		return AmazonSQSClientBuilder.standard()
				.withRegion(parseHost(uri))
				.withClientConfiguration(config)
				.withCredentials(credentialsProvider)
				.build();
	}


	private static AmazonS3 buildS3Client(URI uri, ClientConfiguration config, AWSCredentialsProvider credentialsProvider) throws CommunicationException {
		return AmazonS3ClientBuilder.standard()
				.withRegion(parseHost(uri))
				.withClientConfiguration(config)
				.withCredentials(credentialsProvider)
				.build();
	}


	private static ClientConfiguration buildClientConfiguration(MessageTargetConfig messageTargetConfig) {
		ClientConfiguration config = new ClientConfiguration();
		messageTargetConfig.getIntProperty(PROPERTY_CONNECT_TIMEOUT_MS).ifPresent(config::setConnectionTimeout);
		messageTargetConfig.getIntProperty(PROPERTY_READ_TIMEOUT_MS).ifPresent(config::setSocketTimeout);
		return config;
	}


	private static String parseHost(URI uri) throws CommunicationException {
		String host = uri.getHost();
		if (host == null || host.isEmpty())
			throw new CommunicationException("URI (%s) missing required host information", uri);
		return host;
	}


	private static long getInputLength(Message message) throws IOException {
		CountingInputStream counter = new CountingInputStream(SendUtils.createDataStream(message));
		try (FinishingInputStream finisher = new FinishingInputStream(counter)) {
			/* Nothing to do */
		}
		return counter.getByteCount();
	}


	@Override
	public Set getSupportedProtocols() {
		return PROTOCOLS;
	}

}