com.amazon.sqs.javamessaging.AmazonSQSExtendedClient Maven / Gradle / Ivy
Show all versions of amazon-sqs-java-extended-client-lib Show documentation
/*
* Copyright 2010-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.sqs.javamessaging;
import java.lang.UnsupportedOperationException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import software.amazon.awssdk.awscore.AwsRequest;
import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.core.ApiName;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.core.exception.SdkClientException;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.util.VersionInfo;
import software.amazon.awssdk.services.sqs.SqsClient;
import software.amazon.awssdk.services.sqs.model.BatchEntryIdsNotDistinctException;
import software.amazon.awssdk.services.sqs.model.BatchRequestTooLongException;
import software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityBatchRequest;
import software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityBatchRequestEntry;
import software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityBatchResponse;
import software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityRequest;
import software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityResponse;
import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchRequest;
import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchRequestEntry;
import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchResponse;
import software.amazon.awssdk.services.sqs.model.DeleteMessageRequest;
import software.amazon.awssdk.services.sqs.model.DeleteMessageResponse;
import software.amazon.awssdk.services.sqs.model.EmptyBatchRequestException;
import software.amazon.awssdk.services.sqs.model.InvalidBatchEntryIdException;
import software.amazon.awssdk.services.sqs.model.InvalidIdFormatException;
import software.amazon.awssdk.services.sqs.model.InvalidMessageContentsException;
import software.amazon.awssdk.services.sqs.model.Message;
import software.amazon.awssdk.services.sqs.model.MessageAttributeValue;
import software.amazon.awssdk.services.sqs.model.MessageNotInflightException;
import software.amazon.awssdk.services.sqs.model.OverLimitException;
import software.amazon.awssdk.services.sqs.model.PurgeQueueInProgressException;
import software.amazon.awssdk.services.sqs.model.PurgeQueueRequest;
import software.amazon.awssdk.services.sqs.model.PurgeQueueResponse;
import software.amazon.awssdk.services.sqs.model.QueueDoesNotExistException;
import software.amazon.awssdk.services.sqs.model.ReceiptHandleIsInvalidException;
import software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest;
import software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse;
import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequest;
import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequestEntry;
import software.amazon.awssdk.services.sqs.model.SendMessageBatchResponse;
import software.amazon.awssdk.services.sqs.model.SendMessageRequest;
import software.amazon.awssdk.services.sqs.model.SendMessageResponse;
import software.amazon.awssdk.services.sqs.model.SqsException;
import software.amazon.awssdk.services.sqs.model.TooManyEntriesInBatchRequestException;
import software.amazon.awssdk.utils.StringUtils;
import software.amazon.payloadoffloading.PayloadS3Pointer;
import software.amazon.payloadoffloading.PayloadStore;
import software.amazon.payloadoffloading.S3BackedPayloadStore;
import software.amazon.payloadoffloading.S3Dao;
import software.amazon.payloadoffloading.Util;
/**
* Amazon SQS Extended Client extends the functionality of Amazon SQS client.
* All service calls made using this client are blocking, and will not return
* until the service call completes.
*
*
* The Amazon SQS extended client enables sending and receiving large messages
* via Amazon S3. You can use this library to:
*
*
*
* - Specify whether messages are always stored in Amazon S3 or only when a
* message size exceeds 256 KB.
* - Send a message that references a single message object stored in an
* Amazon S3 bucket.
* - Get the corresponding message object from an Amazon S3 bucket.
* - Delete the corresponding message object from an Amazon S3 bucket.
*
*/
public class AmazonSQSExtendedClient extends AmazonSQSExtendedClientBase implements SqsClient {
static final String USER_AGENT_NAME = AmazonSQSExtendedClient.class.getSimpleName();
static final String USER_AGENT_VERSION = VersionInfo.SDK_VERSION;
private static final Log LOG = LogFactory.getLog(AmazonSQSExtendedClient.class);
static final String LEGACY_RESERVED_ATTRIBUTE_NAME = "SQSLargePayloadSize";
static final List RESERVED_ATTRIBUTE_NAMES = Arrays.asList(LEGACY_RESERVED_ATTRIBUTE_NAME,
SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME);
private ExtendedClientConfiguration clientConfiguration;
private PayloadStore payloadStore;
/**
* Constructs a new Amazon SQS extended client to invoke service methods on
* Amazon SQS with extended functionality using the specified Amazon SQS
* client object.
*
*
* All service calls made using this new client object are blocking, and
* will not return until the service call completes.
*
* @param sqsClient
* The Amazon SQS client to use to connect to Amazon SQS.
*/
public AmazonSQSExtendedClient(SqsClient sqsClient) {
this(sqsClient, new ExtendedClientConfiguration());
}
/**
* Constructs a new Amazon SQS extended client to invoke service methods on
* Amazon SQS with extended functionality using the specified Amazon SQS
* client object.
*
*
* All service calls made using this new client object are blocking, and
* will not return until the service call completes.
*
* @param sqsClient
* The Amazon SQS client to use to connect to Amazon SQS.
* @param extendedClientConfig
* The extended client configuration options controlling the
* functionality of this client.
*/
public AmazonSQSExtendedClient(SqsClient sqsClient, ExtendedClientConfiguration extendedClientConfig) {
super(sqsClient);
this.clientConfiguration = new ExtendedClientConfiguration(extendedClientConfig);
S3Dao s3Dao = new S3Dao(clientConfiguration.getS3Client(),
clientConfiguration.getServerSideEncryptionStrategy(),
clientConfiguration.getObjectCannedACL());
this.payloadStore = new S3BackedPayloadStore(s3Dao, clientConfiguration.getS3BucketName());
}
/**
*
* Delivers a message to the specified queue.
*
*
*
* A message can include only XML, JSON, and unformatted text. The following Unicode characters are allowed:
*
*
* #x9 | #xA | #xD | #x20 to #xD7FF |
* #xE000 to #xFFFD | #x10000 to #x10FFFF
*
*
* Any characters not included in this list will be rejected. For more information, see the W3C specification for characters.
*
*
*
* @param sendMessageRequest
* @return Result of the SendMessage operation returned by the service.
* @throws InvalidMessageContentsException
* The message contains characters outside the allowed set.
* @throws UnsupportedOperationException
* Error code 400. Unsupported operation.
* @throws SdkException
* Base class for all exceptions that can be thrown by the SDK (both service and client). Can be used for
* catch all scenarios.
* @throws SdkClientException
* If any client side error occurs such as an IO related failure, failure to get credentials, etc.
* @throws SqsException
* Base class for all service exceptions. Unknown exceptions will be thrown as an instance of this type.
* @sample SqsClient.SendMessage
* @see AWS API
* Documentation
*/
public SendMessageResponse sendMessage(SendMessageRequest sendMessageRequest) {
//TODO: Clone request since it's modified in this method and will cause issues if the client reuses request object.
if (sendMessageRequest == null) {
String errorMessage = "sendMessageRequest cannot be null.";
LOG.error(errorMessage);
throw SdkClientException.create(errorMessage);
}
SendMessageRequest.Builder sendMessageRequestBuilder = sendMessageRequest.toBuilder();
sendMessageRequest = appendUserAgent(sendMessageRequestBuilder).build();
if (!clientConfiguration.isPayloadSupportEnabled()) {
return super.sendMessage(sendMessageRequest);
}
if (StringUtils.isEmpty(sendMessageRequest.messageBody())) {
String errorMessage = "messageBody cannot be null or empty.";
LOG.error(errorMessage);
throw SdkClientException.create(errorMessage);
}
//Check message attributes for ExtendedClient related constraints
checkMessageAttributes(sendMessageRequest.messageAttributes());
if (clientConfiguration.isAlwaysThroughS3() || isLarge(sendMessageRequest)) {
sendMessageRequest = storeMessageInS3(sendMessageRequest);
}
return super.sendMessage(sendMessageRequest);
}
/**
*
* Retrieves one or more messages (up to 10), from the specified queue. Using the WaitTimeSeconds
* parameter enables long-poll support. For more information, see Amazon
* SQS Long Polling in the Amazon Simple Queue Service Developer Guide.
*
*
* Short poll is the default behavior where a weighted random set of machines is sampled on a
* ReceiveMessage call. Thus, only the messages on the sampled machines are returned. If the number of
* messages in the queue is small (fewer than 1,000), you most likely get fewer messages than you requested per
* ReceiveMessage call. If the number of messages in the queue is extremely small, you might not
* receive any messages in a particular ReceiveMessage response. If this happens, repeat the request.
*
*
* For each message returned, the response includes the following:
*
*
* -
*
* The message body.
*
*
* -
*
* An MD5 digest of the message body. For information about MD5, see RFC1321.
*
*
* -
*
* The MessageId you received when you sent the message to the queue.
*
*
* -
*
* The receipt handle.
*
*
* -
*
* The message attributes.
*
*
* -
*
* An MD5 digest of the message attributes.
*
*
*
*
* The receipt handle is the identifier you must provide when deleting the message. For more information, see Queue and Message Identifiers in the Amazon Simple Queue Service Developer Guide.
*
*
* You can provide the VisibilityTimeout parameter in your request. The parameter is applied to the
* messages that Amazon SQS returns in the response. If you don't include the parameter, the overall visibility
* timeout for the queue is used for the returned messages. For more information, see Visibility Timeout in the Amazon Simple Queue Service Developer Guide.
*
*
* A message that isn't deleted or a message whose visibility isn't extended before the visibility timeout expires
* counts as a failed receive. Depending on the configuration of the queue, the message might be sent to the
* dead-letter queue.
*
*
*
* In the future, new attributes might be added. If you write code that calls this action, we recommend that you
* structure your code so that it can handle new attributes gracefully.
*
*
*
* @param receiveMessageRequest
* @return Result of the ReceiveMessage operation returned by the service.
* @throws OverLimitException
* The specified action violates a limit. For example, ReceiveMessage returns this error if the
* maximum number of inflight messages is reached and AddPermission returns this error if the
* maximum number of permissions for the queue is reached.
* @throws SdkException
* Base class for all exceptions that can be thrown by the SDK (both service and client). Can be used for
* catch all scenarios.
* @throws SdkClientException
* If any client side error occurs such as an IO related failure, failure to get credentials, etc.
* @throws SqsException
* Base class for all service exceptions. Unknown exceptions will be thrown as an instance of this type.
* @sample SqsClient.ReceiveMessage
* @see AWS API
* Documentation
*/
public ReceiveMessageResponse receiveMessage(ReceiveMessageRequest receiveMessageRequest) {
//TODO: Clone request since it's modified in this method and will cause issues if the client reuses request object.
if (receiveMessageRequest == null) {
String errorMessage = "receiveMessageRequest cannot be null.";
LOG.error(errorMessage);
throw SdkClientException.create(errorMessage);
}
ReceiveMessageRequest.Builder receiveMessageRequestBuilder = receiveMessageRequest.toBuilder();
appendUserAgent(receiveMessageRequestBuilder);
if (!clientConfiguration.isPayloadSupportEnabled()) {
return super.receiveMessage(receiveMessageRequestBuilder.build());
}
//Remove before adding to avoid any duplicates
List messageAttributeNames = new ArrayList<>(receiveMessageRequest.messageAttributeNames());
messageAttributeNames.removeAll(RESERVED_ATTRIBUTE_NAMES);
messageAttributeNames.addAll(RESERVED_ATTRIBUTE_NAMES);
receiveMessageRequestBuilder.messageAttributeNames(messageAttributeNames);
receiveMessageRequest = receiveMessageRequestBuilder.build();
ReceiveMessageResponse receiveMessageResponse = super.receiveMessage(receiveMessageRequest);
ReceiveMessageResponse.Builder receiveMessageResponseBuilder = receiveMessageResponse.toBuilder();
List messages = receiveMessageResponse.messages();
List modifiedMessages = new ArrayList<>(messages.size());
for (Message message : messages) {
Message.Builder messageBuilder = message.toBuilder();
// for each received message check if they are stored in S3.
Optional largePayloadAttributeName = getReservedAttributeNameIfPresent(message.messageAttributes());
if (largePayloadAttributeName.isPresent()) {
String largeMessagePointer = message.body();
largeMessagePointer = largeMessagePointer.replace("com.amazon.sqs.javamessaging.MessageS3Pointer", "software.amazon.payloadoffloading.PayloadS3Pointer");
messageBuilder.body(payloadStore.getOriginalPayload(largeMessagePointer));
// remove the additional attribute before returning the message
// to user.
Map messageAttributes = new HashMap<>(message.messageAttributes());
messageAttributes.keySet().removeAll(RESERVED_ATTRIBUTE_NAMES);
messageBuilder.messageAttributes(messageAttributes);
// Embed s3 object pointer in the receipt handle.
String modifiedReceiptHandle = embedS3PointerInReceiptHandle(
message.receiptHandle(),
largeMessagePointer);
messageBuilder.receiptHandle(modifiedReceiptHandle);
}
modifiedMessages.add(messageBuilder.build());
}
receiveMessageResponseBuilder.messages(modifiedMessages);
return receiveMessageResponseBuilder.build();
}
/**
*
* Deletes the specified message from the specified queue. To select the message to delete, use the
* ReceiptHandle of the message (not the MessageId which you receive when you send
* the message). Amazon SQS can delete a message from a queue even if a visibility timeout setting causes the
* message to be locked by another consumer. Amazon SQS automatically deletes messages left in a queue longer than
* the retention period configured for the queue.
*
*
*
* The ReceiptHandle is associated with a specific instance of receiving a message. If you
* receive a message more than once, the ReceiptHandle is different each time you receive a message.
* When you use the DeleteMessage action, you must provide the most recently received
* ReceiptHandle for the message (otherwise, the request succeeds, but the message might not be
* deleted).
*
*
* For standard queues, it is possible to receive a message even after you delete it. This might happen on rare
* occasions if one of the servers which stores a copy of the message is unavailable when you send the request to
* delete the message. The copy remains on the server and might be returned to you during a subsequent receive
* request. You should ensure that your application is idempotent, so that receiving a message more than once does
* not cause issues.
*
*
*
* @param deleteMessageRequest
* @return Result of the DeleteMessage operation returned by the service.
* @throws InvalidIdFormatException
* The specified receipt handle isn't valid for the current version.
* @throws ReceiptHandleIsInvalidException
* The specified receipt handle isn't valid.
* @throws SdkException
* Base class for all exceptions that can be thrown by the SDK (both service and client). Can be used for
* catch all scenarios.
* @throws SdkClientException
* If any client side error occurs such as an IO related failure, failure to get credentials, etc.
* @throws SqsException
* Base class for all service exceptions. Unknown exceptions will be thrown as an instance of this type.
* @sample SqsClient.DeleteMessage
* @see AWS API
* Documentation
*/
public DeleteMessageResponse deleteMessage(DeleteMessageRequest deleteMessageRequest) {
if (deleteMessageRequest == null) {
String errorMessage = "deleteMessageRequest cannot be null.";
LOG.error(errorMessage);
throw SdkClientException.create(errorMessage);
}
DeleteMessageRequest.Builder deleteMessageRequestBuilder = deleteMessageRequest.toBuilder();
appendUserAgent(deleteMessageRequestBuilder);
if (!clientConfiguration.isPayloadSupportEnabled()) {
return super.deleteMessage(deleteMessageRequestBuilder.build());
}
String receiptHandle = deleteMessageRequest.receiptHandle();
String origReceiptHandle = receiptHandle;
// Update original receipt handle if needed
if (isS3ReceiptHandle(receiptHandle)) {
origReceiptHandle = getOrigReceiptHandle(receiptHandle);
// Delete pay load from S3 if needed
if (clientConfiguration.doesCleanupS3Payload()) {
String messagePointer = getMessagePointerFromModifiedReceiptHandle(receiptHandle);
payloadStore.deleteOriginalPayload(messagePointer);
}
}
deleteMessageRequestBuilder.receiptHandle(origReceiptHandle);
return super.deleteMessage(deleteMessageRequestBuilder.build());
}
/**
*
* Changes the visibility timeout of a specified message in a queue to a new value. The default visibility timeout
* for a message is 30 seconds. The minimum is 0 seconds. The maximum is 12 hours. For more information, see
* Visibility Timeout in the Amazon Simple Queue Service Developer Guide.
*
*
* For example, you have a message with a visibility timeout of 5 minutes. After 3 minutes, you call
* ChangeMessageVisibility with a timeout of 10 minutes. You can continue to call
* ChangeMessageVisibility to extend the visibility timeout to the maximum allowed time. If you try to
* extend the visibility timeout beyond the maximum, your request is rejected.
*
*
* An Amazon SQS message has three basic states:
*
*
* -
*
* Sent to a queue by a producer.
*
*
* -
*
* Received from the queue by a consumer.
*
*
* -
*
* Deleted from the queue.
*
*
*
*
* A message is considered to be stored after it is sent to a queue by a producer, but not yet received from
* the queue by a consumer (that is, between states 1 and 2). There is no limit to the number of stored messages. A
* message is considered to be in flight after it is received from a queue by a consumer, but not yet deleted
* from the queue (that is, between states 2 and 3). There is a limit to the number of inflight messages.
*
*
* Limits that apply to inflight messages are unrelated to the unlimited number of stored messages.
*
*
* For most standard queues (depending on queue traffic and message backlog), there can be a maximum of
* approximately 120,000 inflight messages (received from a queue by a consumer, but not yet deleted from the
* queue). If you reach this limit, Amazon SQS returns the OverLimit error message. To avoid reaching
* the limit, you should delete messages from the queue after they're processed. You can also increase the number of
* queues you use to process your messages. To request a limit increase, file a support request.
*
*
* For FIFO queues, there can be a maximum of 20,000 inflight messages (received from a queue by a consumer, but not
* yet deleted from the queue). If you reach this limit, Amazon SQS returns no error messages.
*
*
*
* If you attempt to set the VisibilityTimeout to a value greater than the maximum time left, Amazon
* SQS returns an error. Amazon SQS doesn't automatically recalculate and increase the timeout to the maximum
* remaining time.
*
*
* Unlike with a queue, when you change the visibility timeout for a specific message the timeout value is applied
* immediately but isn't saved in memory for that message. If you don't delete a message after it is received, the
* visibility timeout for the message reverts to the original timeout value (not to the value you set using the
* ChangeMessageVisibility action) the next time the message is received.
*
*
*
* @param changeMessageVisibilityRequest
* @return Result of the ChangeMessageVisibility operation returned by the service.
* @throws MessageNotInflightException
* The specified message isn't in flight.
* @throws ReceiptHandleIsInvalidException
* The specified receipt handle isn't valid.
* @throws SdkException
* Base class for all exceptions that can be thrown by the SDK (both service and client). Can be used for
* catch all scenarios.
* @throws SdkClientException
* If any client side error occurs such as an IO related failure, failure to get credentials, etc.
* @throws SqsException
* Base class for all service exceptions. Unknown exceptions will be thrown as an instance of this type.
* @sample SqsClient.ChangeMessageVisibility
* @see AWS
* API Documentation
*/
public ChangeMessageVisibilityResponse changeMessageVisibility(ChangeMessageVisibilityRequest changeMessageVisibilityRequest)
throws AwsServiceException, SdkClientException {
ChangeMessageVisibilityRequest.Builder changeMessageVisibilityRequestBuilder = changeMessageVisibilityRequest.toBuilder();
if (isS3ReceiptHandle(changeMessageVisibilityRequest.receiptHandle())) {
changeMessageVisibilityRequestBuilder.receiptHandle(
getOrigReceiptHandle(changeMessageVisibilityRequest.receiptHandle()));
}
return amazonSqsToBeExtended.changeMessageVisibility(changeMessageVisibilityRequestBuilder.build());
}
/**
*
* Delivers up to ten messages to the specified queue. This is a batch version of SendMessage.
* For a FIFO queue, multiple messages within a single batch are enqueued in the order they are sent.
*
*
* The result of sending each message is reported individually in the response. Because the batch request can result
* in a combination of successful and unsuccessful actions, you should check for batch errors even when the call
* returns an HTTP status code of 200.
*
*
* The maximum allowed individual message size and the maximum total payload size (the sum of the individual lengths
* of all of the batched messages) are both 256 KB (262,144 bytes).
*
*
*
* A message can include only XML, JSON, and unformatted text. The following Unicode characters are allowed:
*
*
* #x9 | #xA | #xD | #x20 to #xD7FF |
* #xE000 to #xFFFD | #x10000 to #x10FFFF
*
*
* Any characters not included in this list will be rejected. For more information, see the W3C specification for characters.
*
*
*
* If you don't specify the DelaySeconds parameter for an entry, Amazon SQS uses the default value for
* the queue.
*
*
* Some actions take lists of parameters. These lists are specified using the param.n notation. Values
* of n are integers starting from 1. For example, a parameter list with two elements looks like this:
*
*
* &AttributeName.1=first
*
*
* &AttributeName.2=second
*
*
* @param sendMessageBatchRequest
* @return Result of the SendMessageBatch operation returned by the service.
* @throws TooManyEntriesInBatchRequestException
* The batch request contains more entries than permissible.
* @throws EmptyBatchRequestException
* The batch request doesn't contain any entries.
* @throws BatchEntryIdsNotDistinctException
* Two or more batch entries in the request have the same Id.
* @throws BatchRequestTooLongException
* The length of all the messages put together is more than the limit.
* @throws InvalidBatchEntryIdException
* The Id of a batch entry in a batch request doesn't abide by the specification.
* @throws UnsupportedOperationException
* Error code 400. Unsupported operation.
* @throws SdkException
* Base class for all exceptions that can be thrown by the SDK (both service and client). Can be used for
* catch all scenarios.
* @throws SdkClientException
* If any client side error occurs such as an IO related failure, failure to get credentials, etc.
* @throws SqsException
* Base class for all service exceptions. Unknown exceptions will be thrown as an instance of this type.
* @sample SqsClient.SendMessageBatch
* @see AWS API
* Documentation
*/
public SendMessageBatchResponse sendMessageBatch(SendMessageBatchRequest sendMessageBatchRequest) {
if (sendMessageBatchRequest == null) {
String errorMessage = "sendMessageBatchRequest cannot be null.";
LOG.error(errorMessage);
throw SdkClientException.create(errorMessage);
}
SendMessageBatchRequest.Builder sendMessageBatchRequestBuilder = sendMessageBatchRequest.toBuilder();
appendUserAgent(sendMessageBatchRequestBuilder);
sendMessageBatchRequest = sendMessageBatchRequestBuilder.build();
if (!clientConfiguration.isPayloadSupportEnabled()) {
return super.sendMessageBatch(sendMessageBatchRequest);
}
List batchEntries = new ArrayList<>(sendMessageBatchRequest.entries().size());
boolean hasS3Entries = false;
for (SendMessageBatchRequestEntry entry : sendMessageBatchRequest.entries()) {
//Check message attributes for ExtendedClient related constraints
checkMessageAttributes(entry.messageAttributes());
if (clientConfiguration.isAlwaysThroughS3() || isLarge(entry)) {
entry = storeMessageInS3(entry);
hasS3Entries = true;
}
batchEntries.add(entry);
}
if (hasS3Entries) {
sendMessageBatchRequest = sendMessageBatchRequest.toBuilder().entries(batchEntries).build();
}
return super.sendMessageBatch(sendMessageBatchRequest);
}
/**
*
* Deletes up to ten messages from the specified queue. This is a batch version of
* DeleteMessage. The result of the action on each message is reported individually in the
* response.
*
*
*
* Because the batch request can result in a combination of successful and unsuccessful actions, you should check
* for batch errors even when the call returns an HTTP status code of 200.
*
*
*
* Some actions take lists of parameters. These lists are specified using the param.n notation. Values
* of n are integers starting from 1. For example, a parameter list with two elements looks like this:
*
*
* &AttributeName.1=first
*
*
* &AttributeName.2=second
*
*
* @param deleteMessageBatchRequest
* @return Result of the DeleteMessageBatch operation returned by the service.
* @throws TooManyEntriesInBatchRequestException
* The batch request contains more entries than permissible.
* @throws EmptyBatchRequestException
* The batch request doesn't contain any entries.
* @throws BatchEntryIdsNotDistinctException
* Two or more batch entries in the request have the same Id.
* @throws InvalidBatchEntryIdException
* The Id of a batch entry in a batch request doesn't abide by the specification.
* @throws SdkException
* Base class for all exceptions that can be thrown by the SDK (both service and client). Can be used for
* catch all scenarios.
* @throws SdkClientException
* If any client side error occurs such as an IO related failure, failure to get credentials, etc.
* @throws SqsException
* Base class for all service exceptions. Unknown exceptions will be thrown as an instance of this type.
* @sample SqsClient.DeleteMessageBatch
* @see AWS API
* Documentation
*/
public DeleteMessageBatchResponse deleteMessageBatch(DeleteMessageBatchRequest deleteMessageBatchRequest) {
if (deleteMessageBatchRequest == null) {
String errorMessage = "deleteMessageBatchRequest cannot be null.";
LOG.error(errorMessage);
throw SdkClientException.create(errorMessage);
}
DeleteMessageBatchRequest.Builder deleteMessageBatchRequestBuilder = deleteMessageBatchRequest.toBuilder();
appendUserAgent(deleteMessageBatchRequestBuilder);
if (!clientConfiguration.isPayloadSupportEnabled()) {
return super.deleteMessageBatch(deleteMessageBatchRequest);
}
List entries = new ArrayList<>(deleteMessageBatchRequest.entries().size());
for (DeleteMessageBatchRequestEntry entry : deleteMessageBatchRequest.entries()) {
DeleteMessageBatchRequestEntry.Builder entryBuilder = entry.toBuilder();
String receiptHandle = entry.receiptHandle();
String origReceiptHandle = receiptHandle;
// Update original receipt handle if needed
if (isS3ReceiptHandle(receiptHandle)) {
origReceiptHandle = getOrigReceiptHandle(receiptHandle);
// Delete s3 payload if needed
if (clientConfiguration.doesCleanupS3Payload()) {
String messagePointer = getMessagePointerFromModifiedReceiptHandle(receiptHandle);
payloadStore.deleteOriginalPayload(messagePointer);
}
}
entryBuilder.receiptHandle(origReceiptHandle);
entries.add(entryBuilder.build());
}
deleteMessageBatchRequestBuilder.entries(entries);
return super.deleteMessageBatch(deleteMessageBatchRequestBuilder.build());
}
/**
*
* Changes the visibility timeout of multiple messages. This is a batch version of
* ChangeMessageVisibility. The result of the action on each message is reported individually
* in the response. You can send up to 10 ChangeMessageVisibility requests with each
* ChangeMessageVisibilityBatch action.
*
*
*
* Because the batch request can result in a combination of successful and unsuccessful actions, you should check
* for batch errors even when the call returns an HTTP status code of 200.
*
*
*
* Some actions take lists of parameters. These lists are specified using the param.n notation. Values
* of n are integers starting from 1. For example, a parameter list with two elements looks like this:
*
*
* &AttributeName.1=first
*
*
* &AttributeName.2=second
*
*
* @param changeMessageVisibilityBatchRequest
* @return Result of the ChangeMessageVisibilityBatch operation returned by the service.
* @throws TooManyEntriesInBatchRequestException
* The batch request contains more entries than permissible.
* @throws EmptyBatchRequestException
* The batch request doesn't contain any entries.
* @throws BatchEntryIdsNotDistinctException
* Two or more batch entries in the request have the same Id.
* @throws InvalidBatchEntryIdException
* The Id of a batch entry in a batch request doesn't abide by the specification.
* @throws SdkException
* Base class for all exceptions that can be thrown by the SDK (both service and client). Can be used for
* catch all scenarios.
* @throws SdkClientException
* If any client side error occurs such as an IO related failure, failure to get credentials, etc.
* @throws SqsException
* Base class for all service exceptions. Unknown exceptions will be thrown as an instance of this type.
* @sample SqsClient.ChangeMessageVisibilityBatch
* @see AWS API Documentation
*/
public ChangeMessageVisibilityBatchResponse changeMessageVisibilityBatch(
ChangeMessageVisibilityBatchRequest changeMessageVisibilityBatchRequest) throws AwsServiceException,
SdkClientException {
List entries = new ArrayList<>(changeMessageVisibilityBatchRequest.entries().size());
for (ChangeMessageVisibilityBatchRequestEntry entry : changeMessageVisibilityBatchRequest.entries()) {
ChangeMessageVisibilityBatchRequestEntry.Builder entryBuilder = entry.toBuilder();
if (isS3ReceiptHandle(entry.receiptHandle())) {
entryBuilder.receiptHandle(getOrigReceiptHandle(entry.receiptHandle()));
}
entries.add(entryBuilder.build());
}
return amazonSqsToBeExtended.changeMessageVisibilityBatch(
changeMessageVisibilityBatchRequest.toBuilder().entries(entries).build());
}
/**
*
* Deletes the messages in a queue specified by the QueueURL parameter.
*
*
*
* When you use the PurgeQueue action, you can't retrieve any messages deleted from a queue.
*
*
* The message deletion process takes up to 60 seconds. We recommend waiting for 60 seconds regardless of your
* queue's size.
*
*
*
* Messages sent to the queue before you call PurgeQueue might be received but are deleted
* within the next minute.
*
*
* Messages sent to the queue after you call PurgeQueue might be deleted while the queue is
* being purged.
*
*
* @param purgeQueueRequest
* @return Result of the PurgeQueue operation returned by the service.
* @throws QueueDoesNotExistException
* The specified queue doesn't exist.
* @throws PurgeQueueInProgressException
* Indicates that the specified queue previously received a PurgeQueue request within the last
* 60 seconds (the time it can take to delete the messages in the queue).
* @throws SdkException
* Base class for all exceptions that can be thrown by the SDK (both service and client). Can be used for
* catch all scenarios.
* @throws SdkClientException
* If any client side error occurs such as an IO related failure, failure to get credentials, etc.
* @throws SqsException
* Base class for all service exceptions. Unknown exceptions will be thrown as an instance of this type.
* @sample SqsClient.PurgeQueue
* @see AWS API
* Documentation
*/
public PurgeQueueResponse purgeQueue(PurgeQueueRequest purgeQueueRequest)
throws AwsServiceException, SdkClientException {
LOG.warn("Calling purgeQueue deletes SQS messages without deleting their payload from S3.");
if (purgeQueueRequest == null) {
String errorMessage = "purgeQueueRequest cannot be null.";
LOG.error(errorMessage);
throw SdkClientException.create(errorMessage);
}
PurgeQueueRequest.Builder purgeQueueRequestBuilder = purgeQueueRequest.toBuilder();
appendUserAgent(purgeQueueRequestBuilder);
return super.purgeQueue(purgeQueueRequestBuilder.build());
}
private void checkMessageAttributes(Map messageAttributes) {
int msgAttributesSize = getMsgAttributesSize(messageAttributes);
if (msgAttributesSize > clientConfiguration.getPayloadSizeThreshold()) {
String errorMessage = "Total size of Message attributes is " + msgAttributesSize
+ " bytes which is larger than the threshold of " + clientConfiguration.getPayloadSizeThreshold()
+ " Bytes. Consider including the payload in the message body instead of message attributes.";
LOG.error(errorMessage);
throw SdkClientException.create(errorMessage);
}
int messageAttributesNum = messageAttributes.size();
if (messageAttributesNum > SQSExtendedClientConstants.MAX_ALLOWED_ATTRIBUTES) {
String errorMessage = "Number of message attributes [" + messageAttributesNum
+ "] exceeds the maximum allowed for large-payload messages ["
+ SQSExtendedClientConstants.MAX_ALLOWED_ATTRIBUTES + "].";
LOG.error(errorMessage);
throw SdkClientException.create(errorMessage);
}
Optional largePayloadAttributeName = getReservedAttributeNameIfPresent(messageAttributes);
if (largePayloadAttributeName.isPresent()) {
String errorMessage = "Message attribute name " + largePayloadAttributeName.get()
+ " is reserved for use by SQS extended client.";
LOG.error(errorMessage);
throw SdkClientException.create(errorMessage);
}
}
/**
* TODO: Wrap the message pointer as-is to the receiptHandle so that it can be generic
* and does not use any LargeMessageStore implementation specific details.
*/
private String embedS3PointerInReceiptHandle(String receiptHandle, String pointer) {
PayloadS3Pointer s3Pointer = PayloadS3Pointer.fromJson(pointer);
String s3MsgBucketName = s3Pointer.getS3BucketName();
String s3MsgKey = s3Pointer.getS3Key();
String modifiedReceiptHandle = SQSExtendedClientConstants.S3_BUCKET_NAME_MARKER + s3MsgBucketName
+ SQSExtendedClientConstants.S3_BUCKET_NAME_MARKER + SQSExtendedClientConstants.S3_KEY_MARKER
+ s3MsgKey + SQSExtendedClientConstants.S3_KEY_MARKER + receiptHandle;
return modifiedReceiptHandle;
}
private String getOrigReceiptHandle(String receiptHandle) {
int secondOccurence = receiptHandle.indexOf(SQSExtendedClientConstants.S3_KEY_MARKER,
receiptHandle.indexOf(SQSExtendedClientConstants.S3_KEY_MARKER) + 1);
return receiptHandle.substring(secondOccurence + SQSExtendedClientConstants.S3_KEY_MARKER.length());
}
private String getFromReceiptHandleByMarker(String receiptHandle, String marker) {
int firstOccurence = receiptHandle.indexOf(marker);
int secondOccurence = receiptHandle.indexOf(marker, firstOccurence + 1);
return receiptHandle.substring(firstOccurence + marker.length(), secondOccurence);
}
private boolean isS3ReceiptHandle(String receiptHandle) {
return receiptHandle.contains(SQSExtendedClientConstants.S3_BUCKET_NAME_MARKER)
&& receiptHandle.contains(SQSExtendedClientConstants.S3_KEY_MARKER);
}
private String getMessagePointerFromModifiedReceiptHandle(String receiptHandle) {
String s3MsgBucketName = getFromReceiptHandleByMarker(receiptHandle, SQSExtendedClientConstants.S3_BUCKET_NAME_MARKER);
String s3MsgKey = getFromReceiptHandleByMarker(receiptHandle, SQSExtendedClientConstants.S3_KEY_MARKER);
PayloadS3Pointer payloadS3Pointer = new PayloadS3Pointer(s3MsgBucketName, s3MsgKey);
return payloadS3Pointer.toJson();
}
private boolean isLarge(SendMessageRequest sendMessageRequest) {
int msgAttributesSize = getMsgAttributesSize(sendMessageRequest.messageAttributes());
long msgBodySize = Util.getStringSizeInBytes(sendMessageRequest.messageBody());
long totalMsgSize = msgAttributesSize + msgBodySize;
return (totalMsgSize > clientConfiguration.getPayloadSizeThreshold());
}
private boolean isLarge(SendMessageBatchRequestEntry batchEntry) {
int msgAttributesSize = getMsgAttributesSize(batchEntry.messageAttributes());
long msgBodySize = Util.getStringSizeInBytes(batchEntry.messageBody());
long totalMsgSize = msgAttributesSize + msgBodySize;
return (totalMsgSize > clientConfiguration.getPayloadSizeThreshold());
}
private Optional getReservedAttributeNameIfPresent(Map msgAttributes) {
String reservedAttributeName = null;
if (msgAttributes.containsKey(SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME)) {
reservedAttributeName = SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME;
} else if (msgAttributes.containsKey(LEGACY_RESERVED_ATTRIBUTE_NAME)) {
reservedAttributeName = LEGACY_RESERVED_ATTRIBUTE_NAME;
}
return Optional.ofNullable(reservedAttributeName);
}
private int getMsgAttributesSize(Map msgAttributes) {
int totalMsgAttributesSize = 0;
for (Map.Entry entry : msgAttributes.entrySet()) {
totalMsgAttributesSize += Util.getStringSizeInBytes(entry.getKey());
MessageAttributeValue entryVal = entry.getValue();
if (entryVal.dataType() != null) {
totalMsgAttributesSize += Util.getStringSizeInBytes(entryVal.dataType());
}
String stringVal = entryVal.stringValue();
if (stringVal != null) {
totalMsgAttributesSize += Util.getStringSizeInBytes(entryVal.stringValue());
}
SdkBytes binaryVal = entryVal.binaryValue();
if (binaryVal != null) {
totalMsgAttributesSize += binaryVal.asByteArray().length;
}
}
return totalMsgAttributesSize;
}
private SendMessageBatchRequestEntry storeMessageInS3(SendMessageBatchRequestEntry batchEntry) {
// Read the content of the message from message body
String messageContentStr = batchEntry.messageBody();
Long messageContentSize = Util.getStringSizeInBytes(messageContentStr);
SendMessageBatchRequestEntry.Builder batchEntryBuilder = batchEntry.toBuilder();
batchEntryBuilder.messageAttributes(
updateMessageAttributePayloadSize(batchEntry.messageAttributes(), messageContentSize));
// Store the message content in S3.
String largeMessagePointer = storeOriginalPayload(messageContentStr);
batchEntryBuilder.messageBody(largeMessagePointer);
return batchEntryBuilder.build();
}
private SendMessageRequest storeMessageInS3(SendMessageRequest sendMessageRequest) {
// Read the content of the message from message body
String messageContentStr = sendMessageRequest.messageBody();
Long messageContentSize = Util.getStringSizeInBytes(messageContentStr);
SendMessageRequest.Builder sendMessageRequestBuilder = sendMessageRequest.toBuilder();
sendMessageRequestBuilder.messageAttributes(
updateMessageAttributePayloadSize(sendMessageRequest.messageAttributes(), messageContentSize));
// Store the message content in S3.
String largeMessagePointer = storeOriginalPayload(messageContentStr);
sendMessageRequestBuilder.messageBody(largeMessagePointer);
return sendMessageRequestBuilder.build();
}
private String storeOriginalPayload(String messageContentStr) {
String s3KeyPrefix = clientConfiguration.getS3KeyPrefix();
if (StringUtils.isBlank(s3KeyPrefix)) {
return payloadStore.storeOriginalPayload(messageContentStr);
}
return payloadStore.storeOriginalPayload(messageContentStr, s3KeyPrefix + UUID.randomUUID());
}
private Map updateMessageAttributePayloadSize(
Map messageAttributes, Long messageContentSize) {
Map updatedMessageAttributes = new HashMap<>(messageAttributes);
// Add a new message attribute as a flag
MessageAttributeValue.Builder messageAttributeValueBuilder = MessageAttributeValue.builder();
messageAttributeValueBuilder.dataType("Number");
messageAttributeValueBuilder.stringValue(messageContentSize.toString());
MessageAttributeValue messageAttributeValue = messageAttributeValueBuilder.build();
if (!clientConfiguration.usesLegacyReservedAttributeName()) {
updatedMessageAttributes.put(SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME, messageAttributeValue);
} else {
updatedMessageAttributes.put(LEGACY_RESERVED_ATTRIBUTE_NAME, messageAttributeValue);
}
return updatedMessageAttributes;
}
@SuppressWarnings("unchecked")
private static T appendUserAgent(final T builder) {
return (T) builder
.overrideConfiguration(
AwsRequestOverrideConfiguration.builder()
.addApiName(ApiName.builder().name(USER_AGENT_NAME)
.version(USER_AGENT_VERSION).build())
.build());
}
@Override
public void close() {
super.close();
this.clientConfiguration.getS3Client().close();
}
}