com.amazon.sqs.javamessaging.AmazonSQSExtendedClient Maven / Gradle / Ivy
Show all versions of amazon-sqs-java-extended-client-lib Show documentation
/*
* Copyright 2010-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazon.sqs.javamessaging;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.Map.Entry;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.model.BatchEntryIdsNotDistinctException;
import com.amazonaws.services.sqs.model.BatchRequestTooLongException;
import com.amazonaws.services.sqs.model.DeleteMessageBatchRequest;
import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry;
import com.amazonaws.services.sqs.model.DeleteMessageBatchResult;
import com.amazonaws.services.sqs.model.DeleteMessageRequest;
import com.amazonaws.services.sqs.model.EmptyBatchRequestException;
import com.amazonaws.services.sqs.model.InvalidBatchEntryIdException;
import com.amazonaws.services.sqs.model.InvalidIdFormatException;
import com.amazonaws.services.sqs.model.InvalidMessageContentsException;
import com.amazonaws.services.sqs.model.Message;
import com.amazonaws.services.sqs.model.MessageAttributeValue;
import com.amazonaws.services.sqs.model.OverLimitException;
import com.amazonaws.services.sqs.model.PurgeQueueRequest;
import com.amazonaws.services.sqs.model.ReceiptHandleIsInvalidException;
import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
import com.amazonaws.services.sqs.model.ReceiveMessageResult;
import com.amazonaws.services.sqs.model.SendMessageBatchRequest;
import com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry;
import com.amazonaws.services.sqs.model.SendMessageBatchResult;
import com.amazonaws.services.sqs.model.SendMessageRequest;
import com.amazonaws.services.sqs.model.SendMessageResult;
import com.amazonaws.services.sqs.model.TooManyEntriesInBatchRequestException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* Amazon SQS Extended Client extends the functionality of Amazon SQS client.
* All service calls made using this client are blocking, and will not return
* until the service call completes.
*
*
* The Amazon SQS extended client enables sending and receiving large messages
* via Amazon S3. You can use this library to:
*
*
*
* - Specify whether messages are always stored in Amazon S3 or only when a
* message size exceeds 256 KB.
* - Send a message that references a single message object stored in an
* Amazon S3 bucket.
* - Get the corresponding message object from an Amazon S3 bucket.
* - Delete the corresponding message object from an Amazon S3 bucket.
*
*/
public class AmazonSQSExtendedClient extends AmazonSQSExtendedClientBase implements AmazonSQS {
private static final Log LOG = LogFactory.getLog(AmazonSQSExtendedClient.class);
private ExtendedClientConfiguration clientConfiguration;
/**
* Constructs a new Amazon SQS extended client to invoke service methods on
* Amazon SQS with extended functionality using the specified Amazon SQS
* client object.
*
*
* All service calls made using this new client object are blocking, and
* will not return until the service call completes.
*
* @param sqsClient
* The Amazon SQS client to use to connect to Amazon SQS.
*/
public AmazonSQSExtendedClient(AmazonSQS sqsClient) {
this(sqsClient, new ExtendedClientConfiguration());
}
/**
* Constructs a new Amazon SQS extended client to invoke service methods on
* Amazon SQS with extended functionality using the specified Amazon SQS
* client object.
*
*
* All service calls made using this new client object are blocking, and
* will not return until the service call completes.
*
* @param sqsClient
* The Amazon SQS client to use to connect to Amazon SQS.
* @param extendedClientConfig
* The extended client configuration options controlling the
* functionality of this client.
*/
public AmazonSQSExtendedClient(AmazonSQS sqsClient, ExtendedClientConfiguration extendedClientConfig) {
super(sqsClient);
this.clientConfiguration = new ExtendedClientConfiguration(extendedClientConfig);
}
/**
*
* Delivers a message to the specified queue and uploads the message payload
* to Amazon S3 if necessary.
*
*
* IMPORTANT: The following list shows the characters (in Unicode)
* allowed in your message, according to the W3C XML specification. For more
* information, go to http://www.w3.org/TR/REC-xml/#charsets If you send any
* characters not included in the list, your request will be rejected. #x9 |
* #xA | #xD | [#x20 to #xD7FF] | [#xE000 to #xFFFD] | [#x10000 to #x10FFFF]
*
*
* IMPORTANT: The input object may be modified by the method.
*
* @param sendMessageRequest
* Container for the necessary parameters to execute the
* SendMessage service method on AmazonSQS.
*
* @return The response from the SendMessage service method, as returned by
* AmazonSQS.
*
* @throws InvalidMessageContentsException
* @throws UnsupportedOperationException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client
* while attempting to make the request or handle the response.
* For example if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonSQS indicating
* either a problem with the data in the request, or a server
* side issue.
*/
public SendMessageResult sendMessage(SendMessageRequest sendMessageRequest) {
if (sendMessageRequest == null) {
String errorMessage = "sendMessageRequest cannot be null.";
LOG.error(errorMessage);
throw new AmazonClientException(errorMessage);
}
sendMessageRequest.getRequestClientOptions().appendUserAgent(SQSExtendedClientConstants.USER_AGENT_HEADER);
if (!clientConfiguration.isLargePayloadSupportEnabled()) {
return super.sendMessage(sendMessageRequest);
}
if (sendMessageRequest.getMessageBody() == null || "".equals(sendMessageRequest.getMessageBody())) {
String errorMessage = "messageBody cannot be null or empty.";
LOG.error(errorMessage);
throw new AmazonClientException(errorMessage);
}
if (clientConfiguration.isAlwaysThroughS3() || isLarge(sendMessageRequest)) {
sendMessageRequest = storeMessageInS3(sendMessageRequest);
}
return super.sendMessage(sendMessageRequest);
}
/**
*
* Delivers a message to the specified queue and uploads the message payload
* to Amazon S3 if necessary.
*
*
* IMPORTANT: The following list shows the characters (in Unicode)
* allowed in your message, according to the W3C XML specification. For more
* information, go to http://www.w3.org/TR/REC-xml/#charsets If you send any
* characters not included in the list, your request will be rejected. #x9 |
* #xA | #xD | [#x20 to #xD7FF] | [#xE000 to #xFFFD] | [#x10000 to #x10FFFF]
*
*
* @param queueUrl
* The URL of the Amazon SQS queue to take action on.
* @param messageBody
* The message to send. For a list of allowed characters, see the
* preceding important note.
*
* @return The response from the SendMessage service method, as returned by
* AmazonSQS.
*
* @throws InvalidMessageContentsException
* @throws UnsupportedOperationException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client
* while attempting to make the request or handle the response.
* For example if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonSQS indicating
* either a problem with the data in the request, or a server
* side issue.
*/
public SendMessageResult sendMessage(String queueUrl, String messageBody) {
SendMessageRequest sendMessageRequest = new SendMessageRequest(queueUrl, messageBody);
return sendMessage(sendMessageRequest);
}
/**
*
* Retrieves one or more messages, with a maximum limit of 10 messages, from
* the specified queue. Downloads the message payloads from Amazon S3 when
* necessary. Long poll support is enabled by using the
* WaitTimeSeconds parameter. For more information, see Amazon SQS Long Poll in the Amazon SQS Developer Guide .
*
*
* Short poll is the default behavior where a weighted random set of
* machines is sampled on a ReceiveMessage call. This means
* only the messages on the sampled machines are returned. If the number of
* messages in the queue is small (less than 1000), it is likely you will
* get fewer messages than you requested per ReceiveMessage
* call. If the number of messages in the queue is extremely small, you
* might not receive any messages in a particular
* ReceiveMessage response; in which case you should repeat the
* request.
*
*
* For each message returned, the response includes the following:
*
*
*
* -
*
* Message body
*
*
* -
*
* MD5 digest of the message body. For information about MD5, go to
* http://www.faqs.org/rfcs/rfc1321.html .
*
*
* -
*
* Message ID you received when you sent the message to the queue.
*
*
* -
*
* Receipt handle.
*
*
* -
*
* Message attributes.
*
*
* -
*
* MD5 digest of the message attributes.
*
*
*
*
*
* The receipt handle is the identifier you must provide when deleting the
* message. For more information, see Queue and Message Identifiers in the Amazon SQS Developer
* Guide .
*
*
* You can provide the VisibilityTimeout parameter in your
* request, which will be applied to the messages that Amazon SQS returns in
* the response. If you do not include the parameter, the overall visibility
* timeout for the queue is used for the returned messages. For more
* information, see Visibility Timeout in the Amazon SQS Developer Guide .
*
*
* NOTE: Going forward, new attributes might be added. If you are
* writing code that calls this action, we recommend that you structure your
* code so that it can handle new attributes gracefully.
*
*
* @param receiveMessageRequest
* Container for the necessary parameters to execute the
* ReceiveMessage service method on AmazonSQS.
*
* @return The response from the ReceiveMessage service method, as returned
* by AmazonSQS.
*
* @throws OverLimitException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client
* while attempting to make the request or handle the response.
* For example if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonSQS indicating
* either a problem with the data in the request, or a server
* side issue.
*/
public ReceiveMessageResult receiveMessage(ReceiveMessageRequest receiveMessageRequest) {
if (receiveMessageRequest == null) {
String errorMessage = "receiveMessageRequest cannot be null.";
LOG.error(errorMessage);
throw new AmazonClientException(errorMessage);
}
receiveMessageRequest.getRequestClientOptions().appendUserAgent(SQSExtendedClientConstants.USER_AGENT_HEADER);
if (!clientConfiguration.isLargePayloadSupportEnabled()) {
return super.receiveMessage(receiveMessageRequest);
}
receiveMessageRequest.getMessageAttributeNames().add(SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME);
ReceiveMessageResult receiveMessageResult = super.receiveMessage(receiveMessageRequest);
List messages = receiveMessageResult.getMessages();
for (Message message : messages) {
// for each received message check if they are stored in S3.
MessageAttributeValue largePayloadAttributeValue = message.getMessageAttributes().get(
SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME);
if (largePayloadAttributeValue != null) {
String messageBody = message.getBody();
// read the S3 pointer from the message body JSON string.
MessageS3Pointer s3Pointer = readMessageS3PointerFromJSON(messageBody);
String s3MsgBucketName = s3Pointer.getS3BucketName();
String s3MsgKey = s3Pointer.getS3Key();
String origMsgBody = getTextFromS3(s3MsgBucketName, s3MsgKey);
LOG.info("S3 object read, Bucket name: " + s3MsgBucketName + ", Object key: " + s3MsgKey + ".");
message.setBody(origMsgBody);
// remove the additional attribute before returning the message
// to user.
message.getMessageAttributes().remove(SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME);
// Embed s3 object pointer in the receipt handle.
String modifiedReceiptHandle = embedS3PointerInReceiptHandle(message.getReceiptHandle(),
s3MsgBucketName, s3MsgKey);
message.setReceiptHandle(modifiedReceiptHandle);
}
}
return receiveMessageResult;
}
/**
*
* Retrieves one or more messages, with a maximum limit of 10 messages, from
* the specified queue. Downloads the message payloads from Amazon S3 when
* necessary. Long poll support is enabled by using the
* WaitTimeSeconds parameter. For more information, see Amazon SQS Long Poll in the Amazon SQS Developer Guide .
*
*
* Short poll is the default behavior where a weighted random set of
* machines is sampled on a ReceiveMessage call. This means
* only the messages on the sampled machines are returned. If the number of
* messages in the queue is small (less than 1000), it is likely you will
* get fewer messages than you requested per ReceiveMessage
* call. If the number of messages in the queue is extremely small, you
* might not receive any messages in a particular
* ReceiveMessage response; in which case you should repeat the
* request.
*
*
* For each message returned, the response includes the following:
*
*
*
* -
*
* Message body
*
*
* -
*
* MD5 digest of the message body. For information about MD5, go to
* http://www.faqs.org/rfcs/rfc1321.html .
*
*
* -
*
* Message ID you received when you sent the message to the queue.
*
*
* -
*
* Receipt handle.
*
*
* -
*
* Message attributes.
*
*
* -
*
* MD5 digest of the message attributes.
*
*
*
*
*
* The receipt handle is the identifier you must provide when deleting the
* message. For more information, see Queue and Message Identifiers in the Amazon SQS Developer
* Guide .
*
*
* You can provide the VisibilityTimeout parameter in your
* request, which will be applied to the messages that Amazon SQS returns in
* the response. If you do not include the parameter, the overall visibility
* timeout for the queue is used for the returned messages. For more
* information, see Visibility Timeout in the Amazon SQS Developer Guide .
*
*
* NOTE: Going forward, new attributes might be added. If you are
* writing code that calls this action, we recommend that you structure your
* code so that it can handle new attributes gracefully.
*
*
* @param queueUrl
* The URL of the Amazon SQS queue to take action on.
*
* @return The response from the ReceiveMessage service method, as returned
* by AmazonSQS.
*
* @throws OverLimitException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client
* while attempting to make the request or handle the response.
* For example if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonSQS indicating
* either a problem with the data in the request, or a server
* side issue.
*/
public ReceiveMessageResult receiveMessage(String queueUrl) {
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(queueUrl);
return receiveMessage(receiveMessageRequest);
}
/**
*
* Deletes the specified message from the specified queue and deletes the
* message payload from Amazon S3 when necessary. You specify the message by
* using the message's receipt handle and not the
* message ID you received when you sent the message. Even if
* the message is locked by another reader due to the visibility timeout
* setting, it is still deleted from the queue. If you leave a message in
* the queue for longer than the queue's configured retention period, Amazon
* SQS automatically deletes it.
*
*
* NOTE: The receipt handle is associated with a specific instance of
* receiving the message. If you receive a message more than once, the
* receipt handle you get each time you receive the message is different.
* When you request DeleteMessage, if you don't provide the most recently
* received receipt handle for the message, the request will still succeed,
* but the message might not be deleted.
*
*
* IMPORTANT: It is possible you will receive a message even after
* you have deleted it. This might happen on rare occasions if one of the
* servers storing a copy of the message is unavailable when you request to
* delete the message. The copy remains on the server and might be returned
* to you again on a subsequent receive request. You should create your
* system to be idempotent so that receiving a particular message more than
* once is not a problem.
*
*
* @param deleteMessageRequest
* Container for the necessary parameters to execute the
* DeleteMessage service method on AmazonSQS.
*
*
* @throws ReceiptHandleIsInvalidException
* @throws InvalidIdFormatException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client
* while attempting to make the request or handle the response.
* For example if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonSQS indicating
* either a problem with the data in the request, or a server
* side issue.
*/
public void deleteMessage(DeleteMessageRequest deleteMessageRequest) {
if (deleteMessageRequest == null) {
String errorMessage = "deleteMessageRequest cannot be null.";
LOG.error(errorMessage);
throw new AmazonClientException(errorMessage);
}
deleteMessageRequest.getRequestClientOptions().appendUserAgent(SQSExtendedClientConstants.USER_AGENT_HEADER);
if (!clientConfiguration.isLargePayloadSupportEnabled()) {
super.deleteMessage(deleteMessageRequest);
return;
}
String receiptHandle = deleteMessageRequest.getReceiptHandle();
String origReceiptHandle = receiptHandle;
if (isS3ReceiptHandle(receiptHandle)) {
deleteMessagePayloadFromS3(receiptHandle);
origReceiptHandle = getOrigReceiptHandle(receiptHandle);
}
deleteMessageRequest.setReceiptHandle(origReceiptHandle);
super.deleteMessage(deleteMessageRequest);
}
/**
*
* Deletes the specified message from the specified queue and deletes the
* message payload from Amazon S3 when necessary. You specify the message by
* using the message's receipt handle and not the
* message ID you received when you sent the message. Even if
* the message is locked by another reader due to the visibility timeout
* setting, it is still deleted from the queue. If you leave a message in
* the queue for longer than the queue's configured retention period, Amazon
* SQS automatically deletes it.
*
*
* NOTE: The receipt handle is associated with a specific instance of
* receiving the message. If you receive a message more than once, the
* receipt handle you get each time you receive the message is different.
* When you request DeleteMessage, if you don't provide the most recently
* received receipt handle for the message, the request will still succeed,
* but the message might not be deleted.
*
*
* IMPORTANT: It is possible you will receive a message even after
* you have deleted it. This might happen on rare occasions if one of the
* servers storing a copy of the message is unavailable when you request to
* delete the message. The copy remains on the server and might be returned
* to you again on a subsequent receive request. You should create your
* system to be idempotent so that receiving a particular message more than
* once is not a problem.
*
*
* @param queueUrl
* The URL of the Amazon SQS queue to take action on.
* @param receiptHandle
* The receipt handle associated with the message to delete.
*
* @return The response from the DeleteMessage service method, as returned
* by AmazonSQS.
*
* @throws ReceiptHandleIsInvalidException
* @throws InvalidIdFormatException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client
* while attempting to make the request or handle the response.
* For example if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonSQS indicating
* either a problem with the data in the request, or a server
* side issue.
*/
public void deleteMessage(String queueUrl, String receiptHandle) {
DeleteMessageRequest deleteMessageRequest = new DeleteMessageRequest(queueUrl, receiptHandle);
deleteMessage(deleteMessageRequest);
}
/**
*
* Delivers up to ten messages to the specified queue. This is a batch
* version of SendMessage. The result of the send action on each message is
* reported individually in the response. Uploads message payloads to Amazon
* S3 when necessary.
*
*
* If the DelaySeconds parameter is not specified for an entry,
* the default for the queue is used.
*
*
* IMPORTANT:The following list shows the characters (in Unicode)
* that are allowed in your message, according to the W3C XML specification.
* For more information, go to http://www.faqs.org/rfcs/rfc1321.html. If you
* send any characters that are not included in the list, your request will
* be rejected. #x9 | #xA | #xD | [#x20 to #xD7FF] | [#xE000 to #xFFFD] |
* [#x10000 to #x10FFFF]
*
*
* IMPORTANT: Because the batch request can result in a combination
* of successful and unsuccessful actions, you should check for batch errors
* even when the call returns an HTTP status code of 200.
*
* IMPORTANT: The input object may be modified by the method.
*
* NOTE:Some API actions take lists of parameters. These lists are
* specified using the param.n notation. Values of n are integers starting
* from 1. For example, a parameter list with two elements looks like this:
*
*
* &Attribute.1=this
*
*
* &Attribute.2=that
*
*
* @param sendMessageBatchRequest
* Container for the necessary parameters to execute the
* SendMessageBatch service method on AmazonSQS.
*
* @return The response from the SendMessageBatch service method, as
* returned by AmazonSQS.
*
* @throws BatchEntryIdsNotDistinctException
* @throws TooManyEntriesInBatchRequestException
* @throws BatchRequestTooLongException
* @throws UnsupportedOperationException
* @throws InvalidBatchEntryIdException
* @throws EmptyBatchRequestException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client
* while attempting to make the request or handle the response.
* For example if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonSQS indicating
* either a problem with the data in the request, or a server
* side issue.
*/
public SendMessageBatchResult sendMessageBatch(SendMessageBatchRequest sendMessageBatchRequest) {
if (sendMessageBatchRequest == null) {
String errorMessage = "sendMessageBatchRequest cannot be null.";
LOG.error(errorMessage);
throw new AmazonClientException(errorMessage);
}
sendMessageBatchRequest.getRequestClientOptions().appendUserAgent(SQSExtendedClientConstants.USER_AGENT_HEADER);
if (!clientConfiguration.isLargePayloadSupportEnabled()) {
return super.sendMessageBatch(sendMessageBatchRequest);
}
List batchEntries = sendMessageBatchRequest.getEntries();
int index = 0;
for (SendMessageBatchRequestEntry entry : batchEntries) {
if (clientConfiguration.isAlwaysThroughS3() || isLarge(entry)) {
batchEntries.set(index, storeMessageInS3(entry));
}
++index;
}
return super.sendMessageBatch(sendMessageBatchRequest);
}
/**
*
* Delivers up to ten messages to the specified queue. This is a batch
* version of SendMessage. The result of the send action on each message is
* reported individually in the response. Uploads message payloads to Amazon
* S3 when necessary.
*
*
* If the DelaySeconds parameter is not specified for an entry,
* the default for the queue is used.
*
*
* IMPORTANT:The following list shows the characters (in Unicode)
* that are allowed in your message, according to the W3C XML specification.
* For more information, go to http://www.faqs.org/rfcs/rfc1321.html. If you
* send any characters that are not included in the list, your request will
* be rejected. #x9 | #xA | #xD | [#x20 to #xD7FF] | [#xE000 to #xFFFD] |
* [#x10000 to #x10FFFF]
*
*
* IMPORTANT: Because the batch request can result in a combination
* of successful and unsuccessful actions, you should check for batch errors
* even when the call returns an HTTP status code of 200.
*
*
* NOTE:Some API actions take lists of parameters. These lists are
* specified using the param.n notation. Values of n are integers starting
* from 1. For example, a parameter list with two elements looks like this:
*
*
* &Attribute.1=this
*
*
* &Attribute.2=that
*
*
* @param queueUrl
* The URL of the Amazon SQS queue to take action on.
* @param entries
* A list of SendMessageBatchRequestEntry items.
*
* @return The response from the SendMessageBatch service method, as
* returned by AmazonSQS.
*
* @throws BatchEntryIdsNotDistinctException
* @throws TooManyEntriesInBatchRequestException
* @throws BatchRequestTooLongException
* @throws UnsupportedOperationException
* @throws InvalidBatchEntryIdException
* @throws EmptyBatchRequestException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client
* while attempting to make the request or handle the response.
* For example if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonSQS indicating
* either a problem with the data in the request, or a server
* side issue.
*/
public SendMessageBatchResult sendMessageBatch(String queueUrl, List entries) {
SendMessageBatchRequest sendMessageBatchRequest = new SendMessageBatchRequest(queueUrl, entries);
return sendMessageBatch(sendMessageBatchRequest);
}
/**
*
* Deletes up to ten messages from the specified queue. This is a batch
* version of DeleteMessage. The result of the delete action on each message
* is reported individually in the response. Also deletes the message
* payloads from Amazon S3 when necessary.
*
*
* IMPORTANT: Because the batch request can result in a combination
* of successful and unsuccessful actions, you should check for batch errors
* even when the call returns an HTTP status code of 200.
*
*
* NOTE:Some API actions take lists of parameters. These lists are
* specified using the param.n notation. Values of n are integers starting
* from 1. For example, a parameter list with two elements looks like this:
*
*
* &Attribute.1=this
*
*
* &Attribute.2=that
*
*
* @param deleteMessageBatchRequest
* Container for the necessary parameters to execute the
* DeleteMessageBatch service method on AmazonSQS.
*
* @return The response from the DeleteMessageBatch service method, as
* returned by AmazonSQS.
*
* @throws BatchEntryIdsNotDistinctException
* @throws TooManyEntriesInBatchRequestException
* @throws InvalidBatchEntryIdException
* @throws EmptyBatchRequestException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client
* while attempting to make the request or handle the response.
* For example if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonSQS indicating
* either a problem with the data in the request, or a server
* side issue.
*/
public DeleteMessageBatchResult deleteMessageBatch(DeleteMessageBatchRequest deleteMessageBatchRequest) {
if (deleteMessageBatchRequest == null) {
String errorMessage = "deleteMessageBatchRequest cannot be null.";
LOG.error(errorMessage);
throw new AmazonClientException(errorMessage);
}
deleteMessageBatchRequest.getRequestClientOptions().appendUserAgent(
SQSExtendedClientConstants.USER_AGENT_HEADER);
if (!clientConfiguration.isLargePayloadSupportEnabled()) {
return super.deleteMessageBatch(deleteMessageBatchRequest);
}
for (DeleteMessageBatchRequestEntry entry : deleteMessageBatchRequest.getEntries()) {
String receiptHandle = entry.getReceiptHandle();
String origReceiptHandle = receiptHandle;
if (isS3ReceiptHandle(receiptHandle)) {
deleteMessagePayloadFromS3(receiptHandle);
origReceiptHandle = getOrigReceiptHandle(receiptHandle);
}
entry.setReceiptHandle(origReceiptHandle);
}
return super.deleteMessageBatch(deleteMessageBatchRequest);
}
/**
*
* Deletes up to ten messages from the specified queue. This is a batch
* version of DeleteMessage. The result of the delete action on each message
* is reported individually in the response. Also deletes the message
* payloads from Amazon S3 when necessary.
*
*
* IMPORTANT: Because the batch request can result in a combination
* of successful and unsuccessful actions, you should check for batch errors
* even when the call returns an HTTP status code of 200.
*
*
* NOTE:Some API actions take lists of parameters. These lists are
* specified using the param.n notation. Values of n are integers starting
* from 1. For example, a parameter list with two elements looks like this:
*
*
* &Attribute.1=this
*
*
* &Attribute.2=that
*
*
* @param queueUrl
* The URL of the Amazon SQS queue to take action on.
* @param entries
* A list of receipt handles for the messages to be deleted.
*
* @return The response from the DeleteMessageBatch service method, as
* returned by AmazonSQS.
*
* @throws BatchEntryIdsNotDistinctException
* @throws TooManyEntriesInBatchRequestException
* @throws InvalidBatchEntryIdException
* @throws EmptyBatchRequestException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client
* while attempting to make the request or handle the response.
* For example if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonSQS indicating
* either a problem with the data in the request, or a server
* side issue.
*/
public DeleteMessageBatchResult deleteMessageBatch(String queueUrl, List entries) {
DeleteMessageBatchRequest deleteMessageBatchRequest = new DeleteMessageBatchRequest(queueUrl, entries);
return deleteMessageBatch(deleteMessageBatchRequest);
}
/**
*
* Deletes the messages in a queue specified by the queue URL .
*
*
* IMPORTANT:When you use the PurgeQueue API, the deleted messages in
* the queue cannot be retrieved.
*
*
* IMPORTANT: This does not delete the message payloads from Amazon S3.
*
*
* When you purge a queue, the message deletion process takes up to 60
* seconds. All messages sent to the queue before calling
* PurgeQueue will be deleted; messages sent to the queue while
* it is being purged may be deleted. While the queue is being purged,
* messages sent to the queue before PurgeQueue was called may
* be received, but will be deleted within the next minute.
*
*
* @param purgeQueueRequest
* Container for the necessary parameters to execute the
* PurgeQueue service method on AmazonSQS.
*
*
* @throws PurgeQueueInProgressException
* @throws QueueDoesNotExistException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client
* while attempting to make the request or handle the response.
* For example if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonSQS indicating
* either a problem with the data in the request, or a server
* side issue.
*/
public void purgeQueue(PurgeQueueRequest purgeQueueRequest) throws AmazonServiceException, AmazonClientException {
LOG.warn("Calling purgeQueue deletes SQS messages without deleting their payload from S3.");
if (purgeQueueRequest == null) {
String errorMessage = "purgeQueueRequest cannot be null.";
LOG.error(errorMessage);
throw new AmazonClientException(errorMessage);
}
purgeQueueRequest.getRequestClientOptions().appendUserAgent(SQSExtendedClientConstants.USER_AGENT_HEADER);
super.purgeQueue(purgeQueueRequest);
}
private void deleteMessagePayloadFromS3(String receiptHandle) {
String s3MsgBucketName = getFromReceiptHandleByMarker(receiptHandle,
SQSExtendedClientConstants.S3_BUCKET_NAME_MARKER);
String s3MsgKey = getFromReceiptHandleByMarker(receiptHandle, SQSExtendedClientConstants.S3_KEY_MARKER);
try {
clientConfiguration.getAmazonS3Client().deleteObject(s3MsgBucketName, s3MsgKey);
} catch (AmazonServiceException e) {
String errorMessage = "Failed to delete the S3 object which contains the SQS message payload. SQS message was not deleted.";
LOG.error(errorMessage, e);
throw new AmazonServiceException(errorMessage, e);
} catch (AmazonClientException e) {
String errorMessage = "Failed to delete the S3 object which contains the SQS message payload. SQS message was not deleted.";
LOG.error(errorMessage, e);
throw new AmazonClientException(errorMessage, e);
}
LOG.info("S3 object deleted, Bucket name: " + s3MsgBucketName + ", Object key: " + s3MsgKey + ".");
}
private void checkMessageAttributes(Map messageAttributes) {
int msgAttributesSize = getMsgAttributesSize(messageAttributes);
if (msgAttributesSize > clientConfiguration.getMessageSizeThreshold()) {
String errorMessage = "Total size of Message attributes is " + msgAttributesSize
+ " bytes which is larger than the threshold of " + clientConfiguration.getMessageSizeThreshold()
+ " Bytes. Consider including the payload in the message body instead of message attributes.";
LOG.error(errorMessage);
throw new AmazonClientException(errorMessage);
}
int messageAttributesNum = messageAttributes.size();
if (messageAttributesNum > SQSExtendedClientConstants.MAX_ALLOWED_ATTRIBUTES) {
String errorMessage = "Number of message attributes [" + messageAttributesNum
+ "] exceeds the maximum allowed for large-payload messages ["
+ SQSExtendedClientConstants.MAX_ALLOWED_ATTRIBUTES + "].";
LOG.error(errorMessage);
throw new AmazonClientException(errorMessage);
}
MessageAttributeValue largePayloadAttributeValue = messageAttributes
.get(SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME);
if (largePayloadAttributeValue != null) {
String errorMessage = "Message attribute name " + SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME
+ " is reserved for use by SQS extended client.";
LOG.error(errorMessage);
throw new AmazonClientException(errorMessage);
}
}
private String embedS3PointerInReceiptHandle(String receiptHandle, String s3MsgBucketName, String s3MsgKey) {
String modifiedReceiptHandle = SQSExtendedClientConstants.S3_BUCKET_NAME_MARKER + s3MsgBucketName
+ SQSExtendedClientConstants.S3_BUCKET_NAME_MARKER + SQSExtendedClientConstants.S3_KEY_MARKER
+ s3MsgKey + SQSExtendedClientConstants.S3_KEY_MARKER + receiptHandle;
return modifiedReceiptHandle;
}
private MessageS3Pointer readMessageS3PointerFromJSON(String messageBody) {
MessageS3Pointer s3Pointer = null;
try {
JsonDataConverter jsonDataConverter = new JsonDataConverter();
s3Pointer = jsonDataConverter.deserializeFromJson(messageBody, MessageS3Pointer.class);
} catch (Exception e) {
String errorMessage = "Failed to read the S3 object pointer from an SQS message. Message was not received.";
LOG.error(errorMessage, e);
throw new AmazonClientException(errorMessage, e);
}
return s3Pointer;
}
private String getOrigReceiptHandle(String receiptHandle) {
int secondOccurence = receiptHandle.indexOf(SQSExtendedClientConstants.S3_KEY_MARKER,
receiptHandle.indexOf(SQSExtendedClientConstants.S3_KEY_MARKER) + 1);
return receiptHandle.substring(secondOccurence + SQSExtendedClientConstants.S3_KEY_MARKER.length());
}
private String getFromReceiptHandleByMarker(String receiptHandle, String marker) {
int firstOccurence = receiptHandle.indexOf(marker);
int secondOccurence = receiptHandle.indexOf(marker, firstOccurence + 1);
return receiptHandle.substring(firstOccurence + marker.length(), secondOccurence);
}
private boolean isS3ReceiptHandle(String receiptHandle) {
return receiptHandle.contains(SQSExtendedClientConstants.S3_BUCKET_NAME_MARKER)
&& receiptHandle.contains(SQSExtendedClientConstants.S3_KEY_MARKER);
}
private String getTextFromS3(String s3BucketName, String s3Key) {
GetObjectRequest getObjectRequest = new GetObjectRequest(s3BucketName, s3Key);
String embeddedText = null;
S3Object obj = null;
try {
obj = clientConfiguration.getAmazonS3Client().getObject(getObjectRequest);
} catch (AmazonServiceException e) {
String errorMessage = "Failed to get the S3 object which contains the message payload. Message was not received.";
LOG.error(errorMessage, e);
throw new AmazonServiceException(errorMessage, e);
} catch (AmazonClientException e) {
String errorMessage = "Failed to get the S3 object which contains the message payload. Message was not received.";
LOG.error(errorMessage, e);
throw new AmazonClientException(errorMessage, e);
}
try {
InputStream objContent = obj.getObjectContent();
java.util.Scanner objContentScanner = new java.util.Scanner(objContent, "UTF-8");
objContentScanner.useDelimiter("\\A");
embeddedText = objContentScanner.hasNext() ? objContentScanner.next() : "";
objContentScanner.close();
objContent.close();
} catch (IOException e) {
String errorMessage = "Failure when handling the message which was read from S3 object. Message was not received.";
LOG.error(errorMessage, e);
throw new AmazonClientException(errorMessage, e);
}
return embeddedText;
}
private boolean isLarge(SendMessageRequest sendMessageRequest) {
int msgAttributesSize = getMsgAttributesSize(sendMessageRequest.getMessageAttributes());
long msgBodySize = getStringSizeInBytes(sendMessageRequest.getMessageBody());
long totalMsgSize = msgAttributesSize + msgBodySize;
return (totalMsgSize > clientConfiguration.getMessageSizeThreshold());
}
private boolean isLarge(SendMessageBatchRequestEntry batchEntry) {
int msgAttributesSize = getMsgAttributesSize(batchEntry.getMessageAttributes());
long msgBodySize = getStringSizeInBytes(batchEntry.getMessageBody());
long totalMsgSize = msgAttributesSize + msgBodySize;
return (totalMsgSize > clientConfiguration.getMessageSizeThreshold());
}
private int getMsgAttributesSize(Map msgAttributes) {
int totalMsgAttributesSize = 0;
for (Entry entry : msgAttributes.entrySet()) {
totalMsgAttributesSize += getStringSizeInBytes(entry.getKey());
MessageAttributeValue entryVal = entry.getValue();
if (entryVal.getDataType() != null) {
totalMsgAttributesSize += getStringSizeInBytes(entryVal.getDataType());
}
String stringVal = entryVal.getStringValue();
if (stringVal != null) {
totalMsgAttributesSize += getStringSizeInBytes(entryVal.getStringValue());
}
ByteBuffer binaryVal = entryVal.getBinaryValue();
if (binaryVal != null) {
totalMsgAttributesSize += binaryVal.array().length;
}
}
return totalMsgAttributesSize;
}
private SendMessageBatchRequestEntry storeMessageInS3(SendMessageBatchRequestEntry batchEntry) {
checkMessageAttributes(batchEntry.getMessageAttributes());
String s3Key = UUID.randomUUID().toString();
// Read the content of the message from message body
String messageContentStr = batchEntry.getMessageBody();
Long messageContentSize = getStringSizeInBytes(messageContentStr);
// Add a new message attribute as a flag
MessageAttributeValue messageAttributeValue = new MessageAttributeValue();
messageAttributeValue.setDataType("Number");
messageAttributeValue.setStringValue(messageContentSize.toString());
batchEntry.addMessageAttributesEntry(SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME, messageAttributeValue);
// Store the message content in S3.
storeTextInS3(s3Key, messageContentStr, messageContentSize);
LOG.info("S3 object created, Bucket name: " + clientConfiguration.getS3BucketName() + ", Object key: " + s3Key
+ ".");
// Convert S3 pointer (bucket name, key, etc) to JSON string
MessageS3Pointer s3Pointer = new MessageS3Pointer(clientConfiguration.getS3BucketName(), s3Key);
String s3PointerStr = getJSONFromS3Pointer(s3Pointer);
// Storing S3 pointer in the message body.
batchEntry.setMessageBody(s3PointerStr);
return batchEntry;
}
private SendMessageRequest storeMessageInS3(SendMessageRequest sendMessageRequest) {
checkMessageAttributes(sendMessageRequest.getMessageAttributes());
String s3Key = UUID.randomUUID().toString();
// Read the content of the message from message body
String messageContentStr = sendMessageRequest.getMessageBody();
Long messageContentSize = getStringSizeInBytes(messageContentStr);
// Add a new message attribute as a flag
MessageAttributeValue messageAttributeValue = new MessageAttributeValue();
messageAttributeValue.setDataType("Number");
messageAttributeValue.setStringValue(messageContentSize.toString());
sendMessageRequest.addMessageAttributesEntry(SQSExtendedClientConstants.RESERVED_ATTRIBUTE_NAME,
messageAttributeValue);
// Store the message content in S3.
storeTextInS3(s3Key, messageContentStr, messageContentSize);
LOG.info("S3 object created, Bucket name: " + clientConfiguration.getS3BucketName() + ", Object key: " + s3Key
+ ".");
// Convert S3 pointer (bucket name, key, etc) to JSON string
MessageS3Pointer s3Pointer = new MessageS3Pointer(clientConfiguration.getS3BucketName(), s3Key);
String s3PointerStr = getJSONFromS3Pointer(s3Pointer);
// Storing S3 pointer in the message body.
sendMessageRequest.setMessageBody(s3PointerStr);
return sendMessageRequest;
}
private String getJSONFromS3Pointer(MessageS3Pointer s3Pointer) {
String s3PointerStr = null;
try {
JsonDataConverter jsonDataConverter = new JsonDataConverter();
s3PointerStr = jsonDataConverter.serializeToJson(s3Pointer);
} catch (Exception e) {
String errorMessage = "Failed to convert S3 object pointer to text. Message was not sent.";
LOG.error(errorMessage, e);
throw new AmazonClientException(errorMessage, e);
}
return s3PointerStr;
}
private void storeTextInS3(String s3Key, String messageContentStr, Long messageContentSize) {
InputStream messageContentStream = new ByteArrayInputStream(messageContentStr.getBytes(StandardCharsets.UTF_8));
ObjectMetadata messageContentStreamMetadata = new ObjectMetadata();
messageContentStreamMetadata.setContentLength(messageContentSize);
PutObjectRequest putObjectRequest = new PutObjectRequest(clientConfiguration.getS3BucketName(), s3Key,
messageContentStream, messageContentStreamMetadata);
try {
clientConfiguration.getAmazonS3Client().putObject(putObjectRequest);
} catch (AmazonServiceException e) {
String errorMessage = "Failed to store the message content in an S3 object. SQS message was not sent.";
LOG.error(errorMessage, e);
throw new AmazonServiceException(errorMessage, e);
} catch (AmazonClientException e) {
String errorMessage = "Failed to store the message content in an S3 object. SQS message was not sent.";
LOG.error(errorMessage, e);
throw new AmazonClientException(errorMessage, e);
}
}
private static long getStringSizeInBytes(String str) {
CountingOutputStream counterOutputStream = new CountingOutputStream();
try {
Writer writer = new OutputStreamWriter(counterOutputStream, "UTF-8");
writer.write(str);
writer.flush();
writer.close();
} catch (IOException e) {
String errorMessage = "Failed to calculate the size of message payload.";
LOG.error(errorMessage, e);
throw new AmazonClientException(errorMessage, e);
}
return counterOutputStream.getTotalSize();
}
}