com.nimbusds.infinispan.persistence.dynamodb.DynamoDBStore Maven / Gradle / Ivy
package com.nimbusds.infinispan.persistence.dynamodb;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicInteger;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Index;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.model.DeleteTableResult;
import com.amazonaws.services.dynamodbv2.model.ResourceInUseException;
import com.codahale.metrics.Timer;
import com.nimbusds.infinispan.persistence.common.InfinispanEntry;
import com.nimbusds.infinispan.persistence.common.InfinispanStore;
import com.nimbusds.infinispan.persistence.common.query.QueryExecutor;
import com.nimbusds.infinispan.persistence.dynamodb.config.DynamoDBStoreConfiguration;
import com.nimbusds.infinispan.persistence.dynamodb.logging.Loggers;
import com.nimbusds.infinispan.persistence.dynamodb.query.DynamoDBQueryExecutor;
import com.nimbusds.infinispan.persistence.dynamodb.query.DynamoDBQueryExecutorInitContext;
import net.jcip.annotations.ThreadSafe;
import org.infinispan.commons.configuration.ConfiguredBy;
import org.infinispan.filter.KeyFilter;
import org.infinispan.marshall.core.MarshalledEntry;
import org.infinispan.marshall.core.MarshalledEntryFactory;
import org.infinispan.persistence.TaskContextImpl;
import org.infinispan.persistence.spi.InitializationContext;
import org.infinispan.persistence.spi.PersistenceException;
import org.kohsuke.MetaInfServices;
/**
* AWS DynamoDB store for Infinispan 9+ caches and maps.
*/
@ThreadSafe
@MetaInfServices
@ConfiguredBy(DynamoDBStoreConfiguration.class)
public class DynamoDBStore extends InfinispanStore {
/**
* The DynamoDB configuration.
*/
private DynamoDBStoreConfiguration config;
/**
* The DynamoDB HTTP client.
*/
private AmazonDynamoDB client;
/**
* The DynamoDB table API client.
*/
private Table table;
/**
* The DynamoDB item transformer (to / from Infinispan entries).
*/
private DynamoDBItemTransformer itemTransformer;
/**
* The optional DynamoDB query executor.
*/
private DynamoDBQueryExecutor queryExecutor;
/**
* The DynamoDB request factory.
*/
private RequestFactory requestFactory;
/**
* The marshalled Infinispan entry factory.
*/
private MarshalledEntryFactory marshalledEntryFactory;
/**
* Purges expired entries found in the DynamoDB store, as indicated by
* their persisted metadata (optional, may be ignored / not stored).
*/
private ExpiredEntryReaper reaper;
/**
* DynamoDB operation timers.
*/
private DynamoDBTimers timers;
/**
* Loads a DynamoDB item transformer with the specified class name.
*
* @param clazz The class. Must not be {@code null}.
*
* @return The DynamoDB entry transformer.
*/
@SuppressWarnings( "unchecked" )
private DynamoDBItemTransformer loadItemTransformerClass(final Class clazz) {
try {
Class> genClazz = (Class>)clazz;
return genClazz.newInstance();
} catch (Exception e) {
throw new PersistenceException("Couldn't load DynamoDB item transformer class: " + e.getMessage(), e);
}
}
/**
* Loads a DynamoDB query executor with the specified class name.
*
* @param clazz The class. Must not be {@code null}.
*
* @return The DynamoDB query executor.
*/
@SuppressWarnings( "unchecked" )
private DynamoDBQueryExecutor loadQueryExecutorClass(final Class clazz) {
try {
Class> genClazz = (Class>)clazz;
return genClazz.newInstance();
} catch (Exception e) {
throw new PersistenceException("Couldn't load DynamoDB query executor class: " + e.getMessage(), e);
}
}
/**
* Returns the DynamoDB store configuration.
*
* @return The DynamoDB store configuration, {@code null} if not
* initialised.
*/
public DynamoDBStoreConfiguration getConfiguration() {
return config;
}
@Override
public QueryExecutor getQueryExecutor() {
return queryExecutor;
}
@Override
@SuppressWarnings("unchecked")
public void init(final InitializationContext ctx) {
// This method will be invoked by the PersistenceManager during initialization. The InitializationContext
// contains:
// - this CacheLoader's configuration
// - the cache to which this loader is applied. Your loader might want to use the cache's name to construct
// cache-specific identifiers
// - the StreamingMarshaller that needs to be used to marshall/unmarshall the entries
// - a TimeService which the loader can use to determine expired entries
// - a ByteBufferFactory which needs to be used to construct ByteBuffers
// - a MarshalledEntryFactory which needs to be used to construct entries from the data retrieved by the loader
super.init(ctx);
this.config = ctx.getConfiguration();
// Log the configuration
Loggers.MAIN_LOG.info("[DS0100] DynamoDB store: Infinispan cache store configuration for {}:", getCacheName());
config.log();
// Create a DynamoDB client
AmazonDynamoDBClientBuilder builder = AmazonDynamoDBClientBuilder
.standard()
.withCredentials(DefaultAWSCredentialsProviderChain.getInstance());
if (config.getEndpoint() != null && ! config.getEndpoint().trim().isEmpty()) {
// If endpoint is set, override region
builder = builder.withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration(
config.getEndpoint(),
null // inferred from endpoint
)
);
} else {
builder.withRegion(config.getRegion());
}
client = builder.build();
Loggers.MAIN_LOG.info("[DS0140] DynamoDB store: Expiration thread wake up interval for cache {}: {}", getCacheName(),
ctx.getCache().getCacheConfiguration().expiration().wakeUpInterval());
// Load and initialise the DynamoDB item transformer
Loggers.MAIN_LOG.debug("[DS0101] Loading DynamoDB item transformer class {} for cache {}...",
config.getItemTransformerClass(),
getCacheName());
itemTransformer = loadItemTransformerClass(config.getItemTransformerClass());
requestFactory = new RequestFactory<>(
itemTransformer,
config.getIndexedAttributes(),
config.getProvisionedThroughput(),
config.getTablePrefix(),
config.getApplyRangeKey(),
config.getRangeKeyValue());
table = new DynamoDB(client).getTable(requestFactory.getTableName());
// Load and initialise the optional SQL query executor
if (config.getQueryExecutorClass() != null) {
Loggers.MAIN_LOG.debug("[DS0130] Loading optional DynamoDB query executor class {} for cache {}...",
config.getQueryExecutorClass(),
getCacheName());
queryExecutor = loadQueryExecutorClass(config.getQueryExecutorClass());
queryExecutor.init(new DynamoDBQueryExecutorInitContext() {
@Override
public DynamoDBItemTransformer getDynamoDBItemTransformer() {
return itemTransformer;
}
@Override
public Index getDynamoDBIndex(final String attributeName) {
return table.getIndex(requestFactory.getGSIName(attributeName));
}
});
}
marshalledEntryFactory = (MarshalledEntryFactory)ctx.getMarshalledEntryFactory();
timers = new DynamoDBTimers(ctx.getCache().getName() + ".");
Loggers.MAIN_LOG.info("[DS0102] Initialized DynamoDB store for cache {} with table {}",
getCacheName(),
table.getTableName());
}
@Override
public void start() {
// This method will be invoked by the PersistenceManager to start the CacheLoader. At this stage configuration
// is complete and the loader can perform operations such as opening a connection to the external storage,
// initialize internal data structures, etc.
try {
table = new DynamoDB(client).createTable(requestFactory
.resolveCreateTableRequest()
.withProvisionedThroughput(
config.getProvisionedThroughput()));
Loggers.MAIN_LOG.info("[DS0129] DynamoDB store: Created table {} for cache {}", table.getTableName(), getCacheName());
} catch (ResourceInUseException e) {
// table exists
} catch (Exception e) {
Loggers.MAIN_LOG.fatal("[DS0103] DynamoDB store: Couldn't create table: {}: {}", e.getMessage(), e);
throw new PersistenceException(e.getMessage(), e);
}
try {
table.waitForActive();
} catch (InterruptedException e) {
throw new PersistenceException("Interrupted while awaiting DynamoDB table to become active: " + e.getMessage(), e);
}
Loggers.MAIN_LOG.info("[DS0141] DynamoDB store: Table properties: " + table.getDescription());
reaper = new ExpiredEntryReaper<>(requestFactory);
Loggers.MAIN_LOG.info("[DS0104] Started DynamoDB external store connector for cache {} with table {}", getCacheName(), table.getTableName());
}
@Override
public void stop() {
super.stop();
if (client != null) {
client.shutdown();
}
Loggers.MAIN_LOG.info("[DS0105] Stopped DynamoDB store connector for cache {}", getCacheName());
}
@Override
public boolean contains(final Object key) {
// This method will be invoked by the PersistenceManager to determine if the loader contains the specified key.
// The implementation should be as fast as possible, e.g. it should strive to transfer the least amount of data possible
// from the external storage to perform the check. Also, if possible, make sure the field is indexed on the external storage
// so that its existence can be determined as quickly as possible.
//
// Note that keys will be in the cache's native format, which means that if the cache is being used by a remoting protocol
// such as HotRod or REST and compatibility mode has not been enabled, then they will be encoded in a byte[].
Loggers.DYNAMODB_LOG.trace("[DS0106] DynamoDB store: Checking {} cache key {}", getCacheName(), key);
Timer.Context timerCtx = timers.getTimer.time();
try {
return table.getItem(requestFactory.resolveGetItemSpec(key)) != null;
} catch (Exception e) {
Loggers.DYNAMODB_LOG.error("[DS0107] {}: {}", e.getMessage(), e);
throw new PersistenceException(e.getMessage(), e);
} finally {
timerCtx.stop();
}
}
@Override
@SuppressWarnings("unchecked")
public MarshalledEntry load(final Object key) {
// Fetches an entry from the storage using the specified key. The CacheLoader should retrieve from the external storage all
// of the data that is needed to reconstruct the entry in memory, i.e. the value and optionally the metadata. This method
// needs to return a MarshalledEntry which can be constructed as follows:
//
// ctx.getMarshalledEntryFactory().new MarshalledEntry(key, value, metadata);
//
// If the entry does not exist or has expired, this method should return null.
// If an error occurs while retrieving data from the external storage, this method should throw a PersistenceException
//
// Note that keys and values will be in the cache's native format, which means that if the cache is being used by a remoting protocol
// such as HotRod or REST and compatibility mode has not been enabled, then they will be encoded in a byte[].
// If the loader needs to have knowledge of the key/value data beyond their binary representation, then it needs access to the key's and value's
// classes and the marshaller used to encode them.
Loggers.DYNAMODB_LOG.trace("[DS0108] DynamoDB store: Loading {} cache entry with key {}", getCacheName(), key);
final Item item;
Timer.Context timerCtx = timers.getTimer.time();
try {
item = table.getItem(requestFactory.resolveGetItemSpec(key));
} catch (Exception e) {
Loggers.DYNAMODB_LOG.error("[DS0109] {}, {}", e.getMessage(), e);
throw new PersistenceException(e.getMessage(), e);
} finally {
timerCtx.stop();
}
if (item == null) {
// Not found
Loggers.DYNAMODB_LOG.trace("[DS0110] DynamoDB store: Item with key {} not found", key);
return null;
}
if (Loggers.DYNAMODB_LOG.isTraceEnabled()) {
Loggers.DYNAMODB_LOG.trace("[DS0111] DynamoDB store: Retrieved item: {}", item);
}
// Transform DynamoDB entry to Infinispan entry
InfinispanEntry infinispanEntry = itemTransformer.toInfinispanEntry(item);
return marshalledEntryFactory.newMarshalledEntry(
infinispanEntry.getKey(),
infinispanEntry.getValue(),
infinispanEntry.getMetadata());
}
@Override
public boolean delete(final Object key) {
// The CacheWriter should remove from the external storage the entry identified by the specified key.
// Note that keys will be in the cache's native format, which means that if the cache is being used by a remoting protocol
// such as HotRod or REST and compatibility mode has not been enabled, then they will be encoded in a byte[].
Loggers.DYNAMODB_LOG.trace("[DS0112] DynamoDB store: Deleting {} cache entry with key {}", getCacheName(), key);
final boolean deleted;
Timer.Context timerCtx = timers.deleteTimer.time();
try {
deleted = table.deleteItem(requestFactory.resolveDeleteItemSpec(key)).getItem() != null;
} catch (Exception e) {
Loggers.DYNAMODB_LOG.error("[DS0113] {}, {}", e.getMessage(), e);
throw new PersistenceException(e.getMessage(), e);
} finally {
timerCtx.stop();
}
if (deleted) {
Loggers.DYNAMODB_LOG.trace("[DS0113] DynamoDB store: Deleted item with key {}", key);
}
return deleted;
}
@Override
public void write(final MarshalledEntry marshalledEntry) {
// The CacheWriter should write the specified entry to the external storage.
//
// The PersistenceManager uses MarshalledEntry as the default format so that CacheWriters can efficiently store data coming
// from a remote node, thus avoiding any additional transformation steps.
//
// Note that keys and values will be in the cache's native format, which means that if the cache is being used by a remoting protocol
// such as HotRod or REST and compatibility mode has not been enabled, then they will be encoded in a byte[].
Loggers.DYNAMODB_LOG.trace("[DS0115] DynamoDB store: Writing {} cache entry {}", getCacheName(), marshalledEntry);
Timer.Context timerCtx = timers.putTimer.time();
try {
table.putItem(requestFactory.resolveItem(
new InfinispanEntry<>(
marshalledEntry.getKey(),
marshalledEntry.getValue(),
marshalledEntry.getMetadata())));
} catch (Exception e) {
Loggers.DYNAMODB_LOG.error("[DS0117] {}: {}", e.getMessage(), e);
throw new PersistenceException(e.getMessage(), e);
} finally {
timerCtx.stop();
}
}
@Override
public void process(final KeyFilter keyFilter,
final CacheLoaderTask cacheLoaderTask,
final Executor executor,
final boolean fetchValue,
final boolean fetchMetadata) {
Loggers.DYNAMODB_LOG.trace("[DS0118] DynamoDB store: Processing key filter for {} cache: fetchValue={} fetchMetadata=",
getCacheName(), fetchValue, fetchMetadata);
final TaskContext taskContext = new TaskContextImpl();
executor.execute(() -> {
Timer.Context timerCtx = timers.processTimer.time();
try {
requestFactory.getAllItems(table).forEach(item -> {
if (taskContext.isStopped()) {
return;
}
InfinispanEntry infinispanEntry = itemTransformer.toInfinispanEntry(item);
if (keyFilter.accept(infinispanEntry.getKey())) {
MarshalledEntry marshalledEntry = marshalledEntryFactory.newMarshalledEntry(
infinispanEntry.getKey(),
infinispanEntry.getValue(),
infinispanEntry.getMetadata());
try {
cacheLoaderTask.processEntry(marshalledEntry, taskContext);
} catch (InterruptedException e) {
throw new PersistenceException(e.getMessage(), e);
}
}
});
} catch (Exception e) {
Loggers.DYNAMODB_LOG.error("[DS0119] {}: {}", e.getMessage(), e);
throw new PersistenceException(e.getMessage(), e);
} finally {
timerCtx.stop();
}
});
}
@Override
public int size() {
// Infinispan code analysis on 8.2 shows that this method is never called in practice, and
// is not wired to the data / cache container API
// TODO inaccurate when a range key is applied!
Loggers.DYNAMODB_LOG.trace("[DS0120] DynamoDB store: Counting {} records", getCacheName());
final int count;
try {
count = table.describe().getItemCount().intValue();
} catch (Exception e) {
Loggers.DYNAMODB_LOG.error("[DS0121] {}: {}", e.getMessage(), e);
throw new PersistenceException(e.getMessage(), e);
}
Loggers.DYNAMODB_LOG.trace("[DS0122] DynamoDB store: Reported approximately {} {} items", count, getCacheName());
return count;
}
@Override
public void clear() {
Loggers.DYNAMODB_LOG.trace("[DS0123] DynamoDB store: Clearing {} items", getCacheName());
if (requestFactory.appliesRangeKey()) {
throw new PersistenceException("DynamoDB clear operation not supported with applied range key");
}
try {
DeleteTableResult result = table.delete();
int numDeleted = result.getTableDescription().getItemCount().intValue();
Loggers.DYNAMODB_LOG.info("[DS0125] DynamoDB store: Cleared {} {} items", numDeleted, table.getTableName());
table.waitForDelete();
client.createTable(requestFactory.resolveCreateTableRequest());
table.waitForActive();
} catch (Exception e) {
Loggers.DYNAMODB_LOG.error("[DS0124] {}: {}", e.getMessage(), e);
throw new PersistenceException(e.getMessage(), e);
}
}
@Override
public void purge(final Executor executor, final PurgeListener purgeListener) {
Loggers.DYNAMODB_LOG.trace("[DS0126] DynamoDB store: Purging {} cache entries", getCacheName());
final AtomicInteger numPurged = new AtomicInteger();
Timer.Context timerCtx = timers.purgeTimer.time();
try {
executor.execute(() -> numPurged.set(reaper.purge(table, purgeListener)));
} catch (Exception e) {
Loggers.DYNAMODB_LOG.error("[DS0127] {}: {}", e.getMessage(), e);
throw new PersistenceException("Purge exception: " + e.getMessage(), e);
} finally {
timerCtx.stop();
}
Loggers.DYNAMODB_LOG.debug("[DS0128] DynamoDB store: Purged {} expired {} cache entries", numPurged.get(), getCacheName());
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy