com.scalar.db.storage.dynamo.DynamoAdmin Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of scalardb Show documentation
Show all versions of scalardb Show documentation
A universal transaction manager that achieves database-agnostic transactions and distributed transactions that span multiple databases
package com.scalar.db.storage.dynamo;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.Uninterruptibles;
import com.google.inject.Inject;
import com.scalar.db.api.DistributedStorageAdmin;
import com.scalar.db.api.Scan.Ordering.Order;
import com.scalar.db.api.TableMetadata;
import com.scalar.db.config.DatabaseConfig;
import com.scalar.db.exception.storage.ExecutionException;
import com.scalar.db.io.DataType;
import com.scalar.db.util.ScalarDbUtils;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.concurrent.ThreadSafe;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.applicationautoscaling.ApplicationAutoScalingClient;
import software.amazon.awssdk.services.applicationautoscaling.ApplicationAutoScalingClientBuilder;
import software.amazon.awssdk.services.applicationautoscaling.model.ApplicationAutoScalingException;
import software.amazon.awssdk.services.applicationautoscaling.model.DeleteScalingPolicyRequest;
import software.amazon.awssdk.services.applicationautoscaling.model.DeregisterScalableTargetRequest;
import software.amazon.awssdk.services.applicationautoscaling.model.MetricType;
import software.amazon.awssdk.services.applicationautoscaling.model.ObjectNotFoundException;
import software.amazon.awssdk.services.applicationautoscaling.model.PolicyType;
import software.amazon.awssdk.services.applicationautoscaling.model.PredefinedMetricSpecification;
import software.amazon.awssdk.services.applicationautoscaling.model.PutScalingPolicyRequest;
import software.amazon.awssdk.services.applicationautoscaling.model.RegisterScalableTargetRequest;
import software.amazon.awssdk.services.applicationautoscaling.model.ScalableDimension;
import software.amazon.awssdk.services.applicationautoscaling.model.ServiceNamespace;
import software.amazon.awssdk.services.applicationautoscaling.model.TargetTrackingScalingPolicyConfiguration;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.ContinuousBackupsStatus;
import software.amazon.awssdk.services.dynamodb.model.CreateGlobalSecondaryIndexAction;
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
import software.amazon.awssdk.services.dynamodb.model.DeleteGlobalSecondaryIndexAction;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.DeleteTableRequest;
import software.amazon.awssdk.services.dynamodb.model.DescribeContinuousBackupsRequest;
import software.amazon.awssdk.services.dynamodb.model.DescribeContinuousBackupsResponse;
import software.amazon.awssdk.services.dynamodb.model.DescribeTableRequest;
import software.amazon.awssdk.services.dynamodb.model.DescribeTableResponse;
import software.amazon.awssdk.services.dynamodb.model.DynamoDbException;
import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
import software.amazon.awssdk.services.dynamodb.model.GlobalSecondaryIndex;
import software.amazon.awssdk.services.dynamodb.model.GlobalSecondaryIndexDescription;
import software.amazon.awssdk.services.dynamodb.model.GlobalSecondaryIndexUpdate;
import software.amazon.awssdk.services.dynamodb.model.IndexStatus;
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
import software.amazon.awssdk.services.dynamodb.model.KeyType;
import software.amazon.awssdk.services.dynamodb.model.ListTablesRequest;
import software.amazon.awssdk.services.dynamodb.model.ListTablesResponse;
import software.amazon.awssdk.services.dynamodb.model.PointInTimeRecoverySpecification;
import software.amazon.awssdk.services.dynamodb.model.Projection;
import software.amazon.awssdk.services.dynamodb.model.ProjectionType;
import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.ResourceNotFoundException;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
import software.amazon.awssdk.services.dynamodb.model.TableStatus;
import software.amazon.awssdk.services.dynamodb.model.UpdateContinuousBackupsRequest;
import software.amazon.awssdk.services.dynamodb.model.UpdateTableRequest;
/**
* Manages table creating, dropping and truncating in Dynamo DB
*
* @author Pham Ba Thong
*/
@ThreadSafe
public class DynamoAdmin implements DistributedStorageAdmin {
public static final String NO_SCALING = "no-scaling";
public static final String NO_BACKUP = "no-backup";
public static final String REQUEST_UNIT = "ru";
public static final String DEFAULT_NO_SCALING = "false";
public static final String DEFAULT_NO_BACKUP = "false";
public static final String DEFAULT_REQUEST_UNIT = "10";
private static final int DEFAULT_WAITING_DURATION_SECS = 3;
@VisibleForTesting static final String PARTITION_KEY = "concatenatedPartitionKey";
@VisibleForTesting static final String CLUSTERING_KEY = "concatenatedClusteringKey";
private static final String GLOBAL_INDEX_NAME_PREFIX = "global_index";
private static final int COOL_DOWN_DURATION_SECS = 60;
private static final double TARGET_USAGE_RATE = 70.0;
private static final int DELETE_BATCH_SIZE = 100;
private static final String SCALING_TYPE_READ = "read";
private static final String SCALING_TYPE_WRITE = "write";
private static final String SCALING_TYPE_INDEX_READ = "index-read";
private static final String SCALING_TYPE_INDEX_WRITE = "index-write";
public static final String METADATA_NAMESPACE = "scalardb";
public static final String METADATA_TABLE = "metadata";
@VisibleForTesting static final String METADATA_ATTR_PARTITION_KEY = "partitionKey";
@VisibleForTesting static final String METADATA_ATTR_CLUSTERING_KEY = "clusteringKey";
@VisibleForTesting static final String METADATA_ATTR_CLUSTERING_ORDERS = "clusteringOrders";
@VisibleForTesting static final String METADATA_ATTR_SECONDARY_INDEX = "secondaryIndex";
@VisibleForTesting static final String METADATA_ATTR_COLUMNS = "columns";
@VisibleForTesting static final String METADATA_ATTR_TABLE = "table";
private static final long METADATA_TABLE_REQUEST_UNIT = 1;
private static final ImmutableMap SECONDARY_INDEX_DATATYPE_MAP =
ImmutableMap.builder()
.put(DataType.INT, ScalarAttributeType.N)
.put(DataType.BIGINT, ScalarAttributeType.N)
.put(DataType.FLOAT, ScalarAttributeType.N)
.put(DataType.DOUBLE, ScalarAttributeType.N)
.put(DataType.TEXT, ScalarAttributeType.S)
.put(DataType.BLOB, ScalarAttributeType.B)
.build();
private static final ImmutableSet TABLE_SCALING_TYPE_SET =
ImmutableSet.builder().add(SCALING_TYPE_READ).add(SCALING_TYPE_WRITE).build();
private static final ImmutableSet SECONDARY_INDEX_SCALING_TYPE_SET =
ImmutableSet.builder()
.add(SCALING_TYPE_INDEX_READ)
.add(SCALING_TYPE_INDEX_WRITE)
.build();
private static final ImmutableMap SCALABLE_DIMENSION_MAP =
ImmutableMap.builder()
.put(SCALING_TYPE_READ, ScalableDimension.DYNAMODB_TABLE_READ_CAPACITY_UNITS)
.put(SCALING_TYPE_WRITE, ScalableDimension.DYNAMODB_TABLE_WRITE_CAPACITY_UNITS)
.put(SCALING_TYPE_INDEX_READ, ScalableDimension.DYNAMODB_INDEX_READ_CAPACITY_UNITS)
.put(SCALING_TYPE_INDEX_WRITE, ScalableDimension.DYNAMODB_INDEX_WRITE_CAPACITY_UNITS)
.build();
private static final ImmutableMap SCALING_POLICY_METRIC_TYPE_MAP =
ImmutableMap.builder()
.put(SCALING_TYPE_READ, MetricType.DYNAMO_DB_READ_CAPACITY_UTILIZATION)
.put(SCALING_TYPE_WRITE, MetricType.DYNAMO_DB_WRITE_CAPACITY_UTILIZATION)
.put(SCALING_TYPE_INDEX_READ, MetricType.DYNAMO_DB_READ_CAPACITY_UTILIZATION)
.put(SCALING_TYPE_INDEX_WRITE, MetricType.DYNAMO_DB_WRITE_CAPACITY_UTILIZATION)
.build();
private final DynamoDbClient client;
private final ApplicationAutoScalingClient applicationAutoScalingClient;
private final String metadataNamespace;
private final String namespacePrefix;
private final int waitingDurationSecs;
@Inject
public DynamoAdmin(DatabaseConfig databaseConfig) {
DynamoConfig config = new DynamoConfig(databaseConfig);
AwsCredentialsProvider credentialsProvider = createCredentialsProvider(config);
DynamoDbClientBuilder builder = DynamoDbClient.builder();
config.getEndpointOverride().ifPresent(e -> builder.endpointOverride(URI.create(e)));
client =
builder
.credentialsProvider(credentialsProvider)
.region(Region.of(config.getRegion()))
.build();
applicationAutoScalingClient = createApplicationAutoScalingClient(config);
metadataNamespace =
config.getNamespacePrefix().orElse("")
+ config.getTableMetadataNamespace().orElse(METADATA_NAMESPACE);
namespacePrefix = config.getNamespacePrefix().orElse("");
waitingDurationSecs = DEFAULT_WAITING_DURATION_SECS;
}
@SuppressFBWarnings("EI_EXPOSE_REP2")
DynamoAdmin(DynamoDbClient client, DynamoConfig config) {
this.client = client;
applicationAutoScalingClient = createApplicationAutoScalingClient(config);
metadataNamespace =
config.getNamespacePrefix().orElse("")
+ config.getTableMetadataNamespace().orElse(METADATA_NAMESPACE);
namespacePrefix = config.getNamespacePrefix().orElse("");
waitingDurationSecs = DEFAULT_WAITING_DURATION_SECS;
}
@VisibleForTesting
DynamoAdmin(
DynamoDbClient client,
ApplicationAutoScalingClient applicationAutoScalingClient,
DynamoConfig config) {
this.client = client;
this.applicationAutoScalingClient = applicationAutoScalingClient;
metadataNamespace =
config.getNamespacePrefix().orElse("")
+ config.getTableMetadataNamespace().orElse(METADATA_NAMESPACE);
namespacePrefix = config.getNamespacePrefix().orElse("");
waitingDurationSecs = 0;
}
private AwsCredentialsProvider createCredentialsProvider(DynamoConfig config) {
return StaticCredentialsProvider.create(
AwsBasicCredentials.create(config.getAccessKeyId(), config.getSecretAccessKey()));
}
private ApplicationAutoScalingClient createApplicationAutoScalingClient(DynamoConfig config) {
ApplicationAutoScalingClientBuilder builder = ApplicationAutoScalingClient.builder();
config.getEndpointOverride().ifPresent(e -> builder.endpointOverride(URI.create(e)));
return builder
.credentialsProvider(createCredentialsProvider(config))
.region(Region.of(config.getRegion()))
.build();
}
@Override
public void createNamespace(String nonPrefixedNamespace, Map options) {
// In Dynamo DB storage, namespace will be added to table name as prefix along with dot
// separator.
}
@Override
public void createTable(
String nonPrefixedNamespace,
String table,
TableMetadata metadata,
Map options)
throws ExecutionException {
Namespace namespace = Namespace.of(namespacePrefix, nonPrefixedNamespace);
checkMetadata(metadata);
long ru = Long.parseLong(options.getOrDefault(REQUEST_UNIT, DEFAULT_REQUEST_UNIT));
CreateTableRequest.Builder requestBuilder = CreateTableRequest.builder();
buildAttributeDefinitions(requestBuilder, metadata);
buildPrimaryKey(requestBuilder, metadata);
buildSecondaryIndexes(namespace, table, requestBuilder, metadata, ru);
requestBuilder.provisionedThroughput(
ProvisionedThroughput.builder().readCapacityUnits(ru).writeCapacityUnits(ru).build());
requestBuilder.tableName(getFullTableName(namespace, table));
try {
client.createTable(requestBuilder.build());
} catch (Exception e) {
throw new ExecutionException("creating the table failed", e);
}
waitForTableCreation(namespace, table);
boolean noScaling = Boolean.parseBoolean(options.getOrDefault(NO_SCALING, DEFAULT_NO_SCALING));
if (!noScaling) {
enableAutoScaling(namespace, table, metadata.getSecondaryIndexNames(), ru);
}
boolean noBackup = Boolean.parseBoolean(options.getOrDefault(NO_BACKUP, DEFAULT_NO_BACKUP));
if (!noBackup) {
enableContinuousBackup(namespace, table);
}
createMetadataTableIfNotExists(noBackup);
putTableMetadata(namespace, table, metadata);
}
private void checkMetadata(TableMetadata metadata) {
Iterator partitionKeyNameIterator = metadata.getPartitionKeyNames().iterator();
while (partitionKeyNameIterator.hasNext()) {
String partitionKeyName = partitionKeyNameIterator.next();
if (!partitionKeyNameIterator.hasNext()) {
break;
}
if (metadata.getColumnDataType(partitionKeyName) == DataType.BLOB) {
throw new IllegalArgumentException(
"BLOB type is supported only for the last column in partition key in DynamoDB: "
+ partitionKeyName);
}
}
for (String clusteringKeyName : metadata.getClusteringKeyNames()) {
if (metadata.getColumnDataType(clusteringKeyName) == DataType.BLOB) {
throw new IllegalArgumentException(
"Currently, BLOB type is not supported for clustering keys in DynamoDB: "
+ clusteringKeyName);
}
}
for (String secondaryIndexName : metadata.getSecondaryIndexNames()) {
if (metadata.getColumnDataType(secondaryIndexName) == DataType.BOOLEAN) {
throw new IllegalArgumentException(
"Currently, BOOLEAN type is not supported for a secondary index in DynamoDB: "
+ secondaryIndexName);
}
}
}
private void buildAttributeDefinitions(
CreateTableRequest.Builder requestBuilder, TableMetadata metadata) {
List columnsToAttributeDefinitions = new ArrayList<>();
// for partition key
columnsToAttributeDefinitions.add(
AttributeDefinition.builder()
.attributeName(PARTITION_KEY)
.attributeType(ScalarAttributeType.B)
.build());
// for clustering key
if (!metadata.getClusteringKeyNames().isEmpty()) {
columnsToAttributeDefinitions.add(
AttributeDefinition.builder()
.attributeName(CLUSTERING_KEY)
.attributeType(ScalarAttributeType.B)
.build());
}
// for secondary indexes
if (!metadata.getSecondaryIndexNames().isEmpty()) {
for (String secondaryIndex : metadata.getSecondaryIndexNames()) {
columnsToAttributeDefinitions.add(
AttributeDefinition.builder()
.attributeName(secondaryIndex)
.attributeType(
SECONDARY_INDEX_DATATYPE_MAP.get(metadata.getColumnDataType(secondaryIndex)))
.build());
}
}
requestBuilder.attributeDefinitions(columnsToAttributeDefinitions);
}
private void buildPrimaryKey(CreateTableRequest.Builder requestBuilder, TableMetadata metadata) {
List keySchemaElementList = new ArrayList<>();
keySchemaElementList.add(
KeySchemaElement.builder().attributeName(PARTITION_KEY).keyType(KeyType.HASH).build());
if (!metadata.getClusteringKeyNames().isEmpty()) {
keySchemaElementList.add(
KeySchemaElement.builder().attributeName(CLUSTERING_KEY).keyType(KeyType.RANGE).build());
}
requestBuilder.keySchema(keySchemaElementList);
}
private void buildSecondaryIndexes(
Namespace namespace,
String table,
CreateTableRequest.Builder requestBuilder,
TableMetadata metadata,
long ru) {
if (!metadata.getSecondaryIndexNames().isEmpty()) {
List globalSecondaryIndexList = new ArrayList<>();
for (String secondaryIndex : metadata.getSecondaryIndexNames()) {
globalSecondaryIndexList.add(
GlobalSecondaryIndex.builder()
.indexName(getGlobalIndexName(namespace, table, secondaryIndex))
.keySchema(
KeySchemaElement.builder()
.attributeName(secondaryIndex)
.keyType(KeyType.HASH)
.build())
.projection(Projection.builder().projectionType(ProjectionType.ALL).build())
.provisionedThroughput(
ProvisionedThroughput.builder()
.readCapacityUnits(ru)
.writeCapacityUnits(ru)
.build())
.build());
}
requestBuilder.globalSecondaryIndexes(globalSecondaryIndexList);
}
}
private String getGlobalIndexName(Namespace namespace, String tableName, String keyName) {
return getFullTableName(namespace, tableName) + "." + GLOBAL_INDEX_NAME_PREFIX + "." + keyName;
}
private void putTableMetadata(Namespace namespace, String table, TableMetadata metadata)
throws ExecutionException {
// Add metadata
Map itemValues = new HashMap<>();
itemValues.put(
METADATA_ATTR_TABLE,
AttributeValue.builder().s(getFullTableName(namespace, table)).build());
Map columns = new HashMap<>();
for (String columnName : metadata.getColumnNames()) {
columns.put(
columnName,
AttributeValue.builder()
.s(metadata.getColumnDataType(columnName).name().toLowerCase())
.build());
}
itemValues.put(METADATA_ATTR_COLUMNS, AttributeValue.builder().m(columns).build());
itemValues.put(
METADATA_ATTR_PARTITION_KEY,
AttributeValue.builder()
.l(
metadata.getPartitionKeyNames().stream()
.map(pKey -> AttributeValue.builder().s(pKey).build())
.collect(Collectors.toList()))
.build());
if (!metadata.getClusteringKeyNames().isEmpty()) {
itemValues.put(
METADATA_ATTR_CLUSTERING_KEY,
AttributeValue.builder()
.l(
metadata.getClusteringKeyNames().stream()
.map(pKey -> AttributeValue.builder().s(pKey).build())
.collect(Collectors.toList()))
.build());
Map clusteringOrders = new HashMap<>();
for (String clusteringKeyName : metadata.getClusteringKeyNames()) {
clusteringOrders.put(
clusteringKeyName,
AttributeValue.builder()
.s(metadata.getClusteringOrder(clusteringKeyName).name())
.build());
}
itemValues.put(
METADATA_ATTR_CLUSTERING_ORDERS, AttributeValue.builder().m(clusteringOrders).build());
}
if (!metadata.getSecondaryIndexNames().isEmpty()) {
itemValues.put(
METADATA_ATTR_SECONDARY_INDEX,
AttributeValue.builder().ss(metadata.getSecondaryIndexNames()).build());
}
try {
client.putItem(
PutItemRequest.builder()
.tableName(ScalarDbUtils.getFullTableName(metadataNamespace, METADATA_TABLE))
.item(itemValues)
.build());
} catch (Exception e) {
throw new ExecutionException(
"adding the meta data for table " + getFullTableName(namespace, table) + " failed", e);
}
}
private void createMetadataTableIfNotExists(boolean noBackup) throws ExecutionException {
if (metadataTableExists()) {
return;
}
List columnsToAttributeDefinitions = new ArrayList<>();
columnsToAttributeDefinitions.add(
AttributeDefinition.builder()
.attributeName(METADATA_ATTR_TABLE)
.attributeType(ScalarAttributeType.S)
.build());
try {
client.createTable(
CreateTableRequest.builder()
.attributeDefinitions(columnsToAttributeDefinitions)
.keySchema(
KeySchemaElement.builder()
.attributeName(METADATA_ATTR_TABLE)
.keyType(KeyType.HASH)
.build())
.provisionedThroughput(
ProvisionedThroughput.builder()
.readCapacityUnits(METADATA_TABLE_REQUEST_UNIT)
.writeCapacityUnits(METADATA_TABLE_REQUEST_UNIT)
.build())
.tableName(ScalarDbUtils.getFullTableName(metadataNamespace, METADATA_TABLE))
.build());
} catch (Exception e) {
throw new ExecutionException("creating the metadata table failed", e);
}
waitForTableCreation(Namespace.of(metadataNamespace), METADATA_TABLE);
if (!noBackup) {
enableContinuousBackup(Namespace.of(metadataNamespace), METADATA_TABLE);
}
}
private boolean metadataTableExists() throws ExecutionException {
try {
client.describeTable(
DescribeTableRequest.builder()
.tableName(ScalarDbUtils.getFullTableName(metadataNamespace, METADATA_TABLE))
.build());
return true;
} catch (Exception e) {
if (e instanceof ResourceNotFoundException) {
return false;
} else {
throw new ExecutionException("checking the metadata table existence failed", e);
}
}
}
private void waitForTableCreation(Namespace namespace, String table) throws ExecutionException {
try {
while (true) {
Uninterruptibles.sleepUninterruptibly(waitingDurationSecs, TimeUnit.SECONDS);
DescribeTableResponse describeTableResponse =
client.describeTable(
DescribeTableRequest.builder()
.tableName(getFullTableName(namespace, table))
.build());
if (describeTableResponse.table().tableStatus() == TableStatus.ACTIVE) {
break;
}
}
} catch (Exception e) {
throw new ExecutionException("waiting for the table creation failed", e);
}
}
private void enableAutoScaling(
Namespace namespace, String table, Set secondaryIndexes, long ru)
throws ExecutionException {
List registerScalableTargetRequestList = new ArrayList<>();
List putScalingPolicyRequestList = new ArrayList<>();
// write, read scaling of table
for (String scalingType : TABLE_SCALING_TYPE_SET) {
registerScalableTargetRequestList.add(
buildRegisterScalableTargetRequest(
getTableResourceID(namespace, table), scalingType, (int) ru));
putScalingPolicyRequestList.add(
buildPutScalingPolicyRequest(getTableResourceID(namespace, table), scalingType));
}
// write, read scaling of global indexes (secondary indexes)
for (String secondaryIndex : secondaryIndexes) {
for (String scalingType : SECONDARY_INDEX_SCALING_TYPE_SET) {
registerScalableTargetRequestList.add(
buildRegisterScalableTargetRequest(
getGlobalIndexResourceID(namespace, table, secondaryIndex), scalingType, (int) ru));
putScalingPolicyRequestList.add(
buildPutScalingPolicyRequest(
getGlobalIndexResourceID(namespace, table, secondaryIndex), scalingType));
}
}
registerScalableTarget(registerScalableTargetRequestList);
putScalingPolicy(putScalingPolicyRequestList);
}
private RegisterScalableTargetRequest buildRegisterScalableTargetRequest(
String resourceID, String type, int ruValue) {
return RegisterScalableTargetRequest.builder()
.serviceNamespace(ServiceNamespace.DYNAMODB)
.resourceId(resourceID)
.scalableDimension(SCALABLE_DIMENSION_MAP.get(type))
.minCapacity(ruValue > 10 ? ruValue / 10 : ruValue)
.maxCapacity(ruValue)
.build();
}
private PutScalingPolicyRequest buildPutScalingPolicyRequest(String resourceID, String type) {
return PutScalingPolicyRequest.builder()
.serviceNamespace(ServiceNamespace.DYNAMODB)
.resourceId(resourceID)
.scalableDimension(SCALABLE_DIMENSION_MAP.get(type))
.policyName(getPolicyName(resourceID, type))
.policyType(PolicyType.TARGET_TRACKING_SCALING)
.targetTrackingScalingPolicyConfiguration(getScalingPolicyConfiguration(type))
.build();
}
private String getTableResourceID(Namespace namespace, String table) {
return "table/" + getFullTableName(namespace, table);
}
private String getGlobalIndexResourceID(Namespace namespace, String table, String globalIndex) {
return "table/"
+ getFullTableName(namespace, table)
+ "/index/"
+ getGlobalIndexName(namespace, table, globalIndex);
}
private String getPolicyName(String resourceID, String type) {
return resourceID + "-" + type;
}
private TargetTrackingScalingPolicyConfiguration getScalingPolicyConfiguration(String type) {
return TargetTrackingScalingPolicyConfiguration.builder()
.predefinedMetricSpecification(
PredefinedMetricSpecification.builder()
.predefinedMetricType(SCALING_POLICY_METRIC_TYPE_MAP.get(type))
.build())
.scaleInCooldown(COOL_DOWN_DURATION_SECS)
.scaleOutCooldown(COOL_DOWN_DURATION_SECS)
.targetValue(TARGET_USAGE_RATE)
.build();
}
private void enableContinuousBackup(Namespace namespace, String table) throws ExecutionException {
waitForTableBackupEnabledAtCreation(namespace, table);
try {
client.updateContinuousBackups(buildUpdateContinuousBackupsRequest(namespace, table));
} catch (Exception e) {
throw new ExecutionException(
"Unable to enable continuous backup for " + getFullTableName(namespace, table), e);
}
}
private void waitForTableBackupEnabledAtCreation(Namespace namespace, String table)
throws ExecutionException {
try {
while (true) {
Uninterruptibles.sleepUninterruptibly(waitingDurationSecs, TimeUnit.SECONDS);
DescribeContinuousBackupsResponse describeContinuousBackupsResponse =
client.describeContinuousBackups(
DescribeContinuousBackupsRequest.builder()
.tableName(getFullTableName(namespace, table))
.build());
if (describeContinuousBackupsResponse
.continuousBackupsDescription()
.continuousBackupsStatus()
== ContinuousBackupsStatus.ENABLED) {
break;
}
}
} catch (Exception e) {
throw new ExecutionException("waiting for the table backup enabled at creation failed", e);
}
}
private PointInTimeRecoverySpecification buildPointInTimeRecoverySpecification() {
return PointInTimeRecoverySpecification.builder().pointInTimeRecoveryEnabled(true).build();
}
private UpdateContinuousBackupsRequest buildUpdateContinuousBackupsRequest(
Namespace namespace, String table) {
return UpdateContinuousBackupsRequest.builder()
.tableName(getFullTableName(namespace, table))
.pointInTimeRecoverySpecification(buildPointInTimeRecoverySpecification())
.build();
}
@Override
public void dropTable(String nonPrefixedNamespace, String table) throws ExecutionException {
Namespace namespace = Namespace.of(namespacePrefix, nonPrefixedNamespace);
disableAutoScaling(namespace, table);
String fullTableName = getFullTableName(namespace, table);
try {
client.deleteTable(DeleteTableRequest.builder().tableName(fullTableName).build());
} catch (Exception e) {
throw new ExecutionException("deleting table " + fullTableName + " failed", e);
}
waitForTableDeletion(namespace, table);
deleteTableMetadata(namespace, table);
}
private void disableAutoScaling(Namespace namespace, String table) throws ExecutionException {
TableMetadata tableMetadata = getTableMetadata(namespace.nonPrefixed(), table);
if (tableMetadata == null) {
return;
}
List deleteScalingPolicyRequestList = new ArrayList<>();
List deregisterScalableTargetRequestList = new ArrayList<>();
// write, read scaling of table
for (String scalingType : TABLE_SCALING_TYPE_SET) {
deleteScalingPolicyRequestList.add(
buildDeleteScalingPolicyRequest(getTableResourceID(namespace, table), scalingType));
deregisterScalableTargetRequestList.add(
buildDeregisterScalableTargetRequest(getTableResourceID(namespace, table), scalingType));
}
// write, read scaling of global indexes (secondary indexes)
Set secondaryIndexes = tableMetadata.getSecondaryIndexNames();
for (String secondaryIndex : secondaryIndexes) {
for (String scalingType : SECONDARY_INDEX_SCALING_TYPE_SET) {
deleteScalingPolicyRequestList.add(
buildDeleteScalingPolicyRequest(
getGlobalIndexResourceID(namespace, table, secondaryIndex), scalingType));
deregisterScalableTargetRequestList.add(
buildDeregisterScalableTargetRequest(
getGlobalIndexResourceID(namespace, table, secondaryIndex), scalingType));
}
}
deleteScalingPolicy(deleteScalingPolicyRequestList);
deregisterScalableTarget(deregisterScalableTargetRequestList);
}
private DeregisterScalableTargetRequest buildDeregisterScalableTargetRequest(
String resourceID, String type) {
return DeregisterScalableTargetRequest.builder()
.serviceNamespace(ServiceNamespace.DYNAMODB)
.resourceId(resourceID)
.scalableDimension(SCALABLE_DIMENSION_MAP.get(type))
.build();
}
private DeleteScalingPolicyRequest buildDeleteScalingPolicyRequest(
String resourceID, String type) {
return DeleteScalingPolicyRequest.builder()
.serviceNamespace(ServiceNamespace.DYNAMODB)
.resourceId(resourceID)
.scalableDimension(SCALABLE_DIMENSION_MAP.get(type))
.policyName(getPolicyName(resourceID, type))
.build();
}
private void deleteTableMetadata(Namespace namespace, String table) throws ExecutionException {
String metadataTable = ScalarDbUtils.getFullTableName(metadataNamespace, METADATA_TABLE);
Map keyToDelete = new HashMap<>();
keyToDelete.put(
METADATA_ATTR_TABLE,
AttributeValue.builder().s(getFullTableName(namespace, table)).build());
try {
client.deleteItem(
DeleteItemRequest.builder().tableName(metadataTable).key(keyToDelete).build());
} catch (Exception e) {
throw new ExecutionException("deleting the metadata failed", e);
}
ScanResponse scanResponse;
try {
scanResponse = client.scan(ScanRequest.builder().tableName(metadataTable).limit(1).build());
} catch (Exception e) {
throw new ExecutionException("scanning the metadata table failed", e);
}
if (scanResponse.count() == 0) {
try {
client.deleteTable(DeleteTableRequest.builder().tableName(metadataTable).build());
} catch (Exception e) {
throw new ExecutionException("deleting the empty metadata table failed", e);
}
waitForTableDeletion(Namespace.of(metadataNamespace), METADATA_TABLE);
}
}
private void waitForTableDeletion(Namespace namespace, String tableName)
throws ExecutionException {
try {
while (true) {
Uninterruptibles.sleepUninterruptibly(waitingDurationSecs, TimeUnit.SECONDS);
Set tableSet = getNamespaceTableNames(namespace.nonPrefixed());
if (!tableSet.contains(tableName)) {
break;
}
}
} catch (Exception e) {
throw new ExecutionException("waiting for the table deletion failed", e);
}
}
@Override
public void dropNamespace(String nonPrefixedNamespace) {
// Do nothing since DynamoDB does not support namespace
}
@Override
public void truncateTable(String nonPrefixedNamespace, String table) throws ExecutionException {
String fullTableName =
getFullTableName(Namespace.of(namespacePrefix, nonPrefixedNamespace), table);
Map lastKeyEvaluated = null;
do {
ScanResponse scanResponse;
try {
scanResponse =
client.scan(
ScanRequest.builder()
.tableName(fullTableName)
.limit(DELETE_BATCH_SIZE)
.exclusiveStartKey(lastKeyEvaluated)
.build());
} catch (Exception e) {
throw new ExecutionException("scanning items from table " + fullTableName + " failed.", e);
}
for (Map item : scanResponse.items()) {
Map keyToDelete = new HashMap<>();
keyToDelete.put(PARTITION_KEY, item.get(PARTITION_KEY));
if (item.containsKey(CLUSTERING_KEY)) {
keyToDelete.put(CLUSTERING_KEY, item.get(CLUSTERING_KEY));
}
try {
client.deleteItem(
DeleteItemRequest.builder().tableName(fullTableName).key(keyToDelete).build());
} catch (Exception e) {
throw new ExecutionException("deleting item from table " + fullTableName + " failed.", e);
}
}
lastKeyEvaluated = scanResponse.lastEvaluatedKey();
} while (!lastKeyEvaluated.isEmpty());
}
@Override
public void createIndex(
String nonPrefixedNamespace, String table, String columnName, Map options)
throws ExecutionException {
Namespace namespace = Namespace.of(namespacePrefix, nonPrefixedNamespace);
TableMetadata metadata = getTableMetadata(nonPrefixedNamespace, table);
if (metadata == null) {
throw new IllegalArgumentException(
"Table " + getFullTableName(namespace, table) + " does not exist.");
}
if (metadata.getColumnDataType(columnName) == DataType.BOOLEAN) {
throw new IllegalArgumentException(
"Currently, BOOLEAN type is not supported for a secondary index in DynamoDB: "
+ columnName);
}
long ru = Long.parseLong(options.getOrDefault(REQUEST_UNIT, DEFAULT_REQUEST_UNIT));
try {
client.updateTable(
UpdateTableRequest.builder()
.tableName(getFullTableName(namespace, table))
.attributeDefinitions(
AttributeDefinition.builder()
.attributeName(columnName)
.attributeType(
SECONDARY_INDEX_DATATYPE_MAP.get(metadata.getColumnDataType(columnName)))
.build())
.globalSecondaryIndexUpdates(
GlobalSecondaryIndexUpdate.builder()
.create(
CreateGlobalSecondaryIndexAction.builder()
.indexName(getGlobalIndexName(namespace, table, columnName))
.keySchema(
KeySchemaElement.builder()
.attributeName(columnName)
.keyType(KeyType.HASH)
.build())
.projection(
Projection.builder().projectionType(ProjectionType.ALL).build())
.provisionedThroughput(
ProvisionedThroughput.builder()
.readCapacityUnits(ru)
.writeCapacityUnits(ru)
.build())
.build())
.build())
.build());
} catch (Exception e) {
throw new ExecutionException("creating the secondary index failed", e);
}
waitForIndexCreation(namespace, table, columnName);
// enable auto scaling
boolean noScaling = Boolean.parseBoolean(options.getOrDefault(NO_SCALING, DEFAULT_NO_SCALING));
if (!noScaling) {
List registerScalableTargetRequestList = new ArrayList<>();
List putScalingPolicyRequestList = new ArrayList<>();
// write, read scaling of global indexes (secondary indexes)
for (String scalingType : SECONDARY_INDEX_SCALING_TYPE_SET) {
registerScalableTargetRequestList.add(
buildRegisterScalableTargetRequest(
getGlobalIndexResourceID(namespace, table, columnName), scalingType, (int) ru));
putScalingPolicyRequestList.add(
buildPutScalingPolicyRequest(
getGlobalIndexResourceID(namespace, table, columnName), scalingType));
}
registerScalableTarget(registerScalableTargetRequestList);
putScalingPolicy(putScalingPolicyRequestList);
}
// update metadata
TableMetadata tableMetadata = getTableMetadata(nonPrefixedNamespace, table);
putTableMetadata(
namespace,
table,
TableMetadata.newBuilder(tableMetadata).addSecondaryIndex(columnName).build());
}
private void waitForIndexCreation(Namespace namespace, String table, String columnName)
throws ExecutionException {
try {
String indexName = getGlobalIndexName(namespace, table, columnName);
while (true) {
Uninterruptibles.sleepUninterruptibly(waitingDurationSecs, TimeUnit.SECONDS);
DescribeTableResponse response =
client.describeTable(
DescribeTableRequest.builder()
.tableName(getFullTableName(namespace, table))
.build());
for (GlobalSecondaryIndexDescription globalSecondaryIndex :
response.table().globalSecondaryIndexes()) {
if (globalSecondaryIndex.indexName().equals(indexName)) {
if (globalSecondaryIndex.indexStatus() == IndexStatus.ACTIVE) {
return;
}
}
}
}
} catch (Exception e) {
throw new ExecutionException("waiting for the secondary index creation failed", e);
}
}
private void registerScalableTarget(
List registerScalableTargetRequestList)
throws ExecutionException {
for (RegisterScalableTargetRequest registerScalableTargetRequest :
registerScalableTargetRequestList) {
try {
applicationAutoScalingClient.registerScalableTarget(registerScalableTargetRequest);
} catch (Exception e) {
throw new ExecutionException(
"Unable to register scalable target for " + registerScalableTargetRequest.resourceId(),
e);
}
}
}
private void putScalingPolicy(List putScalingPolicyRequestList)
throws ExecutionException {
for (PutScalingPolicyRequest putScalingPolicyRequest : putScalingPolicyRequestList) {
try {
applicationAutoScalingClient.putScalingPolicy(putScalingPolicyRequest);
} catch (Exception e) {
throw new ExecutionException(
"Unable to put scaling policy request for " + putScalingPolicyRequest.resourceId(), e);
}
}
}
@Override
public void dropIndex(String nonPrefixedNamespace, String table, String columnName)
throws ExecutionException {
Namespace namespace = Namespace.of(namespacePrefix, nonPrefixedNamespace);
try {
client.updateTable(
UpdateTableRequest.builder()
.tableName(getFullTableName(namespace, table))
.globalSecondaryIndexUpdates(
GlobalSecondaryIndexUpdate.builder()
.delete(
DeleteGlobalSecondaryIndexAction.builder()
.indexName(getGlobalIndexName(namespace, table, columnName))
.build())
.build())
.build());
} catch (Exception e) {
throw new ExecutionException("dropping the secondary index failed", e);
}
waitForIndexDeletion(namespace, table, columnName);
// disable auto scaling
List deleteScalingPolicyRequestList = new ArrayList<>();
List deregisterScalableTargetRequestList = new ArrayList<>();
for (String scalingType : SECONDARY_INDEX_SCALING_TYPE_SET) {
deleteScalingPolicyRequestList.add(
buildDeleteScalingPolicyRequest(
getGlobalIndexResourceID(namespace, table, columnName), scalingType));
deregisterScalableTargetRequestList.add(
buildDeregisterScalableTargetRequest(
getGlobalIndexResourceID(namespace, table, columnName), scalingType));
}
deleteScalingPolicy(deleteScalingPolicyRequestList);
deregisterScalableTarget(deregisterScalableTargetRequestList);
// update metadata
TableMetadata tableMetadata = getTableMetadata(nonPrefixedNamespace, table);
putTableMetadata(
namespace,
table,
TableMetadata.newBuilder(tableMetadata).removeSecondaryIndex(columnName).build());
}
private void waitForIndexDeletion(Namespace namespace, String table, String columnName)
throws ExecutionException {
try {
String indexName = getGlobalIndexName(namespace, table, columnName);
while (true) {
Uninterruptibles.sleepUninterruptibly(waitingDurationSecs, TimeUnit.SECONDS);
DescribeTableResponse response =
client.describeTable(
DescribeTableRequest.builder()
.tableName(getFullTableName(namespace, table))
.build());
boolean deleted = true;
for (GlobalSecondaryIndexDescription globalSecondaryIndex :
response.table().globalSecondaryIndexes()) {
if (globalSecondaryIndex.indexName().equals(indexName)) {
deleted = false;
break;
}
}
if (deleted) {
break;
}
}
} catch (Exception e) {
throw new ExecutionException("waiting for the secondary index deletion failed", e);
}
}
private void deleteScalingPolicy(
List deleteScalingPolicyRequestList) {
for (DeleteScalingPolicyRequest deleteScalingPolicyRequest : deleteScalingPolicyRequestList) {
try {
applicationAutoScalingClient.deleteScalingPolicy(deleteScalingPolicyRequest);
// Suppress exceptions when the scaling policy does not exist
} catch (ObjectNotFoundException ignored) {
// ObjectNotFoundException is thrown when using a regular Dynamo DB instance
} catch (ApplicationAutoScalingException e) {
// The auto-scaling service is not supported with Dynamo DB local. Any API call to the
// 'applicationAutoScalingClient' will raise an ApplicationAutoScalingException
if (!(e.awsErrorDetails().errorCode().equals("InvalidAction") && e.statusCode() == 400)) {
throw e;
}
}
}
}
private void deregisterScalableTarget(
List deregisterScalableTargetRequestList) {
for (DeregisterScalableTargetRequest deregisterScalableTargetRequest :
deregisterScalableTargetRequestList) {
try {
applicationAutoScalingClient.deregisterScalableTarget(deregisterScalableTargetRequest);
// Suppress exceptions when the scalable target does not exist
} catch (ObjectNotFoundException ignored) {
// ObjectNotFoundException is thrown when using a regular Dynamo DB instance
} catch (ApplicationAutoScalingException e) {
// The auto-scaling service is not supported with Dynamo DB local. Any API call to the
// 'applicationAutoScalingClient' will raise an ApplicationAutoScalingException
if (!(e.awsErrorDetails().errorCode().equals("InvalidAction") && e.statusCode() == 400)) {
throw e;
}
}
}
}
@Override
public TableMetadata getTableMetadata(String nonPrefixedNamespace, String table)
throws ExecutionException {
try {
String fullName =
getFullTableName(Namespace.of(namespacePrefix, nonPrefixedNamespace), table);
return readMetadata(fullName);
} catch (RuntimeException e) {
throw new ExecutionException("getting a table metadata failed", e);
}
}
private TableMetadata readMetadata(String fullName) throws ExecutionException {
Map key = new HashMap<>();
key.put(METADATA_ATTR_TABLE, AttributeValue.builder().s(fullName).build());
try {
Map metadata =
client
.getItem(
GetItemRequest.builder()
.tableName(ScalarDbUtils.getFullTableName(metadataNamespace, METADATA_TABLE))
.key(key)
.consistentRead(true)
.build())
.item();
if (metadata.isEmpty()) {
// The specified table is not found
return null;
}
return createTableMetadata(metadata);
} catch (Exception e) {
throw new ExecutionException("Failed to read the table metadata", e);
}
}
private TableMetadata createTableMetadata(Map metadata)
throws ExecutionException {
TableMetadata.Builder builder = TableMetadata.newBuilder();
for (Entry entry : metadata.get(METADATA_ATTR_COLUMNS).m().entrySet()) {
builder.addColumn(entry.getKey(), convertDataType(entry.getValue().s()));
}
metadata.get(METADATA_ATTR_PARTITION_KEY).l().stream()
.map(AttributeValue::s)
.forEach(builder::addPartitionKey);
if (metadata.containsKey(METADATA_ATTR_CLUSTERING_KEY)) {
Map clusteringOrders =
metadata.get(METADATA_ATTR_CLUSTERING_ORDERS).m();
metadata.get(METADATA_ATTR_CLUSTERING_KEY).l().stream()
.map(AttributeValue::s)
.forEach(n -> builder.addClusteringKey(n, Order.valueOf(clusteringOrders.get(n).s())));
}
if (metadata.containsKey(METADATA_ATTR_SECONDARY_INDEX)) {
metadata.get(METADATA_ATTR_SECONDARY_INDEX).ss().forEach(builder::addSecondaryIndex);
}
return builder.build();
}
private DataType convertDataType(String columnType) throws ExecutionException {
switch (columnType) {
case "int":
return DataType.INT;
case "bigint":
return DataType.BIGINT;
case "float":
return DataType.FLOAT;
case "double":
return DataType.DOUBLE;
case "text":
return DataType.TEXT;
case "boolean":
return DataType.BOOLEAN;
case "blob":
return DataType.BLOB;
default:
throw new ExecutionException("unknown column type: " + columnType);
}
}
@Override
public Set getNamespaceTableNames(String nonPrefixedNamespace) throws ExecutionException {
try {
Set tableSet = new HashSet<>();
String lastEvaluatedTableName = null;
do {
ListTablesRequest listTablesRequest =
ListTablesRequest.builder().exclusiveStartTableName(lastEvaluatedTableName).build();
ListTablesResponse listTablesResponse = client.listTables(listTablesRequest);
lastEvaluatedTableName = listTablesResponse.lastEvaluatedTableName();
List tableNames = listTablesResponse.tableNames();
Namespace namespace = Namespace.of(namespacePrefix, nonPrefixedNamespace);
String prefix = namespace.prefixed() + ".";
for (String tableName : tableNames) {
if (tableName.startsWith(prefix)) {
tableSet.add(tableName.substring(prefix.length()));
}
}
} while (lastEvaluatedTableName != null);
return tableSet;
} catch (Exception e) {
throw new ExecutionException("getting list of tables failed", e);
}
}
@Override
public boolean namespaceExists(String nonPrefixedNamespace) throws ExecutionException {
try {
boolean namespaceExists = false;
String lastEvaluatedTableName = null;
do {
ListTablesRequest listTablesRequest =
ListTablesRequest.builder().exclusiveStartTableName(lastEvaluatedTableName).build();
ListTablesResponse listTablesResponse = client.listTables(listTablesRequest);
lastEvaluatedTableName = listTablesResponse.lastEvaluatedTableName();
List tableNames = listTablesResponse.tableNames();
Namespace namespace = Namespace.of(namespacePrefix, nonPrefixedNamespace);
for (String tableName : tableNames) {
if (tableName.startsWith(namespace.prefixed() + ".")) {
namespaceExists = true;
break;
}
}
} while (lastEvaluatedTableName != null);
return namespaceExists;
} catch (DynamoDbException e) {
throw new ExecutionException("checking the namespace existence failed", e);
}
}
@Override
public void repairTable(
String nonPrefixedNamespace,
String table,
TableMetadata metadata,
Map options)
throws ExecutionException {
Namespace namespace = Namespace.of(namespacePrefix, nonPrefixedNamespace);
try {
if (!tableExists(nonPrefixedNamespace, table)) {
throw new IllegalArgumentException(
"The table " + getFullTableName(namespace, table) + " does not exist");
}
boolean noBackup = Boolean.parseBoolean(options.getOrDefault(NO_BACKUP, DEFAULT_NO_BACKUP));
createMetadataTableIfNotExists(noBackup);
putTableMetadata(namespace, table, metadata);
} catch (IllegalArgumentException e) {
throw e;
} catch (RuntimeException e) {
throw new ExecutionException(
String.format("repairing the table %s.%s failed", namespace, table), e);
}
}
@Override
public void addNewColumnToTable(
String nonPrefixedNamespace, String table, String columnName, DataType columnType)
throws ExecutionException {
try {
TableMetadata currentTableMetadata = getTableMetadata(nonPrefixedNamespace, table);
TableMetadata updatedTableMetadata =
TableMetadata.newBuilder(currentTableMetadata).addColumn(columnName, columnType).build();
Namespace namespace = Namespace.of(namespacePrefix, nonPrefixedNamespace);
putTableMetadata(namespace, table, updatedTableMetadata);
} catch (ExecutionException e) {
throw new ExecutionException(
String.format(
"Adding the new column %s to the %s.%s table failed",
columnName, nonPrefixedNamespace, table),
e);
}
}
@Override
public TableMetadata getImportTableMetadata(String namespace, String table) {
throw new UnsupportedOperationException(
"import-related functionality is not supported in DynamoDB");
}
@Override
public void addRawColumnToTable(
String namespace, String table, String columnName, DataType columnType) {
throw new UnsupportedOperationException(
"import-related functionality is not supported in DynamoDB");
}
@Override
public void importTable(String namespace, String table) {
throw new UnsupportedOperationException(
"import-related functionality is not supported in DynamoDB");
}
private String getFullTableName(Namespace namespace, String table) {
return ScalarDbUtils.getFullTableName(namespace.prefixed(), table);
}
@Override
public void close() {
client.close();
applicationAutoScalingClient.close();
}
}