com.bazaarvoice.emodb.sor.DataStoreModule Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of emodb-sor Show documentation
There is a newer version: 6.5.190
package com.bazaarvoice.emodb.sor;

import com.bazaarvoice.emodb.cachemgr.api.CacheRegistry;
import com.bazaarvoice.emodb.common.cassandra.CassandraConfiguration;
import com.bazaarvoice.emodb.common.cassandra.CassandraFactory;
import com.bazaarvoice.emodb.common.cassandra.CassandraKeyspace;
import com.bazaarvoice.emodb.common.cassandra.CqlDriverConfiguration;
import com.bazaarvoice.emodb.common.cassandra.cqldriver.HintsPollerCQLSession;
import com.bazaarvoice.emodb.common.dropwizard.guice.SystemTablePlacement;
import com.bazaarvoice.emodb.common.dropwizard.healthcheck.HealthCheckRegistry;
import com.bazaarvoice.emodb.common.dropwizard.leader.LeaderServiceTask;
import com.bazaarvoice.emodb.common.dropwizard.lifecycle.LifeCycleRegistry;
import com.bazaarvoice.emodb.common.dropwizard.log.DefaultRateLimitedLogFactory;
import com.bazaarvoice.emodb.common.dropwizard.log.RateLimitedLogFactory;
import com.bazaarvoice.emodb.common.dropwizard.service.EmoServiceMode;
import com.bazaarvoice.emodb.common.dropwizard.task.TaskRegistry;
import com.bazaarvoice.emodb.common.zookeeper.store.MapStore;
import com.bazaarvoice.emodb.common.zookeeper.store.ValueStore;
import com.bazaarvoice.emodb.common.zookeeper.store.ZkBooleanSerializer;
import com.bazaarvoice.emodb.common.zookeeper.store.ZkDoubleSerializer;
import com.bazaarvoice.emodb.common.zookeeper.store.ZkDurationSerializer;
import com.bazaarvoice.emodb.common.zookeeper.store.ZkMapStore;
import com.bazaarvoice.emodb.common.zookeeper.store.ZkTimestampSerializer;
import com.bazaarvoice.emodb.common.zookeeper.store.ZkValueStore;
import com.bazaarvoice.emodb.datacenter.DataCenterConfiguration;
import com.bazaarvoice.emodb.datacenter.api.DataCenters;
import com.bazaarvoice.emodb.datacenter.api.KeyspaceDiscovery;
import com.bazaarvoice.emodb.sor.admin.RowKeyTask;
import com.bazaarvoice.emodb.sor.api.CompactionControlSource;
import com.bazaarvoice.emodb.sor.api.DataStore;
import com.bazaarvoice.emodb.sor.audit.AuditFlusher;
import com.bazaarvoice.emodb.sor.audit.AuditStore;
import com.bazaarvoice.emodb.sor.audit.AuditWriter;
import com.bazaarvoice.emodb.sor.audit.DiscardingAuditWriter;
import com.bazaarvoice.emodb.sor.audit.s3.AthenaAuditWriter;
import com.bazaarvoice.emodb.sor.condition.Condition;
import com.bazaarvoice.emodb.sor.condition.Conditions;
import com.bazaarvoice.emodb.sor.core.DataProvider;
import com.bazaarvoice.emodb.sor.core.DataStoreMinSplitSize;
import com.bazaarvoice.emodb.sor.core.DataStoreProviderProxy;
import com.bazaarvoice.emodb.sor.core.DataTools;
import com.bazaarvoice.emodb.sor.core.DataWriteCloser;
import com.bazaarvoice.emodb.sor.core.DatabusEventWriterRegistry;
import com.bazaarvoice.emodb.sor.core.DefaultDataStore;
import com.bazaarvoice.emodb.sor.core.DefaultHistoryStore;
import com.bazaarvoice.emodb.sor.core.DeltaHistoryTtl;
import com.bazaarvoice.emodb.sor.core.GracefulShutdownManager;
import com.bazaarvoice.emodb.sor.core.HistoryStore;
import com.bazaarvoice.emodb.sor.core.LocalDataStore;
import com.bazaarvoice.emodb.sor.core.ManagedDataStoreDelegate;
import com.bazaarvoice.emodb.sor.core.ManagedTableBackingStoreDelegate;
import com.bazaarvoice.emodb.sor.core.MinSplitSizeCleanupMonitor;
import com.bazaarvoice.emodb.sor.core.MinSplitSizeMap;
import com.bazaarvoice.emodb.sor.core.StashRoot;
import com.bazaarvoice.emodb.sor.core.SystemDataStore;
import com.bazaarvoice.emodb.sor.core.WriteCloseableDataStore;
import com.bazaarvoice.emodb.sor.core.ZKDataStoreMinSplitSizeSerializer;
import com.bazaarvoice.emodb.sor.db.astyanax.DAOModule;
import com.bazaarvoice.emodb.sor.db.astyanax.DeltaPlacementFactory;
import com.bazaarvoice.emodb.sor.db.cql.CqlForMultiGets;
import com.bazaarvoice.emodb.sor.db.cql.CqlForScans;
import com.bazaarvoice.emodb.sor.db.cql.SorCqlSettingsTask;
import com.bazaarvoice.emodb.sor.log.LogbackSlowQueryLogProvider;
import com.bazaarvoice.emodb.sor.log.SlowQueryLog;
import com.bazaarvoice.emodb.sor.log.SlowQueryLogConfiguration;
import com.bazaarvoice.emodb.table.db.ClusterInfo;
import com.bazaarvoice.emodb.table.db.Placements;
import com.bazaarvoice.emodb.table.db.ShardsPerTable;
import com.bazaarvoice.emodb.table.db.StashBlackListTableCondition;
import com.bazaarvoice.emodb.table.db.StashTableDAO;
import com.bazaarvoice.emodb.table.db.TableBackingStore;
import com.bazaarvoice.emodb.table.db.TableChangesEnabled;
import com.bazaarvoice.emodb.table.db.TableDAO;
import com.bazaarvoice.emodb.table.db.astyanax.AstyanaxKeyspaceDiscovery;
import com.bazaarvoice.emodb.table.db.astyanax.AstyanaxTableDAO;
import com.bazaarvoice.emodb.table.db.astyanax.BootstrapTables;
import com.bazaarvoice.emodb.table.db.astyanax.CQLSessionForHintsPollerMap;
import com.bazaarvoice.emodb.table.db.astyanax.CQLStashTableDAO;
import com.bazaarvoice.emodb.table.db.astyanax.CurrentDataCenter;
import com.bazaarvoice.emodb.table.db.astyanax.FullConsistencyTimeProvider;
import com.bazaarvoice.emodb.table.db.astyanax.KeyspaceMap;
import com.bazaarvoice.emodb.table.db.astyanax.Maintenance;
import com.bazaarvoice.emodb.table.db.astyanax.MaintenanceDAO;
import com.bazaarvoice.emodb.table.db.astyanax.MaintenanceRateLimitTask;
import com.bazaarvoice.emodb.table.db.astyanax.MaintenanceSchedulerManager;
import com.bazaarvoice.emodb.table.db.astyanax.MoveTableTask;
import com.bazaarvoice.emodb.table.db.astyanax.PlacementCache;
import com.bazaarvoice.emodb.table.db.astyanax.PlacementFactory;
import com.bazaarvoice.emodb.table.db.astyanax.PlacementsUnderMove;
import com.bazaarvoice.emodb.table.db.astyanax.RateLimiterCache;
import com.bazaarvoice.emodb.table.db.astyanax.SystemTableNamespace;
import com.bazaarvoice.emodb.table.db.astyanax.TableChangesEnabledTask;
import com.bazaarvoice.emodb.table.db.astyanax.ValidTablePlacements;
import com.bazaarvoice.emodb.table.db.consistency.CassandraClusters;
import com.bazaarvoice.emodb.table.db.consistency.ClusterHintsPoller;
import com.bazaarvoice.emodb.table.db.consistency.CompositeConsistencyTimeProvider;
import com.bazaarvoice.emodb.table.db.consistency.DatabusClusterInfo;
import com.bazaarvoice.emodb.table.db.consistency.GlobalFullConsistencyZooKeeper;
import com.bazaarvoice.emodb.table.db.consistency.HintsConsistencyTimeProvider;
import com.bazaarvoice.emodb.table.db.consistency.HintsConsistencyTimeTask;
import com.bazaarvoice.emodb.table.db.consistency.HintsConsistencyTimeValues;
import com.bazaarvoice.emodb.table.db.consistency.HintsPollerManager;
import com.bazaarvoice.emodb.table.db.consistency.MinLagConsistencyTimeProvider;
import com.bazaarvoice.emodb.table.db.consistency.MinLagDurationTask;
import com.bazaarvoice.emodb.table.db.consistency.MinLagDurationValues;
import com.bazaarvoice.emodb.table.db.curator.TableMutexManager;
import com.bazaarvoice.emodb.table.db.eventregistry.TableEventRegistry;
import com.bazaarvoice.emodb.table.db.eventregistry.TableEventTools;
import com.bazaarvoice.emodb.table.db.generic.CachingTableDAO;
import com.bazaarvoice.emodb.table.db.generic.CachingTableDAODelegate;
import com.bazaarvoice.emodb.table.db.generic.CachingTableDAORegistry;
import com.bazaarvoice.emodb.table.db.generic.MutexTableDAO;
import com.bazaarvoice.emodb.table.db.generic.MutexTableDAODelegate;
import com.codahale.metrics.MetricRegistry;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.inject.Exposed;
import com.google.inject.Key;
import com.google.inject.PrivateModule;
import com.google.inject.Provider;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.name.Names;
import com.sun.jersey.api.client.Client;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.utils.ZKPaths;

import java.net.URI;
import java.time.Clock;
import java.time.Duration;
import java.util.Collection;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;

/**
 * Guice module for constructing a {@link DataStore}.
 * 
 * Requires the following external references:
 * 

 *  {@link DataStoreConfiguration}
 * 
 {@link DataCenterConfiguration}
 * 
 {@link DataCenters}
 * 
 {@link CacheRegistry}
 * 
 {@link HealthCheckRegistry}
 * 
 {@link LeaderServiceTask}
 * 
 {@link LifeCycleRegistry}
 * 
 {@link TaskRegistry}
 * 
 Curator {@link CuratorFramework}
 * 
 Jersey {@link Client}
 * 
 @{@link CqlForMultiGets} Supplier<Boolean>
 * 
 @{@link CqlForScans} Supplier<Boolean>
 * 
 {@link CqlDriverConfiguration}
 * 
 {@link Clock}
 * 
 {@link CompactionControlSource}
 * 
 * Exports the following:
 * 
 *  {@link DataStore}
 * 
 {@link DataProvider}
 * 
 {@link DataTools}
 * 
 */
public class DataStoreModule extends PrivateModule {
    private final EmoServiceMode _serviceMode;

    public DataStoreModule(EmoServiceMode serviceMode) {
        _serviceMode = serviceMode;
    }

    @Override
    protected void configure() {

        requireBinding(Key.get(String.class, SystemTablePlacement.class));

        // Note: we only use ZooKeeper if this is the data center that is allowed to edit table metadata (create/drop table)
        // Chain TableDAO -> MutexTableDAO -> CachingTableDAO -> AstyanaxTableDAO.
        bind(TableDAO.class).to(MutexTableDAO.class).asEagerSingleton();
        bind(TableDAO.class).annotatedWith(MutexTableDAODelegate.class).to(CachingTableDAO.class).asEagerSingleton();
        bind(TableDAO.class).annotatedWith(CachingTableDAODelegate.class).to(AstyanaxTableDAO.class).asEagerSingleton();
        bind(String.class).annotatedWith(SystemTableNamespace.class).toInstance("__system_sor");
        bind(PlacementFactory.class).to(DeltaPlacementFactory.class).asEagerSingleton();
        bind(DeltaPlacementFactory.class).asEagerSingleton();
        bind(PlacementCache.class).asEagerSingleton();
        bind(AstyanaxTableDAO.class).asEagerSingleton();
        bind(CassandraFactory.class).asEagerSingleton();
        bind(SlowQueryLog.class).toProvider(LogbackSlowQueryLogProvider.class);
        bind(HintsConsistencyTimeProvider.class).asEagerSingleton();
        bind(MinLagConsistencyTimeProvider.class).asEagerSingleton();

        // The web servers are responsible for updating the ZooKeeper full consistency data.  CLI tools don't need to.
        // Enable updating the ZooKeeper full consistency data if specified
        if (_serviceMode.specifies(EmoServiceMode.Aspect.sor_zookeeper_full_consistency)) {
            bind(HintsPollerManager.class).asEagerSingleton();
            bind(HintsConsistencyTimeTask.class).asEagerSingleton();
            bind(MinLagDurationTask.class).asEagerSingleton();
            bind(ClusterHintsPoller.class).asEagerSingleton();
        }

        if (_serviceMode.specifies(EmoServiceMode.Aspect.dataStore_web)) {
            bind(MinSplitSizeCleanupMonitor.class).asEagerSingleton();
        }

        // The web servers are responsible for performing background table background_table_maintenance.
        // Enable background table background_table_maintenance if specified.
        if (_serviceMode.specifies(EmoServiceMode.Aspect.background_table_maintenance)) {
            bind(MaintenanceSchedulerManager.class).asEagerSingleton();
            bind(MaintenanceDAO.class).to(AstyanaxTableDAO.class).asEagerSingleton();
            bind(CuratorFramework.class).annotatedWith(Maintenance.class)
                    .to(Key.get(CuratorFramework.class, DataStoreZooKeeper.class)).asEagerSingleton();
            bind(String.class).annotatedWith(Maintenance.class).toInstance("sor");
            bind(TableChangesEnabledTask.class).asEagerSingleton();
            bind(MaintenanceRateLimitTask.class).asEagerSingleton();
            bind(MoveTableTask.class).asEagerSingleton();
        }

        if (_serviceMode.specifies(EmoServiceMode.Aspect.megabus)) {
            bind(TableEventTools.class).to(AstyanaxTableDAO.class).asEagerSingleton();
            bind(TableEventRegistry.class).to(AstyanaxTableDAO.class).asEagerSingleton();
            expose(TableEventTools.class);
            expose(TableEventRegistry.class);
        }

        // Stash requires an additional DAO for storing Stash artifacts and provides a custom interface for access.
        if (_serviceMode.specifies(EmoServiceMode.Aspect.scanner) || _serviceMode.specifies(EmoServiceMode.Aspect.megabus)) {
            bind(CQLStashTableDAO.class).asEagerSingleton();
            bind(StashTableDAO.class).to(AstyanaxTableDAO.class).asEagerSingleton();
            expose(StashTableDAO.class);
        }

        // The system of record requires two bootstrap tables in which it stores its metadata about tables.
        // The 64-bit UUID values were chosen at random.
        bind(new TypeLiteral>() {}).annotatedWith(BootstrapTables.class).toInstance(ImmutableMap.of(
                "__system_sor:table", 0x09d7f33f08984b67L,
                "__system_sor:table_uuid", 0xab33556547b99d25L,
                "__system_sor:data_center", 0x33f1f082cffc2c2fL,
                "__system_sor:table_unpublished_databus_events", 0x44ab556547b99dffL,
                "__system_sor:table_event_registry", 0xb0b2716384a93cc4L));

        // Bind all DAOs from the DAO module
        install(new DAOModule());

        bind(HistoryStore.class).to(DefaultHistoryStore.class).asEagerSingleton();

        bind(RateLimitedLogFactory.class).to(DefaultRateLimitedLogFactory.class).asEagerSingleton();

        bind(AuditWriter.class).to(AuditStore.class);
        bind(AuditFlusher.class).to(AuditStore.class);

        // The LocalDataStore annotation binds to the default implementation
        // The unannotated version of DataStore provided below is what the rest of the application will consume
        bind(DefaultDataStore.class).asEagerSingleton();
        bind(WriteCloseableDataStore.class).asEagerSingleton();
        bind(DataStore.class).annotatedWith(ManagedDataStoreDelegate.class).to(DefaultDataStore.class);
        bind(TableBackingStore.class).annotatedWith(ManagedTableBackingStoreDelegate.class).to(DefaultDataStore.class);
        bind(DataStore.class).annotatedWith(LocalDataStore.class).to(WriteCloseableDataStore.class);
        expose(DataStore.class);

        // The AstyanaxTableDAO class uses the DataStore (recursively) to store table metadata
        bind(TableBackingStore.class).to(WriteCloseableDataStore.class);
        expose(TableBackingStore.class);

        // Publish events to listeners like the Databus via an instance of EventBus
        bind(DatabusEventWriterRegistry.class).asEagerSingleton();
        expose(DatabusEventWriterRegistry.class);
        // The Databus uses a back-door API to the DataStore.
        bind(DataProvider.class).to(DefaultDataStore.class);
        expose(DataProvider.class);

        // The Databus monitors each placement via a "databus canary" subscription.  It needs placement metadata.
        bind(Placements.class).to(DeltaPlacementFactory.class).asEagerSingleton();
        expose(Placements.class);

        // The DataCenter discovery expects to be able to enumerate SoR keyspaces
        if (_serviceMode.specifies(EmoServiceMode.Aspect.dataCenter_announce)) {
            bind(KeyspaceDiscovery.class).annotatedWith(Names.named("sor")).to(AstyanaxKeyspaceDiscovery.class).asEagerSingleton();
            expose(KeyspaceDiscovery.class).annotatedWith(Names.named("sor"));
        }

        // Administration tasks
        bind(RowKeyTask.class).asEagerSingleton();
        bind(SorCqlSettingsTask.class).asEagerSingleton();

        // Data tools used to generate reports
        bind(DataTools.class).to(DefaultDataStore.class);
        expose(DataTools.class);

        bind(DataWriteCloser.class).to(WriteCloseableDataStore.class);
        bind(GracefulShutdownManager.class).asEagerSingleton();
    }

    @Provides @Singleton
    AuditStore provideAuditStore(DataStoreConfiguration dataStoreConfiguration, ObjectMapper objectMapper, Clock clock,
                                 RateLimitedLogFactory rateLimitedLogFactory, MetricRegistry metricRegistry) {
        if (dataStoreConfiguration.getAuditWriterConfiguration() != null) {
            return new AthenaAuditWriter(dataStoreConfiguration.getAuditWriterConfiguration(), objectMapper, clock,
                    rateLimitedLogFactory, metricRegistry);
        }

        return new DiscardingAuditWriter();
    }

    @Provides @Singleton
    Optional provideTableMutexManager(DataCenterConfiguration dataCenterConfiguration, @DataStoreZooKeeper CuratorFramework curator) {
        // We only use ZooKeeper if this is the data center that is allowed to edit table metadata (create/drop table)
        if (dataCenterConfiguration.isSystemDataCenter()) {
            return Optional.of(new TableMutexManager(curator, "/lock/table-partitions"));
        }
        return Optional.empty();
    }

    @Provides @Singleton
    DataStore provideDataStore(@LocalDataStore Provider localDataStoreProvider,
                               @SystemDataStore Provider systemDataStoreProvider,
                               DataCenterConfiguration dataCenterConfiguration) {
        // Provides the unannotated version of the DataStore
        // If this is the system data center, return the local DataStore implementation
        // Otherwise return a proxy that delegates to local or remote system DataStores
        if (dataCenterConfiguration.isSystemDataCenter()) {
            return localDataStoreProvider.get();
        } else {
            return new DataStoreProviderProxy(localDataStoreProvider, systemDataStoreProvider);
        }
    }

    @Provides @Singleton @DeltaHistoryTtl
    Duration provideDeltaHistoryTtl(DataStoreConfiguration configuration) {
        return configuration.getHistoryTtl();
    }

    @Provides @Singleton @CurrentDataCenter
    String provideDataCenters(DataCenterConfiguration dataCenterConfiguration) {
        return dataCenterConfiguration.getCurrentDataCenter();
    }

    @Provides @Singleton @ValidTablePlacements
    Set provideValidTablePlacements(DataStoreConfiguration configuration) {
        return configuration.getValidTablePlacements();
    }

    @Provides @Singleton @ShardsPerTable
    int provideShardsPerTable(DataStoreConfiguration configuration) {
        return configuration.getMinimumSplitsPerTable();
    }

    @Provides @Singleton @KeyspaceMap
    Map provideKeyspaces(DataStoreConfiguration configuration, CassandraFactory factory) {
        Map keyspaceMap = Maps.newHashMap();
        for (CassandraConfiguration cassandraConfig : configuration.getCassandraClusters().values()) {
            Map keyspacesInCluster = factory.build(cassandraConfig);
            keyspaceMap.putAll(keyspacesInCluster);
        }
        return ImmutableMap.copyOf(keyspaceMap);
    }

    @Provides @Singleton @PlacementsUnderMove
    Map providePlacementsUnderMove(DataStoreConfiguration configuration,
                                                   @ValidTablePlacements Set validPlacements) {
        return configuration.getPlacementsUnderMove() == null ? ImmutableMap.of()
                : validateMoveMap(ImmutableMap.copyOf(configuration.getPlacementsUnderMove()), validPlacements);
    }

    @Provides @Singleton @CachingTableDAORegistry
    CacheRegistry provideCacheRegistry(CacheRegistry cacheRegistry) {
        return cacheRegistry.withNamespace("sor");
    }

    @Provides @Singleton
    SlowQueryLogConfiguration provideSlowQueryLogConfiguration(DataStoreConfiguration configuration) {
        return configuration.getSlowQueryLogConfiguration();
    }

    @Provides @Singleton @CassandraClusters
    Collection provideCassandraClusterNames(DataStoreConfiguration configuration) {
        Set clusters = Sets.newLinkedHashSet();
        for (CassandraConfiguration config : configuration.getCassandraClusters().values()) {
            clusters.add(config.getCluster());
        }
        return clusters;
    }

    @Provides @Singleton @CQLSessionForHintsPollerMap
    Map provideCQLSessionsForHintsPoller(DataStoreConfiguration configuration, CassandraFactory factory) {
        Map cqlSessionMap = Maps.newHashMap();
        for (CassandraConfiguration config : configuration.getCassandraClusters().values()) {
            String clusterName = config.getCluster();
            if(!cqlSessionMap.containsKey(clusterName)) {
                cqlSessionMap.put(clusterName, factory.cqlSessionForHintsPoller(config));
            }
        }
        return ImmutableMap.copyOf(cqlSessionMap);
    }

    @Provides @Singleton
    Collection provideCassandraClusterInfoForConsistency(DataStoreConfiguration configuration) {
        return getClusterInfos(configuration);
    }

    // Expose this provider so that DatabusModule can wire in this annotation, which only cares about SoR clusters
    @Provides @Singleton @Exposed @DatabusClusterInfo
    Collection provideCassandraClusterInfo(Collection clusterInfos) {
        return clusterInfos;
    }

    @Provides @Singleton
    FullConsistencyTimeProvider provideFullConsistencyTimeProvider(Collection cassandraClusterInfo,
                                                                   HintsConsistencyTimeProvider hintsConsistencyTimeProvider,
                                                                   MinLagConsistencyTimeProvider minLagConsistencyTimeProvider,
                                                                   MetricRegistry metricRegistry) {
        return new CompositeConsistencyTimeProvider(cassandraClusterInfo, ImmutableList.of(hintsConsistencyTimeProvider,
                minLagConsistencyTimeProvider), metricRegistry);
    }

    @Provides @Singleton @HintsConsistencyTimeValues
    Map> provideHintsTimestampValues(@CassandraClusters Collection cassandraClusters,
                                                              @GlobalFullConsistencyZooKeeper CuratorFramework curator,
                                                              LifeCycleRegistry lifeCycle)
            throws Exception {
        // Create a timestamp holder for each Cassandra cluster.
        Map> valuesByCluster = Maps.newLinkedHashMap();
        for (String cluster : cassandraClusters) {
            String zkPath = ZKPaths.makePath("/consistency/max-timestamp", cluster);
            ZkValueStore holder = new ZkValueStore<>(curator, zkPath, new ZkTimestampSerializer());
            valuesByCluster.put(cluster, lifeCycle.manage(holder));
        }
        return valuesByCluster;
    }

    @Provides @Singleton @MinLagDurationValues
    Map> provideMinLagDurationValues(@CassandraClusters Collection cassandraClusters,
                                                                  @GlobalFullConsistencyZooKeeper final CuratorFramework curator,
                                                                  final LifeCycleRegistry lifeCycle) {
        final ConcurrentMap> valuesByCluster = Maps.newConcurrentMap();
        for (String cluster : cassandraClusters) {
            String zkPath = ZKPaths.makePath("/consistency/min-lag", cluster);
            ZkValueStore holder = new ZkValueStore<>(curator, zkPath, new ZkDurationSerializer());
            valuesByCluster.put(cluster, lifeCycle.manage(holder));
        }
        return valuesByCluster;
    }

    @Provides @Singleton @TableChangesEnabled
    ValueStore provideTableChangesEnabled(@DataStoreZooKeeper CuratorFramework curator,
                                                   LifeCycleRegistry lifeCycle) {
        return lifeCycle.manage(new ZkValueStore<>(curator, "settings/table-changes-enabled", new ZkBooleanSerializer(), true));
    }

    @Provides @Singleton @Maintenance
    MapStore provideRateLimiterSettings(@DataStoreZooKeeper CuratorFramework curator,
                                                LifeCycleRegistry lifeCycle) {
        return lifeCycle.manage(new ZkMapStore<>(curator, "background_table_maintenance-rate-limits", new ZkDoubleSerializer()));
    }

    @Provides @Singleton @Maintenance
    RateLimiterCache provideRateLimiterCache(@Maintenance MapStore rateLimits) {
        return new RateLimiterCache(rateLimits, 1000);
    }

    @Provides @Singleton @MinSplitSizeMap
    MapStore provideMinSplitSizeMap(@DataStoreZooKeeper CuratorFramework curator,
                                                LifeCycleRegistry lifeCycle) {
        return lifeCycle.manage(new ZkMapStore<>(curator, "min-split-size", new ZKDataStoreMinSplitSizeSerializer()));
    }

    @Provides @Singleton @StashRoot
    Optional provideStashRootDirectory(DataStoreConfiguration configuration) {
        if (configuration.getStashRoot().isPresent()) {
            return Optional.of(URI.create(configuration.getStashRoot().get()));
        }
        return Optional.empty();
    }

    @Provides @Singleton @StashBlackListTableCondition
    protected Condition provideStashBlackListTableCondition(DataStoreConfiguration configuration) {
        return configuration.getStashBlackListTableCondition()
                .map(Conditions::fromString)
                .orElse(Conditions.alwaysFalse());
    }

    private Collection getClusterInfos(DataStoreConfiguration configuration) {
        Map clusterInfoMap = Maps.newLinkedHashMap();
        for (CassandraConfiguration config : configuration.getCassandraClusters().values()) {
            ClusterInfo clusterInfo = new ClusterInfo(config.getCluster(), config.getClusterMetric());
            ClusterInfo old = clusterInfoMap.put(config.getCluster(), clusterInfo);
            checkState(old == null || old.getClusterMetric().equals(clusterInfo.getClusterMetric()),
                    "Cluster %s is configured with multiple distinct names for the cluster metric.", config.getCluster());
        }
        return ImmutableList.copyOf(clusterInfoMap.values());
    }

    private Map validateMoveMap(Map moveMap, Set validPlacements) {
        for (Map.Entry entry : moveMap.entrySet()) {
            checkArgument(validPlacements.contains(entry.getKey()), "Invalid move-from placement");
            checkArgument(validPlacements.contains(entry.getValue()), "Invalid move-to placement");
            // Prevent chained moves
            checkArgument(!moveMap.containsKey(entry.getValue()), "Chained moves are not allowed");
        }
        return moveMap;
    }
}