All Downloads are FREE. Search and download functionalities are using the official Maven repository.

alluxio.inode.RocksBenchConfig Maven / Gradle / Ivy

There is a newer version: 313
Show newest version
/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio.inode;

import static org.apache.commons.io.FileUtils.writeStringToFile;

import alluxio.conf.InstancedConfiguration;
import alluxio.conf.PropertyKey;
import alluxio.exception.status.InvalidArgumentException;
import alluxio.master.metastore.rocks.DataBlockIndexType;
import alluxio.master.metastore.rocks.IndexType;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;

/**
 * This class is used to set different RocksDB configurations for the
 * benchmarks through the mRocksConfig option, as follows:
 * mRocksConfig - if using rocks, or rocksCache for mType, then the type
 *   of RocksDB configuration to use (if using heap then this option must
 *   be set to javaConfig). This can be one of the following:
 *     - javaConfig - this uses the configuration defined in the java code in
 *     {@link alluxio.master.metastore.rocks.RocksInodeStore} (i.e. no configuration
 *     file is used)
 *     - emptyConfig - this uses a configuration file with no configurations, i.e.
 *     it uses all RocksDB defaults.
 *     - bloomConfig - this uses a configuration with bloom filters enabled, both
 *     for the memtable and the block files. Additionally, it uses larger caches
 *     and uses block has indices for faster point lookups.
 *     - baseConfig - this is the same as javaConfig, except defined in a string
 *     representing a config file, allowing easy modifications.

 */
public class RocksBenchConfig {

  static final String JAVA_CONFIG = "javaConfig";
  static final String EMPTY_CONFIG = "emptyConfig";
  static final String BLOOM_CONFIG = "bloomConfig";
  static final String BASE_CONFIG = "baseConfig";

  static void setRocksConfig(String confType, String dir,
                             InstancedConfiguration conf) throws IOException {
    switch (confType) {
      case EMPTY_CONFIG:
        resetConfig(false, dir, conf);
        break;
      case JAVA_CONFIG:
        resetConfig(true, null, conf);
        break;
      case BLOOM_CONFIG:
        setBloomConfig(dir, conf);
        break;
      case BASE_CONFIG:
        setBaseConfig(dir, conf);
        break;
      default:
        throw new InvalidArgumentException(String.format(
            "Invalid RocksDB config type %s", confType));
    }
  }

  private static void resetConfig(boolean javaConfig,
      String dir, InstancedConfiguration conf) throws IOException {
    if (!javaConfig) {
      File confFile = new File(dir + "conf");
      writeStringToFile(confFile, NO_CONFIG_FILE, (Charset) null);
      conf.set(PropertyKey.ROCKS_INODE_CONF_FILE,
          (Object) confFile.getAbsolutePath());
    } else {
      // if the config file is unset, then the java configuration will be used
      conf.unset(PropertyKey.ROCKS_INODE_CONF_FILE);
    }
    conf.unset(PropertyKey.MASTER_METASTORE_ROCKS_EDGE_BLOCK_INDEX);
    conf.unset(PropertyKey.MASTER_METASTORE_ROCKS_EDGE_INDEX);
    conf.unset(PropertyKey.MASTER_METASTORE_ROCKS_EDGE_BLOOM_FILTER);
    conf.unset(PropertyKey.MASTER_METASTORE_ROCKS_INODE_CACHE_SIZE);
    conf.unset(PropertyKey.MASTER_METASTORE_ROCKS_INODE_BLOCK_INDEX);
    conf.unset(PropertyKey.MASTER_METASTORE_ROCKS_INODE_INDEX);
    conf.unset(PropertyKey.MASTER_METASTORE_ROCKS_INODE_BLOOM_FILTER);
    conf.unset(PropertyKey.MASTER_METASTORE_ROCKS_INODE_CACHE_SIZE);
  }

  private static void setBloomConfig(
      String dir, InstancedConfiguration conf) throws IOException {
    File confFile = new File(dir + "conf");
    writeStringToFile(confFile, BLOOM_CONFIG_FILE, (Charset) null);
    conf.set(PropertyKey.ROCKS_INODE_CONF_FILE,
        (Object) confFile.getAbsolutePath());
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_EDGE_BLOCK_INDEX,
        DataBlockIndexType.kDataBlockBinaryAndHash);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_EDGE_INDEX, IndexType.kHashSearch);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_EDGE_BLOOM_FILTER, true);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_INODE_CACHE_SIZE, 64 * 1024 * 1024);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_INODE_BLOCK_INDEX,
        DataBlockIndexType.kDataBlockBinaryAndHash);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_INODE_INDEX, IndexType.kHashSearch);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_INODE_BLOOM_FILTER, true);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_INODE_CACHE_SIZE, 64 * 1024 * 1024);
  }

  private static void setBaseConfig(String dir,
                            InstancedConfiguration conf) throws IOException {
    File confFile = new File(dir + "conf");
    writeStringToFile(confFile, BASE_CONFIG_FILE, (Charset) null);
    conf.set(PropertyKey.ROCKS_INODE_CONF_FILE,
        (Object) confFile.getAbsolutePath());
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_EDGE_BLOCK_INDEX,
        DataBlockIndexType.kDataBlockBinarySearch);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_EDGE_INDEX, IndexType.kBinarySearch);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_EDGE_BLOOM_FILTER, false);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_INODE_CACHE_SIZE, 8 * 1024 * 1024);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_INODE_BLOCK_INDEX,
        DataBlockIndexType.kDataBlockBinarySearch);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_INODE_INDEX, IndexType.kBinarySearch);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_INODE_BLOOM_FILTER, false);
    conf.set(PropertyKey.MASTER_METASTORE_ROCKS_INODE_CACHE_SIZE, 8 * 1024 * 1024);
  }

  private static final String NO_CONFIG_FILE = "\n"
      + "[Version]\n"
      + "  rocksdb_version=7.0.3\n"
      + "  options_file_version=1.1\n"
      + "\n"
      + "[DBOptions]\n"
      + "  create_if_missing=true\n"
      + "  create_missing_column_families=true\n"
      + "  \n"
      + "\n"
      + "[CFOptions \"default\"]\n"
      + "  \n"
      + "[TableOptions/BlockBasedTable \"default\"]\n"
      + "  \n"
      + "\n"
      + "[CFOptions \"inodes\"]\n"
      + "  \n"
      + "[TableOptions/BlockBasedTable \"inodes\"]\n"
      + "  \n"
      + "\n"
      + "[CFOptions \"edges\"]\n"
      + "  \n"
      + "[TableOptions/BlockBasedTable \"edges\"]\n"
      + "  \n";

  private static final String BASE_CONFIG_FILE = "\n"
      + "[Version]\n"
      + "  rocksdb_version=7.0.3\n"
      + "  options_file_version=1.1\n"
      + "\n"
      + "[DBOptions]\n"
      + "  allow_concurrent_memtable_write=false\n"
      + "  create_if_missing=true\n"
      + "  create_missing_column_families=true\n"
      + "  \n"
      + "\n"
      + "[CFOptions \"default\"]\n"
      + "  \n"
      + "[TableOptions/BlockBasedTable \"default\"]\n"
      + "  \n"
      + "\n"
      + "[CFOptions \"inodes\"]\n"
      + "   compression=kNoCompression\n"
      + "   prefix_extractor=rocksdb.FixedPrefix.8\n"
      + "   memtable_factory={id=HashLinkListRepFactory;logging_threshold=4096;log_when_flash=true;"
      + "huge_page_size=0;threshold=256;bucket_count=50000;}\n"
      + "  \n"
      + "[TableOptions/BlockBasedTable \"inodes\"]\n"
      + "  \n"
      + "\n"
      + "[CFOptions \"edges\"]\n"
      + "   compression=kNoCompression\n"
      + "   prefix_extractor=rocksdb.FixedPrefix.8\n"
      + "   memtable_factory={id=HashLinkListRepFactory;logging_threshold=4096;log_when_flash=true;"
      + "huge_page_size=0;threshold=256;bucket_count=50000;}\n"
      + "  \n"
      + "[TableOptions/BlockBasedTable \"edges\"]\n"
      + "  \n";

  private static final String BLOOM_CONFIG_FILE =  "\n"
      + "[Version]\n"
      + "  rocksdb_version=7.0.3\n"
      + "  options_file_version=1.1\n"
      + "\n"
      + "[DBOptions]\n"
      + "  allow_concurrent_memtable_write=false\n"
      + "  create_if_missing=true\n"
      + "  create_missing_column_families=true\n"
      + "  \n"
      + "\n"
      + "[CFOptions \"default\"]\n"
      + "  \n"
      + "[TableOptions/BlockBasedTable \"default\"]\n"
      + "  \n"
      + "\n"
      + "[CFOptions \"inodes\"]\n"
      + "  max_bytes_for_level_base=536870912\n"
      + "  memtable_whole_key_filtering=true\n"
      + "  max_write_buffer_number=6\n"
      + "  compression=kNoCompression\n"
      + "  level0_file_num_compaction_trigger=2\n"
      + "  prefix_extractor=rocksdb.FixedPrefix.8\n"
      + "  write_buffer_size=134217728\n"
      + "  memtable_prefix_bloom_size_ratio=0.020000\n"
      + "  memtable_factory={id=HashLinkListRepFactory;logging_threshold=4096;log_when_flash=true;"
      + "huge_page_size=0;threshold=256;bucket_count=50000;}\n"
      + "  min_write_buffer_number_to_merge=2      \n"
      + "  \n"
      + "[TableOptions/BlockBasedTable \"inodes\"]\n"
      + "  index_type=kHashSearch\n"
      + "  filter_policy=bloomfilter:10:false\n"
      + "  data_block_index_type=kDataBlockBinarySearch\n"
      + "  \n"
      + "\n"
      + "[CFOptions \"edges\"]\n"
      + "  max_bytes_for_level_base=536870912\n"
      + "  memtable_whole_key_filtering=true\n"
      + "  max_write_buffer_number=6\n"
      + "  compression=kNoCompression\n"
      + "  level0_file_num_compaction_trigger=2\n"
      + "  prefix_extractor=rocksdb.FixedPrefix.8\n"
      + "  write_buffer_size=134217728\n"
      + "  memtable_prefix_bloom_size_ratio=0.020000\n"
      + "  memtable_factory={id=HashLinkListRepFactory;logging_threshold=4096;log_when_flash=true;"
      + "huge_page_size=0;threshold=256;bucket_count=50000;}\n"
      + "  min_write_buffer_number_to_merge=2      \n"
      + "  \n"
      + "[TableOptions/BlockBasedTable \"edges\"]\n"
      + "  index_type=kHashSearch\n"
      + "  filter_policy=bloomfilter:10:false\n"
      + "  data_block_index_type=kDataBlockBinarySearch\n"
      + "  \n";
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy