Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.voovan.tools.collection.RocksMap Maven / Gradle / Ivy
package org.voovan.tools.collection;
import org.rocksdb.*;
import org.voovan.Global;
import org.voovan.tools.TByte;
import org.voovan.tools.TDateTime;
import org.voovan.tools.TFile;
import org.voovan.tools.Varint;
import org.voovan.tools.exception.ParseException;
import org.voovan.tools.exception.RocksMapException;
import org.voovan.tools.log.Logger;
import org.voovan.tools.serialize.TSerialize;
import java.io.Closeable;
import java.io.File;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.*;
import java.util.Comparator;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
/**
* RocksDB 的 Map 封装
*
* @author: helyho
* ignite-test Framework.
* WebSite: https://github.com/helyho/ignite-test
* Licence: Apache v2 License
*/
public class RocksMap implements SortedMap, Closeable {
//--------------------- 公共静态变量 --------------------
static {
RocksDB.loadLibrary();
}
public final static String DEFAULT_COLUMN_FAMILY_NAME = "Default";
private static byte[] DATA_BYTES = "data".getBytes();
//缓存 db 和他对应的 TransactionDB
private static Map ROCKSDB_MAP = new ConcurrentHashMap();
//缓存 TransactionDB 和列族句柄的关系
private static Map> COLUMN_FAMILY_HANDLE_MAP = new ConcurrentHashMap>();
//数据文件的默认保存路径
private static String DEFAULT_DB_PATH = ".rocksdb"+ File.separator;
private static String DEFAULT_WAL_PATH = DEFAULT_DB_PATH + ".wal"+ File.separator;
/**
* 获取默认数据存储路径
* @return 默认数存储据路径
*/
public static String getDefaultDbPath() {
return DEFAULT_DB_PATH;
}
/**
* 设置默认数存储据路径
* @param defaultDbPath 默认数存储据路径
*/
public static void setDefaultDbPath(String defaultDbPath) {
DEFAULT_DB_PATH = defaultDbPath.endsWith(File.separator) ? defaultDbPath : defaultDbPath + File.separator;
}
/**
* 默认WAL数存储据路径
* @return WAL数存储据路径
*/
public static String getDefaultWalPath() {
return DEFAULT_WAL_PATH;
}
/**
* 设置WAL数存储据路径
* @param defaultWalPath WAL数存储据路径
*/
public static void setDefaultWalPath(String defaultWalPath) {
DEFAULT_WAL_PATH = defaultWalPath.endsWith(File.separator) ? defaultWalPath : defaultWalPath + File.separator;;
}
/**
* 根据名称获取列族
* @param rocksDB RocksDB 对象
* @param columnFamilyName 列族名称
* @return 列族句柄
*/
private static ColumnFamilyHandle getColumnFamilyHandler(RocksDB rocksDB, String columnFamilyName) {
return COLUMN_FAMILY_HANDLE_MAP.get(rocksDB).get(columnFamilyName);
}
/**
* 关闭 RocksDB 极其句柄
* @param rocksDB RocksDB 对象
*/
private static void closeRocksDB(RocksDB rocksDB) {
for (ColumnFamilyHandle columnFamilyHandle : COLUMN_FAMILY_HANDLE_MAP.get(rocksDB).values()) {
columnFamilyHandle.close();
}
rocksDB.close();
COLUMN_FAMILY_HANDLE_MAP.remove(rocksDB);
}
//--------------------- 成员变量 --------------------
public transient DBOptions dbOptions;
public transient ReadOptions readOptions;
public transient WriteOptions writeOptions;
public transient ColumnFamilyOptions columnFamilyOptions;
private transient RocksDB rocksDB;
private transient ColumnFamilyHandle dataColumnFamilyHandle;
private transient ThreadLocal threadLocalTransaction = new ThreadLocal();
private transient ThreadLocal threadLocalSavePointCount = ThreadLocal.withInitial(()->new Integer(0));
private transient ThreadLocal threadLocalBuilder = ThreadLocal.withInitial(()->new StringBuilder());
private transient String dbname;
private transient String dataPath;
private transient String walPath;
private transient String columnFamilyName;
private transient Boolean readOnly;
private transient Boolean isDuplicate = false;
private transient int transactionLockTimeout = 5000;
/**
* 构造方法
*/
public RocksMap() {
this(null, null, null, null, null, null, null);
}
/**
* 构造方法
* @param columnFamilyName 列族名称
*/
public RocksMap(String columnFamilyName) {
this(null, columnFamilyName, null, null, null, null, null);
}
/**
* 构造方法
* @param dbname 数据库的名称, 基于数据保存目录的相对路径
* @param columnFamilyName 列族名称
*/
public RocksMap(String dbname, String columnFamilyName) {
this(dbname, columnFamilyName, null, null, null, null, null);
}
/**
* 构造方法
* @param readOnly 是否以只读模式打开
*/
public RocksMap(boolean readOnly) {
this(null, null, null, null, null, null, readOnly);
}
/**
* 构造方法
* @param columnFamilyName 列族名称
* @param readOnly 是否以只读模式打开
*/
public RocksMap(String columnFamilyName, boolean readOnly) {
this(null, columnFamilyName, null, null, null, null, readOnly);
}
/**
* 构造方法
* @param dbname 数据库的名称, 基于数据保存目录的相对路径
* @param columnFamilyName 列族名称
* @param readOnly 是否以只读模式打开
*/
public RocksMap(String dbname, String columnFamilyName, boolean readOnly) {
this(dbname, columnFamilyName, null, null, null, null, readOnly);
}
/**
* 构造方法
* @param dbname 数据库的名称, 基于数据保存目录的相对路径
* @param columnFamilyName 列族名称
* @param dbOptions DBOptions 配置对象
* @param readOptions ReadOptions 配置对象
* @param writeOptions WriteOptions 配置对象
* @param columnFamilyOptions 列族配置对象
* @param readOnly 是否以只读模式打开
*/
public RocksMap(String dbname, String columnFamilyName, ColumnFamilyOptions columnFamilyOptions, DBOptions dbOptions, ReadOptions readOptions, WriteOptions writeOptions, Boolean readOnly) {
this.dbname = dbname == null ? DEFAULT_COLUMN_FAMILY_NAME : dbname;
this.columnFamilyName = columnFamilyName == null ? "voovan_default" : columnFamilyName;
this.readOptions = readOptions == null ? new ReadOptions() : readOptions;
this.writeOptions = writeOptions == null ? new WriteOptions() : writeOptions;
this.columnFamilyOptions = columnFamilyOptions == null ? new ColumnFamilyOptions() : columnFamilyOptions;
this.readOnly = readOnly == null ? false : readOnly;
Options options = new Options();
options.setCreateIfMissing(true);
options.setCreateMissingColumnFamilies(true);
if(dbOptions == null) {
//Rocksdb 数据库配置
this.dbOptions = new DBOptions(options);
} else {
this.dbOptions = dbOptions;
}
this.dataPath = DEFAULT_DB_PATH + this.dbname;
this.walPath = DEFAULT_WAL_PATH +this.dbname;
this.dbOptions.useDirectIoForFlushAndCompaction();
this.dbOptions.setWalDir(walPath);
TFile.mkdir(dataPath);
TFile.mkdir(this.dbOptions.walDir());
rocksDB = ROCKSDB_MAP.get(this.dbname);
try {
if (rocksDB == null || this.readOnly) {
//默认列族列表
List DEFAULT_CF_DESCRIPTOR_LIST = new ArrayList();
//加载已经存在的所有列族
{
List columnFamilyNameBytes = RocksDB.listColumnFamilies(new Options(), DEFAULT_DB_PATH + this.dbname);
if (columnFamilyNameBytes.size() > 0) {
for (byte[] columnFamilyNameByte : columnFamilyNameBytes) {
ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(columnFamilyNameByte, this.columnFamilyOptions);
DEFAULT_CF_DESCRIPTOR_LIST.add(columnFamilyDescriptor);
}
}
//如果为空创建默认列族
if (DEFAULT_CF_DESCRIPTOR_LIST.size() == 0) {
DEFAULT_CF_DESCRIPTOR_LIST.add( new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, this.columnFamilyOptions));
}
}
//用来接收ColumnFamilyHandle
List columnFamilyHandleList = new ArrayList();
//打开 Rocksdb
if (this.readOnly) {
rocksDB = TransactionDB.openReadOnly(this.dbOptions, DEFAULT_DB_PATH + this.dbname, DEFAULT_CF_DESCRIPTOR_LIST, columnFamilyHandleList);
} else {
rocksDB = TransactionDB.open(this.dbOptions, new TransactionDBOptions(), DEFAULT_DB_PATH + this.dbname, DEFAULT_CF_DESCRIPTOR_LIST, columnFamilyHandleList);
ROCKSDB_MAP.put(this.dbname, rocksDB);
}
Map columnFamilyHandleMap = new ConcurrentHashMap();
for(ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) {
columnFamilyHandleMap.put(new String(columnFamilyHandle.getName()), columnFamilyHandle);
}
COLUMN_FAMILY_HANDLE_MAP.put(rocksDB, columnFamilyHandleMap);
}
choseColumnFamily(this.columnFamilyName);
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap initilize failed, " + e.getMessage(), e);
}
}
private RocksMap(RocksMap rocksMap, String columnFamilyName, boolean shareTransaction){
this.dbOptions = new DBOptions(rocksMap.dbOptions);
this.readOptions = new ReadOptions(rocksMap.readOptions);
this.writeOptions = new WriteOptions(rocksMap.writeOptions);
this.columnFamilyOptions = new ColumnFamilyOptions(rocksMap.columnFamilyOptions);
this.rocksDB = rocksMap.rocksDB;
//是否使用父对象的实物对象
if(shareTransaction) {
this.threadLocalTransaction = rocksMap.threadLocalTransaction;
this.threadLocalSavePointCount = rocksMap.threadLocalSavePointCount;
} else {
this.threadLocalTransaction = ThreadLocal.withInitial(()->null);
this.threadLocalSavePointCount = ThreadLocal.withInitial(()->new Integer(0));
}
this.dbname = rocksMap.dbname;
this.columnFamilyName = columnFamilyName;
this.readOnly = rocksMap.readOnly;
this.transactionLockTimeout = rocksMap.transactionLockTimeout;
this.isDuplicate = true;
this.choseColumnFamily(columnFamilyName);
}
/**
* 复制出一个列族不同,但事务共享的 RocksMap
* @param cfName 列族名称
* @return 事务共享的 RocksMap
*/
public RocksMap duplicate(String cfName){
return new RocksMap(this, cfName, true);
}
/**
* 复制出一个列族不同的 RocksMAp
* @param cfName 列族名称
* @param shareTransaction true: 共享事务, false: 不共享事务
* @return 事务共享的 RocksMap
*/
public RocksMap duplicate(String cfName, boolean shareTransaction){
return new RocksMap(this, cfName, shareTransaction);
}
public String getDbname() {
return dbname;
}
public String getDataPath() {
return dataPath;
}
public String getWalPath() {
return walPath;
}
public String getColumnFamilyName() {
return columnFamilyName;
}
public Boolean getReadOnly() {
return readOnly;
}
public int savePointCount() {
return threadLocalSavePointCount.get();
}
public RocksDB getRocksDB(){
return rocksDB;
}
/**
* 创建一个备份选项
* @param dbName db 名称
* @param backupPath 备份路径, null 使用默认路径
* @return BackupableDBOptions 对象
*/
public static BackupableDBOptions createBackupableOption(String dbName, String backupPath) {
String defaultBackPath = backupPath==null ? DEFAULT_DB_PATH+".backups"+File.separator+dbName+File.separator : backupPath;
TFile.mkdir(defaultBackPath);
return new BackupableDBOptions(defaultBackPath);
}
public static BackupableDBOptions createBackupableOption(String dbName) {
return createBackupableOption(dbName, null);
}
public static BackupableDBOptions createBackupableOption(RocksMap rocksMap) {
return createBackupableOption(rocksMap.getDbname(), null);
}
/**
* 创建一个备份
* @param backupableDBOptions 备份选项
* @param beforeFlush 是否在备份前执行 flush
* @return 备份路径
* @throws RocksDBException 异常
*/
public String createBackup(BackupableDBOptions backupableDBOptions, boolean beforeFlush) throws RocksDBException {
if(backupableDBOptions==null) {
backupableDBOptions = createBackupableOption(this.dbname);
}
BackupEngine backupEngine = BackupEngine.open(rocksDB.getEnv(), backupableDBOptions);
backupEngine.createNewBackup(this.rocksDB, beforeFlush);
return backupableDBOptions.backupDir();
}
public String createBackup(BackupableDBOptions backupableDBOptions) throws RocksDBException {
return createBackup(backupableDBOptions, false);
}
public String createBackup() throws RocksDBException {
return createBackup(null, false);
}
/**
* 获取备份信息
* @param backupableDBOptions 备份选线
* @return 备份信息清单
* @throws RocksDBException 异常
*/
public List getBackupInfo(BackupableDBOptions backupableDBOptions) throws RocksDBException {
if(backupableDBOptions==null) {
backupableDBOptions = createBackupableOption(this.dbname);
}
BackupEngine backupEngine = BackupEngine.open(RocksEnv.getDefault(), backupableDBOptions);
return backupEngine.getBackupInfo();
}
public List getBackupInfo() throws RocksDBException {
return getBackupInfo(null);
}
/**
* 从最后一次备份恢复数据
* @param dbName 数据库名, 用以确定恢复路径
* @param backupableDBOptions 备份选项
* @param keepLogfile 是否覆盖原有 wal 日志
* @throws RocksDBException 异常
*/
public static void restoreFromLatestBackup(String dbName, BackupableDBOptions backupableDBOptions, Boolean keepLogfile) throws RocksDBException {
if(backupableDBOptions==null) {
backupableDBOptions = createBackupableOption(dbName);
}
String dataPath = DEFAULT_DB_PATH + dbName;
String walPath = DEFAULT_WAL_PATH + dbName;
RestoreOptions restoreOptions = new RestoreOptions(keepLogfile);
BackupEngine backupEngine = BackupEngine.open(RocksEnv.getDefault(), backupableDBOptions);
backupEngine.restoreDbFromLatestBackup(dataPath, walPath, restoreOptions);
}
public static void restoreFromLatestBackup() throws RocksDBException {
restoreFromLatestBackup(DEFAULT_COLUMN_FAMILY_NAME, null, true);
}
/**
* 从指定备份恢复数据
* @param backupId 备份 id 标识
* @param dbName 数据库名, 用以确定恢复路径
* @param backupableDBOptions 备份选项
* @param keepLogfile 是否覆盖原有 wal 日志
* @throws RocksDBException 异常
*/
public static void restore(int backupId, String dbName, BackupableDBOptions backupableDBOptions, Boolean keepLogfile) throws RocksDBException {
if(backupableDBOptions==null) {
backupableDBOptions = createBackupableOption(dbName);
}
String dataPath = DEFAULT_DB_PATH + dbName;
String walPath = DEFAULT_WAL_PATH + dbName;
RestoreOptions restoreOptions = new RestoreOptions(keepLogfile);
BackupEngine backupEngine = BackupEngine.open(RocksEnv.getDefault(), backupableDBOptions);
backupEngine.restoreDbFromBackup(backupId, dataPath, walPath, restoreOptions);
}
public static void restore(int backupId) throws RocksDBException {
restoreFromLatestBackup(DEFAULT_COLUMN_FAMILY_NAME, null, true);
}
public int getColumnFamilyId(){
return getColumnFamilyId(this.columnFamilyName);
}
public void compact(){
try {
rocksDB.compactRange(dataColumnFamilyHandle);
} catch (RocksDBException e) {
throw new RocksMapException("compact failed", e);
}
}
public void compactRange(K start, K end){
try {
rocksDB.compactRange(dataColumnFamilyHandle, TSerialize.serialize(start), TSerialize.serialize(end));
} catch (RocksDBException e) {
throw new RocksMapException("compact failed", e);
}
}
private String getProperty(ColumnFamilyHandle columnFamilyHandle, String name) {
try {
return rocksDB.getProperty(columnFamilyHandle, "rocksdb." + name);
} catch (RocksDBException e) {
throw new RocksMapException("getProperty failed", e);
}
}
public String getProperty(String columnFamilyName, String name) {
try {
ColumnFamilyHandle columnFamilyHandle = getColumnFamilyHandler(rocksDB, columnFamilyName);
if(columnFamilyHandle != null) {
return rocksDB.getProperty(columnFamilyHandle, "rocksdb." + name);
} else {
return "ColumnFamily: " + columnFamilyName + " not exists";
}
} catch (RocksDBException e) {
throw new RocksMapException("getProperty failed", e);
}
}
public String getProperty(String name) {
try {
return rocksDB.getProperty(this.dataColumnFamilyHandle, "rocksdb." + name);
} catch (RocksDBException e) {
throw new RocksMapException("getProperty failed", e);
}
}
/**
* 获取最后的序号
* @return 返回最后的日志序号
*/
public Long getLastSequence() {
return rocksDB.getLatestSequenceNumber();
}
/**
* 获取某个序号以后的更新操作记录
* @param sequenceNumber 序号
* @param withSerial 是否进行序列化行为
* @return 日志记录集合
*/
public List getWalSince(Long sequenceNumber, boolean withSerial) {
return getWalBetween(sequenceNumber, null, null, withSerial);
}
/**
* 获取某个序号以后的更新操作记录
* @param startSequence 起始序号
* @param filter 过滤器,用来过滤可用的操作类型和列族
* @param withSerial 是否进行反序列化
* @return 日志记录集合
*/
public List getWalSince(Long startSequence, BiFunction filter, boolean withSerial) {
return getWalBetween(startSequence, null, filter, withSerial);
}
/**
* 获取某个序号以后的更新操作记录
* @param startSequence 起始序号
* @param endSequence 结束序号
* @param withSerial 是否进行反序列化
* @return 日志记录集合
*/
public List getWalSince(Long startSequence, Long endSequence, boolean withSerial) {
return getWalBetween(startSequence, endSequence, null, withSerial);
}
/**
* 获取某个序号以后的更新操作记录
* @param startSequence 起始序号
* @param endSequence 结束序号
* @param filter 过滤器,用来过滤可用的操作类型和列族
* @param withSerial 是否进行反序列化
* @return 日志记录集合
*/
public List getWalBetween(Long startSequence, Long endSequence, BiFunction filter, boolean withSerial) {
try (TransactionLogIterator transactionLogIterator = rocksDB.getUpdatesSince(startSequence)) {
ArrayList rocksWalRecords = new ArrayList();
if(startSequence > getLastSequence()) {
return rocksWalRecords;
}
if(endSequence!=null && startSequence > endSequence) {
throw new RocksMapException("startSequence is large than endSequence");
}
long seq = 0l;
while (transactionLogIterator.isValid()) {
TransactionLogIterator.BatchResult batchResult = transactionLogIterator.getBatch();
if (batchResult.sequenceNumber() < startSequence) {
transactionLogIterator.next();
continue;
}
//不包含 endSequence 指定的日志
if(endSequence!=null && batchResult.sequenceNumber() >= endSequence) {
break;
}
seq = batchResult.sequenceNumber();
try (WriteBatch writeBatch = batchResult.writeBatch()) {
List rocksWalRecordBySeq = RocksWalRecord.parse(ByteBuffer.wrap(writeBatch.data()), filter, withSerial);
rocksWalRecords.addAll(rocksWalRecordBySeq);
writeBatch.clear();
}
transactionLogIterator.next();
}
if(rocksWalRecords.size() > 0)
Logger.debug("wal between: " + startSequence + "->" + endSequence + ", " + rocksWalRecords.get(0).getSequence() + "->" + rocksWalRecords.get(rocksWalRecords.size()-1).getSequence());
return rocksWalRecords;
} catch (RocksDBException e) {
throw new RocksMapException("getUpdatesSince failed, " + e.getMessage(), e);
}
}
public int getColumnFamilyId(String columnFamilyName){
ColumnFamilyHandle columnFamilyHandle = getColumnFamilyHandler(rocksDB, columnFamilyName);
if(columnFamilyHandle!=null){
return columnFamilyHandle.getID();
} else {
throw new RocksMapException("ColumnFamily [" + columnFamilyName +"] not found.");
}
}
/**
* 在多个 Columnt 之间切换
* @param cfName columnFamily 的名称
* @return RocksMap 对象
*/
public RocksMap choseColumnFamily(String cfName) {
try {
//设置列族
dataColumnFamilyHandle = getColumnFamilyHandler(rocksDB, cfName);
//如果没有则创建一个列族
if (dataColumnFamilyHandle == null) {
ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(cfName.getBytes(), columnFamilyOptions);
dataColumnFamilyHandle = rocksDB.createColumnFamily(columnFamilyDescriptor);
COLUMN_FAMILY_HANDLE_MAP.get(rocksDB).putIfAbsent(new String(dataColumnFamilyHandle.getName()), dataColumnFamilyHandle);
}
this.columnFamilyName = cfName;
return this;
} catch(RocksDBException e){
throw new RocksMapException("RocksMap initilize failed, " + e.getMessage(), e);
}
}
/**
* 获取事务锁超时时间
* @return 事务锁超时时间
*/
public int getTransactionLockTimeout() {
return transactionLockTimeout;
}
/**
* 设置事务锁超时时间
* @param transactionLockTimeout 事务锁超时时间
*/
public void setTransactionLockTimeout(int transactionLockTimeout) {
this.transactionLockTimeout = transactionLockTimeout;
}
/**
* 开启式事务模式 (立即失败, 死锁检测,无快照)
* 同一个线程共线给一个事务
* 使用内置公共事务通过 savepoint 来失败回滚, 但统一提交, 性能会好很多, 但是由于很多层嵌套的 savepont 在高并发时使用这种方式时回导致提交会慢很多
* @param transFunction 事务执行器, 返回 Null 则事务回滚, 其他则事务提交
* @param 范型
* @return 非 null: 事务成功, null: 事务失败
*/
public T withTransaction(Function, T> transFunction) {
return withTransaction(-1, true, false, transFunction);
}
/**
* 开启式事务模式
* 同一个线程共线给一个事务
* 使用内置公共事务通过 savepoint 来失败回滚, 但统一提交, 性能会好很多, 但是由于很多层嵌套的 savepont 在高并发时使用这种方式时回导致提交会慢很多
* @param expire 提交时锁超时时间
* @param deadlockDetect 死锁检测是否打开
* @param withSnapShot 是否启用快照事务
* @param transFunction 事务执行器, 返回 Null 则事务回滚, 其他则事务提交
* @param 范型
* @return 非 null: 事务成功, null: 事务失败
*/
public T withTransaction(long expire, boolean deadlockDetect, boolean withSnapShot, Function, T> transFunction) {
beginTransaction(expire, deadlockDetect, withSnapShot);
try {
T object = transFunction.apply(this);
if (object == null) {
rollback();
} else {
commit();
}
return object;
} catch (Exception e) {
rollback();
throw e;
}
}
/**
* 开启事务
* 同一个线程共线给一个事务
* 默认: 锁提交等待时间-1, 死锁检测:true, 是否启用快照事务: false
*/
public void beginTransaction() {
beginTransaction(-1, true, false);
}
/**
* 开启事务
* 同一个线程共线给一个事务
* 事务都是读事务，无论操作的记录间是否有交集，都不会锁定。
* 事务包含读、写事务：
* 所有的读事务不会锁定，读到的数据取决于snapshot设置。
* 写事务之间如果不存在记录交集，不会锁定。
* 写事务之间如果存在记录交集，此时如果未设置snapshot，则交集部分的记录是可以串行提交的。如果设置了snapshot，则第一个写事务(写锁队列的head)会成功，其他写事务会失败(之前的事务修改了该记录的情况下)。
* @param expire 提交时锁超时时间
* @param deadlockDetect 死锁检测是否打开
* @param withSnapShot 是否启用快照事务
*/
public void beginTransaction(long expire, boolean deadlockDetect, boolean withSnapShot) {
Transaction transaction = threadLocalTransaction.get();
if(transaction==null) {
transaction = createTransaction(expire, deadlockDetect, withSnapShot);
threadLocalTransaction.set(transaction);
} else {
savePoint();
}
}
private Transaction createTransaction(long expire, boolean deadlockDetect, boolean withSnapShot) {
if(readOnly){
throw new RocksMapException("RocksMap Not supported operation in read only mode");
}
TransactionOptions transactionOptions = new TransactionOptions();
//事务超时时间
transactionOptions.setExpiration(expire);
//是否执行死锁检测
transactionOptions.setDeadlockDetect(deadlockDetect);
//是否启用快照事务模式
transactionOptions.setSetSnapshot(withSnapShot);
//设置快照超时时间
transactionOptions.setLockTimeout(transactionLockTimeout);
Transaction transaction = ((TransactionDB) rocksDB).beginTransaction(writeOptions, transactionOptions);
transactionOptions.close();
return transaction;
}
public Transaction getTransaction(){
Transaction transaction = threadLocalTransaction.get();
if(transaction==null){
throw new RocksMapException("RocksMap is not in transaction model");
}
return transaction;
}
public void savePoint() {
Transaction transaction = getTransaction();
try {
transaction.setSavePoint();
threadLocalSavePointCount.set(threadLocalSavePointCount.get()+1);
} catch (RocksDBException e) {
throw new RocksMapException("commit failed, " + e.getMessage(), e);
}
}
public void rollbackSavePoint(){
Transaction transaction = getTransaction();
try {
transaction.rollbackToSavePoint();
threadLocalSavePointCount.set(threadLocalSavePointCount.get()-1);
} catch (RocksDBException e) {
throw new RocksMapException("commit failed, " + e.getMessage(), e);
}
}
/**
* 事务提交
*/
public void commit() {
Transaction transaction = getTransaction();
if(threadLocalSavePointCount.get() == 0) {
commit(transaction);
threadLocalTransaction.set(null);
} else {
threadLocalSavePointCount.set(threadLocalSavePointCount.get()-1);
}
}
/**
* 事务提交
*/
private void commit(Transaction transaction) {
if (transaction != null) {
try {
transaction.commit();
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap commit failed, " + e.getMessage(), e);
} finally {
transaction.close();
}
} else {
throw new RocksMapException("RocksMap is not in transaction model");
}
}
/**
* 事务回滚
*/
public void rollback() {
Transaction transaction = getTransaction();
if(threadLocalSavePointCount.get()!=0) {
rollbackSavePoint();
} else {
rollback(transaction);
threadLocalTransaction.set(null);
}
}
/**
* 事务回滚
*/
private void rollback(Transaction transaction) {
if (transaction != null) {
try {
transaction.rollback();
} catch(RocksDBException e){
throw new RocksMapException("RocksMap rollback failed, " + e.getMessage(), e);
} finally{
transaction.close();
}
} else {
throw new RocksMapException("RocksMap is not in transaction model");
}
}
@Override
public Comparator comparator() {
return null;
}
private RocksIterator getIterator(){
Transaction transaction = threadLocalTransaction.get();
if(transaction!=null) {
return transaction.getIterator(readOptions, dataColumnFamilyHandle);
} else {
return rocksDB.newIterator(dataColumnFamilyHandle, readOptions);
}
}
@Override
public SortedMap subMap(K fromKey, K toKey) {
TreeMap subMap = new TreeMap();
try (RocksIterator iterator = getIterator()){
byte[] fromKeyBytes = TSerialize.serialize(fromKey);
byte[] toKeyBytes = TSerialize.serialize(toKey);
if (fromKeyBytes == null) {
iterator.seekToFirst();
} else {
iterator.seek(fromKeyBytes);
}
while (iterator.isValid()) {
byte[] key = iterator.key();
if (toKey == null || !Arrays.equals(toKeyBytes, key)) {
subMap.put((K) TSerialize.unserialize(iterator.key()), (V) TSerialize.unserialize(iterator.value()));
} else {
subMap.put((K) TSerialize.unserialize(iterator.key()), (V) TSerialize.unserialize(iterator.value()));
break;
}
iterator.next();
}
return subMap;
}
}
@Override
public SortedMap tailMap(K fromKey){
if(fromKey==null){
return null;
}
return subMap(fromKey, null);
}
@Override
public SortedMap headMap(K toKey){
if(toKey == null){
return null;
}
return subMap(null, toKey);
}
@Override
public K firstKey() {
try (RocksIterator iterator = getIterator()) {
iterator.seekToFirst();
if (iterator.isValid()) {
return (K) TSerialize.unserialize(iterator.key());
}
return null;
}
}
@Override
public K lastKey() {
try (RocksIterator iterator = getIterator()) {
iterator.seekToLast();
if (iterator.isValid()) {
return (K) TSerialize.unserialize(iterator.key());
}
return null;
}
}
@Override
/**
* 遍历所有数据来获取 kv 记录的数量, 会消耗很多性能
*/
public int size() {
// try {
// return Integer.valueOf(rocksDB.getProperty(dataColumnFamilyHandle, "rocksdb.estimate-num-keys"));
// } catch (RocksDBException e) {
// e.printStackTrace();
// }
int count = 0;
RocksIterator iterator = null;
try {
iterator = getIterator();
iterator.seekToFirst();
while (iterator.isValid()) {
iterator.next();
count++;
}
return count;
} finally {
if(iterator!=null){
iterator.close();
}
}
}
@Override
public boolean isEmpty() {
try (RocksIterator iterator = getIterator()){
iterator.seekToFirst();
return !iterator.isValid();
}
}
@Override
public boolean containsKey(Object key) {
byte[] values = null;
try {
Transaction transaction = threadLocalTransaction.get();
if(transaction!=null) {
values = transaction.get(dataColumnFamilyHandle, readOptions, TSerialize.serialize(key));
} else {
values = rocksDB.get(dataColumnFamilyHandle, readOptions, TSerialize.serialize(key));
}
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap containsKey " + key + " failed, " + e.getMessage(), e);
}
return values!=null;
}
@Override
public boolean containsValue(Object value) {
throw new UnsupportedOperationException();
}
/**
* 获取并锁定, 默认独占模式
* @param key 将被加锁的 key
* @return key 对应的 value
*/
public V lock(Object key){
return lock(key, true);
}
/**
* 释放锁
* @param key 将被释放锁的 key
*/
public void unlock(Object key) {
Transaction transaction = getTransaction();
transaction.undoGetForUpdate(TSerialize.serialize(key));
}
/**
* 获取并加锁
* @param key 将被加锁的 key
* @param exclusive 是否独占模式
* @return key 对应的 value
*/
public V lock(Object key, boolean exclusive){
Transaction transaction = getTransaction();
try {
byte[] values = transaction.getForUpdate(readOptions, dataColumnFamilyHandle, TSerialize.serialize(key), exclusive);
return values==null ? null : (V) TSerialize.unserialize(values);
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap lock " + key + " failed, " + e.getMessage(), e);
}
}
private byte[] get(byte[] keyBytes) {
try {
byte[] values = null;
Transaction transaction = threadLocalTransaction.get();
if (transaction != null) {
values = transaction.getForUpdate(readOptions, dataColumnFamilyHandle, keyBytes, true);
} else {
values = rocksDB.get(dataColumnFamilyHandle, readOptions, keyBytes);
}
return values;
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap get failed, " + e.getMessage(), e);
}
}
@Override
public V get(Object key) {
if(key == null){
throw new NullPointerException();
}
byte[] values = get(TSerialize.serialize(key));
return values==null ? null : (V) TSerialize.unserialize(values);
}
public List getAll(Collection keys) {
try {
ArrayList keysBytes = new ArrayList();
ArrayList columnFamilyHandles = new ArrayList();
Iterator keysIterator = keys.iterator();
for (int i = 0; i < keys.size(); i++) {
keysBytes.add(TSerialize.serialize(keysIterator.next()));
columnFamilyHandles.add(dataColumnFamilyHandle);
}
Transaction transaction = threadLocalTransaction.get();
List valuesBytes;
if (transaction != null) {
valuesBytes = Arrays.asList(transaction.multiGet(readOptions, columnFamilyHandles, keysBytes.toArray(new byte[0][0])));
} else {
valuesBytes = rocksDB.multiGetAsList(readOptions, columnFamilyHandles, keysBytes);
}
ArrayList values = new ArrayList();
for (byte[] valueByte : valuesBytes) {
values.add((V) TSerialize.unserialize(valueByte));
}
return values;
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap getAll failed, " + e.getMessage(), e);
}
}
private void put(byte[] keyBytes, byte[] valueBytes) {
try {
Transaction transaction = threadLocalTransaction.get();
if (transaction != null) {
transaction.put(dataColumnFamilyHandle, keyBytes, valueBytes);
} else {
rocksDB.put(dataColumnFamilyHandle, writeOptions, keyBytes, valueBytes);
}
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap put failed, " + e.getMessage(), e);
}
}
@Override
public Object put(Object key, Object value) {
if(key == null || value == null){
throw new NullPointerException();
}
put(TSerialize.serialize(key), TSerialize.serialize(value));
return value;
}
@Override
public V putIfAbsent(K key, V value) {
byte[] keyBytes = TSerialize.serialize(key);
byte[] valueBytes = TSerialize.serialize(value);
//这里使用独立的事务是未了防止默认事务提交导致失效
Transaction innerTransaction = createTransaction(-1, false, false);
try {
byte[] oldValueBytes = innerTransaction.getForUpdate(readOptions, dataColumnFamilyHandle, keyBytes, true);
if(oldValueBytes == null){
innerTransaction.put(dataColumnFamilyHandle, keyBytes, valueBytes);
return null;
} else {
return (V) TSerialize.unserialize(oldValueBytes);
}
} catch (RocksDBException e) {
rollback(innerTransaction);
throw new RocksMapException("RocksMap putIfAbsent error, " + e.getMessage(), e);
} finally {
commit(innerTransaction);
}
}
/**
* 判断 key 是否不存在
* @param key key对象
* @return 当返回 false 的时候 key 一定不存在, 当返回 true 的时候, key 有可能不存在, 参考 boomfilter
*/
public boolean keyMayExists(K key) {
return keyMayExists(TSerialize.serialize(key));
}
private boolean keyMayExists(byte[] keyBytes) {
boolean result = rocksDB.keyMayExist(dataColumnFamilyHandle, keyBytes, threadLocalBuilder.get());
threadLocalBuilder.get().setLength(0);
return result;
}
@Override
public boolean replace(K key, V oldValue, V newValue) {
byte[] keyBytes = TSerialize.serialize(key);
byte[] newValueBytes = TSerialize.serialize(newValue);
byte[] oldValueBytes = TSerialize.serialize(oldValue);
//这里使用独立的事务是未了防止默认事务提交导致失效
Transaction innerTransaction = createTransaction(-1, false, false);
try {
byte[] oldDbValueBytes = innerTransaction.getForUpdate(readOptions, dataColumnFamilyHandle, keyBytes, true);
if(oldDbValueBytes!=null && Arrays.equals(oldDbValueBytes, oldValueBytes)){
innerTransaction.put(dataColumnFamilyHandle, keyBytes, newValueBytes);
return true;
} else {
return false;
}
} catch (RocksDBException e) {
rollback(innerTransaction);
throw new RocksMapException("RocksMap replace failed , " + e.getMessage(), e);
} finally {
commit(innerTransaction);
}
}
/**
* 删除某个 key
* @param keyBytes key 对象
* @param isRetVal 是否返回被移除的 value
* @return 返回值, 在 isRetVal=false 时, 总是为 null
*/
private byte[] remove(byte[] keyBytes, boolean isRetVal) throws RocksDBException {
if(keyBytes == null){
throw new NullPointerException();
}
byte[] valueBytes = null;
if(isRetVal) {
valueBytes = get(keyBytes);
}
if(!isRetVal || valueBytes != null) {
Transaction transaction = threadLocalTransaction.get();
if(transaction!=null) {
transaction.delete(dataColumnFamilyHandle, keyBytes);
} else {
rocksDB.delete(dataColumnFamilyHandle, writeOptions, keyBytes);
}
}
return valueBytes;
}
/**
* 删除某个 key
* @param key key 对象
* @param isRetVal 是否返回被移除的 value
* @return 返回值, 在 isRetVal=false 时, 总是为 null
*/
public V remove(Object key, boolean isRetVal) {
if(key == null){
throw new NullPointerException();
}
try {
byte[] valuesByte = remove(TSerialize.serialize(key), isRetVal);
return (V) TSerialize.unserialize(valuesByte);
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap remove " + key + " failed", e);
}
}
@Override
public V remove(Object key) {
return remove(key, true);
}
public void removeAll(Collection keys) {
if(keys == null){
throw new NullPointerException();
}
try {
Transaction transaction = threadLocalTransaction.get();
WriteBatch writeBatch = null;
if (transaction == null) {
writeBatch = THREAD_LOCAL_WRITE_BATCH.get();
writeBatch.clear();
for(K key : keys) {
if(key == null){
continue;
}
try {
writeBatch.delete(dataColumnFamilyHandle, TSerialize.serialize(key));
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap removeAll " + key + " failed", e);
}
}
rocksDB.write(writeOptions, writeBatch);
} else {
for(K key : keys) {
if(key == null){
continue;
}
try {
transaction.delete(dataColumnFamilyHandle, TSerialize.serialize(key));
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap removeAll " + key + " failed", e);
}
}
}
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap removeAll write failed", e);
}
}
/**
* Removes the database entries in the range ["beginKey", "endKey"), i.e.,
* including "beginKey" and excluding "endKey". a non-OK status on error. It
* is not an error if no keys exist in the range ["beginKey", "endKey").
* @param fromKey 其实 key
* @param toKey 结束 key
*
*/
public void removeRange(K fromKey, K toKey){
removeRange(fromKey, toKey, true);
}
/**
* Removes the database entries in the range ["beginKey", "endKey"), i.e.,
* including "beginKey" and excluding "endKey". a non-OK status on error. It
* is not an error if no keys exist in the range ["beginKey", "endKey").
* @param fromKey 其实 key
* @param toKey 结束 key
* @param useTransaction 是否嵌套至已有事务
*
*/
public void removeRange(K fromKey, K toKey, boolean useTransaction) {
Transaction transaction = useTransaction ? threadLocalTransaction.get() : null;
byte[] fromKeyBytes = TSerialize.serialize(fromKey);
byte[] toKeyBytes = TSerialize.serialize(toKey);
try {
if(transaction==null) {
rocksDB.deleteRange(dataColumnFamilyHandle, writeOptions, fromKeyBytes, toKeyBytes);
} else {
try (RocksIterator iterator = getIterator()) {
iterator.seek(fromKeyBytes);
while (iterator.isValid()) {
if (!Arrays.equals(iterator.key(), toKeyBytes)) {
transaction.delete(dataColumnFamilyHandle, iterator.key());
iterator.next();
} else {
break;
}
}
}
}
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap removeAll failed", e);
}
}
public static ThreadLocal THREAD_LOCAL_WRITE_BATCH = ThreadLocal.withInitial(()->new WriteBatch());
@Override
public void putAll(Map m) {
try {
Transaction transaction = threadLocalTransaction.get();
WriteBatch writeBatch = null;
if (transaction == null) {
writeBatch = THREAD_LOCAL_WRITE_BATCH.get();
writeBatch.clear();
Iterator iterator = m.entrySet().iterator();
while (iterator.hasNext()) {
Entry entry = iterator.next();
Object key = entry.getKey();
Object value = entry.getValue();
writeBatch.put(dataColumnFamilyHandle, TSerialize.serialize(key), TSerialize.serialize(value));
}
rocksDB.write(writeOptions, writeBatch);
} else {
Iterator iterator = m.entrySet().iterator();
while (iterator.hasNext()) {
Entry entry = iterator.next();
Object key = entry.getKey();
Object value = entry.getValue();
transaction.put(dataColumnFamilyHandle, TSerialize.serialize(key), TSerialize.serialize(value));
}
}
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap putAll failed", e);
}
}
/**
* 刷新数据到文件
* @param sync 同步刷新
* @param allowStall 是否允许写入暂停
*/
public void flush(boolean sync, boolean allowStall){
try {
FlushOptions flushOptions = new FlushOptions();
flushOptions.setWaitForFlush(sync);
flushOptions.setAllowWriteStall(allowStall);
rocksDB.flush(flushOptions, this.dataColumnFamilyHandle);
if(!sync) {
Global.getHashWheelTimer().addTask(()->{
flushOptions.waitForFlush();
}, 1, true);
}
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap flush failed", e);
}
}
/**
* 刷新数据到文件
* @param sync 同步刷新
*/
public void flush(boolean sync){
flush(sync, true);
}
/**
* 刷新数据到文件
*/
public void flush(){
flush(true, true);
}
/**
* 刷新WAL数据到文件
* @param sync 同步刷新
*/
public void flushWal(boolean sync){
try {
rocksDB.flushWal(sync);
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap flushWal failed", e);
}
}
public void flushWal() {
flushWal(true);
}
@Override
public void clear() {
if(readOnly){
Logger.error("Clear failed, ", new RocksDBException("Not supported operation in read only mode"));
return;
}
try {
drop();
ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(columnFamilyName.getBytes(), columnFamilyOptions);
dataColumnFamilyHandle = rocksDB.createColumnFamily(columnFamilyDescriptor);
COLUMN_FAMILY_HANDLE_MAP.get(rocksDB).put(new String(dataColumnFamilyHandle.getName()), dataColumnFamilyHandle);
//设置列族
dataColumnFamilyHandle = getColumnFamilyHandler(rocksDB, this.columnFamilyName);
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap clear failed", e);
}
}
/**
* 删除这个列族
*/
public void drop(){
try {
rocksDB.dropColumnFamily(dataColumnFamilyHandle);
dataColumnFamilyHandle.close();
COLUMN_FAMILY_HANDLE_MAP.get(rocksDB).remove(new String(dataColumnFamilyHandle.getName()));
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap drop failed", e);
}
}
/**
* 数据拷贝到内存中, 所以对这个 Set 的修改不会在 Rocksdb 中生效
* @return 保存了 Key 的 set
*/
@Override
public Set keySet() {
TreeSet keySet = new TreeSet();
try (RocksIterator iterator = getIterator()){
iterator.seekToFirst();
while (iterator.isValid()) {
K k = (K) TSerialize.unserialize(iterator.key());
keySet.add(k);
iterator.next();
}
return keySet;
}
}
@Override
public Collection values() {
ArrayList values = new ArrayList();
try (RocksIterator iterator = getIterator()){
iterator.seekToFirst();
while (iterator.isValid()) {
V value = (V) TSerialize.unserialize(iterator.value());
values.add(value);
iterator.next();
}
return values;
}
}
/**
* 数据拷贝到内存中, 所以对这个 Set 的修改不会在 Rocksdb 中生效
* @return 保存了 Entry 的 set
*/
@Override
public Set> entrySet() {
TreeSet> entrySet = new TreeSet>();
try (RocksIterator iterator = getIterator()) {
iterator.seekToFirst();
while (iterator.isValid()) {
entrySet.add(new RocksMapEntry(this, iterator.key(), iterator.value()));
iterator.next();
}
return entrySet;
}
}
/**
* 按前缀查找关联的 key
* @param key key 的前缀
* @return 找到的 Map 数据
*/
public Map startWith(K key) {
return startWith(key, 0,0);
}
/**
* 按前缀查找关联的 key
* @param key key 的前缀
* @param skipSize 跳过记录数
* @param size 返回记录数
* @return 找到的 Map 数据
*/
public Map startWith(K key, int skipSize, int size) {
byte[] keyBytes = TSerialize.serialize(key);
TreeMap entryMap = new TreeMap();
try (RocksMapIterator iterator = new RocksMapIterator(this, key, null, skipSize, size)) {
while (iterator.directNext(true)) {
if (TByte.byteArrayStartWith(iterator.keyBytes(), keyBytes)) {
entryMap.put((K) iterator.key(), (V) iterator.value());
} else {
break;
}
}
return entryMap;
}
}
@Override
public void close() {
//关闭事务
Transaction transaction = threadLocalTransaction.get();
if(transaction!=null){
try {
transaction.rollback();
} catch (RocksDBException e) {
throw new RocksMapException("RocksMap rollback on close failed", e);
} finally {
transaction.close();
}
}
dbOptions.close();
readOptions.close();
writeOptions.close();
columnFamilyOptions.close();
dbOptions = null;
readOptions = null;
writeOptions = null;
columnFamilyOptions = null;
if(!isDuplicate) {
RocksMap.closeRocksDB(rocksDB);
}
}
/**
* 构造一个有范围的迭代器
* @param fromKey 起始 key
* @param toKey 结束 key
* @param size 迭代的记录数
* @param skipSize 跳过记录数
* @param size 返回记录数
* @return 迭代器对象
*/
public RocksMapIterator iterator(K fromKey, K toKey, int skipSize, int size){
return new RocksMapIterator(this, fromKey, toKey, skipSize, size);
}
/**
* 构造一个有范围的迭代器
* @param fromKey 起始 key
* @param toKey 结束 key
* @return 迭代器对象
*/
public RocksMapIterator iterator(K fromKey, K toKey){
return new RocksMapIterator(this, fromKey, toKey, 0, 0);
}
/**
* 构造一个有范围的迭代器
* @param skipSize 跳过记录数
* @param size 返回记录数
* @return 迭代器对象
*/
public RocksMapIterator iterator(int skipSize, int size){
return new RocksMapIterator(this, null, null, skipSize, size);
}
/**
* 构造一个有范围的迭代器
* @param size 迭代的记录数
* @return 迭代器对象
*/
public RocksMapIterator iterator(int size){
return new RocksMapIterator(this, null, null, 0, size);
}
/**
* 构造一个迭代器
* @return 迭代器对象
*/
public RocksMapIterator iterator(){
return new RocksMapIterator(this, null, null, 0, 0);
}
/**
* 数据清理执行器
* @param checker 数据清理逻辑, true: 继续扫描, false: 停止扫描
*/
public void scan(Function.RocksMapEntry, Boolean> checker) {
scan(null, null, checker, false);
}
/**
* 数据清理执行器
* @param checker 数据清理逻辑, true: 继续扫描, false: 停止扫描
* @param disableWal 是否屏蔽 wal
*/
public void scan(Function.RocksMapEntry, Boolean> checker, boolean disableWal) {
scan(null, null, checker, disableWal);
}
/**
* 数据清理执行器
* @param fromKey 起始 key
* @param toKey 结束 key
* @param checker 数据清理逻辑, true: 继续扫描, false: 停止扫描
*/
public void scan(K fromKey, K toKey, Function.RocksMapEntry, Boolean> checker) {
scan(fromKey, toKey, checker, false);
}
/**
* 数据清理执行器
* @param fromKey 起始 key
* @param toKey 结束 key
* @param checker 数据清理逻辑, true: 继续扫描, false: 停止扫描
* @param disableWal 是否屏蔽 wal
*/
public void scan(K fromKey, K toKey, Function.RocksMapEntry, Boolean> checker, boolean disableWal) {
RocksMap innerRocksMap = this.duplicate(this.getColumnFamilyName(), false);
innerRocksMap.writeOptions.setDisableWAL(disableWal);
try(RocksMap.RocksMapIterator iterator = innerRocksMap.iterator(fromKey, toKey)) {
RocksMap.RocksMapEntry rocksMapEntry = null;
while ((rocksMapEntry = iterator.nextAndValid(true))!=null) {
if (checker.apply(rocksMapEntry)) {
continue;
} else {
break;
}
}
}
innerRocksMap.close();
}
public class RocksMapEntry implements Map.Entry, Comparable {
private RocksMap rocksMap;
private byte[] keyBytes;
private K k;
private byte[] valueBytes;
private V v;
protected RocksMapEntry(RocksMap rocksMap, byte[] keyBytes, byte[] valueBytes) {
this.rocksMap = rocksMap;
this.keyBytes = keyBytes;
this.valueBytes = valueBytes;
}
@Override
public K getKey() {
if(k==null){
this.k = (K) TSerialize.unserialize(keyBytes);
}
return k;
}
@Override
public V getValue() {
if(v==null) {
this.v = (V) TSerialize.unserialize(valueBytes);
}
return v;
}
public byte[] getKeyBytes() {
return keyBytes;
}
public byte[] getValueBytes() {
return valueBytes;
}
@Override
public V setValue(V value) {
rocksMap.put(keyBytes, TSerialize.serialize(value));
return value;
}
@Override
public int compareTo(RocksMapEntry o) {
return TByte.byteArrayCompare(this.keyBytes, o.keyBytes);
}
@Override
public String toString(){
return getKey() + "=" + getValue();
}
public RocksMap getRocksMap() {
return rocksMap;
}
public void remove(){
try {
rocksMap.remove(keyBytes, false);
} catch (RocksDBException e) {
throw new RocksMapException("RocksMapEntry remove failed", e);
}
}
}
public class RocksMapIterator implements Iterator>, Closeable{
private RocksMap rocksMap;
private RocksIterator iterator;
private byte[] fromKeyBytes;
private byte[] toKeyBytes;
private int skipSize;
private int size = 0;
private int count=0;
//["beginKey", "endKey")
protected RocksMapIterator(RocksMap rocksMap, K fromKey, K toKey, int skipSize, int size) {
this.rocksMap = rocksMap;
this.iterator = rocksMap.getIterator();
this.fromKeyBytes = TSerialize.serialize(fromKey);
this.toKeyBytes = TSerialize.serialize(toKey);
this.skipSize = skipSize;
this.size = size;
if(fromKeyBytes==null) {
iterator.seekToFirst();
} else {
iterator.seek(fromKeyBytes);
}
if(skipSize >0) {
for(int i=0;i<=this.skipSize;i++) {
if(!directNext(false)) {
break;
}
}
}
count = 0;
}
/**
* 获取迭代器当前位置的 Entry
* @return RocksMapEntry对象
*/
public RocksMapEntry getEntry() {
return new RocksMapEntry(rocksMap, iterator.key(), iterator.value());
}
@Override
public boolean hasNext() {
boolean ret = false;
if(count == 0 && isValid()) {
return true;
}
if(!iterator.isValid()) {
return false;
}
try {
iterator.next();
ret = isValid();
} finally {
iterator.prev();
if(size!=0 && count > size - 1){
ret = false;
}
}
return ret;
}
/**
* 迭代器当前位数数据是否有效
* @return true: 有效, false: 无效
*/
public boolean isValid(){
boolean ret;
if (toKeyBytes == null) {
ret = iterator.isValid();
} else {
ret = iterator.isValid() && TByte.byteArrayCompare(iterator.key(), toKeyBytes) < 0;
}
return ret;
}
/**
* 获取 Key 的值
* @return Key 的值
*/
public K key(){
return (K)TSerialize.unserialize(iterator.key());
}
/**
* 获取 value 的值
* @return value 的值
*/
public V value(){
return (V)TSerialize.unserialize(iterator.value());
}
/**
* 获取 Key 的值
* @return Key 的值
*/
public byte[] keyBytes(){
return iterator.key();
}
/**
* 获取 value 的值
* @return value 的值
*/
public byte[] valueBytes(){
return iterator.value();
}
/**
* 只是执行 next 不反序列化数据
* @param valid 是否验证当前迭代可用
* @return true: 成功, false: 失败
*/
public boolean directNext(boolean valid) {
if(count != 0) {
iterator.next();
}
boolean flag = false;
if(valid) {
flag = isValid();
} else {
flag = iterator.isValid();
}
if(flag) {
count++;
return true;
} else {
return false;
}
}
public RocksMapEntry nextAndValid(boolean valid) {
if(directNext(true)) {
return getEntry();
} else {
return null;
}
}
@Override
public RocksMapEntry next() {
return nextAndValid(false);
}
@Override
public void remove() {
try {
rocksMap.rocksDB.delete(rocksMap.dataColumnFamilyHandle, rocksMap.writeOptions, iterator.key());
} catch (RocksDBException e) {
throw new RocksMapException("RocksMapIterator remove failed", e);
}
}
@Override
public void forEachRemaining(Consumer> action) {
throw new UnsupportedOperationException();
}
public void close(){
iterator.close();
}
}
public static class RocksWalRecord {
// WriteBatch::rep_ :=
// sequence: fixed64
// count: fixed32
// data: record[count]
// record :=
// kTypeValue varstring varstring
// kTypeDeletion varstring
// kTypeSingleDeletion varstring
// kTypeRangeDeletion varstring varstring
// kTypeMerge varstring varstring
// kTypeColumnFamilyValue varint32 varstring varstring
// kTypeColumnFamilyDeletion varint32 varstring
// kTypeColumnFamilySingleDeletion varint32 varstring
// kTypeColumnFamilyRangeDeletion varint32 varstring varstring
// kTypeColumnFamilyMerge varint32 varstring varstring
// kTypeBeginPrepareXID varstring
// kTypeEndPrepareXID
// kTypeCommitXID varstring
// kTypeRollbackXID varstring
// kTypeBeginPersistedPrepareXID varstring
// kTypeBeginUnprepareXID varstring
// kTypeNoop
// varstring :=
// len: varint32
// data: uint8[len]
public static int TYPE_DELETION = 0x0;
public static int TYPE_VALUE = 0x1;
public static int TYPE_MERGE = 0x2;
public static int TYPE_LOGDATA = 0x3; // WAL only.
public static int TYPE_COLUMNFAMILY_DELETION = 0x4; // WAL only. <-----
public static int TYPE_COLUMNFAMILY_VALUE = 0x5; // WAL only. <-----
public static int TYPE_COLUMNFAMILY_MERGE = 0x6; // WAL only.
public static int TYPE_SINGLE_DELETION = 0x7;
public static int TYPE_COLUMNFAMILY_SINGLE_DELETION = 0x8; // WAL only.
public static int TYPE_BEGIN_PREPARE_XID = 0x9; // WAL only.
public static int TYPE_END_PREPARE_XID = 0xA; // WAL only.
public static int TYPE_COMMIT_XID = 0xB; // WAL only.
public static int TYPE_ROLLBACK_XID = 0xC; // WAL only.
public static int TYPE_NOOP = 0xD; // WAL only.
public static int TYPE_COLUMNFAMILY_RANGE_DELETION = 0xE; // WAL only.
public static int TYPE_RANGE_DELETION = 0xF; // meta block
public static int TYPE_COLUMNFAMILY_BLOB_INDEX = 0x10; // Blob DB only
public static int TYPE_BLOB_INDEX = 0x11; // Blob DB only
public static int TYPE_BEGIN_PERSISTED_PREPARE_XID = 0x12; // WAL only
public static int TYPE_BEGIN_UNPREPARE_XID = 0x13; // WAL only.
//定义每种操作的数据项数据
public static int[] TYPE_ELEMENT_COUNT = new int[]{
1, //0 kTypeDeletion varstring
2, //1 kTypeValue varstring varstring
2, //2 kTypeMerge varstring varstring
0, //3 ?
1, //4 kTypeColumnFamilyDeletion varint32 varstring
2, //5 kTypeColumnFamilyValue varint32 varstring varstring
2, //6 kTypeColumnFamilyMerge varint32 varstring varstring
1, //7 kTypeSingleDeletion varstring
1, //8 kTypeColumnFamilySingleDeletion varint32 varstring
1, //9 kTypeBeginPrepareXID varstring
0, //A kTypeEndPrepareXID
1, //B kTypeCommitXID varstring
1, //C kTypeRollbackXID varstring
0, //D kTypeNoop
2, //E kTypeColumnFamilyRangeDeletion varint32 varstring varstring
0, //F kTypeRangeDeletion varstring varstring
0, //10 kTypeColumnFamilyBlobIndex
2, //11 kTypeBlobIndex varstring varstring
1, //12 kTypeBeginPersistedPrepareXID varstring
1 //13 kTypeBeginUnprepareXID varstring
};
private long sequence;
private int type;
private int columnFamilyId = 0;
private ArrayList chunks;
private RocksWalRecord(long sequence, int type, int columnFamilyId) {
this.sequence = sequence;
this.type = type;
this.columnFamilyId = columnFamilyId;
chunks = new ArrayList();
}
public long getSequence() {
return sequence;
}
private void setSequence(long sequence) {
this.sequence = sequence;
}
public int getType() {
return type;
}
private void setType(int type) {
this.type = type;
}
public int getColumnFamilyId() {
return columnFamilyId;
}
private void setColumnFamilyId(int columnFamilyId) {
this.columnFamilyId = columnFamilyId;
}
public ArrayList getChunks() {
return chunks;
}
/**
* 解析某个序号以后的更新操作记录
* @param byteBuffer 字节缓冲器
* @param withSerial 是否进行反序列化
* @return 日志记录集合
*/
public static List parse(ByteBuffer byteBuffer, boolean withSerial) {
return parse(byteBuffer, null, withSerial);
}
/**
* 解析某个序号以后的更新操作记录
* @param byteBuffer 字节缓冲器
* @param filter 过滤器,用来过滤可用的操作类型和列族
* @param withSerial 是否进行反序列化
* @return 日志记录集合
*/
public static List parse(ByteBuffer byteBuffer, BiFunction filter, boolean withSerial) {
ByteOrder originByteOrder = byteBuffer.order();
byteBuffer.order(ByteOrder.LITTLE_ENDIAN);
if(byteBuffer.remaining() < 13) {
throw new ParseException("not a correct recorder");
}
//序号
Long sequence = byteBuffer.getLong();
//本次Wal操作包含键值数
int recordCount = byteBuffer.getInt();
List rocksWalRecords = new ArrayList();
for(int count=0;byteBuffer.hasRemaining();count++) {
RocksWalRecord rocksWalRecord = parseOperation(byteBuffer, sequence, filter, withSerial);
if(rocksWalRecord !=null) {
rocksWalRecords.add(rocksWalRecord);
}
}
byteBuffer.order(originByteOrder);
return rocksWalRecords;
}
/**
* 解析某个序号的操作记录
* @param byteBuffer 字节缓冲器
* @param sequence 序号
* @param withSerial 是否进行反序列化
* @return 日志记录集合
*/
public static RocksWalRecord parseOperation(ByteBuffer byteBuffer, long sequence, boolean withSerial){
return parseOperation(byteBuffer, sequence, withSerial);
}
/**
* 解析某个序号的操作记录
* @param byteBuffer 字节缓冲器
* @param sequence 序号
* @param filter 过滤器,用来过滤可用的操作类型和列族
* @param withSerial 是否进行反序列化
* @return 日志记录集合
*/
public static RocksWalRecord parseOperation(ByteBuffer byteBuffer, long sequence, BiFunction filter, boolean withSerial){
//操作类型
int type = byteBuffer.get();
if (type == TYPE_NOOP) {
if(byteBuffer.hasRemaining()) {
type = byteBuffer.get();
} else {
return null;
}
}
int columnFamilyId=0;
if (type == TYPE_COLUMNFAMILY_DELETION ||
type == TYPE_COLUMNFAMILY_VALUE ||
type == TYPE_COLUMNFAMILY_MERGE ||
type == TYPE_COLUMNFAMILY_SINGLE_DELETION ||
type == TYPE_COLUMNFAMILY_RANGE_DELETION) {
columnFamilyId = byteBuffer.get();
}
RocksWalRecord rocksWalRecord = null;
if (type < TYPE_ELEMENT_COUNT.length) {
//应用过滤器
boolean isEnable = filter==null || filter.apply(columnFamilyId, type);
if(isEnable) {
rocksWalRecord = new RocksWalRecord(sequence, type, columnFamilyId);
}
for (int i = 0; i < TYPE_ELEMENT_COUNT[type] && byteBuffer.hasRemaining(); i++) {
int chunkSize = Varint.varintToInt(byteBuffer);
if(isEnable) {
byte[] chunkBytes = new byte[chunkSize];
byteBuffer.get(chunkBytes);
Object chunk = chunkBytes;
if (withSerial) {
chunk = TSerialize.unserialize(chunkBytes);
} else {
chunk = chunkBytes;
}
rocksWalRecord.getChunks().add(chunk);
} else {
byteBuffer.position(byteBuffer.position() + chunkSize);
}
}
}
return rocksWalRecord;
}
}
/**
* 按批次读取Wal, wal 读取器
*/
public static class RocksWalReader {
private Object mark;
private RocksMap rocksMap;
private Long lastSequence;
private int batchSeqsize;
private List columnFamilys;
private List walTypes;
public RocksWalReader(Object mark, RocksMap rocksMap, int batchSeqsize) {
this.mark = mark;
this.rocksMap = rocksMap;
this.batchSeqsize = batchSeqsize;
this.rocksMap = rocksMap;
this.lastSequence = (Long) this.rocksMap.get(mark);
this.lastSequence = this.lastSequence==null ? 0 : this.lastSequence;
Logger.debug("Start sequence: " + this.lastSequence);
}
public long getLastSequence() {
return lastSequence;
}
public List getColumnFamily() {
return columnFamilys;
}
public void setColumnFamily(List columnFamilys) {
this.columnFamilys = columnFamilys;
}
public int getBatchSeqsize() {
return batchSeqsize;
}
public List getWalTypes() {
return walTypes;
}
public void setWalTypes(List walTypes) {
this.walTypes = walTypes;
}
public void processing(RocksWalProcessor rocksWalProcessor){
//wal 日志同步至数据库
Long endSequence = rocksMap.getLastSequence();
//数量控制
if(lastSequence + batchSeqsize < endSequence){
endSequence = lastSequence + batchSeqsize;
}
List rocksWalRecords = rocksMap.getWalBetween(lastSequence, endSequence, (columnFamilyId, type)-> {
return (walTypes == null ? true : walTypes.contains(type)) &&
(columnFamilys==null ? true : columnFamilys.contains(columnFamilyId));
}, true);
if(rocksWalRecords.size() > 0) {
//调用处理器
rocksWalProcessor.process(endSequence, rocksWalRecords);
}
rocksMap.put(mark, endSequence);
lastSequence = endSequence;
}
/**
* 日志处理器
*/
public static interface RocksWalProcessor {
public void process(Long endSequence, List rocksWalRecords);
}
}
}
