com.devsmart.microdb.MapDBDriver Maven / Gradle / Ivy
package com.devsmart.microdb;
import com.devsmart.ubjson.*;
import com.google.common.reflect.Reflection;
import org.mapdb.*;
import java.io.*;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
public class MapDBDriver implements Driver {
DB mMapDB;
Atomic.Var mMetadata;
BTreeMap mObjects;
private Map mIndicies = new HashMap();
public static class UBValueSerializer implements Serializer, Serializable {
@Override
public void serialize(DataOutput out, UBValue value) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
UBWriter writer = new UBWriter(bout);
writer.write(value);
writer.close();
byte[] buff = bout.toByteArray();
out.writeInt(buff.length);
out.write(buff);
}
@Override
public UBValue deserialize(DataInput in, int available) throws IOException {
final int size = in.readInt();
byte[] buff = new byte[size];
in.readFully(buff);
UBReader reader = new UBReader(new ByteArrayInputStream(buff));
UBValue retval = reader.read();
reader.close();
return retval;
}
@Override
public int fixedSize() {
return -1;
}
}
public static final Serializer SERIALIZER_UBVALUE = new UBValueSerializer();
public MapDBDriver(DB mapdb) {
init(mapdb);
}
private void init(DB mapdb) {
mMapDB = mapdb;
mObjects = mMapDB.createTreeMap("objects")
.keySerializerWrap(Serializer.UUID)
.valueSerializer(SERIALIZER_UBVALUE)
.valuesOutsideNodesEnable()
.comparator(BTreeMap.COMPARABLE_COMPARATOR)
.makeOrGet();
if (mMapDB.exists("metadata")) {
mMetadata = mMapDB.getAtomicVar("metadata");
} else {
Atomic.Var metadata = mMapDB.createAtomicVar("metadata", UBValueFactory.createObject(), SERIALIZER_UBVALUE);
mMetadata = (Atomic.Var) metadata;
}
}
public DB getDB() {
return mMapDB;
}
@Override
public void close() {
mMapDB.close();
}
@Override
public UBObject getMeta() throws IOException {
return mMetadata.get().asObject();
}
@Override
public void saveMeta(UBObject obj) throws IOException {
mMetadata.set(obj);
}
@Override
public UBValue get(UUID key) throws IOException {
return mObjects.get(key);
}
@Override
public UUID genId() {
UUID key = UUID.randomUUID();
while (mObjects.containsKey(key)) {
key = UUID.randomUUID();
}
return key;
}
@Override
public void insert(UUID id, UBValue value) throws IOException {
mObjects.put(id, value);
}
@Override
public void update(UUID id, UBValue value) throws IOException {
mObjects.put(id, value);
}
@Override
public void delete(UUID key) throws IOException {
mObjects.remove(key);
}
private static final UUID MAX_UUID = new UUID(Long.MAX_VALUE, Long.MAX_VALUE);
private static final UUID MIN_UUID = new UUID(Long.MIN_VALUE, Long.MIN_VALUE);
@Override
public > Cursor queryIndex(String indexName, T min, boolean minInclusive, T max, boolean maxInclusive) throws IOException {
MapDBCursor retval = new MapDBCursor();
retval.mDriver = this;
NavigableSet> index = mMapDB.getTreeSet(indexName);
if (max != null && min != null) {
retval.min = Fun.t2(min, minInclusive ? MIN_UUID : MAX_UUID);
retval.max = Fun.t2(max, maxInclusive ? MAX_UUID : MIN_UUID);
retval.index = index.subSet(retval.min, minInclusive, retval.max, maxInclusive);
} else if (min != null && max == null) {
retval.min = Fun.t2(min, minInclusive ? MIN_UUID : MAX_UUID);
retval.index = index.tailSet(retval.min, minInclusive);
} else if (min == null && max != null) {
retval.max = Fun.t2(max, maxInclusive ? MAX_UUID : MIN_UUID);
retval.index = index.headSet(retval.max, maxInclusive);
} else {
retval.index = index;
}
retval.seekToBegining();
return retval;
}
private static class MapDBCursor> implements Cursor {
MapDBDriver mDriver;
NavigableSet> index;
Fun.Tuple2 min;
Fun.Tuple2 max;
private Fun.Tuple2 mCurrentValue;
private int mPosition;
@Override
public void seekToBegining() {
if(!index.isEmpty()) {
mCurrentValue = index.first();
mPosition = 0;
}
}
@Override
public void seekToEnd() {
mCurrentValue = index.last();
mPosition = getCount();
}
@Override
public int getPosition() {
return mPosition;
}
@Override
public boolean moveToPosition(int pos) {
int currentPos;
while( (currentPos = getPosition()) != pos) {
if(currentPos < pos) {
next();
} else {
prev();
}
}
return true;
}
@Override
public boolean next() {
mCurrentValue = index.higher(mCurrentValue);
mPosition++;
return mCurrentValue != null;
}
@Override
public boolean prev() {
mCurrentValue = index.lower(mCurrentValue);
mPosition--;
return mCurrentValue != null;
}
@Override
public Row get() {
if(mCurrentValue == null) {
return null;
} else {
return new MapDBRow(mDriver, mCurrentValue);
}
}
@Override
public int getCount() {
return index.size();
}
}
private static class MapDBRow> implements Row {
private final MapDBDriver mDriver;
final Fun.Tuple2 mTuple;
UBValue mValue;
public MapDBRow(MapDBDriver driver, Fun.Tuple2 tuple) {
mDriver = driver;
mTuple = tuple;
}
@Override
public UUID getPrimaryKey() {
return mTuple.b;
}
@Override
public T getSecondaryKey() {
return mTuple.a;
}
@Override
public UBValue getValue() {
if (mValue == null) {
try {
mValue = mDriver.get(getPrimaryKey());
} catch (IOException e) {
e.printStackTrace();
}
}
return mValue;
}
}
private class IndexObject> {
public final String name;
MapFunction mapFunction;
private Bind.MapListener mListener;
public IndexObject(String name, final MapFunction mapFunction) {
this.name = name;
this.mapFunction = mapFunction;
}
void install() {
if(mListener != null) {
mObjects.modificationListenerRemove(mListener);
}
NavigableSet> index = mMapDB.createTreeSet(name)
.makeOrGet();
Fun.Function2 microDBMapFunction = new Fun.Function2() {
final MapDBEmitter mEmitter = new MapDBEmitter();
@Override
public T[] run(UUID uuid, UBValue ubValue) {
synchronized (mEmitter) {
mEmitter.clear();
mapFunction.map(ubValue, mEmitter);
return mEmitter.getKeys();
}
}
};
mListener = createIndexListener(mObjects, index, microDBMapFunction);
mObjects.modificationListenerAdd(mListener);
}
void reindex() {
NavigableSet> index = mMapDB.createTreeSet(name)
.makeOrGet();
index.clear();
Fun.Function2 fun = createMapDBFunction();
if(index.isEmpty()){
for(Map.Entry e:mObjects.entrySet()){
T[] k2 = fun.run(e.getKey(), e.getValue());
if(k2 != null)
for(T k22 :k2)
index.add(Fun.t2(k22, e.getKey()));
}
}
}
private Fun.Function2 createMapDBFunction() {
return new Fun.Function2() {
final MapDBEmitter mEmitter = new MapDBEmitter();
@Override
public T[] run(UUID uuid, UBValue ubValue) {
synchronized (mEmitter) {
mEmitter.clear();
mapFunction.map(ubValue, mEmitter);
return mEmitter.getKeys();
}
}
};
}
private Bind.MapListener createIndexListener(Bind.MapWithModificationListener map,
final Set> secondary,
final Fun.Function2 fun) {
return new Bind.MapListener() {
@Override
public void update(K key, V oldVal, V newVal) {
if (newVal == null) {
//removal
K2[] k2 = fun.run(key, oldVal);
if (k2 != null)
for (K2 k22 : k2)
secondary.remove(Fun.t2(k22, key));
} else if (oldVal == null) {
//insert
K2[] k2 = fun.run(key, newVal);
if (k2 != null)
for (K2 k22 : k2)
secondary.add(Fun.t2(k22, key));
} else {
//update, must remove old key and insert new
K2[] oldk = fun.run(key, oldVal);
K2[] newk = fun.run(key, newVal);
if (oldk == null) {
//insert new
if (newk != null)
for (K2 k22 : newk)
secondary.add(Fun.t2(k22, key));
return;
}
if (newk == null) {
//remove old
for (K2 k22 : oldk)
secondary.remove(Fun.t2(k22, key));
return;
}
Set hashes = new HashSet();
Collections.addAll(hashes, oldk);
//add new non existing items
for (K2 k2 : newk) {
if (!hashes.contains(k2)) {
secondary.add(Fun.t2(k2, key));
}
}
//remove items which are in old, but not in new
for (K2 k2 : newk) {
hashes.remove(k2);
}
for (K2 k2 : hashes) {
secondary.remove(Fun.t2(k2, key));
}
}
}
};
}
}
@Override
public > void addIndex(String indexName, final MapFunction mapFunction) throws IOException {
synchronized (mIndicies) {
IndexObject index = mIndicies.get(indexName);
if (index == null) {
index = new IndexObject(indexName, mapFunction);
mIndicies.put(indexName, index);
index.install();
}
}
}
@Override
public void recomputeIndex(String indexName) {
synchronized (mIndicies) {
IndexObject index = mIndicies.get(indexName);
if (index != null) {
index.reindex();
}
}
}
private static class MapDBEmitter> implements Emitter {
ArrayList mKeys = new ArrayList(3);
public void clear() {
mKeys.clear();
}
@Override
public void emit(T key) {
mKeys.add(key);
}
public T[] getKeys() {
if(mKeys.isEmpty()) {
return null;
}
T[] retval = (T[]) new Comparable[mKeys.size()];
retval = mKeys.toArray(retval);
return retval;
}
}
@Override
public void deleteIndex(String indexName) {
mMapDB.delete(indexName);
}
@Override
public long incrementLongField(String fieldName) {
return mMapDB.getAtomicLong(fieldName).getAndIncrement();
}
@Override
public void beginTransaction() throws IOException {
}
@Override
public void commitTransaction() throws IOException {
mMapDB.commit();
}
@Override
public void rollbackTransaction() throws IOException {
mMapDB.rollback();
}
@Override
public void compact() throws IOException {
try {
mMapDB.commit();
Store store = Store.forDB(mMapDB);
if(!(store instanceof StoreDirect)) {
store.compact();
return;
}
File indexFile;
File physFile;
Field indexField = StoreDirect.class.getDeclaredField("index");
indexField.setAccessible(true);
Field physField = StoreDirect.class.getDeclaredField("phys");
physField.setAccessible(true);
indexFile = ((Volume) indexField.get(store)).getFile();
physFile = ((Volume) physField.get(store)).getFile();
final File compactFile = new File(indexFile.getPath() + ".comp2" );
if(compactFile.exists()) {
compactFile.delete();
}
{
File compactPhysicalFile = new File(indexFile.getPath() + ".comp2.p");
if(compactPhysicalFile.exists()) {
compactPhysicalFile.delete();
}
}
DB newDB = DBMaker.newFileDB(compactFile)
.transactionDisable()
.make();
BTreeMap newObject = newDB.createTreeMap("objects")
.keySerializerWrap(Serializer.UUID)
.valueSerializer(SERIALIZER_UBVALUE)
.valuesOutsideNodesEnable()
.comparator(BTreeMap.COMPARABLE_COMPARATOR)
.makeOrGet();
ArrayList indicesFun = new ArrayList();
for (Map.Entry entry : mIndicies.entrySet()) {
NavigableSet> index = newDB.createTreeSet(entry.getKey()).makeOrGet();
indicesFun.add(new AddToIndex(entry.getValue(), index));
}
newDB.createAtomicVar("metadata", mMetadata.get(), SERIALIZER_UBVALUE);
for(Map.Entry entry : mMapDB.getAll().entrySet()) {
String name = entry.getKey();
Object value = entry.getValue();
if(value instanceof Atomic.Long) {
newDB.createAtomicLong(name, ((Atomic.Long)value).longValue());
}
}
for (Map.Entry entry : mObjects.entrySet()) {
UUID key = entry.getKey();
UBValue value = entry.getValue();
newObject.put(key, value);
for (AddToIndex addToIndex : indicesFun) {
addToIndex.index(key, value);
}
}
File indexFile2 = ((Volume)indexField.get(Store.forDB(newDB))).getFile();
File physFile2 = ((Volume)physField.get(Store.forDB(newDB))).getFile();
newDB.close();
mMapDB.close();
com.google.common.io.Files.move(physFile2, physFile);
com.google.common.io.Files.move(indexFile2, indexFile);
//Files.move(physFile2.toPath(), physFile.toPath(), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
//Files.move(indexFile2.toPath(), indexFile.toPath(), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
init(DBMaker.newFileDB(indexFile).make());
for(IndexObject indexObject : mIndicies.values()) {
indexObject.install();
}
} catch (Exception e) {
throw new IOException(e);
}
}
private static class AddToIndex {
private final NavigableSet> mIndex;
private final Fun.Function2 mapFun;
public AddToIndex(IndexObject indexObject, NavigableSet> index) {
mIndex = index;
mapFun = indexObject.createMapDBFunction();
}
public void index(UUID uuid, UBValue value) {
Object[] k2 = mapFun.run(uuid, value);
if(k2 != null) {
for(Object k22 : k2) {
mIndex.add(Fun.t2(k22, uuid));
}
}
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy