org.bimserver.database.berkeley.BerkeleyKeyValueStore Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of BimServer Show documentation
There is a newer version: 1.5.9
package org.bimserver.database.berkeley;

/******************************************************************************
 * Copyright (C) 2009-2016  BIMserver.org
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see {@literal}.
 *****************************************************************************/

import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

import org.bimserver.BimserverDatabaseException;
import org.bimserver.database.BimTransaction;
import org.bimserver.database.BimserverLockConflictException;
import org.bimserver.database.DatabaseSession;
import org.bimserver.database.KeyValueStore;
import org.bimserver.database.Record;
import org.bimserver.database.RecordIterator;
import org.bimserver.database.SearchingRecordIterator;
import org.bimserver.utils.PathUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.sleepycat.je.Cursor;
import com.sleepycat.je.CursorConfig;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseConfig;
import com.sleepycat.je.DatabaseEntry;
import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.EnvironmentLockedException;
import com.sleepycat.je.JEVersion;
import com.sleepycat.je.LockConflictException;
import com.sleepycat.je.LockMode;
import com.sleepycat.je.OperationStatus;
import com.sleepycat.je.Transaction;
import com.sleepycat.je.TransactionConfig;

public class BerkeleyKeyValueStore implements KeyValueStore {

	private static final Logger LOGGER = LoggerFactory.getLogger(BerkeleyKeyValueStore.class);
	private Environment environment;
	private long committedWrites;
	private long reads;
	private final Map tables = new HashMap();
	private boolean isNew;
	private TransactionConfig transactionConfig;
	private CursorConfig cursorConfig;
	private long lastPrintedReads = 0;
	private long lastPrintedCommittedWrites = 0;
	private static final boolean MONITOR_CURSOR_STACK_TRACES = false;
	private final AtomicLong cursorCounter = new AtomicLong();
	private final Map openCursors = new ConcurrentHashMap<>();
	private boolean useTransactions = true;
	private boolean defer = false;

	public BerkeleyKeyValueStore(Path dataDir) throws DatabaseInitException {
		if (Files.isDirectory(dataDir)) {
			try {
				if (PathUtils.list(dataDir).size() > 0) {
					LOGGER.info("Non-empty database directory found \"" + dataDir.toString() + "\"");
					isNew = false;
				} else {
					LOGGER.info("Empty database directory found \"" + dataDir.toString() + "\"");
					isNew = true;
				}
			} catch (IOException e) {
				LOGGER.error("", e);
			}
		} else {
			isNew = true;
			LOGGER.info("No database directory found, creating \"" + dataDir.toString() + "\"");
			try {
				Files.createDirectory(dataDir);
				LOGGER.info("Successfully created database dir \"" + dataDir.toString() + "\"");
			} catch (Exception e) {
				LOGGER.error("Error creating database dir \"" + dataDir.toString() + "\"");
			}
		}
		EnvironmentConfig envConfig = new EnvironmentConfig();
		envConfig.setCachePercent(50);
		envConfig.setAllowCreate(true);
		envConfig.setTransactional(useTransactions);
		envConfig.setTxnTimeout(10, TimeUnit.SECONDS);
		envConfig.setLockTimeout(2000, TimeUnit.MILLISECONDS);
		envConfig.setConfigParam(EnvironmentConfig.CHECKPOINTER_HIGH_PRIORITY, "true");
		envConfig.setConfigParam(EnvironmentConfig.CLEANER_THREADS, "5");
		try {
			environment = new Environment(dataDir.toFile(), envConfig);
		} catch (EnvironmentLockedException e) {
			String message = "Environment locked exception. Another process is using the same database, or the current user has no write access (database location: \""
					+ dataDir.toString() + "\")";
			throw new DatabaseInitException(message);
		} catch (DatabaseException e) {
			String message = "A database initialisation error has occured (" + e.getMessage() + ")";
			throw new DatabaseInitException(message);
		}
		
		transactionConfig = new TransactionConfig();
		transactionConfig.setReadCommitted(true);

		cursorConfig = new CursorConfig();
		cursorConfig.setReadCommitted(true);
	}

	public boolean isNew() {
		return isNew;
	}

	public BimTransaction startTransaction() {
		if (useTransactions) {
			try {
				return new BerkeleyTransaction(environment.beginTransaction(null, transactionConfig));
			} catch (DatabaseException e) {
				LOGGER.error("", e);
			}
		}
		return null;
	}

	public boolean createTable(String tableName, DatabaseSession databaseSession) throws BimserverDatabaseException {
		if (tables.containsKey(tableName)) {
			throw new BimserverDatabaseException("Table " + tableName + " already created");
		}
		DatabaseConfig databaseConfig = new DatabaseConfig();
		databaseConfig.setAllowCreate(true);
		databaseConfig.setDeferredWrite(defer);
		databaseConfig.setTransactional(useTransactions);
		databaseConfig.setSortedDuplicates(false);
		Database database = environment.openDatabase(null, tableName, databaseConfig);
		if (database == null) {
			return false;
		}
		tables.put(tableName, database);
		
		return true;
	}

	public boolean createIndexTable(String tableName, DatabaseSession databaseSession) throws BimserverDatabaseException {
		if (tables.containsKey(tableName)) {
			throw new BimserverDatabaseException("Table " + tableName + " already created");
		}
		DatabaseConfig databaseConfig = new DatabaseConfig();
		databaseConfig.setAllowCreate(true);
		databaseConfig.setDeferredWrite(defer);
		databaseConfig.setTransactional(useTransactions);
		databaseConfig.setSortedDuplicates(true);
		Database database = environment.openDatabase(null, tableName, databaseConfig);
		if (database == null) {
			return false;
		}
		tables.put(tableName, database);
		
		return true;
	}
	
	public boolean openTable(String tableName) throws BimserverDatabaseException {
		if (tables.containsKey(tableName)) {
			throw new BimserverDatabaseException("Table " + tableName + " already opened");
		}
		DatabaseConfig databaseConfig = new DatabaseConfig();
		databaseConfig.setAllowCreate(false);
		databaseConfig.setDeferredWrite(defer);
		databaseConfig.setTransactional(useTransactions);
		databaseConfig.setSortedDuplicates(false);
		Database database = environment.openDatabase(null, tableName, databaseConfig);
		if (database == null) {
			throw new BimserverDatabaseException("Table " + tableName + " not found in database");
		}
		tables.put(tableName, database);
		return true;
	}

	public void openIndexTable(String tableName) throws BimserverDatabaseException {
		if (tables.containsKey(tableName)) {
			throw new BimserverDatabaseException("Table " + tableName + " already opened");
		}
		DatabaseConfig databaseConfig = new DatabaseConfig();
		databaseConfig.setAllowCreate(false);
		databaseConfig.setDeferredWrite(defer);
		databaseConfig.setTransactional(useTransactions);
		databaseConfig.setSortedDuplicates(true);
		Database database = environment.openDatabase(null, tableName, databaseConfig);
		if (database == null) {
			throw new BimserverDatabaseException("Table " + tableName + " not found in database");
		}
		tables.put(tableName, database);
	}
	
	private Database getDatabase(String tableName) throws BimserverDatabaseException {
		Database database = tables.get(tableName);
		if (database == null) {
			throw new BimserverDatabaseException("Table " + tableName + " not found");
		}
		return database;
	}

	private Transaction getTransaction(DatabaseSession databaseSession) {
		if (databaseSession != null) {
			BerkeleyTransaction berkeleyTransaction = (BerkeleyTransaction) databaseSession.getBimTransaction();
			if (berkeleyTransaction != null) {
				return berkeleyTransaction.getTransaction();
			}
		}
		return null;
	}

	public void close() {
		for (Database database : tables.values()) {
			try {
				database.close();
			} catch (DatabaseException e) {
				LOGGER.error("", e);
			}
		}
		if (environment != null) {
			try {
				environment.close();
			} catch (DatabaseException e) {
				LOGGER.error("", e);
			}
		}
	}

	@Override
	public byte[] get(String tableName, byte[] keyBytes, DatabaseSession databaseSession) throws BimserverDatabaseException {
		DatabaseEntry key = new DatabaseEntry(keyBytes);
		DatabaseEntry value = new DatabaseEntry();
		try {
			OperationStatus operationStatus = getDatabase(tableName).get(getTransaction(databaseSession), key, value, LockMode.DEFAULT);
			if (operationStatus == OperationStatus.SUCCESS) {
				return value.getData();
			}
		} catch (DatabaseException e) {
			LOGGER.error("", e);
		}
		return null;
	}

	@Override
	public List getDuplicates(String tableName, byte[] keyBytes, DatabaseSession databaseSession) throws BimserverDatabaseException {
		DatabaseEntry key = new DatabaseEntry(keyBytes);
		DatabaseEntry value = new DatabaseEntry();
		try {
			Cursor cursor = getDatabase(tableName).openCursor(getTransaction(databaseSession), cursorConfig);
			try {
				OperationStatus operationStatus = cursor.getSearchKey(key, value, LockMode.DEFAULT);
				List result = new ArrayList();
				while (operationStatus == OperationStatus.SUCCESS) {
					result.add(value.getData());
					operationStatus = cursor.getNextDup(key, value, LockMode.DEFAULT);
				}
				return result;
			} finally {
				cursor.close();
			}
		} catch (DatabaseException e) {
			LOGGER.error("", e);
		}
		return null;
	}

	public long getTotalWrites() {
		return committedWrites;
	}

	public void sync() {
		try {
			environment.flushLog(true);
			environment.evictMemory();
		} catch (DatabaseException e) {
			LOGGER.error("", e);
		}
	}

	@Override
	public boolean containsTable(String tableName) {
		try {
			return environment.getDatabaseNames().contains(tableName);
		} catch (DatabaseException e) {
			LOGGER.error("", e);
		}
		return false;
	}

	@Override
	public RecordIterator getRecordIterator(String tableName, DatabaseSession databaseSession) throws BimserverDatabaseException {
		Cursor cursor = null;
		try {
			cursor = getDatabase(tableName).openCursor(getTransaction(databaseSession), cursorConfig);
			BerkeleyRecordIterator berkeleyRecordIterator = new BerkeleyRecordIterator(cursor, this, cursorCounter.incrementAndGet());
			if (MONITOR_CURSOR_STACK_TRACES) {
				openCursors.put(berkeleyRecordIterator.getCursorId(), new Exception().getStackTrace());
			}
			return berkeleyRecordIterator;
		} catch (DatabaseException e) {
			LOGGER.error("", e);
		}
		return null;
	}

	@Override
	public SearchingRecordIterator getRecordIterator(String tableName, byte[] mustStartWith, byte[] startSearchingAt, DatabaseSession databaseSession) throws BimserverLockConflictException, BimserverDatabaseException {
		Cursor cursor = null;
		try {
			cursor = getDatabase(tableName).openCursor(getTransaction(databaseSession), cursorConfig);
			BerkeleySearchingRecordIterator berkeleySearchingRecordIterator = new BerkeleySearchingRecordIterator(cursor, this, cursorCounter.incrementAndGet(), mustStartWith, startSearchingAt);
			if (MONITOR_CURSOR_STACK_TRACES) {
				openCursors.put(berkeleySearchingRecordIterator.getCursorId(), new Exception().getStackTrace());
			}
			return berkeleySearchingRecordIterator;
		} catch (BimserverLockConflictException e) {
			if (cursor != null) {
				try {
					cursor.close();
					throw e;
				} catch (DatabaseException e1) {
					LOGGER.error("", e1);
				}
			}
		} catch (DatabaseException e1) {
			LOGGER.error("", e1);
		}
		return null;
	}

	@Override
	public long count(String tableName) {
		try {
			return getDatabase(tableName).count();
		} catch (DatabaseException e) {
			LOGGER.error("", e);
		} catch (BimserverDatabaseException e) {
			LOGGER.error("", e);
		}
		return -1;
	}

	@Override
	public byte[] getFirstStartingWith(String tableName, byte[] key, DatabaseSession databaseSession) throws BimserverLockConflictException, BimserverDatabaseException {
		SearchingRecordIterator recordIterator = getRecordIterator(tableName, key, key, databaseSession);
		try {
			Record record = recordIterator.next(key);
			if (record == null) {
				return null;
			}
			return record.getValue();
		} finally {
			recordIterator.close();
		}
	}

	public void delete(String tableName, byte[] key, DatabaseSession databaseSession) throws BimserverLockConflictException {
		DatabaseEntry entry = new DatabaseEntry(key);
		try {
			getDatabase(tableName).delete(getTransaction(databaseSession), entry);
		} catch (LockConflictException e) {
			throw new BimserverLockConflictException(e);
		} catch (DatabaseException e) {
			LOGGER.error("", e);
		} catch (UnsupportedOperationException e) {
			LOGGER.error("", e);
		} catch (IllegalArgumentException e) {
			LOGGER.error("", e);
		} catch (BimserverDatabaseException e) {
			LOGGER.error("", e);
		}
	}
	
	@Override
	public void delete(String indexTableName, byte[] featureBytesOldIndex, byte[] array, DatabaseSession databaseSession) throws BimserverLockConflictException {
		try {
			Cursor cursor = getDatabase(indexTableName).openCursor(getTransaction(databaseSession), cursorConfig);
			try {
				if (cursor.getSearchBoth(new DatabaseEntry(featureBytesOldIndex), new DatabaseEntry(array), LockMode.DEFAULT) == OperationStatus.SUCCESS) {
					cursor.delete();
				}
			} finally {
				cursor.close();
			}
		} catch (LockConflictException e) {
			throw new BimserverLockConflictException(e);
		} catch (DatabaseException e) {
			LOGGER.error("", e);
		} catch (UnsupportedOperationException e) {
			LOGGER.error("", e);
		} catch (IllegalArgumentException e) {
			LOGGER.error("", e);
		} catch (BimserverDatabaseException e) {
			LOGGER.error("", e);
		}
	}

	@Override
	public String getLocation() {
		try {
			return environment.getHome().getAbsolutePath();
		} catch (DatabaseException e) {
			LOGGER.error("", e);
		}
		return "unknown";
	}

	@Override
	public String getStats() {
		try {
			return environment.getStats(null).toString();
		} catch (DatabaseException e) {
			LOGGER.error("", e);
		}
		return null;
	}

	@Override
	public void commit(DatabaseSession databaseSession) throws BimserverLockConflictException, BimserverDatabaseException {
		Transaction bdbTransaction = getTransaction(databaseSession);
		try {
			bdbTransaction.commit();
		} catch (LockConflictException e) {
			throw new BimserverLockConflictException(e);
		} catch (DatabaseException e) {
			throw new BimserverDatabaseException("", e);
		}
	}

	@Override
	public void store(String tableName, byte[] key, byte[] value, DatabaseSession databaseSession) throws BimserverDatabaseException, BimserverLockConflictException {
		store(tableName, key, value, 0, value.length, databaseSession);
	}
	
	@Override
	public void store(String tableName, byte[] key, byte[] value, int offset, int length, DatabaseSession databaseSession) throws BimserverDatabaseException, BimserverLockConflictException {
		DatabaseEntry dbKey = new DatabaseEntry(key);
		DatabaseEntry dbValue = new DatabaseEntry(value, offset, length);
		try {
			Database database = getDatabase(tableName);
			database.put(getTransaction(databaseSession), dbKey, dbValue);
		} catch (LockConflictException e) {
			throw new BimserverLockConflictException(e);
		} catch (DatabaseException e) {
			throw new BimserverDatabaseException("", e);
		}
	}

	@Override
	public void storeNoOverwrite(String tableName, byte[] key, byte[] value, DatabaseSession databaseSession) throws BimserverDatabaseException, BimserverLockConflictException, BimserverConcurrentModificationDatabaseException {
		storeNoOverwrite(tableName, key, value, 0, value.length, databaseSession);
	}
	
	@Override
	public void storeNoOverwrite(String tableName, byte[] key, byte[] value, int index, int length, DatabaseSession databaseSession) throws BimserverDatabaseException, BimserverLockConflictException, BimserverConcurrentModificationDatabaseException {
		DatabaseEntry dbKey = new DatabaseEntry(key);
		DatabaseEntry dbValue = new DatabaseEntry(value, index, length);
		try {
			Database database = getDatabase(tableName);
			OperationStatus putNoOverwrite = database.putNoOverwrite(getTransaction(databaseSession), dbKey, dbValue);
			if (putNoOverwrite == OperationStatus.KEYEXIST) {
				ByteBuffer keyBuffer = ByteBuffer.wrap(key);
				if (key.length == 16) {
					int pid = keyBuffer.getInt();
					long oid = keyBuffer.getLong();
					int rid = -keyBuffer.getInt();
					throw new BimserverConcurrentModificationDatabaseException("Key exists: pid: " + pid + ", oid: " + oid + ", rid: " + rid);
				} else {
					throw new BimserverConcurrentModificationDatabaseException("Key exists: " );
				}
			}
		} catch (LockConflictException e) {
			throw new BimserverLockConflictException(e);
		} catch (DatabaseException e) {
			throw new BimserverDatabaseException("", e);
		}
	}
	
	@Override
	public String getType() {
		return "Berkeley DB Java Edition " + JEVersion.CURRENT_VERSION.toString();
	}

	@Override
	public long getDatabaseSizeInBytes() {
		long size = 0;
		try {
			File home = environment.getHome();
			for (File file : home.listFiles()) {
				size += file.length();
			}
		} catch (DatabaseException e) {
			LOGGER.error("", e);
		}
		return size;
	}
	
	public Set getAllTableNames() {
		return new HashSet(environment.getDatabaseNames());
	}
	
	public synchronized void incrementReads(long reads) {
		this.reads += reads;
		if (this.reads / 100000 != lastPrintedReads) {
			LOGGER.info("reads: " + this.reads);
			lastPrintedReads = this.reads / 100000;
		}
	}
	
	@Override
	public synchronized void incrementCommittedWrites(long committedWrites) {
		this.committedWrites += committedWrites;
		if (this.committedWrites / 100000 != lastPrintedCommittedWrites) {
			LOGGER.info("writes: " + this.committedWrites);
			lastPrintedCommittedWrites = this.committedWrites / 100000;
		}
	}

	public void removeOpenCursor(long cursorId) {
		if (MONITOR_CURSOR_STACK_TRACES) {
			openCursors.remove(cursorId);
		}
	}

	@Override
	public void dumpOpenCursors() {
		for (StackTraceElement[] ste : openCursors.values()) {
			System.out.println("Open cursor");
			for (StackTraceElement stackTraceElement : ste) {
				LOGGER.info("\t" + stackTraceElement.getClassName() + ":" + stackTraceElement.getLineNumber() + "."
						+ stackTraceElement.getMethodName());
			}
		}
	}
}