com.sleepycat.je.util.DbScavenger Maven / Gradle / Ivy
/*-
* Copyright (C) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
*
* This file was distributed by Oracle as part of a version of Oracle Berkeley
* DB Java Edition made available at:
*
* http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html
*
* Please see the LICENSE file included in the top-level directory of the
* appropriate version of Oracle Berkeley DB Java Edition for a copy of the
* license and additional information.
*/
package com.sleepycat.je.util;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import com.sleepycat.je.DatabaseEntry;
import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.DbInternal;
import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentFailureException;
import com.sleepycat.je.EnvironmentLockedException;
import com.sleepycat.je.EnvironmentNotFoundException;
import com.sleepycat.je.config.EnvironmentParams;
import com.sleepycat.je.dbi.DatabaseImpl;
import com.sleepycat.je.dbi.DbConfigManager;
import com.sleepycat.je.dbi.EnvironmentImpl;
import com.sleepycat.je.log.FileManager;
import com.sleepycat.je.log.LastFileReader;
import com.sleepycat.je.log.LogEntryType;
import com.sleepycat.je.log.ScavengerFileReader;
import com.sleepycat.je.log.entry.LNLogEntry;
import com.sleepycat.je.log.entry.LogEntry;
import com.sleepycat.je.tree.LN;
import com.sleepycat.je.tree.MapLN;
import com.sleepycat.je.tree.NameLN;
import com.sleepycat.je.txn.TxnChain.CompareSlot;
import com.sleepycat.je.utilint.BitMap;
import com.sleepycat.je.utilint.DbLsn;
import com.sleepycat.utilint.StringUtils;
/**
* Used to retrieve as much data as possible from a corrupted environment.
* This utility is meant to be used programmatically, and is the equivalent
* to the -R or -r options for {@link DbDump}.
*
* To scavenge a database:
*
* DbScavenger scavenger =
* new DbScavenger(env, outputDirectory, <boolean>, <boolean>, <boolean>);
* scavenger.dump();
*
*
*
* The recovered databases will put placed in the outputDirectory with ".dump"
* file suffixes. The format of the .dump files will be suitable for use with
* DbLoad.
*/
public class DbScavenger extends DbDump {
private static final int FLUSH_INTERVAL = 100;
private int readBufferSize;
private EnvironmentImpl envImpl;
/*
* Set of committed txn ids that have been seen so far. Positive IDs are
* for non-replicated txns, and negative IDs are for replicated txns.
*/
private BitMap positiveCommittedTxnIdsSeen;
private BitMap negativeCommittedTxnIdsSeen;
/*
* Set of LN Node Ids that have been seen so far.
*/
private Set lnNodesSeen;
/*
* Map of database id to database names.
*/
private Map dbIdToName;
/*
* Map of database id to DatabaseImpl.
*/
private Map dbIdToImpl;
/*
* Map of database id to the .dump file output stream for that database.
*/
private Map dbIdToOutputStream;
private boolean dumpCorruptedBounds = false;
private int flushCounter = 0;
private long lastTime;
/**
* Create a DbScavenger object for a specific environment.
*
* @param env The Environment containing the database to dump.
* @param outputDirectory The directory to create the .dump files in.
* @param formatUsingPrintable true if the dump should use printable
* characters.
* @param doAggressiveScavengerRun true if true, then all data records are
* dumped, regardless of whether they are the latest version or not.
* @param verbose true if status output should be written to System.out
* during scavenging.
*/
public DbScavenger(Environment env,
String outputDirectory,
boolean formatUsingPrintable,
boolean doAggressiveScavengerRun,
boolean verbose) {
super(env, null, null, formatUsingPrintable);
this.doAggressiveScavengerRun = doAggressiveScavengerRun;
this.dbIdToName = new HashMap();
this.dbIdToImpl = new HashMap();
this.dbIdToOutputStream = new HashMap();
this.verbose = verbose;
this.outputDirectory = outputDirectory;
}
/**
* Set to true if corrupted boundaries should be dumped out.
*/
public void setDumpCorruptedBounds(boolean dumpCorruptedBounds) {
this.dumpCorruptedBounds = dumpCorruptedBounds;
}
/**
* Start the scavenger run.
*/
@Override
public void dump()
throws EnvironmentNotFoundException,
EnvironmentLockedException,
IOException {
openEnv(false);
envImpl = DbInternal.getNonNullEnvImpl(env);
DbConfigManager cm = envImpl.getConfigManager();
readBufferSize = cm.getInt(EnvironmentParams.LOG_ITERATOR_READ_SIZE);
/*
* Find the end of the log.
*/
LastFileReader reader = new LastFileReader(envImpl, readBufferSize);
while (reader.readNextEntry()) {
}
/* Tell the fileManager where the end of the log is. */
long lastUsedLsn = reader.getLastValidLsn();
long nextAvailableLsn = reader.getEndOfLog();
envImpl.getFileManager().setLastPosition(nextAvailableLsn,
lastUsedLsn,
reader.getPrevOffset());
try {
/* Pass 1: Scavenge the dbtree. */
if (verbose) {
System.out.println("Pass 1: " + new Date());
}
scavengeDbTree(lastUsedLsn, nextAvailableLsn);
/* Pass 2: Scavenge the databases. */
if (verbose) {
System.out.println("Pass 2: " + new Date());
}
scavenge(lastUsedLsn, nextAvailableLsn);
if (verbose) {
System.out.println("End: " + new Date());
}
} finally {
closeOutputStreams();
}
}
/*
* Scan the log looking for records that are relevant for scavenging the db
* tree.
*/
private void scavengeDbTree(long lastUsedLsn, long nextAvailableLsn)
throws DatabaseException {
positiveCommittedTxnIdsSeen = new BitMap();
negativeCommittedTxnIdsSeen = new BitMap();
lnNodesSeen = new TreeSet();
final ScavengerFileReader scavengerReader =
new ScavengerFileReader(envImpl, readBufferSize, lastUsedLsn,
DbLsn.NULL_LSN, nextAvailableLsn) {
protected void processEntryCallback(LogEntry entry,
LogEntryType entryType)
throws DatabaseException {
processDbTreeEntry(entry, entryType);
}
};
scavengerReader.setTargetType(LogEntryType.LOG_MAPLN);
scavengerReader.setTargetType(LogEntryType.LOG_NAMELN_TRANSACTIONAL);
scavengerReader.setTargetType(LogEntryType.LOG_NAMELN);
scavengerReader.setTargetType(LogEntryType.LOG_TXN_COMMIT);
scavengerReader.setTargetType(LogEntryType.LOG_TXN_ABORT);
lastTime = System.currentTimeMillis();
long fileNum = -1;
while (scavengerReader.readNextEntry()) {
fileNum = reportProgress(fileNum,
scavengerReader.getLastLsn());
}
}
private long reportProgress(long fileNum, long lastLsn) {
long currentFile = DbLsn.getFileNumber(lastLsn);
if (verbose) {
if (currentFile != fileNum) {
long now = System.currentTimeMillis();
System.out.println("processing file " +
FileManager.getFileName(currentFile,
".jdb ") +
(now-lastTime) + " ms");
lastTime = now;
}
}
return currentFile;
}
/*
* Look at an entry and determine if it should be processed for scavenging.
*/
private boolean checkProcessEntry(LogEntry entry,
LogEntryType entryType,
boolean pass2) {
boolean isTransactional = entryType.isTransactional();
/*
* If entry is txnal...
* if a commit record, add to committed txn id set
* if an abort record, ignore it and don't process.
* if an LN, check if it's in the committed txn id set.
* If it is, continue processing, otherwise ignore it.
*/
if (isTransactional) {
final long txnId = entry.getTransactionId();
if (entryType.equals(LogEntryType.LOG_TXN_COMMIT)) {
setCommittedTxn(txnId);
/* No need to process this entry further. */
return false;
}
if (entryType.equals(LogEntryType.LOG_TXN_ABORT)) {
/* No need to process this entry further. */
return false;
}
if (!isCommittedTxn(txnId)) {
return false;
}
}
/*
* Check the nodeId to see if we've already seen it or not.
*/
if (entry instanceof LNLogEntry) {
final LNLogEntry lnEntry = (LNLogEntry) entry;
final long dbId = lnEntry.getDbId().getId();
final DatabaseImpl db = dbIdToImpl.get(dbId);
/* Must call postFetchInit if true is returned. */
if (db != null) {
lnEntry.postFetchInit(db);
} else {
lnEntry.postFetchInit(false /*isDupDb*/);
}
/*
* If aggressive or if processing DbTree entries, don't worry about
* whether this node has been processed already.
*/
if (doAggressiveScavengerRun || !pass2) {
return true;
}
if (db == null) {
throw EnvironmentFailureException.unexpectedState
("Database info not available for DB ID: " + dbId);
}
return lnNodesSeen.add(new CompareSlot(db, lnEntry));
}
return false;
}
/*
* Called once for each log entry during the pass 1 (dbtree).
*/
private void processDbTreeEntry(LogEntry entry, LogEntryType entryType)
throws DatabaseException {
boolean processThisEntry =
checkProcessEntry(entry, entryType, false);
if (processThisEntry &&
(entry instanceof LNLogEntry)) {
LNLogEntry lnEntry = (LNLogEntry) entry;
LN ln = lnEntry.getLN();
if (ln instanceof NameLN) {
String name = StringUtils.fromUTF8(lnEntry.getKey());
Long dbId = Long.valueOf(((NameLN) ln).getId().getId());
if (dbIdToName.containsKey(dbId) &&
!dbIdToName.get(dbId).equals(name)) {
throw EnvironmentFailureException.unexpectedState
("Already name mapped for dbId: " + dbId +
" changed from " + dbIdToName.get(dbId) +
" to " + name);
} else {
dbIdToName.put(dbId, name);
}
}
if (ln instanceof MapLN) {
DatabaseImpl db = ((MapLN) ln).getDatabase();
Long dbId = db.getId().getId();
/* Use latest version to get most recent comparators. */
if (!dbIdToImpl.containsKey(dbId)) {
dbIdToImpl.put(dbId, db);
}
}
}
}
/*
* Pass 2: scavenge the regular (non-dbtree) environment.
*/
private void scavenge(long lastUsedLsn, long nextAvailableLsn)
throws DatabaseException {
final ScavengerFileReader scavengerReader =
new ScavengerFileReader(envImpl, readBufferSize, lastUsedLsn,
DbLsn.NULL_LSN, nextAvailableLsn) {
protected void processEntryCallback(LogEntry entry,
LogEntryType entryType)
throws DatabaseException {
processRegularEntry(entry, entryType);
}
};
/*
* Note: committed transaction id map has been created already, no
* need to read TXN_COMMITS on this pass.
*/
for (LogEntryType entryType : LogEntryType.getAllTypes()) {
if (entryType.isUserLNType()) {
scavengerReader.setTargetType(entryType);
}
}
scavengerReader.setDumpCorruptedBounds(dumpCorruptedBounds);
long progressFileNum = -1;
while (scavengerReader.readNextEntry()) {
progressFileNum = reportProgress(progressFileNum,
scavengerReader.getLastLsn());
}
}
/*
* Process an entry during pass 2.
*/
private void processRegularEntry(LogEntry entry, LogEntryType entryType)
throws DatabaseException {
boolean processThisEntry =
checkProcessEntry(entry, entryType, true);
if (processThisEntry) {
LNLogEntry lnEntry = (LNLogEntry) entry;
Long dbId = Long.valueOf(lnEntry.getDbId().getId());
LN ln = lnEntry.getLN();
/* Create output file even if we don't process a deleted entry. */
PrintStream out = getOutputStream(dbId);
if (!ln.isDeleted()) {
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
lnEntry.getUserKeyData(key, data);
dumpOne(out, key.getData(), formatUsingPrintable);
dumpOne(out, data.getData(), formatUsingPrintable);
if ((++flushCounter % FLUSH_INTERVAL) == 0) {
out.flush();
flushCounter = 0;
}
}
}
}
/*
* Return the output stream for the .dump file for database with id dbId.
* If an output stream has not already been created, then create one.
*/
private PrintStream getOutputStream(Long dbId)
throws DatabaseException {
PrintStream ret = dbIdToOutputStream.get(dbId);
if (ret != null) {
return ret;
}
String name = dbIdToName.get(dbId);
if (name == null) {
name = "db" + dbId;
}
File file = new File(outputDirectory, name + ".dump");
try {
ret = new PrintStream(new FileOutputStream(file), false);
} catch (FileNotFoundException e) {
throw EnvironmentFailureException.unexpectedException(e);
}
dbIdToOutputStream.put(dbId, ret);
DatabaseImpl db = dbIdToImpl.get(dbId);
boolean dupSort = (db != null) ? db.getSortedDuplicates() : false;
printHeader(ret, dupSort, formatUsingPrintable);
return ret;
}
private void closeOutputStreams() {
Iterator iter = dbIdToOutputStream.values().iterator();
while (iter.hasNext()) {
PrintStream s = iter.next();
s.println("DATA=END");
s.close();
}
}
private void setCommittedTxn(final long txnId) {
if (txnId >= 0) {
positiveCommittedTxnIdsSeen.set(txnId);
} else {
negativeCommittedTxnIdsSeen.set(0 - txnId);
}
}
private boolean isCommittedTxn(final long txnId) {
if (txnId >= 0) {
return positiveCommittedTxnIdsSeen.get(txnId);
} else {
return negativeCommittedTxnIdsSeen.get(0 - txnId);
}
}
}