All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.persistit.StreamSaver Maven / Gradle / Ivy

Go to download

Java B+Tree Key-Value Store Library

There is a newer version: 3.3.0
Show newest version
/**
 * Copyright © 2005-2012 Akiban Technologies, Inc.  All rights reserved.
 * 
 * This program and the accompanying materials are made available
 * under the terms of the Eclipse Public License v1.0 which
 * accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 * 
 * This program may also be available under different license terms.
 * For more information, see www.akiban.com or contact [email protected].
 * 
 * Contributors:
 * Akiban Technologies, Inc.
 */

package com.persistit;

import java.io.BufferedOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;

import com.persistit.CLI.Arg;
import com.persistit.CLI.Cmd;
import com.persistit.exception.PersistitException;
import com.persistit.util.Util;

/**
 * 

 * Saves Persistit records to a DataOutputStream in the format expected by a
 * {@link StreamLoader} instance.
 * 

 * 
 * @version 1.0
 */
public class StreamSaver extends Task {

    /**
     * Record type marker for FILL records
     */
    public final static int RECORD_TYPE_FILL = ('A' << 8) + 'Z';
    /**
     * Record type marker for DATA records
     */
    public final static int RECORD_TYPE_DATA = ('D' << 8) + 'R';
    /**
     * Record type marker for KEY_FILTER records
     */
    public final static int RECORD_TYPE_KEY_FILTER = ('K' << 8) + 'F';
    /**
     * Record type marker for VOLUME_ID records
     */
    public final static int RECORD_TYPE_VOLUME_ID = ('V' << 8) + 'I';
    /**
     * Record type marker for TREE_ID records
     */
    public final static int RECORD_TYPE_TREE_ID = ('T' << 8) + 'I';
    /**
     * Record type marker for HOSTNAME records
     */
    public final static int RECORD_TYPE_HOSTNAME = ('H' << 8) + 'N';
    /**
     * Record type marker for USER records
     */
    public final static int RECORD_TYPE_USER = ('H' << 8) + 'U';
    /**
     * Record type marker for COMMENT records
     */
    public final static int RECORD_TYPE_COMMENT = ('C' << 8) + 'O';
    /**
     * Record type marker for FILL records
     */
    public final static int RECORD_TYPE_COUNT = ('R' << 8) + 'C';
    /**
     * Record type marker for START records
     */
    public final static int RECORD_TYPE_START = ('X' << 8) + 'S';
    /**
     * Record type marker for END records
     */
    public final static int RECORD_TYPE_END = ('X' << 8) + 'E';
    /**
     * Record type marker for TIMESTAMP records
     */
    public final static int RECORD_TYPE_TIMESTAMP = ('T' << 8) + 'S';
    /**
     * Record type marker for EXCEPTION records
     */
    public final static int RECORD_TYPE_EXCEPTION = ('E' << 8) + 'X';
    /**
     * Record type marker for COMPLETION records
     */
    public final static int RECORD_TYPE_COMPLETION = ('Z' << 8) + 'Z';
    /**
     * Default count of records to written with elided keys.
     */
    public final static int DEFAULT_CYCLE_COUNT = 1024;
    /**
     * Size of the buffer for BufferedOutputStreams created by the File
     * constructors.
     */
    public final static int DEFAULT_BUFFER_SIZE = 65536;

    protected String _filePath;
    protected DataOutputStream _dos;

    protected Key _lastKey;
    protected Volume _lastVolume;
    protected Tree _lastTree;
    protected long _dataRecordCount = 0;
    protected long _otherRecordCount = 0;
    protected int _cycleCount = DEFAULT_CYCLE_COUNT;
    protected boolean _stop;
    protected Exception _lastException;
    protected int _recordCount;
    protected TreeSelector _treeSelector;

    /**
     * Package-private constructor used by {@link ManagementImpl} to instantiate
     * a {@link Task}. The {@link #setupTask} method must be called to specify
     * trees and the output file name.
     * 
     */
    StreamSaver() {
        _lastKey = new Key((Persistit) null);
    }

    /**
     * Construct a StreamSaver from the provided DataOutputStream. The
     * DataOutputStream should be based on a BufferedOutputStream for better
     * performance.
     * 
     * @param dos
     *            The DataOutputStream
     */
    public StreamSaver(final Persistit persistit, final DataOutputStream dos) {
        super(persistit);
        _lastKey = new Key(persistit);
        _dos = dos;
    }

    /**
     * Construct a StreamSaver from the provided File using a default buffer
     * size of 64K bytes.
     * 
     * @param file
     *            The File to which data will be saved
     * @throws FileNotFoundException
     */
    public StreamSaver(final Persistit persistit, final File file) throws FileNotFoundException {
        this(persistit, new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file), DEFAULT_BUFFER_SIZE)));
    }

    @Cmd("save")
    static StreamSaver createTask(@Arg("file|string:|Save to file") final String file,
            @Arg("trees|string:*|Tree selector - specify Volumes/Trees/Keys to save") final String treeSelectorString,
            @Arg("_flag|v|verbose") final boolean verbose,
            @Arg("_flag|r|Use regular expressions in tree selector") final boolean regex) throws Exception {
        final StreamSaver task = new StreamSaver();
        task._filePath = file;
        task._treeSelector = TreeSelector.parseSelector(treeSelectorString, regex, '\\');
        task.setMessageLogVerbosity(verbose ? LOG_VERBOSE : LOG_NORMAL);
        return task;
    }

    /**
     * Construct a StreamSaver from the provided path name using a default
     * buffer size of 64K bytes.
     * 
     * @param pathName
     *            The path name of the file to which data will be saved
     * @throws FileNotFoundException
     */
    public StreamSaver(final Persistit persistit, final String pathName) throws FileNotFoundException {
        this(persistit, new DataOutputStream(new BufferedOutputStream(new FileOutputStream(pathName),
                DEFAULT_BUFFER_SIZE)));
    }

    /**
     * Construct a StreamSaver from the provided File using a specified buffer
     * size.
     * 
     * @param file
     *            The File to which data will be saved
     * @param bufferSize
     *            The buffer size
     * @throws FileNotFoundException
     */
    public StreamSaver(final Persistit persistit, final File file, final int bufferSize) throws FileNotFoundException {
        this(persistit, new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file), bufferSize)));
    }

    /**
     * Construct a StreamSaver from the provided path name using a specified
     * buffer size.
     * 
     * @param pathName
     *            The path name of the file to which data will be saved
     * @param bufferSize
     *            The buffer size
     * @throws FileNotFoundException
     */
    public StreamSaver(final Persistit persistit, final String pathName, final int bufferSize)
            throws FileNotFoundException {
        this(persistit, new DataOutputStream(new BufferedOutputStream(new FileOutputStream(pathName), bufferSize)));
    }

    /**
     * Returns the cycle count, which is the maximum number of records that will
     * be written with elided keys. When this number of records is reached,
     * StreamSaver writes the next record with a full, unelided key
     * value. This permits easier visual inspection of the save file.
     * 
     * @return The cycle count
     */
    public int getCycleCount() {
        return _cycleCount;
    }

    /**
     * Sets the cycle count. See {@link #getCycleCount} for a description of
     * this property.
     * 
     * @param count
     *            The cycle count
     */
    public void setCycleCount(final int count) {
        _cycleCount = count;
    }

    /**
     * Closes this StreamSaver and the underlying DataOutputStream. If the save
     * operation ran to completion without error, this method writes a
     * COMPLETION record to the save file before closing it. This record
     * indicates that the save file represents all the records requested by the
     * save operation.
     * 
     * @throws IOException
     */
    public void close() throws IOException {
        writeTimestamp();
        if (!_stop && _lastException == null)
            _dos.writeChar(RECORD_TYPE_COMPLETION);
        _lastTree = null;
        _lastVolume = null;
        _lastKey.clear();
        _dos.close();
    }

    /**
     * Writes the key/value pair represented by the current state of an
     * Exchange into a DATA record. If the supplied
     * Exchange is based on different volume or tree than the
     * previously written record, this method also writes VOLUME_ID and/or
     * TREE_ID records before the DATA record. This allows the
     * StreamLoader to apply the data record to the correct volume
     * and tree.
     * 
     * @param exchange
     *            The Exchange
     * @throws IOException
     */
    protected void writeData(final Exchange exchange) throws IOException {
        if (_lastVolume != exchange.getVolume()) {
            writeVolumeInfo(exchange);
        }

        if (_lastTree != exchange.getTree()) {
            writeTreeInfo(exchange);
        }

        writeData(exchange.getKey(), exchange.getValue());
        _recordCount++;
        if ((_recordCount % 100) == 0)
            poll();
    }

    /**
     * Writes a key/value pair into a DATA record.
     * 
     * @param key
     *            The Key
     * @param value
     *            The Value
     * @throws IOException
     */
    protected void writeData(final Key key, final Value value) throws IOException {
        final int elisionCount = key.firstUniqueByteIndex(_lastKey);
        _dos.writeChar(RECORD_TYPE_DATA);
        _dos.writeShort(key.getEncodedSize());
        _dos.writeShort(elisionCount);
        _dos.writeInt(value.getEncodedSize());
        _dos.write(key.getEncodedBytes(), elisionCount, key.getEncodedSize() - elisionCount);
        _dos.write(value.getEncodedBytes(), 0, value.getEncodedSize());

        _dataRecordCount++;
        key.copyTo(_lastKey);

        if (_cycleCount != 0 & (_dataRecordCount % _cycleCount) == 0) {
            writeRecordCount(_dataRecordCount, _otherRecordCount);
            _lastKey.clear();
        }
    }

    /**
     * Writes a RECORD_COUNT record. This record conveys the counts of data
     * records and non-data records written to the stream so far, and can be
     * used to check the integrity of the save file. The RECORD_COUNT record is
     * preceded by three FILL records to allow for easier inspection of the save
     * file.
     * 
     * @throws IOException
     */
    protected void writeRecordCount(final long dataRecordCount, final long otherRecordCount) throws IOException {
        _dos.writeChar(RECORD_TYPE_FILL);
        _dos.writeChar(RECORD_TYPE_FILL);
        _dos.writeChar(RECORD_TYPE_FILL);
        _dos.writeChar(RECORD_TYPE_COUNT);
        _dos.writeLong(dataRecordCount);
        _dos.writeLong(otherRecordCount);
        _otherRecordCount++;
    }

    /**
     * Writes a VOLUME_ID record for the Volume currently
     * associated with the supplied Exchange. The information is
     * sufficient to recreate a new, empty Volume having the same
     * path name, original size and growth parameters as the Volume
     * being saved.
     * 
     * @param exchange
     *            The Exchange
     * @throws IOException
     */
    protected void writeVolumeInfo(final Exchange exchange) throws IOException {
        writeVolumeInfo(exchange.getVolume());
        _lastVolume = exchange.getVolume();
    }

    /**
     * Writes a TREE_ID record for the Tree currently associated
     * with the supplied Exchange. The information is sufficient to
     * recreate a new, empty Tree having the same name as the
     * Tree being saved.
     * 
     * @param exchange
     * @throws IOException
     */
    protected void writeTreeInfo(final Exchange exchange) throws IOException {
        writeTreeInfo(exchange.getTree());
        _lastTree = exchange.getTree();
    }

    /**
     * Writes a VOLUME_ID record for the supplied Volume. The saved
     * information is sufficient to recreate a new, empty Volume
     * having the same path name, original size and growth parameters as the
     * Volume being saved.
     * 
     * @param volume
     *            The Volume
     * @throws IOException
     */
    protected void writeVolumeInfo(final Volume volume) throws IOException {
        _dos.writeChar(RECORD_TYPE_VOLUME_ID);
        _dos.writeLong(volume.getId());
        _dos.writeLong(volume.getSpecification().getInitialPages());
        _dos.writeLong(volume.getSpecification().getExtensionPages());
        _dos.writeLong(volume.getSpecification().getMaximumPages());
        _dos.writeInt(volume.getStructure().getPageSize());
        _dos.writeUTF(volume.getPath());
        _dos.writeUTF(volume.getName());
        _lastVolume = volume;
        _lastTree = null;
        _otherRecordCount++;
    }

    /**
     * Writes a TREE_ID record for the supplied Tree. The saved
     * information is sufficient to recreate a new, empty Tree
     * having the same name as the Tree being saved.
     * 
     * @param tree
     *            The Tree
     * @throws IOException
     */
    protected void writeTreeInfo(final Tree tree) throws IOException {
        _dos.writeChar(RECORD_TYPE_TREE_ID);
        _dos.writeUTF(tree.getName());
        _lastTree = tree;
        _otherRecordCount++;
    }

    /**
     * Writes a TIMESTAMP record containing the current system time.
     * 
     * @throws IOException
     */
    protected void writeTimestamp() throws IOException {
        _dos.writeChar(RECORD_TYPE_TIMESTAMP);
        _dos.writeLong(System.currentTimeMillis());
    }

    /**
     * Writes a COMMENT record containing an arbitrary string
     * 
     * @param comment
     *            The comment string
     * @throws IOException
     */
    protected void writeComment(final String comment) throws IOException {
        _dos.writeChar(RECORD_TYPE_COMMENT);
        _dos.writeUTF(Util.NEW_LINE + "//" + comment + "//");
    }

    /**
     * Writes an EXCEPTION record, indicating an Exception that occurred during
     * the save process.
     * 
     * @param exception
     *            The Exception
     * @throws IOException
     */
    protected void writeException(final Exception exception) throws IOException {
        _dos.writeChar(RECORD_TYPE_EXCEPTION);
        _dos.writeUTF(exception.toString());
    }

    /**
     * Save all key/value pairs in the Tree associated with the
     * supplied Exchange, subject to selection by the supplied
     * KeyFilter. If the filter is null
     * then save all records.
     * 
     * @param exchange
     *            The Exchange
     * @param filter
     *            The KeyFilter
     * @throws PersistitException
     * @throws IOException
     */
    public void save(final Exchange exchange, final KeyFilter filter) throws PersistitException, IOException {
        postMessage("Saving Tree " + exchange.getTree().getName() + " in volume " + exchange.getVolume().getPath()
                + (filter == null ? "" : " using KeyFilter: " + filter.toString()), LOG_VERBOSE);

        writeTimestamp();
        _dos.writeChar(RECORD_TYPE_START);
        if (filter != null) {
            _dos.writeChar(RECORD_TYPE_KEY_FILTER);
            _dos.writeUTF(filter.toString());
        }
        final Key key = exchange.getKey();
        key.clear().append(Key.BEFORE);
        while (exchange.traverse(Key.GT, filter, Integer.MAX_VALUE) & !_stop) {
            writeData(exchange);
        }
        writeRecordCount(_dataRecordCount, _otherRecordCount);
        _dos.writeChar(RECORD_TYPE_END);
        writeTimestamp();
        poll();
    }

    /**
     * Saves one or more trees in a named volume. The volume is specified by
     * volumeName, which may either be the full path name of an
     * open volume, or a substring that uniquely matches one open volume.
     * 
     * @param volumeName
     *            The volume name, or a substring that matches only one volume.
     * @param selectedTreeNames
     *            An array names of the trees to be saved.
     * @throws PersistitException
     * @throws IOException
     */
    public void saveTrees(final String volumeName, final String[] selectedTreeNames) throws PersistitException,
            IOException {
        final Volume volume = _persistit.getVolume(volumeName);
        if (volume != null)
            saveTrees(volume, selectedTreeNames);
    }

    /**
     * Saves on or more trees in the specified Volume.
     * 
     * @param volume
     *            The Volume
     * @param selectedTreeNames
     *            An array names of the trees to be saved.
     * @throws PersistitException
     * @throws IOException
     */
    public void saveTrees(final Volume volume, final String[] selectedTreeNames) throws PersistitException, IOException {
        final String[] treeNames = volume.getTreeNames();
        writeComment("Volume " + volume.getPath());
        for (int index = 0; index < treeNames.length & !_stop; index++) {
            boolean selected = true;
            if (selectedTreeNames != null) {
                for (int index2 = 0; selected && index2 < selectedTreeNames.length; index2++) {
                    if (!selectedTreeNames[index2].equals(treeNames[index])) {
                        selected = false;
                    }
                }
            }
            if (!selected) {
                writeComment("Tree " + treeNames[index] + " not selected");
            } else {
                writeComment("Tree " + treeNames[index]);
                try {
                    final Exchange exchange = _persistit.getExchange(volume, treeNames[index], false);
                    save(exchange, null);
                } catch (final PersistitException exception) {
                    _lastException = exception;
                    writeException(exception);
                }
            }
        }
    }

    /**
     * Saves one or more trees.
     * 
     * @param treeSelector
     *            The TreeSelectors to select volumes, trees, and
     *            KeyFilters within trees.
     * @throws PersistitException
     * @throws IOException
     */
    public void saveTrees(final TreeSelector treeSelector) throws PersistitException, IOException {
        final List trees = _persistit.getSelectedTrees(treeSelector);
        for (final Tree tree : trees) {
            if (tree.getVolume().getDirectoryTree() == tree) {
                for (final String treeName : tree.getVolume().getTreeNames()) {
                    final Tree t = tree.getVolume().getTree(treeName, false);
                    try {
                        writeComment("Tree " + treeName + " in " + tree.getVolume().getPath());
                        final Exchange exchange = new Exchange(t);
                        save(exchange, null);
                    } catch (final PersistitException exception) {
                        _lastException = exception;
                        writeException(exception);
                    }
                }
            } else {
                try {
                    writeComment("Tree " + tree.getName() + " in " + tree.getVolume().getPath());
                    final Exchange exchange = new Exchange(tree);
                    save(exchange, treeSelector.keyFilter(tree.getVolume().getName(), tree.getName()));
                } catch (final PersistitException exception) {
                    _lastException = exception;
                    writeException(exception);
                }
            }
        }
    }

    /**
     * Save all trees in all open volumes.
     * 
     * @throws PersistitException
     * @throws IOException
     */
    public void saveAll() throws PersistitException, IOException {

        for (final Volume volume : _persistit.getVolumes()) {
            if (_stop) {
                break;
            }
            saveTrees(volume, null);
        }
    }

    @Override
    protected void runTask() throws PersistitException, IOException {
        _dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(_filePath), DEFAULT_BUFFER_SIZE));
        saveTrees(_treeSelector);
        close();
    }

    @Override
    public String getStatus() {
        if (_lastTree == null) {
            return null;
        } else {
            return "Saving " + _lastTree.getName() + " in " + _lastTree.getVolume().getPath() + " (" + _recordCount
                    + ")";
        }
    }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy