org.mapdb.BTreeMap Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of mapdb-nounsafe Show documentation
MapDB provides concurrent Maps, Sets and Queues backed by disk storage or off-heap memory. It is a fast, scalable and easy to use embedded Java database.
The newest version!
/*
 *  Copyright (c) 2012 Jan Kotek
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

/*
 * NOTE: some code (and javadoc) used in this class
 * comes from JSR-166 group with following copyright:
 *
 * Written by Doug Lea with assistance from members of JCP JSR-166
 * Expert Group and released to the public domain, as explained at
 * http://creativecommons.org/licenses/publicdomain
 */

package org.mapdb;


import java.io.*;
import java.util.*;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.locks.LockSupport;


/**
 * 
 * A scalable concurrent {@link ConcurrentNavigableMap} implementation.
 * The map is sorted according to the {@linkplain Comparable natural
 * ordering} of its keys, or by a {@link Comparator} provided at map
 * creation time.
 * 

 *
 * Insertion, removal,
 * update, and access operations safely execute concurrently by
 * multiple threads.  Iterators are weakly consistent, returning
 * elements reflecting the state of the map at some point at or since
 * the creation of the iterator.  They do not throw {@link
 * ConcurrentModificationException}, and may proceed concurrently with
 * other operations. Ascending key ordered views and their iterators
 * are faster than descending ones.
 * 

 *
 * It is possible to obtain consistent iterator by using snapshot()
 * method.
 * 

 *
 * All Map.Entry pairs returned by methods in this class
 * and its views represent snapshots of mappings at the time they were
 * produced. They do not support the Entry.setValue
 * method. (Note however that it is possible to change mappings in the
 * associated map using put, putIfAbsent, or
 * replace, depending on exactly which effect you need.)
 * 

 *
 * This collection has optional size counter. If this is enabled Map size is
 * kept in {@link Atomic.Long} variable. Keeping counter brings considerable
 * overhead on inserts and removals.
 * If the size counter is not enabled the size method is not a constant-time operation.
 * Determining the current number of elements requires a traversal of the elements.
 * 

 *
 * Additionally, the bulk operations putAll, equals, and
 * clear are not guaranteed to be performed
 * atomically. For example, an iterator operating concurrently with a
 * putAll operation might view only some of the added
 * elements. NOTE: there is an optional 
 * 

 *
 * This class and its views and iterators implement all of the
 * optional methods of the {@link Map} and {@link Iterator}
 * interfaces. Like most other concurrent collections, this class does
 * not permit the use of null keys or values because some
 * null return values cannot be reliably distinguished from the absence of
 * elements.
 * 

 *
 * Theoretical design of BTreeMap is based on 1986 paper
 * 
 * Concurrent operations on B∗-trees with overtaking
 * written by Yehoshua Sagiv.
 * More practical aspects of BTreeMap implementation are based on
 * demo application from Thomas Dinsdale-Young.
 * Also more work from Thomas: A Simple Abstraction for Complex Concurrent Indexes
 * 

 *
 * B-Linked-Tree used here does not require locking for read.
 * Updates and inserts locks only one, two or three nodes.
 * 

 *
 * This B-Linked-Tree structure does not support removal well, entry deletion does not collapse tree nodes. Massive
 * deletion causes empty nodes and performance lost. There is workaround in form of compaction process, but it is not
 * implemented yet.
 * 
 *
 * @author Jan Kotek
 * @author some parts by Doug Lea and JSR-166 group
 *
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public class BTreeMap
        extends AbstractMap
        implements ConcurrentNavigableMap,
        Bind.MapWithModificationListener,
        Closeable, Serializable {

    /** recid under which reference to rootRecid is stored */
    protected final long rootRecidRef;

    /** Serializer used to convert keys from/into binary form. */
    protected final BTreeKeySerializer keySerializer;

    /** Serializer used to convert values from/into binary form*/
    protected final Serializer valueSerializer;

    /** Serializer used to convert values inside nodes from/into binary form
     * If maps has external serializer, this is ValRef serializer*/
    protected final Serializer valueNodeSerializer;

    /** holds node level locks*/
    protected final LongConcurrentHashMap nodeLocks = new LongConcurrentHashMap();

    /** maximal node size allowed in this BTree*/
    protected final int maxNodeSize;

    /** DB Engine in which entries are persisted */
    protected final Engine engine;

    /** is this a Map or Set?  if false, entries do not have values, only keys are allowed*/
    protected final boolean hasValues;

    /** store values as part of BTree nodes */
    protected final boolean valsOutsideNodes;

    protected final List leftEdges;


    private final KeySet keySet;

    private final EntrySet entrySet;

    private final Values values = new Values(this);

    private final ConcurrentNavigableMap descendingMap = new DescendingMap(this, null,true, null, false);

    protected final Atomic.Long counter;

    protected final int numberOfNodeMetas;
    /**
     * Indicates if this collection collection was not made by DB by user.
     * If user can not access DB object, we must shutdown Executor and close Engine ourself in close() method.
     */
    protected final boolean closeEngine;


    /* hack used for DB Catalog*/
    protected static SortedMap preinitCatalog(DB db) {

        Long rootRef = db.getEngine().get(Engine.RECID_NAME_CATALOG, Serializer.RECID);
;
        //$DELAY$
        if(rootRef==null){
            if(db.getEngine().isReadOnly())
                return Collections.unmodifiableSortedMap(new TreeMap());

            NodeSerializer rootSerializer = new NodeSerializer(false,BTreeKeySerializer.STRING,
                    db.getDefaultSerializer(), 0);
            BNode root = new LeafNode(BTreeKeySerializer.STRING.emptyKeys(), true,true,false, new Object[]{}, 0);
            rootRef = db.getEngine().put(root, rootSerializer);
            //$DELAY$
            db.getEngine().update(Engine.RECID_NAME_CATALOG,rootRef, Serializer.RECID);
            db.getEngine().commit();
        }
        Serializer valser = db.getDefaultSerializer();
        if(CC.ASSERT && valser == null)
            throw new AssertionError();
        return new BTreeMap(
                db.engine,
                false,
                Engine.RECID_NAME_CATALOG,
                32,
                false,
                0,
                BTreeKeySerializer.STRING,
                valser,
                0
                );
    }



    /** if valsOutsideNodes is true, this class is used instead of values.
     * It contains reference to actual value. It also supports assertions from preventing it to leak outside of Map*/
    protected static final class ValRef{
        /** reference to actual value */
        final long recid;
        public ValRef(long recid) {
            this.recid = recid;
        }

        @Override
        public boolean equals(Object obj) {
            throw new IllegalAccessError();
        }

        @Override
        public int hashCode() {
            throw new IllegalAccessError();
        }

        @Override
        public String toString() {
            return "BTreeMap-ValRef["+recid+"]";
        }
    }

    protected static final Serializer VALREF_SERIALIZER = new Serializer.EightByteSerializer(){

        @Override
        public void serialize(DataOutput out, ValRef value) throws IOException {
            DataIO.packLong(out,value.recid);
        }

        @Override
        public ValRef deserialize(DataInput in, int available) throws IOException {
            return new ValRef(DataIO.unpackLong(in));
        }

        @Override
        public int fixedSize() {
            return -1;
        }

        @Override
        public boolean equals(ValRef a1, ValRef a2) {
            throw new IllegalAccessError();
        }

        @Override
        public int hashCode(ValRef valRef, int seed) {
            throw new IllegalAccessError();
        }

        @Override
        protected ValRef unpack(long l) {
            return new ValRef(l);
        }

        @Override
        protected long pack(ValRef l) {
            return l.recid;
        }

        @Override
        public void valueArraySerialize(DataOutput out, Object vals) throws IOException {
            for(long o:(long[]) vals){
                DataIO.packLong(out, o);
            }
        }

        @Override
        public Object valueArrayDeserialize(DataInput in, int size) throws IOException {
            //PERF six-byte long[]
            long[] ret = new long[size];
            for(int i=0;i BOOLEAN_PACKED = new Serializer.BooleanSer() {

        @Override
        public void valueArraySerialize(DataOutput out, Object vals) throws IOException {
            SerializerBase.writeBooleanArray(out,(boolean[]) vals);
        }

        @Override
        public Object valueArrayDeserialize(DataInput in, int size) throws IOException {
            return SerializerBase.readBooleanArray(size, in);
        }
    };


    /** common interface for BTree node */
    public abstract static class BNode{

        static final int LEFT_MASK = 1;
        static final int RIGHT_MASK = 1<<1;
        static final int TOO_LARGE_MASK = 1<<2;

        final Object keys;
        final byte flags;


        public BNode(Object keys, boolean leftEdge, boolean rightEdge, boolean tooLarge){
            this.keys = keys;

            this.flags = (byte)(
                           (leftEdge?LEFT_MASK:0)|
                           (rightEdge?RIGHT_MASK:0)|
                           (tooLarge?TOO_LARGE_MASK:0)
            );
        }




        final public Object key(BTreeKeySerializer keyser, int pos) {
            if(isLeftEdge()){
                if(pos--==0)
                    return null;
            }

            if(pos==keyser.length(keys) && isRightEdge())
                return null;
            return keyser.getKey(keys,pos);
        }

        final public  int keysLen(BTreeKeySerializer keyser) {
            return keyser.length(keys) + leftEdgeInc() + rightEdgeInc();
        }

        final public boolean isLeftEdge(){
            return (flags&LEFT_MASK)!=0;
        }


        final public boolean isRightEdge(){
            return (flags&RIGHT_MASK)!=0;
        }


        /** @return 1 if is left edge, or 0*/
        final public int leftEdgeInc(){
            return flags&LEFT_MASK;
        }


        /** @return 1 if is right edge, or 0*/
        final public int rightEdgeInc(){
            return (flags&RIGHT_MASK)>>>1;
        }


        final public boolean isTooLarge(){
            return (flags&TOO_LARGE_MASK)!=0;
        }

        public abstract boolean isLeaf();
        public abstract Object val(int pos, Serializer valueSerializer);

        final public Object highKey(BTreeKeySerializer keyser) {
            if(isRightEdge())
                return null;
            return keyser.getKey(keys,keyser.length(keys)-1);
        }


        public abstract Object childArray();
        public abstract long child(int i);

        public abstract long next();

        public final int compare(final BTreeKeySerializer keyser, int pos1, int pos2){
            if(pos1==pos2)
                return 0;
            //$DELAY$
            if(isLeftEdge()){
                //first position is negative infinity, so everything else is bigger
                //first keys is missing in array, so adjust positions
                if(pos1--==0)
                    return -1;
                if(pos2--==0)
                    return 1;
            }
            //$DELAY$
            if(isRightEdge()){
                int keysLen = keyser.length(keys);
                //last position is positive infinity, so everything else is smaller
                if(pos1==keysLen)
                    return 1;
                if(pos2==keysLen)
                    return -1;
            }

            return keyser.compare(keys,pos1,pos2);
        }

        public final int compare(final BTreeKeySerializer keyser, int pos, Object second){
            if(isLeftEdge()) {
                //first position is negative infinity, so everything else is bigger
                //first keys is missing in array, so adjust positions
                if (pos-- == 0)
                    return -1;
            }
            //$DELAY$
            if(isRightEdge() && pos==keyser.length(keys)){
                //last position is positive infinity, so everything else is smaller
                return 1;
            }
            return keyser.compare(keys,pos,second);
        }


        public void checkStructure(BTreeKeySerializer keyser, Serializer valser){
            //check all keys are sorted;
            if(keyser==null)
                return;

            int keylen = keyser.length(keys);
            int end = keylen-2+rightEdgeInc();
            if(end>1){
                for(int i = 1;i<=end;i++){
                    if(keyser.compare(keys,i-1, i)>=0)
                        throw new DBException.DataCorruption("keys are not sorted: "+Arrays.toString(keyser.keysToArray(keys)));
                }
            }
            //check last key is sorted or null
            if(!isRightEdge() && keylen>2){
                if(keyser.compare(keys,keylen-2, keylen-1)>0){
                    throw new DBException.DataCorruption("Last key is not sorted: "+Arrays.toString(keyser.keysToArray(keys)));
                }
            }
        }

        public abstract BNode copyAddKey(BTreeKeySerializer keyser, Serializer valser, int pos, Object newKey, long newChild, Object newValue);

        public abstract BNode copySplitRight(BTreeKeySerializer keyser, Serializer valser, int splitPos);

        public abstract BNode copySplitLeft(BTreeKeySerializer keyser, Serializer valser, int splitPos, long newNext);

        public abstract int valSize(Serializer valueSerializer);

        public abstract int childArrayLength();

        public int childIndexOf(long child){
            for(int i=0;i0){
                    throw new AssertionError();
                }
            }

            return new LeafNode(keys2, isLeftEdge(), false, false, vals2, newNext);
        }

        @Override
        public int valSize(Serializer valueSerializer) {
            return valueSerializer.valueArraySize(vals);
        }

        @Override
        public int childArrayLength() {
            return -1;
        }

        public LeafNode copyChangeValue(Serializer valser, int pos, Object value) {
            Object vals2 = valser.valueArrayUpdateVal(vals, pos - 1, value);
            //$DELAY$
            return new LeafNode(keys, isLeftEdge(), isRightEdge(), false, vals2, next);
        }

        public LeafNode copyRemoveKey(BTreeKeySerializer keyser, Serializer valser, int pos) {
            int keyPos = pos -leftEdgeInc();
            Object keys2 = keyser.deleteKey(keys,keyPos);
            //$DELAY$
            Object vals2 = valser.valueArrayDeleteValue(vals,pos);
            //$DELAY$
            return new LeafNode(keys2, isLeftEdge(), isRightEdge(), false, vals2, next);
        }

        public LeafNode copyClear(BTreeKeySerializer keyser, Serializer valser) {
            Object[] keys2 = new Object[2-leftEdgeInc()-rightEdgeInc()];
            if(!isLeftEdge())
                keys2[0] = key(keyser,0);
            //$DELAY$
            if(!isRightEdge())
                keys2[1-leftEdgeInc()] = highKey(keyser);
            //$DELAY$
            return new LeafNode (keyser.arrayToKeys(keys2), isLeftEdge(), isRightEdge(), false, valser.valueArrayEmpty(), next);
        }
    }


    protected final Serializer nodeSerializer;

    protected static final class NodeSerializer extends Serializer{

        protected static final int LEAF_MASK = 1<<15;
        protected static final int LEFT_SHIFT = 14;
        protected static final int LEFT_MASK = 1<< LEFT_SHIFT;
        protected static final int RIGHT_SHIFT = 13;
        protected static final int RIGHT_MASK = 1<< RIGHT_SHIFT;
        protected static final int SIZE_MASK = RIGHT_MASK - 1;


        protected final boolean hasValues;
        protected final boolean valsOutsideNodes;
        protected final BTreeKeySerializer keySerializer;
        protected final Serializer