com.sleepycat.je.tree.INKeyRep Maven / Gradle / Ivy
/*-
* Copyright (C) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
*
* This file was distributed by Oracle as part of a version of Oracle Berkeley
* DB Java Edition made available at:
*
* http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html
*
* Please see the LICENSE file included in the top-level directory of the
* appropriate version of Oracle Berkeley DB Java Edition for a copy of the
* license and additional information.
*/
package com.sleepycat.je.tree;
import java.util.Arrays;
import java.util.Comparator;
import com.sleepycat.je.dbi.DupKeyData;
import com.sleepycat.je.dbi.MemoryBudget;
import com.sleepycat.je.evictor.Evictor;
import com.sleepycat.je.utilint.SizeofMarker;
/**
* The abstract class that defines the various formats used to represent
* the keys associated with the IN node. The class is also used to store
* embedded records, where the actual key and the data portion of a record are
* stored together as a single byte sequence.
*
* There are currently two supported representations:
*
* - A default representation
Default that's capable of holding
* any set of keys.
* -
* A compact representation
MaxKeySize that's more efficient for
* holding small keys (LTE 16 bytes) in length. If key prefixing is in use this
* represents the unprefixed part of the key, since that's what is stored in
* this array.
*
*
* The choice of representation is made when an IN node is first read in from
* the log. The MaxKeySize representation is only used when it is
* more storage efficient than the default representation for the set of keys
* currently associated with the IN.
*
* Note that no attempt is currently made to optimize the storage
* representation as keys are added to, or removed from, the
* Default representation to minimize the chances of transitionary
* "back and forth" representation changes that could prove to be expensive.
*/
public abstract class INKeyRep
extends INArrayRep {
/* The different representations for keys. */
public enum Type { DEFAULT, MAX_KEY_SIZE };
private static final byte[][] EMPTY_KEYS_ARRAY = new byte[0][];
public INKeyRep() {
}
public abstract int length();
/**
* Returns true if the key bytes mem usage is accounted for internally
* here, or false if each key has a separate byte array and its mem usage
* is accounted for by the parent.
*/
public abstract boolean accountsForKeyByteMemUsage();
public abstract int size(int idx);
public abstract INKeyRep set(int idx, byte[] key, byte[] data, IN parent);
public abstract INKeyRep setData(int idx, byte[] data, IN parent);
public abstract byte[] getData(int idx);
public abstract byte[] getKey(int idx, boolean embeddedData);
public abstract byte[] getFullKey(
byte[] prefix,
int idx,
boolean embeddedData);
public abstract int compareKeys(
byte[] searchKey,
byte[] prefix,
int idx,
boolean embeddedData,
Comparator comparator);
/**
* The default representation that's capable of storing keys of any size.
*/
public static class Default extends INKeyRep {
private final byte[][] keys;
Default(int nodeMaxEntries) {
this.keys = new byte[nodeMaxEntries][];
}
public Default(@SuppressWarnings("unused") SizeofMarker marker) {
keys = EMPTY_KEYS_ARRAY;
}
private Default(byte[][] keys) {
this.keys = keys;
}
@Override
public INKeyRep resize(int capacity) {
return new Default(Arrays.copyOfRange(keys, 0, capacity));
}
@Override
public Type getType() {
return Type.DEFAULT;
}
@Override
public int length() {
return keys.length;
}
@Override
public INKeyRep set(int idx, byte[] key, IN parent) {
keys[idx] = key;
return this;
}
@Override
public INKeyRep set(int idx, byte[] key, byte[] data, IN parent) {
if (data == null || data.length == 0) {
keys[idx] = key;
} else {
keys[idx] = DupKeyData.combine(key, data);
}
return this;
}
@Override
public INKeyRep setData(int idx, byte[] data, IN parent) {
/*
* TODO #21488: optimize this to avoid creation of new combined
* key, when possible.
*/
return set(idx, getKey(idx, true), data, parent);
}
@Override
public int size(int idx) {
return keys[idx].length;
}
@Override
public byte[] get(int idx) {
return keys[idx];
}
@Override
public byte[] getData(int idx) {
assert(keys[idx] != null);
return DupKeyData.getData(keys[idx], 0, keys[idx].length);
}
@Override
public byte[] getKey(int idx, boolean embeddedData) {
byte[] suffix = keys[idx];
if (suffix == null) {
return Key.EMPTY_KEY;
} else if (embeddedData) {
return DupKeyData.getKey(suffix, 0, suffix.length);
} else {
return suffix;
}
}
@Override
public byte[] getFullKey(
byte[] prefix,
int idx,
boolean embeddedData) {
if (prefix == null || prefix.length == 0) {
return getKey(idx, embeddedData);
}
byte[] suffix = keys[idx];
if (suffix == null) {
assert(!embeddedData);
suffix = Key.EMPTY_KEY;
}
int prefixLen = prefix.length;
int suffixLen;
if (embeddedData) {
suffixLen = DupKeyData.getKeyLength(suffix, 0, suffix.length);
} else {
suffixLen = suffix.length;
}
final byte[] key = new byte[prefixLen + suffixLen];
System.arraycopy(prefix, 0, key, 0, prefixLen);
System.arraycopy(suffix, 0, key, prefixLen, suffixLen);
return key;
}
@Override
public int compareKeys(
byte[] searchKey,
byte[] prefix,
int idx,
boolean embeddedData,
Comparator comparator) {
if (comparator != null) {
byte[] myKey = getFullKey(prefix, idx, embeddedData);
return Key.compareKeys(searchKey, myKey, comparator);
}
int cmp = 0;
if (prefix == null || prefix.length == 0) {
return compareSuffixes(
searchKey, 0, searchKey.length, idx, embeddedData);
}
cmp = Key.compareUnsignedBytes(
searchKey, 0, Math.min(searchKey.length, prefix.length),
prefix, 0, prefix.length);
if (cmp == 0) {
int searchKeyOffset = prefix.length;
int searchKeyLen = searchKey.length - prefix.length;
return compareSuffixes(
searchKey, searchKeyOffset, searchKeyLen,
idx, embeddedData);
}
return cmp;
}
private int compareSuffixes(
byte[] searchKey,
int searchKeyOff,
int searchKeyLen,
int idx,
boolean embeddedData) {
byte[] myKey = keys[idx];
int myKeyLen;
if (myKey == null) {
myKey = Key.EMPTY_KEY;
myKeyLen = 0;
} else if (embeddedData) {
myKeyLen = DupKeyData.getKeyLength(myKey, 0, myKey.length);
} else {
myKeyLen = myKey.length;
}
return Key.compareUnsignedBytes(
searchKey, searchKeyOff, searchKeyLen, myKey, 0, myKeyLen);
}
@Override
public INKeyRep copy(int from, int to, int n, IN parent) {
System.arraycopy(keys, from, keys, to, n);
return this;
}
/**
* Evolves to the MaxKeySize representation if that is more efficient
* for the current set of keys. Note that since all the keys must be
* examined to make the decision, there is a reasonable cost to the
* method and it should not be invoked indiscriminately.
*/
@Override
public INKeyRep compact(IN parent) {
if (keys.length > MaxKeySize.MAX_KEYS) {
return this;
}
final int compactMaxKeyLength = parent.getCompactMaxKeyLength();
if (compactMaxKeyLength <= 0) {
return this;
}
int keyCount = 0;
int maxKeyLength = 0;
int defaultKeyBytes = 0;
for (byte[] key : keys) {
if (key != null) {
keyCount++;
if (key.length > maxKeyLength) {
maxKeyLength = key.length;
if (maxKeyLength > compactMaxKeyLength) {
return this;
}
}
defaultKeyBytes += MemoryBudget.byteArraySize(key.length);
}
}
if (keyCount == 0) {
return this;
}
long defaultSizeWithKeys = calculateMemorySize() + defaultKeyBytes;
if (defaultSizeWithKeys >
MaxKeySize.calculateMemorySize(keys.length, maxKeyLength)) {
return compactToMaxKeySizeRep(maxKeyLength, parent);
}
return this;
}
private MaxKeySize compactToMaxKeySizeRep(
int maxKeyLength,
IN parent) {
MaxKeySize newRep =
new MaxKeySize(keys.length, (short) maxKeyLength);
for (int i = 0; i < keys.length; i++) {
INKeyRep rep = newRep.set(i, keys[i], parent);
assert rep == newRep; /* Rep remains unchanged. */
}
noteRepChange(newRep, parent);
return newRep;
}
@Override
public long calculateMemorySize() {
/*
* Assume empty keys array. The memory consumed by the actual keys
* is accounted for by the IN.getEntryInMemorySize() method.
*/
return MemoryBudget.DEFAULT_KEYVALS_OVERHEAD +
MemoryBudget.objectArraySize(keys.length);
}
@Override
public boolean accountsForKeyByteMemUsage() {
return false;
}
@Override
void updateCacheStats(@SuppressWarnings("unused") boolean increment,
@SuppressWarnings("unused") Evictor evictor) {
/* No stats for the default representation. */
}
}
/**
* The compact representation that can be used to represent keys LTE 16
* bytes in length. The keys are all represented inside a single byte array
* instead of having one byte array per key. Within the array, all keys are
* assigned a storage size equal to that taken up by the longest key, plus
* one byte to hold the actual key length. This makes key retreival fast.
* However, insertion and deletion for larger keys moves bytes proportional
* to the storage length of the keys. This is why the representation is
* restricted to keys LTE 16 bytes in size.
*
* On a 32 bit VM the per key overhead for the Default representation is 4
* bytes for the pointer + 16 bytes for each byte array key object, for a
* total of 20 bytes/key. On a 64 bit machine the overheads are much
* larger: 8 bytes for the pointer plus 24 bytes per key.
*
* The more fully populated the IN the more the savings with this
* representation since the single byte array is sized to hold all the keys
* regardless of the actual number of keys that are present.
*
* It's worth noting that the storage savings here are realized in addition
* to the storage benefits of key prefixing, since the keys stored in the
* key array are the smaller key values after the prefix has been stripped,
* reducing the length of the key and making it more likely that it's small
* enough for this specialized representation.
*/
public static class MaxKeySize extends INKeyRep {
private static final int LENGTH_BYTES = 1;
public static final int MAX_KEYS = 256;
private static final byte NULL_KEY = Byte.MAX_VALUE;
/*
* The array is sized to hold all the keys associated with the IN node.
* Each key is allocated a fixed amount of storage equal to the maximum
* length of all the keys in the IN node + 1 byte to hold the size of
* each key. The length is biased, by -128. That is, a zero length
* key is represented by -128, a 1 byte key by -127, etc.
*/
private final byte[] keys;
/*
* The number of bytes used to store each key ==
* DEFAULT_MAX_KEY_LENGTH (16) + LENGTH_BYTES (1)
*/
private final short fixedKeyLen;
public MaxKeySize(int nodeMaxEntries, short maxKeyLen) {
assert maxKeyLen < 255;
this.fixedKeyLen = (short) (maxKeyLen + LENGTH_BYTES);
this.keys = new byte[fixedKeyLen * nodeMaxEntries];
for (int i = 0; i < nodeMaxEntries; i++) {
INKeyRep rep = set(i, null, null);
assert rep == this; /* Rep remains unchanged. */
}
}
/* Only for use by Sizeof */
public MaxKeySize(@SuppressWarnings("unused") SizeofMarker marker) {
keys = null;
fixedKeyLen = 0;
}
private MaxKeySize(byte[] keys, short fixedKeyLen) {
this.keys = keys;
this.fixedKeyLen = fixedKeyLen;
}
@Override
public INKeyRep resize(int capacity) {
return new MaxKeySize(
Arrays.copyOfRange(keys, 0, capacity * fixedKeyLen),
fixedKeyLen);
}
@Override
public Type getType() {
return Type.MAX_KEY_SIZE;
}
@Override
public int length() {
return keys.length / fixedKeyLen;
}
@Override
public INKeyRep set(int idx, byte[] key, IN parent) {
int slotOff = idx * fixedKeyLen;
if (key == null) {
keys[slotOff] = NULL_KEY;
return this;
}
if (key.length >= fixedKeyLen) {
Default newRep = expandToDefaultRep(parent);
return newRep.set(idx, key, parent);
}
keys[slotOff] = (byte) (key.length + Byte.MIN_VALUE);
slotOff += LENGTH_BYTES;
System.arraycopy(key, 0, keys, slotOff, key.length);
return this;
}
@Override
public INKeyRep set(int idx, byte[] key, byte[] data, IN parent) {
if (data == null || data.length == 0) {
return set(idx, key, parent);
}
byte[] twoPartKey = DupKeyData.combine(key, data);
return set(idx, twoPartKey, parent);
}
@Override
public INKeyRep setData(int idx, byte[] data, IN parent) {
/*
* TODO #21488: optimize this to avoid creation of new combined
* key, when possible.
*/
return set(idx, getKey(idx, true), data, parent);
}
private Default expandToDefaultRep(IN parent) {
final int capacity = length();
final Default newRep = new Default(capacity);
for (int i = 0; i < capacity; i++) {
final byte[] k = get(i);
INKeyRep rep = newRep.set(i, k, parent);
assert rep == newRep; /* Rep remains unchanged. */
}
noteRepChange(newRep, parent);
return newRep;
}
@Override
public int size(int idx) {
int slotOff = idx * fixedKeyLen;
assert keys[slotOff] != NULL_KEY;
return keys[slotOff] - Byte.MIN_VALUE;
}
@Override
public byte[] get(int idx) {
int slotOff = idx * fixedKeyLen;
if (keys[slotOff] == NULL_KEY) {
return null;
}
int slotLen = keys[slotOff] - Byte.MIN_VALUE;
slotOff += LENGTH_BYTES;
byte[] info = new byte[slotLen];
System.arraycopy(keys, slotOff, info, 0, slotLen);
return info;
}
@Override
public byte[] getData(int idx) {
int slotOff = idx * fixedKeyLen;
assert(keys[slotOff] != NULL_KEY);
int slotLen = keys[slotOff] - Byte.MIN_VALUE;
slotOff += LENGTH_BYTES;
return DupKeyData.getData(keys, slotOff, slotLen);
}
@Override
public byte[] getKey(int idx, boolean embeddedData) {
int slotOff = idx * fixedKeyLen;
if (keys[slotOff] == NULL_KEY) {
assert(!embeddedData);
return Key.EMPTY_KEY;
}
int slotLen = keys[slotOff] - Byte.MIN_VALUE;
slotOff += LENGTH_BYTES;
if (embeddedData) {
return DupKeyData.getKey(keys, slotOff, slotLen);
}
byte[] key = new byte[slotLen];
System.arraycopy(keys, slotOff, key, 0, slotLen);
return key;
}
@Override
public byte[] getFullKey(
byte[] prefix,
int idx,
boolean embeddedData) {
if (prefix == null || prefix.length == 0) {
return getKey(idx, embeddedData);
}
int slotOff = idx * fixedKeyLen;
if (keys[slotOff] == NULL_KEY) {
assert(!embeddedData);
return prefix;
}
int slotLen = keys[slotOff] - Byte.MIN_VALUE;
slotOff += LENGTH_BYTES;
int prefixLen = prefix.length;
int suffixLen;
if (embeddedData) {
suffixLen = DupKeyData.getKeyLength(keys, slotOff, slotLen);
} else {
suffixLen = slotLen;
}
byte[] key = new byte[suffixLen + prefixLen];
System.arraycopy(prefix, 0, key, 0, prefixLen);
System.arraycopy(keys, slotOff, key, prefixLen, suffixLen);
return key;
}
@Override
public int compareKeys(
byte[] searchKey,
byte[] prefix,
int idx,
boolean embeddedData,
Comparator comparator) {
if (comparator != null) {
byte[] myKey = getFullKey(prefix, idx, embeddedData);
return Key.compareKeys(searchKey, myKey, comparator);
}
int cmp = 0;
if (prefix == null || prefix.length == 0) {
return compareSuffixes(
searchKey, 0, searchKey.length, idx, embeddedData);
}
cmp = Key.compareUnsignedBytes(
searchKey, 0, Math.min(searchKey.length, prefix.length),
prefix, 0, prefix.length);
if (cmp == 0) {
int searchKeyOff = prefix.length;
int searchKeyLen = searchKey.length - prefix.length;
return compareSuffixes(
searchKey, searchKeyOff, searchKeyLen,
idx, embeddedData);
}
return cmp;
}
private int compareSuffixes(
byte[] searchKey,
int searchKeyOff,
int searchKeyLen,
int idx,
boolean embeddedData) {
int myKeyOff = idx * fixedKeyLen;
int myKeyLen = 0;
if (keys[myKeyOff] != NULL_KEY) {
myKeyLen = keys[myKeyOff] - Byte.MIN_VALUE;
myKeyOff += LENGTH_BYTES;
if (embeddedData) {
myKeyLen = DupKeyData.getKeyLength(
keys, myKeyOff, myKeyLen);
}
} else {
assert(!embeddedData);
myKeyOff += LENGTH_BYTES;
}
return Key.compareUnsignedBytes(
searchKey, searchKeyOff, searchKeyLen,
keys, myKeyOff, myKeyLen);
}
@Override
public INKeyRep copy(int from, int to, int n, IN parent) {
System.arraycopy(keys, (from * fixedKeyLen),
keys, (to * fixedKeyLen),
n * fixedKeyLen);
return this;
}
@Override
public INKeyRep compact(@SuppressWarnings("unused") IN parent) {
/* It's as compact as it gets. */
return this;
}
@Override
public long calculateMemorySize() {
return MemoryBudget.MAX_KEY_SIZE_KEYVALS_OVERHEAD +
MemoryBudget.byteArraySize(keys.length);
}
private static long calculateMemorySize(int maxKeys, int maxKeySize) {
return MemoryBudget.MAX_KEY_SIZE_KEYVALS_OVERHEAD +
MemoryBudget.byteArraySize(maxKeys *
(maxKeySize + LENGTH_BYTES));
}
@Override
public boolean accountsForKeyByteMemUsage() {
return true;
}
@Override
void updateCacheStats(boolean increment, Evictor evictor) {
if (increment) {
evictor.getNINCompactKey().incrementAndGet();
} else {
evictor.getNINCompactKey().decrementAndGet();
}
}
}
}