Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.page.bmap;
import java.nio.ByteBuffer;
/**
* GBinaryHashMap.
* Header 64 bytes
* hash index depends on which index use, such as LongIndexPage(4 bytes count, 4 bytes indicator), IntegerIndex(2 bytes count, 2 bytes indicator).
* hash code 4 bytes * count
* TTL 8 bytes * count
* 4byte keyIndicator //TODO if key is fixed-len, it will be omitted.
* key set
* 4byte valueIndicator //TODO if value is fixed-len, it will be omitted.
* value set
*/
public enum GHashHeaderImpl implements GHashHeader{
LongIndexPage(Long.BYTES) {
public final int fourBytesBits = 32;
public final long fourBytesMask = 0xFFFFFFFFL;
public final long headerIndexInitialValue = -1;
@Override
public void initIndex(int indexUnitLen, ByteBuffer headerAndIndex, int indexLen) {
for (int i = 0; i < indexLen; i++) {
ByteBufferUtils.putLong(headerAndIndex, HEADER_LENGTH + i * indexUnitLen, headerIndexInitialValue);
}
}
@Override
public long getIndexBySlot(int indexUnitLen, ByteBuffer headerAndIndex, int indexSlot) {
return ByteBufferUtils.toLong(headerAndIndex, HEADER_LENGTH + indexSlot * indexUnitLen);
}
@Override
public void writeIndexBySlot(int indexUnitLen, ByteBuffer headerAndIndexBB, int indexSlot, long newIndexValue) {
ByteBufferUtils.putLong(headerAndIndexBB, HEADER_LENGTH + indexSlot * indexUnitLen, newIndexValue);
}
@Override
public int getCountByIndexValue(long indexValue) {
return (int) ((indexValue >>> fourBytesBits) & fourBytesMask);
}
@Override
public int getSlotByIndexValue(long indexValue) {
return (int) (indexValue & fourBytesMask);
}
@Override
public long getInitialIndexValue() {
return headerIndexInitialValue;
}
@Override
public long getNewIndexValue(long oldIndexValue, int slot) {
if (oldIndexValue == headerIndexInitialValue) {
return (1L << fourBytesBits) | (slot & fourBytesMask);
} else {
return ((((oldIndexValue >>> fourBytesBits) & fourBytesMask) + 1) << fourBytesBits) | (oldIndexValue & fourBytesMask);
}
}
},
IntegerIndexPage(Integer.BYTES) {
public final int twoBytesBits = 16;
public final int towBytesMask = 0xFFFF;
public final int headerIndexInitialValue = -1;
@Override
public void initIndex(int indexUnitLen, ByteBuffer headerAndIndex, int indexLen) {
for (int i = 0; i < indexLen; i++) {
ByteBufferUtils.putInt(headerAndIndex, HEADER_LENGTH + i * indexUnitLen, headerIndexInitialValue);
}
}
@Override
public long getIndexBySlot(int indexUnitLen, ByteBuffer headerAndIndex, int indexSlot) {
return ByteBufferUtils.toInt(headerAndIndex, HEADER_LENGTH + indexSlot * indexUnitLen);
}
@Override
public void writeIndexBySlot(int indexUnitLen, ByteBuffer headerAndIndexBB, int indexSlot, long newIndexValue) {
ByteBufferUtils.putInt(headerAndIndexBB, HEADER_LENGTH + indexSlot * indexUnitLen, (int) newIndexValue);
}
@Override
public int getCountByIndexValue(long indexValue) {
return (int) ((indexValue >>> twoBytesBits) & towBytesMask);
}
@Override
public int getSlotByIndexValue(long indexValue) {
return (int) (indexValue & towBytesMask);
}
@Override
public long getInitialIndexValue() {
return headerIndexInitialValue;
}
@Override
public long getNewIndexValue(long oldIndexValue, int slot) {
if (oldIndexValue == headerIndexInitialValue) {
return (1L << twoBytesBits) | (slot & towBytesMask);
} else {
return ((((oldIndexValue >>> twoBytesBits) & towBytesMask) + 1) << twoBytesBits) | (oldIndexValue & towBytesMask);
}
}
};
private final int indexUnitLen;
GHashHeaderImpl(int indexUnitLen) {
this.indexUnitLen = indexUnitLen;
}
public static GHashHeaderImpl getPageHelper(int indexLen) {
if (indexLen <= 65536) {
return IntegerIndexPage;
} else {
return LongIndexPage;
}
}
public abstract void initIndex(int indexUnitLen, ByteBuffer headerAndIndex, int indexLen);
public abstract long getIndexBySlot(int indexUnitLen, ByteBuffer headerAndIndex, int indexSlot);
public abstract void writeIndexBySlot(
int indexUnitLen, ByteBuffer headerAndIndexBB, int indexSlot, long newIndexValue);
@Override
public abstract int getCountByIndexValue(long indexValue);
@Override
public abstract int getSlotByIndexValue(long indexValue);
@Override
public abstract long getInitialIndexValue();
@Override
public abstract long getNewIndexValue(long oldIndexValue, int slot);
public static final int HEADER_LENGTH = 64;
public static final int VALUE_TYPE_INDICATOR_BITS = 28;
public static final int VALUE_TYPE_INDICATOR_MARK = 0xFFFFFFF;
//TODO PageType combine with PAGE ID to a long.
private static final int HEADER_PAGE_TYPE_OFFSET = 0;
private static final int HEADER_PAGE_ID_START_OFFSET = HEADER_PAGE_TYPE_OFFSET + Byte.BYTES;
private static final int HEADER_TOTAL_KEY_START_OFFSET = HEADER_PAGE_ID_START_OFFSET + Integer.BYTES;
private static final int HEADER_INDEX_COUNT_START_OFFSET = HEADER_TOTAL_KEY_START_OFFSET + Integer.BYTES;
private static final int HEADER_KEY_OFFSET_OFFSET = HEADER_INDEX_COUNT_START_OFFSET + Integer.BYTES;
private static final int HEADER_VALUE_OFFSET_OFFSET = HEADER_KEY_OFFSET_OFFSET + Integer.BYTES;
//TODO, now only LZ4
private static final int HEADER_COMPRESS_CODEC_OFFSET = HEADER_VALUE_OFFSET_OFFSET + Integer.BYTES;
private static final int HEADER_UNCOMPRESS_KEY_P_SIZE_OFFSET = HEADER_COMPRESS_CODEC_OFFSET + Byte.BYTES;
private static final int HEADER_UNCOMPRESS_VALUE_P_SIZE_OFFSET = HEADER_UNCOMPRESS_KEY_P_SIZE_OFFSET + Integer.BYTES;
private static final int HEADER_VERSION_OFFSET = HEADER_UNCOMPRESS_VALUE_P_SIZE_OFFSET + Integer.BYTES; //8 bytes
private static final int HEADER_STAT_COMPACTION_COUNT_OFFSET = HEADER_VERSION_OFFSET + Long.BYTES; //8 bytes
//TODO In Future support Fix Len of key and value.
// private static final int HEADER_KEY_FIXED_LEN_OFFSET = HEADER_UNCOMPRESS_VALUE_P_SIZE_OFFSET + Integer.BYTES;
// private static final int HEADER_VALUE_FIXED_LEN_OFFSET = HEADER_KEY_FIXED_LEN_OFFSET + Integer.BYTES;
@Override
public void initIndex(ByteBuffer headerAndIndex, int indexLen) {
initIndex(this.indexUnitLen, headerAndIndex, indexLen);
}
@Override
public long getIndexBySlot(ByteBuffer headerAndIndex, int indexSlot) {
return getIndexBySlot(this.indexUnitLen, headerAndIndex, indexSlot);
}
@Override
public void writeIndexBySlot(ByteBuffer headerAndIndexBB, int indexSlot, long newIndexValue) {
writeIndexBySlot(this.indexUnitLen, headerAndIndexBB, indexSlot, newIndexValue);
}
@Override
public void writeHashCode(ByteBuffer headerAndIndexBB, int indexLen, int keyCursor, int hashCode) {
ByteBufferUtils.putInt(headerAndIndexBB,
HEADER_LENGTH + indexLen * this.indexUnitLen + keyCursor * Integer.BYTES,
hashCode);
}
@Override
public void writeSeqIDBytSlot(
ByteBuffer headerAndIndexBB, int indexLen, int totalKeys, long seqID, int keyCursor) {
ByteBufferUtils.putLong(headerAndIndexBB,
HEADER_LENGTH + indexLen * this.indexUnitLen + totalKeys * Integer.BYTES + keyCursor * Long.BYTES,
seqID);
}
public static void writeHeadPageType(ByteBuffer headerAndIndexBB, byte pageType) {
ByteBufferUtils.putByte(headerAndIndexBB, HEADER_PAGE_TYPE_OFFSET, pageType);
}
public static void writeHeadPageID(ByteBuffer headerAndIndexBB, int logicPageId) {
ByteBufferUtils.putInt(headerAndIndexBB, HEADER_PAGE_ID_START_OFFSET, logicPageId);
}
public static void writeHeaderTotalKeyCount(ByteBuffer headerAndIndexBB, int totalKeys) {
ByteBufferUtils.putInt(headerAndIndexBB, HEADER_TOTAL_KEY_START_OFFSET, totalKeys);
}
public static void writeHeaderIndexCount(ByteBuffer headerAndIndexBB, int indexLen) {
ByteBufferUtils.putInt(headerAndIndexBB, HEADER_INDEX_COUNT_START_OFFSET, indexLen);
}
public static void writeHeaderKeyOffset(ByteBuffer headerAndIndexBB, int offset) {
ByteBufferUtils.putInt(headerAndIndexBB, HEADER_KEY_OFFSET_OFFSET, offset);
}
public static void writeHeaderValueOffset(ByteBuffer headerAndIndexBB, int offset) {
ByteBufferUtils.putInt(headerAndIndexBB, HEADER_VALUE_OFFSET_OFFSET, offset);
}
public static void writeHeaderVersion(ByteBuffer headerAndIndexBB, long version) {
ByteBufferUtils.putLong(headerAndIndexBB, HEADER_VERSION_OFFSET, version);
}
public static void writeHeaderStatCompactionCount(ByteBuffer headerAndIndexBB, long compactionCount) {
ByteBufferUtils.putLong(headerAndIndexBB, HEADER_STAT_COMPACTION_COUNT_OFFSET, compactionCount);
}
public static void writeHeaderCompressCode(ByteBuffer headerAndIndexBB, byte compressCode) {
ByteBufferUtils.putByte(headerAndIndexBB, HEADER_COMPRESS_CODEC_OFFSET, compressCode);
}
public static void writeHeaderKeyUncompressSize(ByteBuffer headerAndIndexBB, int uncompressSize) {
//-1 no compress
ByteBufferUtils.putInt(headerAndIndexBB, HEADER_UNCOMPRESS_KEY_P_SIZE_OFFSET, uncompressSize);
}
public static void writeHeaderValueUncompressSize(ByteBuffer headerAndIndexBB, int uncompressSize) {
//-1 no compress
ByteBufferUtils.putInt(headerAndIndexBB, HEADER_UNCOMPRESS_VALUE_P_SIZE_OFFSET, uncompressSize);
}
public static byte getHeadPageType(ByteBuffer headerAndIndexBB) {
return ByteBufferUtils.toByte(headerAndIndexBB, HEADER_PAGE_TYPE_OFFSET);
}
public static int getHeadPageID(ByteBuffer headerAndIndexBB) {
return ByteBufferUtils.toInt(headerAndIndexBB, HEADER_PAGE_ID_START_OFFSET);
}
public static int getHeaderTotalKeyCount(ByteBuffer headerAndIndexBB) {
return ByteBufferUtils.toInt(headerAndIndexBB, HEADER_TOTAL_KEY_START_OFFSET);
}
public static int getHeaderIndexCount(ByteBuffer headerAndIndexBB) {
return ByteBufferUtils.toInt(headerAndIndexBB, HEADER_INDEX_COUNT_START_OFFSET);
}
public static int getHeaderKeyOffset(ByteBuffer headerAndIndexBB) {
return ByteBufferUtils.toInt(headerAndIndexBB, HEADER_KEY_OFFSET_OFFSET);
}
public static int getHeaderValueOffset(ByteBuffer headerAndIndexBB) {
return ByteBufferUtils.toInt(headerAndIndexBB, HEADER_VALUE_OFFSET_OFFSET);
}
public static byte getHeaderCompressCode(ByteBuffer headerAndIndexBB) {
return ByteBufferUtils.toByte(headerAndIndexBB, HEADER_COMPRESS_CODEC_OFFSET);
}
public static int getHeaderKeyUncompressSize(ByteBuffer headerAndIndexBB) {
//-1 no compress
return ByteBufferUtils.toInt(headerAndIndexBB, HEADER_UNCOMPRESS_KEY_P_SIZE_OFFSET);
}
public static int getHeaderValueUncompressSize(ByteBuffer headerAndIndexBB) {
//-1 no compress
return ByteBufferUtils.toInt(headerAndIndexBB, HEADER_UNCOMPRESS_VALUE_P_SIZE_OFFSET);
}
public static long getHeaderVersion(ByteBuffer headerAndIndexBB) {
return ByteBufferUtils.toLong(headerAndIndexBB, HEADER_VERSION_OFFSET);
}
public static long getHeaderStatCompactionCount(ByteBuffer headerAndIndexBB) {
return ByteBufferUtils.toLong(headerAndIndexBB, HEADER_STAT_COMPACTION_COUNT_OFFSET);
}
@Override
public int getHashCode(ByteBuffer headerAndIndexBB, int indexLen, int keyCursor) {
return ByteBufferUtils.toInt(headerAndIndexBB,
HEADER_LENGTH + indexLen * this.indexUnitLen + keyCursor * Integer.BYTES);
}
public static int getEndOffsetBySlot(ByteBuffer data, int baseOffset, int keySlot) {
return ByteBufferUtils.toInt(data, baseOffset + keySlot * Integer.BYTES);
}
@Override
public long getSeqIDBytSlot(
ByteBuffer headerAndIndexBB, int indexLen, int totalKeys, int keyCursor) {
return ByteBufferUtils.toLong(headerAndIndexBB,
HEADER_LENGTH + indexLen * this.indexUnitLen + totalKeys * Integer.BYTES + keyCursor * Long.BYTES);
}
@Override
public int getHeaderAndIndexLen(int indexLen, int totalKeys) {
return HEADER_LENGTH + indexLen * this.indexUnitLen + totalKeys * Integer.BYTES + totalKeys * Long.BYTES;
}
}
