Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.apache.flink.runtime.state.gemini.engine.page.bmap.GBinaryList Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.page.bmap;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.base.IntSerializer;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiRuntimeException;
import org.apache.flink.runtime.state.gemini.engine.memstore.GSValue;
import org.apache.flink.runtime.state.gemini.engine.page.GValueType;
import org.apache.flink.runtime.state.gemini.engine.rm.Allocator;
import org.apache.flink.runtime.state.gemini.engine.rm.GByteBuffer;
import org.apache.flink.util.Preconditions;
import java.nio.ByteBuffer;
import java.util.AbstractList;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import static org.apache.flink.runtime.state.gemini.engine.page.bmap.GHashHeaderImpl.VALUE_TYPE_INDICATOR_BITS;
import static org.apache.flink.runtime.state.gemini.engine.page.bmap.GHashHeaderImpl.VALUE_TYPE_INDICATOR_MARK;
/**
* GBinaryList.
* NOTES: now no any Meta information or head for this List, so it's can be regard as embedded value in related page.
* count 4 bytes
* TTL 8 bytes * count
* valueIndicator 4 bytes * count //TODO if value is fixed-len, it will be omitted.
* value set
*/
public class GBinaryList extends AbstractList {
public static final GBinaryList EMPTY_G_BINARY_LIST = new GBinaryList(null, null);
private static final int HEADER_LENGTH = 4;
private static final int HEADER_COUNT_OFFSET = 0;
private final ByteBuffer data;
private final TypeSerializer valueTypeSerializer;
public GBinaryList(ByteBuffer data, TypeSerializer valueTypeSerializer) {
this.data = (data != null && data.capacity() == 0) ? null : data;
this.valueTypeSerializer = valueTypeSerializer;
}
@Override
public BinaryValue get(int index) {
int totalValueCount = size();
if (index >= totalValueCount) {
throw new IndexOutOfBoundsException("Index: " + index + ", Size: " + size());
}
BinaryValue binaryValue;
long seqID = getSeqIDBytSlot(data, index);
int baseValueOffset = headerAndIndexLen(totalValueCount);
int valueIndicate = getValueIndicate(data, baseValueOffset, index);
GValueType gValueType = GValueType.valueOf((byte) (valueIndicate >>> VALUE_TYPE_INDICATOR_BITS));
if (gValueType == GValueType.Delete) {
binaryValue = new BinaryValue(null, GValueType.Delete, seqID, -1, -1);
} else {
int endValueOffset = valueIndicate & VALUE_TYPE_INDICATOR_MARK;
int startValueOffset;
if (index == 0) {
startValueOffset = totalValueCount * Integer.BYTES;
} else {
startValueOffset = (getValueIndicate(data,
headerAndIndexLen(totalValueCount),
index - 1)) & VALUE_TYPE_INDICATOR_MARK;
}
binaryValue = new BinaryValue(data,
gValueType,
seqID,
baseValueOffset + startValueOffset,
endValueOffset - startValueOffset);
}
return binaryValue;
}
public int bytesSize() {
return data == null ? 0 : data.capacity();
}
@Override
public int size() {
return data == null ? 0 : ByteBufferUtils.toInt(data, HEADER_COUNT_OFFSET);
}
public ByteBuffer getData() {
return this.data;
}
public byte[] getDataByte() {
Preconditions.checkArgument(data != null, "BUG");
if (data.hasArray()) {
return data.array();
} else {
byte[] result = new byte[data.capacity()];
ByteBufferUtils.copyFromBufferToArray(data, result, 0, 0, data.capacity());
return result;
}
}
protected static int headerAndIndexLen(int valueCount) {
return HEADER_LENGTH + valueCount * Long.BYTES;
}
private static int getValueOffset(int valueCount) {
return HEADER_LENGTH + valueCount * Long.BYTES + valueCount * Integer.BYTES;
}
protected static int getValueIndicate(ByteBuffer data, int valueIndicateOffset, int valueIndex) {
return ByteBufferUtils.toInt(data, valueIndicateOffset + valueIndex * Integer.BYTES);
}
private static long getSeqIDBytSlot(ByteBuffer headerBB, int valueCursor) {
return ByteBufferUtils.toLong(headerBB, HEADER_LENGTH + valueCursor * Long.BYTES);
}
private static void writeSeqIDBytSlot(ByteBuffer headerBB, long seqID, int valueCursor) {
ByteBufferUtils.putLong(headerBB, HEADER_LENGTH + valueCursor * Long.BYTES, seqID);
}
private static void writeHeadCount(ByteBuffer headerBB, int totalValues) {
ByteBufferUtils.putInt(headerBB, HEADER_COUNT_OFFSET, totalValues);
}
public static GBinaryList ofGBinaryList(
List> valueList, TypeSerializer valueSerializer, Allocator allocator) {
try {
int totalValues = valueList.size();
if (totalValues == 0) {
return EMPTY_G_BINARY_LIST;
}
byte[] headerAndIndex = new byte[headerAndIndexLen(totalValues)];
ByteBuffer headerBB = ByteBuffer.wrap(headerAndIndex);
GByteArrayOutputStreamWithPos outputStreamForValue = new GByteArrayOutputStreamWithPos();
DataOutputViewStreamWrapper outputViewForValue = new DataOutputViewStreamWrapper(outputStreamForValue);
int lastValuePosition = totalValues * Integer.BYTES;
Iterator> iterator = valueList.iterator();
int valueCursor = 0;
while (iterator.hasNext()) {
GSValue gsValue = iterator.next();
//write value
if (gsValue.getValueType() != GValueType.Delete) {
outputStreamForValue.setPosition(lastValuePosition);
valueSerializer.serialize(gsValue.getValue(), outputViewForValue);
lastValuePosition = outputStreamForValue.getPosition();
}
//write value indicator
outputStreamForValue.setPosition(valueCursor * Integer.BYTES);
int valueIndicator = (gsValue.getValueType().getCode() << VALUE_TYPE_INDICATOR_BITS) | (lastValuePosition & VALUE_TYPE_INDICATOR_MARK);
IntSerializer.INSTANCE.serialize(valueIndicator, outputViewForValue);
writeSeqIDBytSlot(headerBB, gsValue.getSeqID(), valueCursor);
valueCursor++;
}
outputStreamForValue.setPosition(lastValuePosition);
byte[] valueBytes = outputStreamForValue.getBuf();
//write Header
writeHeadCount(headerBB, totalValues);
GByteBuffer gByteBuffer = allocator.allocate(headerAndIndex.length + lastValuePosition);
ByteBufferUtils.copyFromArrayToBuffer(gByteBuffer.getByteBuffer(), 0, headerAndIndex, 0, headerAndIndex.length);
ByteBufferUtils.copyFromArrayToBuffer(gByteBuffer.getByteBuffer(), headerAndIndex.length, valueBytes, 0, lastValuePosition);
//List is as value, so GByteBuffer is unpool.
return new GBinaryList<>(gByteBuffer.getByteBuffer(), valueSerializer);
} catch (Exception e) {
throw new GeminiRuntimeException("GBinaryHashMap get exception: " + e.getMessage(), e);
}
}
public static GBinaryList mergeGBinaryList(
List> gBinaryListsByOrder, TypeSerializer valueSerializer, Allocator allocator) {
try {
int totalValues = gBinaryListsByOrder.stream().map((value) -> value.size()).reduce(0, (a, b) -> a + b);
if (totalValues == 0) {
return EMPTY_G_BINARY_LIST;
}
byte[] headerAndIndex = new byte[headerAndIndexLen(totalValues)];
ByteBuffer headerBB = ByteBuffer.wrap(headerAndIndex);
GByteArrayOutputStreamWithPos outputStreamForValue = new GByteArrayOutputStreamWithPos();
int lastIndicatorValuePosition = 0;
int lastValuePosition = totalValues * Integer.BYTES;
int lastValuePositionOffsetByMerge = totalValues * Integer.BYTES;
int copyedValueCount = 0;
for (GBinaryList gBinaryList : gBinaryListsByOrder) {
int curValueCount = gBinaryList.size();
if (curValueCount == 0) {
continue;
}
//copy TTL
ByteBufferUtils.copyFromBufferToArray(gBinaryList.data,
headerAndIndex,
HEADER_LENGTH,
HEADER_LENGTH + copyedValueCount * Long.BYTES,
curValueCount * Long.BYTES);
//copy value Indicator
outputStreamForValue.setPosition(lastIndicatorValuePosition);
for (int i = 0; i < curValueCount; i++) {
int valueIndicate = getValueIndicate(gBinaryList.data, headerAndIndexLen(curValueCount), i);
GValueType gValueType = GValueType.valueOf((byte) (valueIndicate >>> VALUE_TYPE_INDICATOR_BITS));
int endValueOffset = valueIndicate & VALUE_TYPE_INDICATOR_MARK;
int startValueOffset;
if (i == 0) {
startValueOffset = curValueCount * Integer.BYTES;
} else {
startValueOffset = (getValueIndicate(gBinaryList.data,
headerAndIndexLen(curValueCount),
i - 1)) & VALUE_TYPE_INDICATOR_MARK;
}
lastValuePositionOffsetByMerge += (endValueOffset - startValueOffset);
int newValueIndicator = (gValueType.getCode() << VALUE_TYPE_INDICATOR_BITS) | (lastValuePositionOffsetByMerge & VALUE_TYPE_INDICATOR_MARK);
outputStreamForValue.writeInt(newValueIndicator);
}
lastIndicatorValuePosition = outputStreamForValue.getPosition();
//copy value
outputStreamForValue.setPosition(lastValuePosition);
outputStreamForValue.write(gBinaryList.data,
getValueOffset(curValueCount),
gBinaryList.bytesSize() - getValueOffset(curValueCount));
lastValuePosition = outputStreamForValue.getPosition();
copyedValueCount += curValueCount;
}
outputStreamForValue.setPosition(lastValuePosition);
byte[] valueBytes = outputStreamForValue.getBuf();
//write Header
writeHeadCount(headerBB, totalValues);
GByteBuffer gByteBuffer = allocator.allocate(headerAndIndex.length + lastValuePosition);
ByteBufferUtils.copyFromArrayToBuffer(gByteBuffer.getByteBuffer(), 0, headerAndIndex, 0, headerAndIndex.length);
ByteBufferUtils.copyFromArrayToBuffer(gByteBuffer.getByteBuffer(), headerAndIndex.length, valueBytes, 0, lastValuePosition);
return new GBinaryList<>(gByteBuffer.getByteBuffer(), valueSerializer);
} catch (Exception e) {
throw new GeminiRuntimeException("GBinaryHashMap get exception: " + e.getMessage(), e);
}
}
public List> toPOJOList() {
List> result = new ArrayList<>();
int totalValueCount = size();
if (totalValueCount <= 0) {
return result;
}
try {
for (int i = 0; i < totalValueCount; i++) {
BinaryValue binaryValue = get(i);
if (binaryValue.getgValueType() == GValueType.Delete) {
result.add(new GSValue<>(null, GValueType.Delete, binaryValue.getSeqID()));
} else {
DataInputView byteBufferDataInputView = new ByteBufferDataInputView(binaryValue.getBb(),
binaryValue.getValueOffset(),
binaryValue.getValueLen());
E value = valueTypeSerializer.deserialize(byteBufferDataInputView);
result.add(new GSValue<>(value, binaryValue.getgValueType(), binaryValue.getSeqID()));
}
}
return result;
} catch (Exception e) {
throw new GeminiRuntimeException("toPOJOList hash Exception: " + e.getMessage(), e);
}
}
}
