org.apache.flink.runtime.state.gemini.engine.page.LocalDataPageUtil Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.page;
import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiRuntimeException;
import org.apache.flink.runtime.state.gemini.engine.fs.FileReader;
import org.apache.flink.runtime.state.gemini.engine.page.DataPage.DataPageType;
import org.apache.flink.runtime.state.gemini.engine.page.compress.CompressorCodec;
import org.apache.flink.runtime.state.gemini.engine.page.compress.GCompressAlgorithm;
import org.apache.flink.runtime.state.gemini.engine.rm.Allocator;
import org.apache.flink.runtime.state.gemini.engine.rm.GByteBuffer;
import org.apache.flink.runtime.state.gemini.engine.rm.UnpoolAllocatorImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.util.zip.CRC32;
/**
* DataPageUtil. be responsible for operator of logical page.
*/
public class LocalDataPageUtil implements DataPageUtil {
private static final Logger LOG = LoggerFactory.getLogger(LocalDataPageUtil.class);
private final Allocator allocator;
private final boolean enableChecksum;
public LocalDataPageUtil() {
this(new UnpoolAllocatorImpl(), true);
}
public LocalDataPageUtil(Allocator allocator, boolean enableChecksum) {
this.enableChecksum = enableChecksum;
this.allocator = allocator;
}
@Override
public DataPage getDataPageFromReader(
PageSerdeFlink pageSerdeFlink, FileReader reader, int offsetInFile, PageAddress pageAddress) {
try {
GCompressAlgorithm compressAlgorithm = pageAddress.getOndiskDataCompressionAlgorithm();
CompressorCodec compressorCodec = compressAlgorithm == null ? null : compressAlgorithm.getCompressorCodec();
GByteBuffer dataGByteBuffer = allocator.allocate(pageAddress.getDataLen());
if (compressorCodec != null) {
//get reused BB from compression. because compressorCodec is threadLocal.
ByteBuffer diskDataBB = compressorCodec.getReuseByteBuffer(pageAddress.getOnDiskDataLen());
if (diskDataBB.position() != 0) {
throw new GeminiRuntimeException("bb pool position should be 0, now is " + diskDataBB.position());
}
reader.readByteBuffer(offsetInFile, diskDataBB, 0, pageAddress.getOnDiskDataLen());
compressorCodec.decompress((ByteBuffer) diskDataBB.flip(), dataGByteBuffer.getByteBuffer());
} else {
reader.readByteBuffer(offsetInFile, dataGByteBuffer.getByteBuffer(), 0, pageAddress.getOnDiskDataLen());
}
dataGByteBuffer.getByteBuffer().position(0);
int crc = 0;
if (enableChecksum) {
CRC32 crc32 = new CRC32();
crc32.update(dataGByteBuffer.getByteBuffer());
crc = (int) crc32.getValue();
if (crc != pageAddress.getChecksum()) {
throw new GeminiRuntimeException("checkSum fail, " + pageAddress + " when reading from file=" + reader.getFileMeta() + " ,expected=" + pageAddress.getChecksum() + " ,now=" + crc);
}
dataGByteBuffer.getByteBuffer().position(0);
}
DataPageType dataPageType = DataPageType.valueOf(dataGByteBuffer.getByteBuffer().get(0));
switch (dataPageType) {
case KV:
return DataPageKVImpl.readKVPageFrom(pageSerdeFlink, dataGByteBuffer, crc);
case KHashMap:
return DataPageKMapImpl.readKMapPageFrom((PageSerdeFlink2Key) pageSerdeFlink, dataGByteBuffer, crc);
case KSortedMap:
return DataPageKSortedMapImpl.readKSortedMapPageFrom((PageSerdeFlink2Key) pageSerdeFlink,
dataGByteBuffer, crc);
case KList:
return DataPageKListImpl.readDataPageKListFrom((PageSerdeFlinkListImpl) pageSerdeFlink,
dataGByteBuffer, crc);
default:
throw new GeminiRuntimeException("error dataType:" + dataPageType + " pageAddress:" + pageAddress);
}
} catch (Exception e) {
LOG.error("PageAddress:{}", pageAddress + " => " + e + " reader=>" + reader.getFileMeta());
throw new GeminiRuntimeException(e);
}
}
}