org.apache.flink.runtime.state.gemini.engine.page.LocalDataPageUtil Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.page;
import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiRuntimeException;
import org.apache.flink.runtime.state.gemini.engine.fs.FileReader;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.ByteBufferUtils;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GHashHeaderImpl;
import org.apache.flink.runtime.state.gemini.engine.page.compress.CompressorCodec;
import org.apache.flink.runtime.state.gemini.engine.page.compress.GCompressAlgorithm;
import org.apache.flink.runtime.state.gemini.engine.page.compress.GCompressHeaderHelper;
import org.apache.flink.runtime.state.gemini.engine.rm.Allocator;
import org.apache.flink.runtime.state.gemini.engine.rm.GByteBuffer;
import org.apache.flink.runtime.state.gemini.engine.rm.UnpoolAllocatorImpl;
import java.nio.ByteBuffer;
import java.util.zip.CRC32;
/**
* DataPageUtil. be responsible for operator of logical page.
*/
public class LocalDataPageUtil extends AbstractDataPageUtil {
private final Allocator allocator;
public LocalDataPageUtil() {
this(new UnpoolAllocatorImpl(), true);
}
public LocalDataPageUtil(Allocator allocator, boolean enableChecksum) {
super(enableChecksum);
this.allocator = allocator;
}
@Override
public GByteBuffer getDataPageFromReader(FileReader reader, int offsetInFile, PageAddress pageAddress) {
GByteBuffer headerGByteBuffer = null;
GByteBuffer dataGByteBuffer = null;
try {
//TODO:#Compress Currently local page compress do not support Snappy.
// because snappy can not support (de)compress on bytebuffer.
int firstReadLength = GCompressHeaderHelper.LENGTH;
// Here we'll always read 16 bytes(Compress header's length),
// we would never encounter an EOFException because page header's length(64) will always bigger than compress header's length(16).
headerGByteBuffer = allocator.allocate(firstReadLength);
ByteBuffer headerBuffer = headerGByteBuffer.getByteBuffer();
reader.readByteBuffer(offsetInFile, headerBuffer, GCompressHeaderHelper.LENGTH);
dataGByteBuffer = allocator.allocate(pageAddress.getDataLen());
if (GCompressHeaderHelper.isPageCompressed(headerBuffer)) {
GCompressHeaderHelper.checkMagicNumber(headerBuffer);
GCompressAlgorithm compressAlgorithm = GCompressHeaderHelper.readCompressAlgorithm(headerBuffer);
CompressorCodec compressorCodec = compressAlgorithm.getCompressorCodec();
int diskLength = GCompressHeaderHelper.readDiskLength(headerBuffer);
int pageHeaderLengthOnDisk = GHashHeaderImpl.HEADER_LENGTH;
//get reused BB from compression. because compressorCodec is threadLocal.
ByteBuffer diskDataBB = compressorCodec.getReuseByteBuffer(diskLength);
if (diskDataBB.position() != 0) {
throw new GeminiRuntimeException("bb pool position should be 0, now is " + diskDataBB.position());
}
ByteBuffer pageByteBuffer = dataGByteBuffer.getByteBuffer();
pageByteBuffer.position(0);
//read the page header
int pageHeaderStartOffset = offsetInFile + GCompressHeaderHelper.LENGTH;
reader.readByteBuffer(pageHeaderStartOffset, pageByteBuffer, pageHeaderLengthOnDisk);
// read the compressed page data
int pageDataStartOffset = pageHeaderStartOffset + pageHeaderLengthOnDisk;
reader.readByteBuffer(pageDataStartOffset, diskDataBB, diskLength);
pageByteBuffer.position(pageHeaderLengthOnDisk);
compressorCodec.decompress((ByteBuffer) diskDataBB.flip(), pageByteBuffer);
} else {
// If page is not compressed before, we should first copy the bytes(16) we read before, then read the left bytes from file
ByteBuffer pageDataBuffer = dataGByteBuffer.getByteBuffer();
ByteBufferUtils.copyFromBufferToBuffer(headerBuffer, pageDataBuffer, 0, 0, firstReadLength);
pageDataBuffer.position(firstReadLength);
reader.readByteBuffer(offsetInFile + firstReadLength,
pageDataBuffer,
pageAddress.getDataLen() - firstReadLength);
}
dataGByteBuffer.getByteBuffer().position(0);
int crc;
if (enableChecksum) {
CRC32 crc32 = new CRC32();
crc32.update(dataGByteBuffer.getByteBuffer());
crc = (int) crc32.getValue();
if (crc != pageAddress.getChecksum()) {
throw new GeminiRuntimeException("checkSum fail, " + pageAddress + " when reading from file=" + reader.getFileMeta() + " ,expected=" + pageAddress.getChecksum() + " ,now=" + crc);
}
dataGByteBuffer.getByteBuffer().position(0);
}
return dataGByteBuffer;
} catch (Exception e) {
if (dataGByteBuffer != null) {
dataGByteBuffer.release();
}
LOG.error("Local PageAddress:{}", pageAddress + " => " + e + " reader=>" + reader.getFileMeta());
throw new GeminiRuntimeException(e);
} finally {
if (headerGByteBuffer != null) {
headerGByteBuffer.release();
}
}
}
}