org.apache.flink.runtime.state.gemini.engine.page.DfsDataPageUtil Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.state.gemini.engine.page;

import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiRuntimeException;
import org.apache.flink.runtime.state.gemini.engine.fs.FileReader;
import org.apache.flink.runtime.state.gemini.engine.page.DataPage.DataPageType;
import org.apache.flink.runtime.state.gemini.engine.rm.GByteBuffer;
import org.apache.flink.runtime.state.gemini.engine.rm.GUnPooledByteBuffer;

import java.nio.ByteBuffer;
import java.util.zip.CRC32;

/**
 * DataPageUtil. be responsible for operator of logical page.
 */
public class DfsDataPageUtil implements DataPageUtil {
	private final boolean enableChecksum;

	public DfsDataPageUtil(boolean enableChecksum) {
		this.enableChecksum = enableChecksum;
	}

	@Override
	public DataPage getDataPageFromReader(
		PageSerdeFlink pageSerdeFlink, FileReader reader, int offsetInFile, PageAddress pageAddress) {
		try {
			//TODO DFS not support compress now
			byte[] diskData = new byte[pageAddress.getDataLen()];
			reader.read(offsetInFile, diskData, 0, pageAddress.getDataLen());

			int crc = 0;
			if (enableChecksum) {
				CRC32 crc32 = new CRC32();
				crc32.update(diskData);
				crc = (int) crc32.getValue();
				if (crc != pageAddress.getChecksum()) {
					throw new GeminiRuntimeException("checkSum fail, " + pageAddress + " when reading from file=" + reader.getFileMeta() + " ,expected=" + pageAddress.getChecksum() + " ,now=" + crc);
				}
			}

			DataPageType dataPageType = DataPageType.valueOf(diskData[0]);
			GByteBuffer gByteBuffer = new GUnPooledByteBuffer(ByteBuffer.wrap(diskData));

			switch (dataPageType) {
				case KV:
					return DataPageKVImpl.readKVPageFrom(pageSerdeFlink, gByteBuffer, crc);
				case KHashMap:
					return DataPageKMapImpl.readKMapPageFrom((PageSerdeFlink2Key) pageSerdeFlink, gByteBuffer, crc);
				case KSortedMap:
					return DataPageKSortedMapImpl.readKSortedMapPageFrom((PageSerdeFlink2Key) pageSerdeFlink,
						gByteBuffer,
						crc);
				case KList:
					return DataPageKListImpl.readDataPageKListFrom((PageSerdeFlinkListImpl) pageSerdeFlink,
						gByteBuffer,
						crc);
				default:
					throw new GeminiRuntimeException("error dataType:" + dataPageType + " pageAddress:" + pageAddress);

			}
		} catch (Exception e) {
			throw new GeminiRuntimeException(e);
		}
	}
}