Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.page;
import org.apache.flink.runtime.state.gemini.engine.GRegionContext;
import org.apache.flink.runtime.state.gemini.engine.filecache.FileCache;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GBinaryHashMap;
import org.apache.flink.runtime.state.gemini.engine.rm.GByteBuffer;
import org.apache.flink.runtime.state.gemini.engine.snapshot.RegionSnapshot;
import org.apache.flink.runtime.state.gemini.engine.snapshot.SnapshotMetaFile;
import org.apache.flink.shaded.guava18.com.google.common.base.MoreObjects;
import org.apache.flink.shaded.netty4.io.netty.util.concurrent.EventExecutor;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import static org.apache.flink.runtime.state.gemini.engine.page.PageConstants.NO_PAGE;
import static org.apache.flink.runtime.state.gemini.engine.snapshot.SnapshotMetaFile.writerFunc;
/**
* The implementation of {@link PageAddress} for normal page.
*/
public class PageAddressSingleImpl implements PageAddress {
private DataPage dataPage;
private volatile long dfsAddress;
private volatile long localAddress;
private final int oriDataLen;
private final int checksum;
/**
* Currently only the 3 low bits are used, and the other 5 bits are reserved.
* 1. bit 0: the status of page. 0 is invalid(discarded), and 1 is valid. The
* semantic of invalid here is that the page is no longer in page index, and
* we can not access it through page index later, but it's not guaranteed that
* no access to it later because someone may reference it before it's discarded
* and has not released the reference after it's discarded.
* 2. bit 1: the status of local address, 0 is invalid, and 1 is valid.
* 3. bit 2: the status of dfs address, 0 is invalid, and 1 is valid.
* The semantic of invalid for local/dfs address is that the data pointed by the
* address exists on some storage. It's the duty of file cache to set the status
* of page address. When the page is flushed successfully, it should be set valid,
* when it was evicted or the file contains the data is deleted, it should be set
* invalid. But actually, we may not explicitly set the status to invalid, at least
* in {@link InfiniteFileCache},
* that's once the status is set valid, it will never been set back to invalid,
* because it's much complicated to find the pages which use the file to be deleted.
* So we need other ways to guarantee the safety:
* 1. if the address status is invalid, do not access the data pointed by the address
* 2. if the status of page and address are both valid, it's safe because we'll not
* release the reference of file and file will not be deleted
* 3. if the status of page is invalid and the address is valid, we use MVCC (access number)
* to delay the deletion of file to ensure the safety.
*/
private volatile byte status;
public PageAddressSingleImpl(DataPage dataPage) {
this.dataPage = dataPage;
this.oriDataLen = dataPage.getSize();
// init status: page is used, local and dfs address are both invalid
this.status = PAGE_VALID_MASK;
this.checksum = dataPage.getCheckSum();
}
public PageAddressSingleImpl(int dataLen, int checksum) {
this.status = PAGE_VALID_MASK;
this.oriDataLen = dataLen;
this.checksum = checksum;
}
@Override
public DataPage getDataPage() {
DataPage tmpDataPage = dataPage;
if (tmpDataPage != null) {
tmpDataPage.retain();
return tmpDataPage;
} else {
return null;
}
}
/**
* Get GByteBuffer with reference added. Note that when the GByteBuffer is freed,
* it will return null.
* Note that main and snapshot thread should not call this method.
* @return the GByteBuffer
*/
@Override
public GByteBuffer getGByteBufferWithReference() {
DataPage tmpDataPage = dataPage;
if (tmpDataPage != null) {
GByteBuffer buffer = tmpDataPage.getGBinaryHashMap().getGByteBuffer();
if (buffer != null) {
buffer.retain();
// Behaves differently between on-heap and off-heap.
if (buffer.isPooled() && buffer.refCnt() == 1) {
buffer.release();
return null;
}
}
return buffer;
} else {
return null;
}
}
@Override
public GByteBuffer getGByteBufferNoReference() {
DataPage tmpDataPage = dataPage;
if (tmpDataPage != null) {
return tmpDataPage.getGBinaryHashMap().getGByteBuffer();
} else {
return null;
}
}
@Override
public DataPage getDataPageNoReference() {
return dataPage;
}
@Override
public boolean hasDataPage() {
return dataPage != null;
}
@Override
public long getDfsAddress() {
return dfsAddress;
}
@Override
public long getLocalAddress() {
return localAddress;
}
@Override
public long getVersion() {
return dataPage != null ? dataPage.getVersion() : NO_PAGE;
}
@Override
public void setDataPage(DataPage dataPage) {
this.dataPage = dataPage;
}
@Override
public void setDfsAddress(long dfsAddress) {
this.dfsAddress = dfsAddress;
}
@Override
public void setLocalAddress(long localAddress) {
this.localAddress = localAddress;
}
@Override
public void setPageStatus(boolean flag) {
byte st = status;
this.status = (byte) (flag ? (st | PAGE_VALID_MASK) : (st & (~PAGE_VALID_MASK)));
}
@Override
public boolean isPageValid() {
return (status & PAGE_VALID_MASK) != 0;
}
@Override
public void setLocalStatus(boolean flag) {
byte st = status;
this.status = (byte) (flag ? (st | LOCAL_VALID_MASK) : (st & (~LOCAL_VALID_MASK)));
}
@Override
public boolean isLocalValid() {
return (status & LOCAL_VALID_MASK) != 0;
}
@Override
public void setDfsStatus(boolean flag) {
byte st = status;
this.status = (byte) (flag ? (st | DFS_VALID_MASK) : (st & (~DFS_VALID_MASK)));
}
@Override
public boolean isDfsValid() {
return (status & DFS_VALID_MASK) != 0;
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this).
add("dataPage", dataPage).
add("dfsAddress", dfsAddress).
add("localAddress", localAddress).
add("oriDataLen", oriDataLen).toString();
}
@Override
public int getDataLen() {
return this.oriDataLen;
}
@Override
public int getMainPageDataLen() {
return this.oriDataLen;
}
@Override
public int getSubPageNum() {
return 0;
}
@Override
public int getSubPageDataLen() {
return 0;
}
@Override
public int getChecksum() {
return checksum;
}
@Override
public void snapshot(Collection regionSnapshots) throws IOException {
int len = getDataLen();
//TODO future support flush-compress.
int checksum = getChecksum();
long localAddress = getLocalAddress();
long dfsAddress = getDfsAddress();
// write data length
writerFunc(regionSnapshots, w -> {
w.writeInt(len);
w.writeInt(checksum);
// write dfs address for both snapshot
w.writeLong(dfsAddress);
});
// for different kinds of region snapshots, we would write different contents.
for (RegionSnapshot regionSnapshot : regionSnapshots) {
if (regionSnapshot.isLocalSnapshot()) {
if (isLocalValid()) {
regionSnapshot.getWriter().writeBoolean(true);
regionSnapshot.getWriter().writeLong(localAddress);
} else {
regionSnapshot.getWriter().writeBoolean(false);
}
regionSnapshot.updateFileMeta(localAddress, 1, len);
} else {
// not write local address for dfs snapshot
regionSnapshot.getWriter().writeBoolean(false);
regionSnapshot.updateFileMeta(dfsAddress, 1, len);
}
}
}
@Override
public byte getPageAddressType() {
return SINGLE_PAGE_ADDRESS;
}
@Override
public void setChainIndex(int chainIndex) {
if (dataPage != null) {
dataPage.setChainIndex(chainIndex);
}
}
/**
* Returns the data page size if this {@link DataPage} is in the memory, otherwise zero.
*
* @return The data page size if this {@link DataPage} is in the memory, otherwise zero.
*/
@Override
public int getMemorySize() {
return dataPage == null ? 0 : oriDataLen;
}
@Override
public void discard(
FileCache fileCache, GRegionContext gRegionContext, EventExecutor eventExecutor) {
fileCache.discardPage(this, gRegionContext, eventExecutor);
//reduce GByteBuffer reference count. when discardPage, if reference is not 0, we will keep looking it.
// make sure it can be removed.
if (dataPage != null) {
gRegionContext.tryToDelHugePage(dataPage);
dataPage.release();
// TODO release useless memory as soon as possible, and
// guarantee data can be read by main thread via local.
// This can reduce memory cost for snapshot
if (isLocalValid()) {
dataPage = null;
}
}
gRegionContext.getGContext().getSupervisor().getBloomFilterManager().removeBloomFilter(this);
}
public static PageAddress restore(
SnapshotMetaFile.Reader reader, PageStoreStats pageStoreStats) throws IOException {
int pageLen = reader.readInt();
int checksum = reader.readInt();
PageAddress pageAddress = new PageAddressSingleImpl(pageLen, checksum);
long dfsAddress = reader.readLong();
pageAddress.setDfsAddress(dfsAddress);
pageAddress.setDfsStatus(true);
if (reader.readBoolean()) {
long localAddress = reader.readLong();
pageAddress.setLocalAddress(localAddress);
pageAddress.setLocalStatus(true);
}
pageStoreStats.addLogicPageSize(pageLen);
return pageAddress;
}
@Override
public GBinaryHashMap toBoxGBinaryHashMap(GBinaryHashMap gBinaryHashMap, GRegionContext gRegionContext, int logicPageChainIndex, int logicPageChainHashCode) {
return gBinaryHashMap;
}
@Override
public Iterator pageIterator() {
return Collections.singleton((PageAddress) this).iterator();
}
@Override
public Iterator pageIteratorOrdered() {
return Collections.singleton((PageAddress) this).iterator();
}
@Override
public void addRequestCountForNewPage(long currentTickTime, int requestCount) {
if (dataPage != null) {
dataPage.addRequestCount(currentTickTime, requestCount);
}
}
@Override
public int getPageNum() {
return 1;
}
}
