Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.page;
import org.apache.flink.runtime.state.gemini.engine.GRegionContext;
import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiRuntimeException;
import org.apache.flink.runtime.state.gemini.engine.filecache.FileCache;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GBinaryHashMap;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GBinarySplitHashMap;
import org.apache.flink.runtime.state.gemini.engine.page.bmap.GBufferAddressMapping;
import org.apache.flink.runtime.state.gemini.engine.rm.GByteBuffer;
import org.apache.flink.runtime.state.gemini.engine.snapshot.RegionSnapshot;
import org.apache.flink.runtime.state.gemini.engine.snapshot.SnapshotMetaFile;
import org.apache.flink.shaded.guava18.com.google.common.base.MoreObjects;
import org.apache.flink.shaded.guava18.com.google.common.base.Preconditions;
import org.apache.flink.shaded.netty4.io.netty.util.concurrent.EventExecutor;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.NoSuchElementException;
import static org.apache.flink.runtime.state.gemini.engine.snapshot.SnapshotMetaFile.writerFunc;
/**
* The implementation of {@link PageAddress} for the pages which exist map split.
*/
public class PageAddressCompositeImpl implements PageAddress {
private final int totalDataLen;
private final PageAddress mainPageAddress;
private final PageAddress[] subPageAddress;
public PageAddressCompositeImpl(DataPage dataPage) {
mainPageAddress = new PageAddressSingleImpl(dataPage);
Preconditions.checkArgument(dataPage.getGBinaryHashMap() instanceof GBinarySplitHashMap);
subPageAddress = ((GBinarySplitHashMap) dataPage.getGBinaryHashMap()).getBufferAddressMapping().getSubPageAddress();
this.totalDataLen = calcTotalDataLen();
}
public PageAddressCompositeImpl(int dataLen, int checksum, GBufferAddressMapping subPageAddress) {
mainPageAddress = new PageAddressSingleImpl(dataLen, checksum);
this.subPageAddress = subPageAddress.getSubPageAddress();
this.totalDataLen = calcTotalDataLen();
}
public PageAddressCompositeImpl(PageAddress mainPageAddress, PageAddress[] subPageAddress) {
this.mainPageAddress = mainPageAddress;
this.subPageAddress = subPageAddress;
this.totalDataLen = calcTotalDataLen();
}
private int calcTotalDataLen() {
return mainPageAddress.getDataLen() + Arrays.stream(subPageAddress).map(PageAddress::getDataLen).reduce(0,
Integer::sum);
}
public PageAddress[] getSubPageAddress() {
return this.subPageAddress;
}
@Override
public int getDataLen() {
return this.totalDataLen;
}
@Override
public int getMainPageDataLen() {
return mainPageAddress.getDataLen();
}
@Override
public int getSubPageNum() {
return this.subPageAddress.length;
}
@Override
public int getSubPageDataLen() {
return Arrays.stream(subPageAddress).map(PageAddress::getDataLen).reduce(0, Integer::sum);
}
@Override
public void setChainIndex(int chainIndex) {
mainPageAddress.setChainIndex(chainIndex);
Arrays.stream(subPageAddress).forEach(pageAddress -> pageAddress.setChainIndex(chainIndex));
}
/**
* Returns the summary of the size of all data pages which are in the memory.
*
* @return The summary of the size of all data pages which are in the memory.
*/
@Override
public int getMemorySize() {
return mainPageAddress.getMemorySize() + Arrays.stream(subPageAddress).map(PageAddress::getMemorySize).reduce(0,
Integer::sum);
}
@Override
public void discard(
FileCache fileCache, GRegionContext gRegionContext, EventExecutor eventExecutor) {
Arrays.stream(subPageAddress).forEach(pageAddress -> pageAddress.discard(fileCache,
gRegionContext,
eventExecutor));
mainPageAddress.discard(fileCache, gRegionContext, eventExecutor);
}
@Override
public GByteBuffer getGByteBufferWithReference() {
throw new GeminiRuntimeException(
"Internal Bug! PageAddressCompositeImpl shouldn't call getGByteBufferWithReference");
}
@Override
public GByteBuffer getGByteBufferNoReference() {
throw new GeminiRuntimeException(
"Internal Bug! PageAddressCompositeImpl shouldn't call getGByteBufferNoReference");
}
@Override
public byte getPageAddressType() {
return COMPOSITE_PAGE_ADDRESS;
}
@Override
public DataPage getDataPage() {
return mainPageAddress.getDataPage();
}
@Override
public DataPage getDataPageNoReference() {
return mainPageAddress.getDataPageNoReference();
}
@Override
public boolean hasDataPage() {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call hasDataPage");
}
@Override
public long getDfsAddress() {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call getDfsAddress");
}
@Override
public long getLocalAddress() {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call getLocalAddress");
}
@Override
public long getVersion() {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call getVersion");
}
@Override
public void setDataPage(DataPage dataPage) {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call setDataPage");
}
@Override
public void setDfsAddress(long dfsAddress) {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call setDfsAddress");
}
@Override
public void setLocalAddress(long localAddress) {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call setLocalAddress");
}
@Override
public void setPageStatus(boolean flag) {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call setPageStatus");
}
@Override
public boolean isPageValid() {
//check main pageAddress.
return mainPageAddress.isPageValid();
}
@Override
public void setLocalStatus(boolean flag) {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call setLocalStatus");
}
@Override
public boolean isLocalValid() {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call isLocalValid");
}
@Override
public void setDfsStatus(boolean flag) {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call setDfsStatus");
}
@Override
public boolean isDfsValid() {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call isDfsValid");
}
@Override
public int getChecksum() {
throw new GeminiRuntimeException("Internal Bug! PageAddressCompositeImpl shouldn't call getChecksum");
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this).
add("mainPageAddress", mainPageAddress).
add("subPageAddress", Arrays.asList(subPageAddress)).toString();
}
@Override
public void snapshot(Collection regionSnapshots) throws IOException {
//write main PageAddress
mainPageAddress.snapshot(regionSnapshots);
Preconditions.checkArgument(subPageAddress != null, "Internal Bug!");
//write sub pagePageAddress
writerFunc(regionSnapshots, w -> w.writeInt(subPageAddress.length));
for (PageAddress pageAddress : subPageAddress) {
pageAddress.snapshot(regionSnapshots);
}
}
public static PageAddress restore(
SnapshotMetaFile.Reader reader, PageStoreStats pageStoreStats) throws IOException {
//main pageAddress
PageAddress mainPageAddress = PageAddressSingleImpl.restore(reader, pageStoreStats);
int subSize = reader.readInt();
PageAddress[] subPageAddress = new PageAddress[subSize];
for (int i = 0; i < subSize; i++) {
subPageAddress[i] = PageAddressSingleImpl.restore(reader, pageStoreStats);
}
return new PageAddressCompositeImpl(mainPageAddress, subPageAddress);
}
@Override
public GBinaryHashMap toBoxGBinaryHashMap(GBinaryHashMap gBinaryHashMap, GRegionContext gRegionContext, int logicPageChainIndex, int logicPageChainHashCode) {
GBufferAddressMapping mapping = new GBufferAddressMapping(subPageAddress, gRegionContext, logicPageChainIndex, logicPageChainHashCode);
return new GBinarySplitHashMap<>(gBinaryHashMap, mapping);
}
@Override
public Iterator pageIterator() {
return createOrderedPage();
}
@Override
public Iterator pageIteratorOrdered() {
return createOrderedPage();
}
private Iterator createOrderedPage() {
return new Iterator() {
/**
* -1 indicates next page is main page, and a non-negative
* value represents the position of next page in the sub-page
* array.
*/
private int nextPageIndex = -1;
@Override
public boolean hasNext() {
return nextPageIndex < subPageAddress.length;
}
@Override
public PageAddress next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
PageAddress page = nextPageIndex == -1 ? mainPageAddress : subPageAddress[nextPageIndex];
nextPageIndex++;
return page;
}
};
}
public PageAddress getMainPageAddress() {
return this.mainPageAddress;
}
@Override
public void addRequestCountForNewPage(long currentTickTime, int requestCount) {
//for new page, we regard all datapage having same old request count.
mainPageAddress.addRequestCountForNewPage(currentTickTime, requestCount);
Arrays.stream(subPageAddress).forEach(pageAddress -> pageAddress.addRequestCountForNewPage(currentTickTime,
requestCount));
}
@Override
public int getPageNum() {
return 1 + subPageAddress.length;
}
}
