org.apache.flink.runtime.state.gemini.engine.page.LogicChainedPageImpl Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.state.gemini.engine.page;

import org.apache.flink.runtime.state.gemini.engine.snapshot.RegionSnapshot;
import org.apache.flink.runtime.state.gemini.engine.snapshot.SnapshotMetaFile;
import org.apache.flink.util.Preconditions;

import org.apache.flink.shaded.guava18.com.google.common.base.MoreObjects;

import javax.annotation.Nullable;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
import java.util.stream.Collectors;

/**
 * LogicPageChain.
 */
public class LogicChainedPageImpl implements LogicChainedPage {
	private volatile int chainIndex = -1;
	private volatile PageAddress[] pageAddresses;
	//1byte indicate status, 3 bytes indicate page size
	private volatile byte pageStatus;
	private volatile int pageSize = 0;

	public LogicChainedPageImpl(PageStatus pageStatus) {
		this.pageStatus = pageStatus.getCode();
	}

	public LogicChainedPageImpl(PageStatus pageStatus, int defaultChainLen) {
		this.pageStatus = pageStatus.getCode();
		initChainPageImpl(defaultChainLen);
	}

	private LogicChainedPageImpl(LogicChainedPageImpl logicChainedPage, Map allAddReferenceDataPage) {
		this.chainIndex = logicChainedPage.chainIndex;
		// not copy the page in WaitSplitting status
		if (logicChainedPage.pageAddresses != null) {
			this.pageAddresses = new PageAddress[logicChainedPage.pageAddresses.length];
			System.arraycopy(logicChainedPage.pageAddresses, 0, this.pageAddresses, 0, this.chainIndex + 1);
		}
		//add Reference count.
		DataPage dataPage;
		for (int i = 0; i <= this.chainIndex; i++) {
			dataPage = this.pageAddresses[i].getDataPage();
			if (dataPage != null) {
				allAddReferenceDataPage.put(this.pageAddresses[i], dataPage);
			}
		}
		this.pageStatus = logicChainedPage.pageStatus;
	}

	private void initChainPageImpl(int defaultChainLen) {
		Preconditions.checkArgument(defaultChainLen >= 3, "defaultChainLen too small");
		pageAddresses = new PageAddress[defaultChainLen];
	}

	@Override
	public boolean setPageStatus(
		PageStatus expectedPageStatus, PageStatus targetStatus) {

		if (expectedPageStatus == null) {
			pageStatus = targetStatus.getCode();
			return true;
		}
		if (pageStatus != expectedPageStatus.getCode()) {
			return false;
		}

		synchronized (this) {
			if (pageStatus != expectedPageStatus.getCode()) {
				return false;
			} else {
				pageStatus = targetStatus.getCode();
				return true;
			}
		}
	}

	@Override
	public void addPageSize(int pageSize) {
		this.pageSize += pageSize;
	}

	@Override
	public int getPageSize() {
		return this.pageSize;
	}

	@Override
	public PageStatus getPageStatus() {
		return PageStatus.valueOf(pageStatus);
	}

	@Override
	public PageAddress getPageAddress(int chainIndex) {
		Preconditions.checkArgument(chainIndex >= 0 && chainIndex <= this.chainIndex, "logicPage overflow");
		return this.pageAddresses[chainIndex];
	}

	@Override
	public PageAddress createPage(long initRequestCount, DataPage dataPage) {
		//no concurrence
		Preconditions.checkArgument(chainIndex < pageAddresses.length, "logicPage overflow");
		checkSpace();
		PageAddress pageAddress = new PageAddress(initRequestCount, dataPage);
		pageAddresses[chainIndex + 1] = pageAddress;
		chainIndex++;
		return pageAddress;
	}

	@Override
	public int insertPage(PageAddress pageAddress) {
		//no concurrence
		Preconditions.checkArgument(chainIndex < pageAddresses.length, "logicPage overflow");
		checkSpace();
		pageAddresses[++chainIndex] = pageAddress;
		return chainIndex;
	}

	private void checkSpace() {
		// TODO: #SR optimize this deepcopy
		// change the deepcopy logic in PageIndexHashImpl().
		// do not new the logichainedpage, reference the chain and log the chainIndex.
		if (chainIndex == pageAddresses.length - 1) {
			PageAddress[] newpageAddresses = new PageAddress[pageAddresses.length + 1];
			System.arraycopy(pageAddresses, 0, newpageAddresses, 0, pageAddresses.length);
			pageAddresses = newpageAddresses;
		}
	}

	@Override
	public int getCurrentPageChainIndex() {
		return this.chainIndex;
	}

	@Override
	public int getPageChainCapacity() {
		return pageAddresses.length;
	}

	@Override
	public LogicChainedPage deepCopy(Map allAddReferenceDataPage) {
		if (this == PageIndexHashImpl.WAIT_SPLITTING) {
			return PageIndexHashImpl.WAIT_SPLITTING;
		}
		return new LogicChainedPageImpl(this, allAddReferenceDataPage);
	}

	@Override
	public Iterator pageIterator() {
		// for WaitSplitting page
		if (pageAddresses == null) {
			return Collections.emptyIterator();
		}
		// collect to support remove
		return Arrays.stream(pageAddresses).filter(pageAddress -> pageAddress != null).collect(Collectors.toList()).iterator();
	}

	@Override
	public void snapshot(@Nullable RegionSnapshot localRegionSnapshot, RegionSnapshot dfsRegionSnapshot) throws IOException {
		writeInt(localRegionSnapshot, dfsRegionSnapshot, chainIndex);
		for (int i = 0; i <= chainIndex; i++) {
			PageAddress pageAddress = pageAddresses[i];
			int len = pageAddress.getDataLen();
			//TODO future support flush-compress.
			int checksum = pageAddress.getChecksum();
			long requesetCount = pageAddress.getRequestCount();
			long localAddress = pageAddress.getLocalAddress();
			long dfsAddress = pageAddress.getDfsAddress();
			// write data length
			writeInt(localRegionSnapshot, dfsRegionSnapshot, len);
			writeInt(localRegionSnapshot, dfsRegionSnapshot, checksum);
			writeLong(localRegionSnapshot, dfsRegionSnapshot, requesetCount);
			// write dfs address for both snapshot
			writeLong(localRegionSnapshot, dfsRegionSnapshot, dfsAddress);

			// write local address for local snapshot
			if (localRegionSnapshot != null) {
				if (pageAddress.isLocalValid()) {
					localRegionSnapshot.getWriter().writeBoolean(true);
					localRegionSnapshot.getWriter().writeLong(localAddress);
				} else {
					localRegionSnapshot.getWriter().writeBoolean(false);
				}
			}
			// not write local address for dfs snapshot
			dfsRegionSnapshot.getWriter().writeBoolean(false);

			// update file meta
			if (localRegionSnapshot != null) {
				localRegionSnapshot.updateFileMeta(localAddress, 1, len, true);
				localRegionSnapshot.updateFileMeta(dfsAddress, 1, len, false);
			}
			dfsRegionSnapshot.updateFileMeta(dfsAddress, 1, len, false);
		}
	}

	private void writeInt(
		RegionSnapshot regionSnapshot1,
		RegionSnapshot regionSnapshot2,
		int value) throws IOException {
		if (regionSnapshot1 != null) {
			regionSnapshot1.getWriter().writeInt(value);
		}
		if (regionSnapshot2 != null) {
			regionSnapshot2.getWriter().writeInt(value);
		}
	}

	private void writeLong(
		RegionSnapshot regionSnapshot1,
		RegionSnapshot regionSnapshot2,
		long value) throws IOException {
		if (regionSnapshot1 != null) {
			regionSnapshot1.getWriter().writeLong(value);
		}
		if (regionSnapshot2 != null) {
			regionSnapshot2.getWriter().writeLong(value);
		}
	}

	@Override
	public void restore(SnapshotMetaFile.Reader reader, PageStoreStats pageStoreStats) throws IOException {
		chainIndex = reader.readInt();
		this.pageAddresses = new PageAddress[chainIndex + 1];
		for (int i = 0; i <= chainIndex; ++i) {
			int pageLen = reader.readInt();
			//TODO for compressing when flush.
//			int onDiskDataLen = reader.readInt();
//			byte code = reader.readByte();
			int onDiskDataLen = pageLen;
			int checksum = reader.readInt();
			byte code = -1;
			long requestCount = reader.readLong();
			pageAddresses[i] = new PageAddress(pageLen, onDiskDataLen, code, checksum, requestCount);
			long dfsAddress = reader.readLong();
			pageAddresses[i].setDfsAddress(dfsAddress);
			pageAddresses[i].setDfsStatus(true);
			if (reader.readBoolean()) {
				long localAddress = reader.readLong();
				pageAddresses[i].setLocalAddress(localAddress);
				pageAddresses[i].setLocalStatus(true);
			}
			addPageSize(pageLen);
			pageStoreStats.addLogicPageSize(pageLen);
		}
		pageStoreStats.addLogicPageChainLen(chainIndex + 1);
		pageStoreStats.addLogicPageChainCapacity(chainIndex + 1);
	}

	@Override
	public String toString() {
		return MoreObjects.toStringHelper(this).
			add("pageAddresses", Arrays.toString(pageAddresses)).
			toString();
	}
}