org.apache.flink.runtime.state.gemini.engine.filecache.PageBatchFlusher Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of flink-runtime_2.11 Show documentation
There is a newer version: 1.5.1
/*
 *
 *  * Licensed to the Apache Software Foundation (ASF) under one
 *  * or more contributor license agreements.  See the NOTICE file
 *  * distributed with this work for additional information
 *  * regarding copyright ownership.  The ASF licenses this file
 *  * to you under the Apache License, Version 2.0 (the
 *  * "License"); you may not use this file except in compliance
 *  * with the License.  You may obtain a copy of the License at
 *  *
 *  *     http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  * Unless required by applicable law or agreed to in writing, software
 *  * distributed under the License is distributed on an "AS IS" BASIS,
 *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  * See the License for the specific language governing permissions and
 *  * limitations under the License.
 *
 */

package org.apache.flink.runtime.state.gemini.engine.filecache;

import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.state.gemini.engine.GRegionContext;
import org.apache.flink.runtime.state.gemini.engine.page.PageAddress;

import org.apache.flink.shaded.netty4.io.netty.util.concurrent.EventExecutorGroup;

import java.util.ArrayList;
import java.util.List;
import java.util.function.BiConsumer;

/**
 * Make pages flush to storage in a batch. This is not thread safe.
 */
public class PageBatchFlusher implements PageBatchIO {

	/**
	 * Number of pages in a batch to flush.
	 */
	private final int batchNumPages;

	/**
	 * Size of data in a batch to flush.
	 */
	private final int batchDataSize;

	/**
	 * File cache used to flush pages.
	 */
	private final FileCache fileCache;

	/**
	 * Event executor group used for flush.
	 */
	private final EventExecutorGroup eventExecutorGroup;

	/**
	 * Whether to force the page to flush again if it already been flushed.
	 */
	private final boolean forceFlush;

	/**
	 * List of pages to flush.
	 */
	private List pages;

	/**
	 * Region context for pages.
	 */
	private List regionContexts;

	/**
	 * List of callbacks to execute after pages are flushed.
	 */
	private List> callBacks;

	private int dataSize;

	public PageBatchFlusher(
		int numPages,
		int dataSize,
		boolean forceFlush,
		FileCache fileCache,
		EventExecutorGroup eventExecutorGroup) {
		this.batchNumPages = numPages;
		this.batchDataSize = dataSize;
		this.forceFlush = forceFlush;
		this.fileCache = fileCache;
		this.eventExecutorGroup = eventExecutorGroup;
		reset();
	}

	@Override
	public void addPage(
		PageAddress page,
		GRegionContext gRegionContext,
		BiConsumer callBack) {
		pages.add(page);
		regionContexts.add(gRegionContext);
		callBacks.add(callBack);
		dataSize += page.getDataLen();
		if (pages.size() >= batchNumPages || dataSize >= batchDataSize) {
			flush();
		}
	}

	@Override
	public void flush() {
		if (pages.isEmpty()) {
			return;
		}
		fileCache.flushBatchPages(
			pages,
			regionContexts,
			eventExecutorGroup.next(),
			forceFlush,
			true,
			callBacks);
		reset();
	}

	private void reset() {
		pages = new ArrayList<>(batchNumPages);
		regionContexts = new ArrayList<>(batchNumPages);
		callBacks = new ArrayList<>(batchNumPages);
		this.dataSize = 0;
	}

	public boolean isForceFlush() {
		return forceFlush;
	}

	public int getBatchDataSize() {
		return batchDataSize;
	}

	public int getBatchNumPages() {
		return batchNumPages;
	}

	public FileCache getFileCache() {
		return fileCache;
	}

	public EventExecutorGroup getEventExecutorGroup() {
		return eventExecutorGroup;
	}

	@VisibleForTesting
	int getDataSize() {
		return dataSize;
	}

	@VisibleForTesting
	List getPages() {
		return pages;
	}

	@VisibleForTesting
	List getRegionContexts() {
		return regionContexts;
	}

	@VisibleForTesting
	List> getCallBacks() {
		return callBacks;
	}
}