Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
*
* * Licensed to the Apache Software Foundation (ASF) under one
* * or more contributor license agreements. See the NOTICE file
* * distributed with this work for additional information
* * regarding copyright ownership. The ASF licenses this file
* * to you under the Apache License, Version 2.0 (the
* * "License"); you may not use this file except in compliance
* * with the License. You may obtain a copy of the License at
* *
* * http://www.apache.org/licenses/LICENSE-2.0
* *
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS,
* * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* * See the License for the specific language governing permissions and
* * limitations under the License.
*
*/
package org.apache.flink.runtime.state.gemini.engine.filecompaction;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.state.gemini.engine.DbPageIterator;
import org.apache.flink.runtime.state.gemini.engine.GConfiguration;
import org.apache.flink.runtime.state.gemini.engine.dbms.GContext;
import org.apache.flink.runtime.state.gemini.engine.fs.FileManager;
import org.apache.flink.runtime.state.gemini.engine.fs.FileMeta;
import org.apache.flink.runtime.state.gemini.engine.metrics.FileCompactionMetrics;
import org.apache.flink.runtime.state.gemini.engine.page.PageAddress;
import org.apache.flink.util.Preconditions;
import org.apache.flink.shaded.guava18.com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.flink.shaded.netty4.io.netty.util.concurrent.EventExecutorGroup;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
/**
* Implementation of {@link FileCompaction}. Space amplification will
* be checked periodically, and if it reach the specified trigger ratio,
* a compaction will be started. A compaction will transfer date in some
* files so that amplification after compaction will reduce to the specified
* target ratio. There will be no concurrent compaction.
*/
public class FileCompactionImpl implements FileCompaction {
private static final Logger LOG = LoggerFactory.getLogger(FileCompactionImpl.class);
private final GContext gContext;
/**
* The page transfer.
*/
private final FileCompactionPageTransfer pageTransfer;
/**
* File manager used by DB runtime.
*/
private final FileManager fileManager;
/**
* A compaction process will be triggered when amplification reaches this ratio.
*/
private float compactionTriggerRatio;
/**
* A compaction process will exit when amplification reduces to this ratio.
*/
private float compactionTargetRatio;
/**
* Interval in milliseconds to check the space amplification periodically.
*/
private long checkInterval;
/**
* Executor used to check the amplification.
*/
private final ScheduledThreadPoolExecutor checkExecutor;
/**
* The Compaction running currently. Because compactions are all triggered in
* a single executor, and there is at most one compaction running, so there is
* no concurrent problem to access it.
*/
private volatile RunningCompaction runningCompaction;
/**
* Number of compaction that has run.
*/
private volatile int compactionCounter;
private final FileCompactionStat fileCompactionStat;
@Nullable
private FileCompactionMetrics metrics;
private volatile boolean closed;
public FileCompactionImpl(GContext gContext, FileCompactionPageTransfer pageTransfer) {
this.gContext = Preconditions.checkNotNull(gContext);
this.pageTransfer = Preconditions.checkNotNull(pageTransfer);
this.fileManager = Preconditions.checkNotNull(pageTransfer.getDbFileManager());
GConfiguration config = gContext.getGConfiguration();
this.compactionTriggerRatio = config.getCompactionTriggerRatio();
this.compactionTargetRatio = config.getCompactionTargetRatio();
this.checkInterval = config.getAmplificationCheckInterval();
Preconditions.checkArgument(compactionTriggerRatio >= 1,
"compaction trigger ratio " + compactionTriggerRatio + " should not be less than 1");
Preconditions.checkArgument(compactionTargetRatio >= 1,
"compaction target ratio " + compactionTargetRatio + " should not be less than 1");
Preconditions.checkArgument(compactionTriggerRatio > compactionTargetRatio,
String.format("compaction trigger ratio %f should be larger than target ratio %f",
compactionTriggerRatio, compactionTargetRatio));
Preconditions.checkArgument(checkInterval > 0,
"amplification check interval " + checkInterval + " should be non-negative");
this.checkExecutor = new ScheduledThreadPoolExecutor(1,
new ThreadFactoryBuilder().setNameFormat(
gContext.getGConfiguration().getExecutorPrefixName() + "FileCompaction-%d").build());
this.checkExecutor.setRemoveOnCancelPolicy(true);
this.checkExecutor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
this.checkExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
this.fileCompactionStat = new FileCompactionStat();
MetricGroup dbMetricGroup = gContext.getDBMetricGroup();
if (dbMetricGroup != null) {
metrics = new FileCompactionMetrics(
dbMetricGroup.addGroup("file_compaction"),
config.getMetricSampleCount(),
config.getMetricHistogramWindowSize());
metrics.register(fileCompactionStat);
}
this.compactionCounter = 0;
this.closed = false;
LOG.info("FileCompaction is created, trigger ratio {}, target ratio {}, check interval {}ms",
compactionTriggerRatio, compactionTargetRatio, checkInterval);
}
@Override
public void start() {
checkExecutor.scheduleWithFixedDelay(new AmplificationCheckRunner(),
checkInterval, checkInterval, TimeUnit.MILLISECONDS);
LOG.info("FileCompaction is started.");
}
@VisibleForTesting
RunningCompaction getRunningCompaction() {
return runningCompaction;
}
@VisibleForTesting
FileCompactionPageTransfer getPageTransfer() {
return pageTransfer;
}
@VisibleForTesting
CompactionCheckResult checkCompaction(Map fileMetas, boolean onlyCheck) {
long totalFileSize = 0L;
long totalDataSize = 0L;
Map infoMap = new HashMap<>();
// Set of files currently opened for write
Set fileOpenedForWrite = new HashSet<>();
for (FileMeta fileMeta : fileMetas.values()) {
long fileSize = fileMeta.getFileSize();
long dataSize = fileMeta.addAndGetDataSize(0);
int numPage = fileMeta.addAndGetDBReference(0);
if (numPage == 0) {
continue;
}
int fileId = fileMeta.getFileId().get();
// because not synchronize with file manager, so a file id visited before
// maybe have been recycled, and is visited again
FileInfo info = infoMap.get(fileId);
if (info == null) {
info = new FileInfo();
infoMap.put(fileId, info);
} else {
// remove the old file size
totalFileSize -= info.fileSize;
totalDataSize -= info.dataSize;
fileOpenedForWrite.remove(fileId);
}
if (fileMeta.getFileWriter() != null) {
fileOpenedForWrite.add(fileId);
}
totalFileSize += fileSize;
totalDataSize += dataSize;
float ratio = (float) fileSize / dataSize;
info.set(fileId, fileSize, dataSize, numPage, ratio);
}
float totalRatio = totalDataSize == 0 ? 1 : ((float) totalFileSize / totalDataSize);
// sort files by ratio in descending order
List infoList = new ArrayList<>(infoMap.values());
infoList.sort((i1, i2) -> Float.compare(i2.ratio, i1.ratio));
if (LOG.isDebugEnabled()) {
LOG.debug("current file statistics: total files {}, total file size {}, total data size {}, "
+ "amplification ratio {}", infoList.size(), totalFileSize, totalDataSize, totalRatio);
LOG.debug("current file details: {}", infoList);
}
boolean isNeedCompaction = totalRatio > compactionTriggerRatio;
if (onlyCheck || !isNeedCompaction) {
return CompactionCheckResult.of(totalFileSize, totalDataSize, totalRatio, isNeedCompaction);
}
// files which should be compacted
Map compactionFiles = new HashMap<>();
long expectedTotalFileSize = totalFileSize;
float expectedTotalRatio = totalRatio;
int i = 0;
while (i < infoList.size() && expectedTotalRatio > compactionTargetRatio) {
FileInfo info = infoList.get(i);
// only consider those files that have been closed for write
if (!fileOpenedForWrite.contains(info.fileId)) {
compactionFiles.put(info.fileId, info);
// minus the useless data from totalFileSize
expectedTotalFileSize -= (info.fileSize - info.dataSize);
expectedTotalRatio = (float) expectedTotalFileSize / totalDataSize;
}
i++;
}
if (LOG.isDebugEnabled()) {
LOG.debug("expected file statistics: compact {} files, total file size {}, total data size {},"
+ " amplification ratio {}", i, expectedTotalFileSize, totalDataSize, expectedTotalRatio);
LOG.debug("expected files to compact: {}", compactionFiles.values());
}
return CompactionCheckResult.of(totalFileSize, totalDataSize, totalRatio, true,
expectedTotalFileSize, expectedTotalRatio, compactionFiles);
}
@VisibleForTesting
void startCompaction(CompactionCheckResult result) {
if (!result.needCompaction()) {
return;
}
compactionCounter++;
RunningCompaction compaction = new RunningCompaction(compactionCounter, result);
this.runningCompaction = compaction;
LOG.info("Start compaction " + compactionCounter);
compaction.startTime = System.currentTimeMillis();
Map compactionFiles = compaction.compactionCheckResult.compactionFiles;
EventExecutorGroup executorGroup = gContext.getSupervisor().getFlushExecutorGroup();
DbPageIterator iterator = gContext.getGeminiDB().getDbPageIterator();
compaction.addTask();
while (iterator.valid()) {
PageAddress pa = iterator.currentPage();
if (pageTransfer.hasDbFileAddress(pa)) {
int fileId = pageTransfer.getDbFileId(pa);
if (compactionFiles.containsKey(fileId)) {
compaction.addTask();
compaction.addPreparePage(pa.getDataLen());
pageTransfer.transferPage(
pa,
iterator.currentRegion().getGRegionContext(),
executorGroup.next(),
(success, throwable) -> {
if (success) {
compaction.addSuccessfulPage(pa.getDataLen());
} else {
compaction.addFailedPage(pa.getDataLen());
// ignore all exceptions
LOG.debug("file compaction failed", throwable);
}
compaction.removeTask();
});
}
}
iterator.next();
}
compaction.removeTask();
}
private void endCompaction() {
Preconditions.checkNotNull(runningCompaction, "There is no running compaction");
Preconditions.checkState(runningCompaction.finished(), "file compaction has not finished");
CompactionCheckResult result = checkCompaction(fileManager.getFileMapping(), true);
runningCompaction.endTime = System.currentTimeMillis();
long duration = runningCompaction.endTime - runningCompaction.startTime;
fileCompactionStat.addAndGetNumberCompaction(1);
fileCompactionStat.setTransferSize(runningCompaction.pageSizePrepareToTransfer);
fileCompactionStat.setCompactionDuration(duration);
fileCompactionStat.setAmplificationRatio(result.amplificationRatio);
LOG.info("end compaction {}, duration {}, transfer size {}, amplification after compaction {}",
runningCompaction.id, duration, runningCompaction.pageSizePrepareToTransfer, result.amplificationRatio);
if (LOG.isDebugEnabled()) {
LOG.debug("completed compaction details {}", runningCompaction);
}
runningCompaction = null;
}
@Override
public void close() {
synchronized (this) {
if (closed) {
LOG.warn("FileCompaction has been closed");
return;
}
closed = true;
}
checkExecutor.shutdownNow();
LOG.info("File compaction is closed.");
}
/**
* Runner to check whether to need compaction and trigger a compaction if needed.
*/
class AmplificationCheckRunner implements Runnable {
@Override
public void run() {
// at most one compaction is running at the same time
if (runningCompaction == null) {
CompactionCheckResult result = checkCompaction(fileManager.getFileMapping(), false);
if (LOG.isDebugEnabled()) {
LOG.debug("Compaction check result: {}", result);
}
fileCompactionStat.setAmplificationRatio(result.amplificationRatio);
if (result.needCompaction()) {
startCompaction(result);
}
}
}
}
/**
* Collect information about file.
*/
static class FileInfo {
int fileId;
long fileSize;
long dataSize;
float ratio;
int numPage;
void set(int fileId, long fileSize, long dataSize, int numPage, float ratio) {
this.fileId = fileId;
this.fileSize = fileSize;
this.dataSize = dataSize;
this.numPage = numPage;
this.ratio = ratio;
}
@Override
public String toString() {
return "FileInfo={" +
"fileId=" + fileId +
", fileSize=" + fileSize +
", dataSize=" + dataSize +
", numPage=" + numPage +
", ratio=" + ratio+
"}";
}
}
/**
* Result of compaction check.
*/
static class CompactionCheckResult {
/**
* Total file size before compaction.
*/
long totalFileSize;
/**
* Total data size.
*/
long totalDataSize;
/**
* Amplification ratio before compaction.
*/
float amplificationRatio;
/**
* Whether to need compaction.
*/
boolean isNeedCompaction;
/**
* Files which needs to compact.
*/
Map compactionFiles;
/**
* Expected total file size after compaction.
*/
long expectedTotalFileSize;
/**
* Expected amplification ratio after compaction.
*/
float expectedAmplificationRatio;
boolean needCompaction() {
return isNeedCompaction;
}
static CompactionCheckResult of(
long totalFileSize,
long totalDataSize,
float amplificationRatio,
boolean isNeedCompaction) {
return of(totalFileSize, totalDataSize, amplificationRatio, isNeedCompaction,
totalFileSize, amplificationRatio, Collections.emptyMap());
}
static CompactionCheckResult of(
long totalFileSize,
long totalDataSize,
float amplificationRatio,
boolean isNeedCompaction,
long expectedTotalFileSize,
float expectedAmplificationRatio,
Map compactionFiles) {
CompactionCheckResult result = new CompactionCheckResult();
result.totalFileSize = totalFileSize;
result.totalDataSize = totalDataSize;
result.amplificationRatio = amplificationRatio;
result.isNeedCompaction = isNeedCompaction;
result.expectedTotalFileSize = expectedTotalFileSize;
result.expectedAmplificationRatio = expectedAmplificationRatio;
result.compactionFiles = compactionFiles;
return result;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("CompactionCheckResult{");
sb.append("totalFileSize=");
sb.append(totalFileSize);
sb.append(", totalDataSize=");
sb.append(totalDataSize);
sb.append(", amplificationRatio=");
sb.append(amplificationRatio);
sb.append(", needCompaction=");
sb.append(isNeedCompaction);
if (!compactionFiles.isEmpty()) {
sb.append(", expectedTotalFileSize=");
sb.append(expectedTotalFileSize);
sb.append(", expectedAmplificationRatio=");
sb.append(expectedAmplificationRatio);
sb.append(", numberOfCompactionFiles=");
sb.append(compactionFiles.size());
}
sb.append("}");
return sb.toString();
}
}
/**
* Information of a running compaction.
*/
class RunningCompaction {
/**
* Id of this compaction.
*/
int id;
/**
* Result of the compaction check before running.
*/
CompactionCheckResult compactionCheckResult;
/**
* Number of running tasks.
*/
AtomicInteger runningTasks = new AtomicInteger(0);
/**
* Number of pages to prepare to transfer.
*/
int pagesPrepareToTransfer = 0;
/**
* Size of pages to prepare to transfer.
*/
long pageSizePrepareToTransfer = 0L;
/**
* Number of pages successfully transferred.
*/
AtomicInteger pagesSuccessfullyTransferred = new AtomicInteger(0);
/**
* Size of pages successfully transferred.
*/
AtomicLong pageSizeSuccessfullyTransferred = new AtomicLong(0L);
/**
* Number of pages failed to transfer.
*/
AtomicInteger pagesFailedToTransfer = new AtomicInteger(0);
/**
* Size of pages failed to transfer.
*/
AtomicLong pageSizeFailedToTransfer = new AtomicLong(0L);
/**
* Start time of this compaction.
*/
long startTime;
/**
* End time of this compaction.
*/
long endTime;
RunningCompaction(int id, CompactionCheckResult result) {
this.id = id;
this.compactionCheckResult = result;
}
boolean finished() {
return runningTasks.get() == 0;
}
void addTask() {
runningTasks.addAndGet(1);
}
void removeTask() {
if (runningTasks.addAndGet(-1) == 0) {
// all tasks have been finished
endCompaction();
}
}
void addPreparePage(long size) {
pagesPrepareToTransfer++;
pageSizePrepareToTransfer += size;
}
void addSuccessfulPage(long size) {
pagesSuccessfullyTransferred.addAndGet(1);
pageSizeSuccessfullyTransferred.addAndGet(size);
}
void addFailedPage(long size) {
pagesFailedToTransfer.addAndGet(1);
pageSizeFailedToTransfer.addAndGet(size);
}
@Override
public String toString() {
return "RunningCompaction{" +
"id=" + id +
", totalDataSize=" + compactionCheckResult.totalDataSize +
", pagesPrepareToTransfer=" + pagesPrepareToTransfer +
", pageSizePrepareToTransfer=" + pageSizePrepareToTransfer +
", pagesSuccessfullyTransferred=" + pagesSuccessfullyTransferred.get() +
", pageSizeSuccessfullyTransferred=" + pageSizeSuccessfullyTransferred.get() +
", pagesFailedToTransfer=" + pagesFailedToTransfer.get() +
", pageSizeFailedToTransfer=" + pageSizeFailedToTransfer.get() +
", startTime=" + startTime +
", endTime=" + endTime +
", duration=" + (endTime - startTime) +
"}";
}
}
}
