Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.state.gemini.engine.dbms.GContext;
import org.apache.flink.runtime.state.gemini.engine.dbms.GContextImpl;
import org.apache.flink.runtime.state.gemini.engine.dbms.Supervisor;
import org.apache.flink.runtime.state.gemini.engine.dbms.SupervisorImpl;
import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiRuntimeException;
import org.apache.flink.runtime.state.gemini.engine.filter.CompositeStateFilter;
import org.apache.flink.runtime.state.gemini.engine.filter.RemoveAllStateFilter;
import org.apache.flink.runtime.state.gemini.engine.filter.TtlStateFilter;
import org.apache.flink.runtime.state.gemini.engine.metrics.CacheMetrics;
import org.apache.flink.runtime.state.gemini.engine.metrics.ExceptionMetrics;
import org.apache.flink.runtime.state.gemini.engine.metrics.FileCacheMetrics;
import org.apache.flink.runtime.state.gemini.engine.metrics.FileCleanerMetrics;
import org.apache.flink.runtime.state.gemini.engine.metrics.HandlerMetrics;
import org.apache.flink.runtime.state.gemini.engine.page.DataPage;
import org.apache.flink.runtime.state.gemini.engine.page.PageAddress;
import org.apache.flink.runtime.state.gemini.engine.snapshot.BackendSnapshotMeta;
import org.apache.flink.runtime.state.gemini.engine.snapshot.DBSnapshotResult;
import org.apache.flink.runtime.state.gemini.time.ProcessingTimeProvider;
import org.apache.flink.runtime.state.gemini.time.TimeProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* GeminiDB is a new multi-model data store, designed for using in Flink with single thread accessing manner.
* it depends on all-in-memory-compaction to avoid file-compaction.
* it supports snapshot/restore,computer-storage-desperation,ttl.
* one instance with same dbDFSPath in a JVM.
*/
public class GeminiDB {
private static final Logger LOG = LoggerFactory.getLogger(GeminiDB.class);
/**
* DB status.
*/
public enum Status {
INITIALIZE, OPENED, CLOSING, CLOSED, INTERNAL_ERROR
}
private String dbName;
private Supervisor geminiSupervisor;
private final Object lock = new Object();
private volatile Status geminiDBStatus = Status.INITIALIZE;
private volatile Throwable internalError;
private final Map geminiTableMap = new ConcurrentHashMap<>();
private GContext gContext;
private MetricGroup dbMetricGroup;
private GConfiguration configuration;
/**
* This constructor is only used for tests.
*/
@VisibleForTesting
public GeminiDB() {
this.geminiDBStatus = Status.OPENED;
}
public GeminiDB(
String dbName, GConfiguration conf, int startRegionId, int endRegionId, MetricGroup metricGroup) {
this.dbName = checkNotNull(dbName);
this.configuration = checkNotNull(conf);
this.gContext = new GContextImpl(this, startRegionId, endRegionId, conf);
// TODO how to choose time provider according to TimeCharacteristic
TimeProvider timeProvider = new ProcessingTimeProvider();
this.gContext.setTimeProvider(timeProvider);
// init metrics
int sampleCount = conf.getMetricSampleCount();
int histogramWindowSize = conf.getMetricHistogramWindowSize();
this.dbMetricGroup = metricGroup;
gContext.setDBMetricGroup(dbMetricGroup);
MetricGroup fileManagerMetricGroup = dbMetricGroup.addGroup("fileManager");
gContext.setFileManagerMetricGroup(fileManagerMetricGroup);
CacheMetrics cacheMetric = new CacheMetrics(dbMetricGroup.addGroup("cache"), sampleCount, histogramWindowSize);
this.gContext.setCacheMetric(cacheMetric);
HandlerMetrics handlerMetric = new HandlerMetrics(dbMetricGroup.addGroup("handler"),
sampleCount,
histogramWindowSize);
this.gContext.setHandlerMetric(handlerMetric);
FileCacheMetrics fileCacheMetrics = new FileCacheMetrics(dbMetricGroup.addGroup("fileCache"), sampleCount, histogramWindowSize);
gContext.setFileCacheMetrics(fileCacheMetrics);
FileCleanerMetrics fileCleanerMetrics = new FileCleanerMetrics(dbMetricGroup.addGroup("fileCleaner"), sampleCount, histogramWindowSize);
gContext.setFileCleanerMetrics(fileCleanerMetrics);
ExceptionMetrics exceptionMetrics = new ExceptionMetrics(dbMetricGroup.addGroup("exception"), sampleCount, histogramWindowSize);
gContext.setExceptionMetrics(exceptionMetrics);
//initialize
CompositeStateFilter stateFilter = new CompositeStateFilter();
if (gContext.hasTtl()) {
stateFilter.addStateFilter(new TtlStateFilter());
}
stateFilter.addStateFilter(new RemoveAllStateFilter());
gContext.setStateFilter(stateFilter);
this.geminiSupervisor = new SupervisorImpl(gContext);
this.gContext.setSupervisor(geminiSupervisor);
handlerMetric.registerMetricsCacheStat(this.geminiSupervisor.getWriteBufferManager());
LOG.info("GeminiDB is created.");
}
public boolean setStatus(Status expected, Status target) {
synchronized (lock) {
if (expected != null && geminiDBStatus != expected) {
return false;
}
geminiDBStatus = target;
return true;
}
}
public Status getStatus() {
return geminiDBStatus;
}
public Throwable getInternalError() {
return internalError;
}
public void setInternalError(Throwable throwable) {
if (setStatus(Status.OPENED, Status.INTERNAL_ERROR)) {
internalError = throwable;
}
}
public synchronized void open() {
if (!setStatus(Status.INITIALIZE, Status.OPENED)) {
throw new GeminiRuntimeException("open db failed, current status is " + geminiDBStatus.toString());
}
geminiSupervisor.start();
LOG.info("GeminiDB is opened");
}
public Future startSnapshot(BackendSnapshotMeta backendSnapshotMeta) {
return this.geminiSupervisor.startSnapshot(backendSnapshotMeta);
}
public GTable getTableOrCreate(GTableDescription tableDescription) throws GeminiRuntimeException {
return geminiTableMap.computeIfAbsent(tableDescription.getTableName(), (nothing) -> {
GTable gTable = tableDescription.createTable(gContext);
this.geminiSupervisor.getCacheManager().addTable(gTable);
this.geminiSupervisor.getWriteBufferManager().addTableNum(tableDescription.getTableName());
return gTable;
});
}
public void close() {
setStatus(null, Status.CLOSED);
// Close sequence:
// 1. stop all the executor, stop modifying the pages.
// 2. clear up pages. Resource release manager will help release.
// 3. stop resource release manager, clear all the remaining resources.
geminiSupervisor.stop();
forceCleanUpPages();
geminiSupervisor.close();
LOG.info("GeminiDB is closed");
}
public GContext getGContext() {
return gContext;
}
public GConfiguration getConfiguration() {
return configuration;
}
public Map getGeminiTableMap() {
return geminiTableMap;
}
public DbPageIterator getDbPageIterator() {
return new DbPageIteratorImpl(this);
}
/**
* Clean the remaining pages forcely. Called when close.
*/
private void forceCleanUpPages() {
DbPageIterator iterator = getDbPageIterator();
while (iterator.valid()) {
PageAddress pa = iterator.currentPage();
if (pa.hasDataPage()) {
DataPage dataPage = pa.getDataPageNoReference();
dataPage.release();
}
iterator.next();
}
}
}
