Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.state.gemini.engine.hashtable;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.runtime.state.gemini.engine.GRegionContext;
import org.apache.flink.runtime.state.gemini.engine.GRegionID;
import org.apache.flink.runtime.state.gemini.engine.GeminiKList;
import org.apache.flink.runtime.state.gemini.engine.dbms.GContext;
import org.apache.flink.runtime.state.gemini.engine.exceptions.GeminiRuntimeException;
import org.apache.flink.runtime.state.gemini.engine.memstore.GSValue;
import org.apache.flink.runtime.state.gemini.engine.memstore.WriteBufferKList;
import org.apache.flink.runtime.state.gemini.engine.memstore.WriteBufferKListHashImpl;
import org.apache.flink.runtime.state.gemini.engine.page.GValueType;
import org.apache.flink.runtime.state.gemini.engine.page.PageIndex;
import org.apache.flink.runtime.state.gemini.engine.page.PageStoreHashKListImpl;
import org.apache.flink.runtime.state.gemini.engine.page.PageStoreKList;
import org.apache.flink.util.Preconditions;
import org.apache.flink.shaded.netty4.io.netty.util.concurrent.EventExecutor;
import javax.annotation.Nonnull;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import static org.apache.flink.runtime.state.gemini.engine.page.GValueType.PutValue;
/**
* GRegionKListImpl.
*/
public class GRegionKListImpl implements GeminiKList, HashGRegion {
private final GContext gContext;
private final GRegionContext gRegionContext;
private WriteBufferKList writeBuffer;
private PageStoreKList pageStore;
private final EventExecutor regionEventExecutor;
private boolean readCopy;
private TypeSerializer keySerializer;
private TypeSerializer elementSerializer;
@SuppressWarnings("unchecked")
public GRegionKListImpl(GRegionContext gRegionContext, PageIndex pageIndex) {
this.gRegionContext = Preconditions.checkNotNull(gRegionContext);
this.gContext = gRegionContext.getGContext();
// TODO how to choose an event executor
this.regionEventExecutor = gContext.getSupervisor().getRegionExecutorGroup().next();
gRegionContext.getPageStoreStats().setRegionExecutor(regionEventExecutor);
gContext.getSupervisor().getCacheManager().addRegionEventExecutor(this.regionEventExecutor);
this.pageStore = new PageStoreHashKListImpl<>(this, pageIndex, regionEventExecutor);
this.writeBuffer = new WriteBufferKListHashImpl<>(this, regionEventExecutor, pageStore);
this.readCopy = gRegionContext.getGContext().getGConfiguration().isReadCopy();
this.keySerializer = gRegionContext.getPageSerdeFlink().getKeySerde();
this.elementSerializer = gRegionContext.getPageSerdeFlink().getValueSerde();
}
@SuppressWarnings("unchecked")
public GRegionKListImpl(GRegionContext gRegionContext) {
this.gRegionContext = Preconditions.checkNotNull(gRegionContext);
this.gContext = gRegionContext.getGContext();
// TODO how to choose an event executor
this.regionEventExecutor = gContext.getSupervisor().getRegionExecutorGroup().next();
gRegionContext.getPageStoreStats().setRegionExecutor(regionEventExecutor);
gContext.getSupervisor().getCacheManager().addRegionEventExecutor(this.regionEventExecutor);
this.pageStore = new PageStoreHashKListImpl<>(this, regionEventExecutor);
this.writeBuffer = new WriteBufferKListHashImpl<>(this, regionEventExecutor, pageStore);
this.readCopy = gRegionContext.getGContext().getGConfiguration().isReadCopy();
this.keySerializer = gRegionContext.getPageSerdeFlink().getKeySerde();
this.elementSerializer = gRegionContext.getPageSerdeFlink().getValueSerde();
}
@Override
public GRegionContext getGRegionContext() {
return this.gRegionContext;
}
@Override
public GRegionID getRegionId() {
return gRegionContext.getRegionId();
}
@Override
public WriteBufferKList getWriteBuffer() {
return this.writeBuffer;
}
@Override
public PageStoreKList getPageStore() {
return this.pageStore;
}
@Override
public EventExecutor getExecutor() {
return this.regionEventExecutor;
}
@Override
public void put(K key, List value) {
gContext.checkDBStatus();
gContext.incAccessNumber();
long seqID = gRegionContext.getNextSeqID();
Set duplicatedElementChecker = new HashSet<>();
writeBuffer.put(key, value.stream().map(v -> {
//List will have same element, GeminiDB will gen different seqID for same Element.
long elementSeqID = seqID;
if (!duplicatedElementChecker.add(v)) {
elementSeqID = gRegionContext.getNextSeqID();
}
return GSValue.of(v, PutValue, elementSeqID);
}).collect(Collectors.toList()));
}
@Override
public List get(K key) {
gContext.checkDBStatus();
gContext.incAccessNumber();
return internalGet(key, true);
}
@Override
public List getOrDefault(K key, List defaultValue) {
gContext.checkDBStatus();
List value = get(key);
return value == null ? defaultValue : value;
}
@Override
public void remove(K key) {
gContext.checkDBStatus();
gContext.incAccessNumber();
writeBuffer.removeKey(key);
}
@Override
public Map> getAll() {
gContext.checkDBStatus();
gContext.incAccessNumber();
Map> results = new HashMap<>();
getAll(results);
return results;
}
@Override
public void getAll(Map> results) {
gContext.checkDBStatus();
//TODO provide better performance.
gContext.incAccessNumber();
Set allKeysIncludeDelete = new HashSet<>();
writeBuffer.allKeysIncludeDeleted(allKeysIncludeDelete);
pageStore.allKeysIncludeDeleted(allKeysIncludeDelete);
for (K key : allKeysIncludeDelete) {
List listResult = get(key);
if (listResult != null) {
results.put(copyKeyIfNeeded(key), listResult);
}
}
}
@Override
public void removeAll() {
gContext.checkDBStatus();
gContext.incAccessNumber();
long lastSeqID = gRegionContext.getLastSeqID();
gRegionContext.setRemoveAllSeqID(lastSeqID);
writeBuffer.reset();
}
@Override
public Iterable keys() {
gContext.checkDBStatus();
return getAll().keySet();
}
@Override
public boolean contains(K key) {
gContext.checkDBStatus();
//TODO: current writeBuffer.contains may return wrong answer
// we get the whole state here to correct the logic,
// but it doesn't have the best performance.
gContext.incAccessNumber();
List result = internalGet(key, false);
if (result == null || result.isEmpty()) {
return false;
} else {
return true;
}
}
@Override
public void add(K key, E element) {
gContext.checkDBStatus();
gContext.incAccessNumber();
writeBuffer.add(key, element);
}
@Override
public void addAll(K key, Collection elements) {
gContext.checkDBStatus();
gContext.incAccessNumber();
writeBuffer.addAll(key, elements);
}
@Override
public void remove(K key, E element) {
gContext.checkDBStatus();
List result = get(key);
if (result != null) {
result.remove(element);
if (result.isEmpty()) {
remove(key);
} else {
put(key, result);
}
}
}
@Override
public void removeAll(K key, Collection elements) {
gContext.checkDBStatus();
List result = get(key);
if (result != null) {
result.removeAll(elements);
if (result.isEmpty()) {
remove(key);
} else {
put(key, result);
}
}
}
@Override
public E poll(K key) {
gContext.checkDBStatus();
gContext.incAccessNumber();
List result = internalGet(key, false);
if (result == null) {
return null;
}
// TODO, can we remove the element directly???
E ret = result.remove(0);
remove(key, ret);
return copyElementIfNeeded(ret);
}
@Override
public E peek(K key) {
gContext.checkDBStatus();
gContext.incAccessNumber();
List result = internalGet(key, false);
if (result == null) {
return null;
}
return copyElementIfNeeded(result.get(0));
}
@VisibleForTesting
public List mergeWriteBufferAndPageStore(
List> writeBuffer,
List> pageStore) {
return mergeWriteBufferAndPageStore(writeBuffer, pageStore, false);
}
@VisibleForTesting
public List mergeWriteBufferAndPageStore(
List> writeBuffer,
List> pageStore,
boolean checkReadCopy) {
List ret = new LinkedList<>();
Set> removeDuplicated = new HashSet<>();
updateListFromGSVList(ret, pageStore, null, false);
if (pageStore != null) {
removeDuplicated.addAll(pageStore);
}
updateListFromGSVList(ret, writeBuffer, removeDuplicated, checkReadCopy);
//if empty, return null
return ret.isEmpty() ? null : ret;
}
private List internalGet(K key, boolean checkReadCopy) {
GSValue>> value = writeBuffer.get(key);
if (value != null) {
if (value.getValueType() == GValueType.Delete) {
return null;
}
if (value.getValueType() == GValueType.PutList) {
return getListFromCompcatedList(value.getValue(), checkReadCopy);
}
}
List> pageStoreRes = pageStore.get(key);
List ret = mergeWriteBufferAndPageStore(value == null ? null : value.getValue(), pageStoreRes, checkReadCopy);
//if empty, return null
return ret == null || ret.isEmpty() ? null : ret;
}
private List getListFromCompcatedList(@Nonnull List> origin, boolean checkReadCopy) {
List ret = new LinkedList<>();
updateListFromGSVList(ret, origin, null, checkReadCopy);
//if empty, return null
return ret.isEmpty() ? null : ret;
}
private void updateListFromGSVList(
List list, List> gsvList,
Set> removeDuplicated,
boolean checkReadCopy) {
if (gsvList != null) {
if (gsvList != null) {
for (GSValue e : gsvList) {
if (gRegionContext.filterState(e.getSeqID())) {
continue;
}
if (removeDuplicated != null) {
if (removeDuplicated.contains(e)) {
continue;
}
}
if (e.getValueType() == GValueType.Delete) {
Iterator iterator = list.iterator();
while (iterator.hasNext()) {
E e1 = iterator.next();
if (e1.equals(e.getValue())) {
iterator.remove();
break;
}
}
} else {
if (e.getValueType() == GValueType.PutValue) {
E element = e.getValue();
list.add(checkReadCopy ? copyElementIfNeeded(element) : element);
} else {
throw new GeminiRuntimeException("Unexpected type for list element: " + e.getValueType());
}
}
}
}
}
}
private K copyKeyIfNeeded(K key) {
return readCopy ? keySerializer.copy(key) : key;
}
private E copyElementIfNeeded(E element) {
return readCopy ? elementSerializer.copy(element) : element;
}
}
