com.palantir.atlasdb.keyvalue.impl.KeyValueServices Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of atlasdb-client Show documentation
Show all versions of atlasdb-client Show documentation
Palantir open source project
/*
* (c) Copyright 2018 Palantir Technologies Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.palantir.atlasdb.keyvalue.impl;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.collect.Maps;
import com.google.common.primitives.UnsignedBytes;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.palantir.atlasdb.keyvalue.api.AsyncKeyValueService;
import com.palantir.atlasdb.keyvalue.api.BatchColumnRangeSelection;
import com.palantir.atlasdb.keyvalue.api.Cell;
import com.palantir.atlasdb.keyvalue.api.ColumnRangeSelection;
import com.palantir.atlasdb.keyvalue.api.ColumnSelection;
import com.palantir.atlasdb.keyvalue.api.KeyValueService;
import com.palantir.atlasdb.keyvalue.api.RangeRequest;
import com.palantir.atlasdb.keyvalue.api.RangeRequests;
import com.palantir.atlasdb.keyvalue.api.RowColumnRangeIterator;
import com.palantir.atlasdb.keyvalue.api.RowResult;
import com.palantir.atlasdb.keyvalue.api.TableReference;
import com.palantir.atlasdb.keyvalue.api.Value;
import com.palantir.common.annotation.Output;
import com.palantir.common.base.ClosableIterator;
import com.palantir.common.base.Throwables;
import com.palantir.common.concurrent.BlockingWorkerPool;
import com.palantir.logsafe.logger.SafeLogger;
import com.palantir.logsafe.logger.SafeLoggerFactory;
import com.palantir.util.crypto.Sha256Hash;
import com.palantir.util.paging.SimpleTokenBackedResultsPage;
import com.palantir.util.paging.TokenBackedBasicResultsPage;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import org.apache.commons.lang3.tuple.Pair;
public final class KeyValueServices {
private static final SafeLogger log = SafeLoggerFactory.get(KeyValueServices.class);
private KeyValueServices() {
/**/
}
public static void getFirstBatchForRangeUsingGetRange(
KeyValueService kv,
TableReference tableRef,
RangeRequest request,
long timestamp,
@Output Map, byte[]>> ret) {
if (ret.containsKey(request)) {
return;
}
RangeRequest requestWithHint = request;
if (request.getBatchHint() == null) {
requestWithHint = request.withBatchHint(100);
}
final ClosableIterator> range = kv.getRange(tableRef, requestWithHint, timestamp);
try {
int batchSize = requestWithHint.getBatchHint();
final Iterator> withLimit = Iterators.limit(range, batchSize);
ImmutableList> results = ImmutableList.copyOf(withLimit);
if (results.size() != batchSize) {
ret.put(request, SimpleTokenBackedResultsPage.create(request.getEndExclusive(), results, false));
return;
}
RowResult last = results.get(results.size() - 1);
byte[] lastRowName = last.getRowName();
if (RangeRequests.isTerminalRow(request.isReverse(), lastRowName)) {
ret.put(request, SimpleTokenBackedResultsPage.create(lastRowName, results, false));
return;
}
byte[] nextStartRow = RangeRequests.getNextStartRow(request.isReverse(), lastRowName);
if (Arrays.equals(request.getEndExclusive(), nextStartRow)) {
ret.put(request, SimpleTokenBackedResultsPage.create(nextStartRow, results, false));
} else {
ret.put(request, SimpleTokenBackedResultsPage.create(nextStartRow, results, true));
}
} finally {
range.close();
}
}
@SuppressWarnings("checkstyle:LineLength")
public static Map, byte[]>>
getFirstBatchForRangesUsingGetRangeConcurrent(
ExecutorService executor,
final KeyValueService kv,
final TableReference tableRef,
Iterable rangeRequests,
final long timestamp,
int maxConcurrentRequests) {
final Map, byte[]>> ret = new ConcurrentHashMap<>();
BlockingWorkerPool pool = new BlockingWorkerPool<>(executor, maxConcurrentRequests);
try {
for (final RangeRequest request : rangeRequests) {
pool.submitTask(() -> getFirstBatchForRangeUsingGetRange(kv, tableRef, request, timestamp, ret));
}
pool.waitForSubmittedTasks();
return ret;
} catch (InterruptedException e) {
throw Throwables.rewrapAndThrowUncheckedException(e);
}
}
@SuppressWarnings("checkstyle:LineLength")
public static Map, byte[]>>
getFirstBatchForRangesUsingGetRange(
KeyValueService kv, TableReference tableRef, Iterable rangeRequests, long timestamp) {
Map, byte[]>> ret = new HashMap<>();
for (final RangeRequest request : rangeRequests) {
getFirstBatchForRangeUsingGetRange(kv, tableRef, request, timestamp, ret);
}
return ret;
}
public static Collection> toConstantTimestampValues(
final Collection> cells, final long timestamp) {
return Collections2.transform(
cells, entry -> Maps.immutableEntry(entry.getKey(), Value.create(entry.getValue(), timestamp)));
}
// TODO(gsheasby): kill this when we can properly implement this on all KVSes
public static Map filterGetRowsToColumnRange(
KeyValueService kvs,
TableReference tableRef,
Iterable rows,
BatchColumnRangeSelection columnRangeSelection,
long timestamp) {
log.warn("Using inefficient postfiltering for getRowsColumnRange because the KVS doesn't support it natively. "
+ "Production environments should use a KVS with a proper implementation.");
Map allValues = kvs.getRows(tableRef, rows, ColumnSelection.all(), timestamp);
Map hashesToBytes = new HashMap<>();
Map> rowsToColumns = new HashMap<>();
for (byte[] row : rows) {
Sha256Hash rowHash = Sha256Hash.computeHash(row);
hashesToBytes.put(rowHash, row);
ImmutableSortedMap.Builder builder =
ImmutableSortedMap.orderedBy(UnsignedBytes.lexicographicalComparator());
rowsToColumns.put(rowHash, builder);
}
for (Map.Entry e : allValues.entrySet()) {
Sha256Hash rowHash = Sha256Hash.computeHash(e.getKey().getRowName());
rowsToColumns.get(rowHash).put(e.getKey().getColumnName(), e.getValue());
}
IdentityHashMap results = new IdentityHashMap<>();
for (Map.Entry> row : rowsToColumns.entrySet()) {
SortedMap map = row.getValue().buildOrThrow();
Set> subMap;
if ((columnRangeSelection.getStartCol().length == 0) && (columnRangeSelection.getEndCol().length == 0)) {
subMap = map.entrySet();
} else if (columnRangeSelection.getStartCol().length == 0) {
subMap = map.headMap(columnRangeSelection.getEndCol()).entrySet();
} else if (columnRangeSelection.getEndCol().length == 0) {
subMap = map.tailMap(columnRangeSelection.getStartCol()).entrySet();
} else {
subMap = map.subMap(columnRangeSelection.getStartCol(), columnRangeSelection.getEndCol())
.entrySet();
}
byte[] rowName = hashesToBytes.get(row.getKey());
results.put(
hashesToBytes.get(row.getKey()),
new LocalRowColumnRangeIterator(Iterators.transform(
subMap.iterator(),
e -> Pair.of(Cell.create(rowName, e.getKey()), e.getValue()))));
}
return results;
}
public static RowColumnRangeIterator mergeGetRowsColumnRangeIntoSingleIterator(
KeyValueService kvs,
TableReference tableRef,
Iterable rows,
ColumnRangeSelection columnRangeSelection,
int batchHint,
long timestamp) {
if (Iterables.isEmpty(rows)) {
return new LocalRowColumnRangeIterator(Collections.emptyIterator());
}
int columnBatchSize = Math.max(1, batchHint / Iterables.size(rows));
BatchColumnRangeSelection batchColumnRangeSelection =
BatchColumnRangeSelection.create(columnRangeSelection, columnBatchSize);
Map rowsColumnRanges =
kvs.getRowsColumnRange(tableRef, rows, batchColumnRangeSelection, timestamp);
// Return results in the same order as the provided rows.
Iterable orderedRanges = Iterables.transform(rows, rowsColumnRanges::get);
return new LocalRowColumnRangeIterator(Iterators.concat(orderedRanges.iterator()));
}
/**
* Constructs an {@link AsyncKeyValueService} such that methods are blocking and return immediate futures.
*
* @param keyValueService on which to call synchronous requests
* @return {@link AsyncKeyValueService} which delegates to synchronous methods
*/
public static AsyncKeyValueService synchronousAsAsyncKeyValueService(KeyValueService keyValueService) {
return new AsyncKeyValueService() {
@Override
public ListenableFuture> getAsync(
TableReference tableRef, Map timestampByCell) {
return Futures.immediateFuture(keyValueService.get(tableRef, timestampByCell));
}
@Override
public void close() {
// NoOp
}
};
}
}