All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.dataloader.DataLoader Maven / Gradle / Ivy

There is a newer version: 2022-09-12T23-25-35-08559ba
Show newest version
/*
 * Copyright (c) 2016 The original author or authors
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * and Apache License v2.0 which accompanies this distribution.
 *
 *      The Eclipse Public License is available at
 *      http://www.eclipse.org/legal/epl-v10.html
 *
 *      The Apache License v2.0 is available at
 *      http://www.opensource.org/licenses/apache2.0.php
 *
 * You may elect to redistribute this code under either of these licenses.
 */

package org.dataloader;

import org.dataloader.annotations.PublicApi;
import org.dataloader.annotations.VisibleForTesting;
import org.dataloader.impl.CompletableFutureKit;
import org.dataloader.stats.Statistics;
import org.dataloader.stats.StatisticsCollector;

import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.function.BiConsumer;

import static org.dataloader.impl.Assertions.nonNull;

/**
 * Data loader is a utility class that allows batch loading of data that is identified by a set of unique keys. For
 * each key that is loaded a separate {@link CompletableFuture} is returned, that completes as the batch function completes.
 * 

* With batching enabled the execution will start after calling {@link DataLoader#dispatch()}, causing the queue of * loaded keys to be sent to the batch function, clears the queue, and returns a promise to the values. *

* As {@link org.dataloader.BatchLoader} batch functions are executed the resulting futures are cached using a cache * implementation of choice, so they will only execute once. Individual cache keys can be cleared, so they will * be re-fetched when referred to again. *

* It is also possible to clear the cache entirely, and prime it with values before they are used. *

* Both caching and batching can be disabled. Configuration of the data loader is done by providing a * {@link DataLoaderOptions} instance on creation. *

* A call to the batch loader might result in individual exception failures for item with the returned list. if * you want to capture these specific item failures then use {@link org.dataloader.Try} as a return value and * create the data loader with {@link #newDataLoaderWithTry(BatchLoader)} form. The Try values will be interpreted * as either success values or cause the {@link #load(Object)} promise to complete exceptionally. * * @param type parameter indicating the type of the data load keys * @param type parameter indicating the type of the data that is returned * * @author Arnold Schrijver * @author Brad Baker */ @PublicApi public class DataLoader { private final DataLoaderHelper helper; private final StatisticsCollector stats; private final CacheMap futureCache; private final ValueCache valueCache; /** * Creates new DataLoader with the specified batch loader function and default options * (batching, caching and unlimited batch size). * * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type * * @return a new DataLoader * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newDataLoader(BatchLoader batchLoadFunction) { return newDataLoader(batchLoadFunction, null); } /** * Creates new DataLoader with the specified batch loader function with the provided options * * @param batchLoadFunction the batch load function to use * @param options the options to use * @param the key type * @param the value type * * @return a new DataLoader * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newDataLoader(BatchLoader batchLoadFunction, DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } /** * Creates new DataLoader with the specified batch loader function and default options * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

* If its important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might * have occurred trying to get a value. . * * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type * * @return a new DataLoader * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction) { return newDataLoaderWithTry(batchLoadFunction, null); } /** * Creates new DataLoader with the specified batch loader function and with the provided options * where the batch loader function returns a list of * {@link org.dataloader.Try} objects. * * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param options the options to use * @param the key type * @param the value type * * @return a new DataLoader * * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction, DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } /** * Creates new DataLoader with the specified batch loader function and default options * (batching, caching and unlimited batch size). * * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type * * @return a new DataLoader * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction) { return newDataLoader(batchLoadFunction, null); } /** * Creates new DataLoader with the specified batch loader function with the provided options * * @param batchLoadFunction the batch load function to use * @param options the options to use * @param the key type * @param the value type * * @return a new DataLoader * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } /** * Creates new DataLoader with the specified batch loader function and default options * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

* If its important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might * have occurred trying to get a value. . * * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type * * @return a new DataLoader * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction) { return newDataLoaderWithTry(batchLoadFunction, null); } /** * Creates new DataLoader with the specified batch loader function and with the provided options * where the batch loader function returns a list of * {@link org.dataloader.Try} objects. * * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param options the options to use * @param the key type * @param the value type * * @return a new DataLoader * * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } /** * Creates new DataLoader with the specified batch loader function and default options * (batching, caching and unlimited batch size). * * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type * * @return a new DataLoader * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction) { return newMappedDataLoader(batchLoadFunction, null); } /** * Creates new DataLoader with the specified batch loader function with the provided options * * @param batchLoadFunction the batch load function to use * @param options the options to use * @param the key type * @param the value type * * @return a new DataLoader * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction, DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } /** * Creates new DataLoader with the specified batch loader function and default options * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

* If its important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might * have occurred trying to get a value. . *

* * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type * * @return a new DataLoader * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction) { return newMappedDataLoaderWithTry(batchLoadFunction, null); } /** * Creates new DataLoader with the specified batch loader function and with the provided options * where the batch loader function returns a list of * {@link org.dataloader.Try} objects. * * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param options the options to use * @param the key type * @param the value type * * @return a new DataLoader * * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction, DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } /** * Creates new DataLoader with the specified mapped batch loader function and default options * (batching, caching and unlimited batch size). * * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type * * @return a new DataLoader * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction) { return newMappedDataLoader(batchLoadFunction, null); } /** * Creates new DataLoader with the specified batch loader function with the provided options * * @param batchLoadFunction the batch load function to use * @param options the options to use * @param the key type * @param the value type * * @return a new DataLoader * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } /** * Creates new DataLoader with the specified batch loader function and default options * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

* If its important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might * have occurred trying to get a value. . * * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type * * @return a new DataLoader * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction) { return newMappedDataLoaderWithTry(batchLoadFunction, null); } /** * Creates new DataLoader with the specified batch loader function and with the provided options * where the batch loader function returns a list of * {@link org.dataloader.Try} objects. * * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param options the options to use * @param the key type * @param the value type * * @return a new DataLoader * * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } /** * Creates a new data loader with the provided batch load function, and default options. * * @param batchLoadFunction the batch load function to use * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public DataLoader(BatchLoader batchLoadFunction) { this((Object) batchLoadFunction, null); } /** * Creates a new data loader with the provided batch load function and options. * * @param batchLoadFunction the batch load function to use * @param options the batch load options * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated public DataLoader(BatchLoader batchLoadFunction, DataLoaderOptions options) { this((Object) batchLoadFunction, options); } @VisibleForTesting DataLoader(Object batchLoadFunction, DataLoaderOptions options) { this(batchLoadFunction, options, Clock.systemUTC()); } @VisibleForTesting DataLoader(Object batchLoadFunction, DataLoaderOptions options, Clock clock) { DataLoaderOptions loaderOptions = options == null ? new DataLoaderOptions() : options; this.futureCache = determineFutureCache(loaderOptions); this.valueCache = determineValueCache(loaderOptions); // order of keys matter in data loader this.stats = nonNull(loaderOptions.getStatisticsCollector()); this.helper = new DataLoaderHelper<>(this, batchLoadFunction, loaderOptions, this.futureCache, this.valueCache, this.stats, clock); } @SuppressWarnings("unchecked") private CacheMap determineFutureCache(DataLoaderOptions loaderOptions) { return (CacheMap) loaderOptions.cacheMap().orElseGet(CacheMap::simpleMap); } @SuppressWarnings("unchecked") private ValueCache determineValueCache(DataLoaderOptions loaderOptions) { return (ValueCache) loaderOptions.valueCache().orElseGet(ValueCache::defaultValueCache); } /** * This returns the last instant the data loader was dispatched. When the data loader is created this value is set to now. * * @return the instant since the last dispatch */ public Instant getLastDispatchTime() { return helper.getLastDispatchTime(); } /** * This returns the {@link Duration} since the data loader was dispatched. When the data loader is created this is zero. * * @return the time duration since the last dispatch */ public Duration getTimeSinceDispatch() { return Duration.between(helper.getLastDispatchTime(), helper.now()); } /** * Requests to load the data with the specified key asynchronously, and returns a future of the resulting value. *

* If batching is enabled (the default), you'll have to call {@link DataLoader#dispatch()} at a later stage to * start batch execution. If you forget this call the future will never be completed (unless already completed, * and returned from cache). * * @param key the key to load * * @return the future of the value */ public CompletableFuture load(K key) { return load(key, null); } /** * This will return an optional promise to a value previously loaded via a {@link #load(Object)} call or empty if not call has been made for that key. *

* If you do get a present CompletableFuture it does not mean it has been dispatched and completed yet. It just means * its at least pending and in cache. *

* If caching is disabled there will never be a present Optional returned. *

* NOTE : This will NOT cause a data load to happen. You must called {@link #load(Object)} for that to happen. * * @param key the key to check * * @return an Optional to the future of the value */ public Optional> getIfPresent(K key) { return helper.getIfPresent(key); } /** * This will return an optional promise to a value previously loaded via a {@link #load(Object)} call that has in fact been completed or empty * if no call has been made for that key or the promise has not completed yet. *

* If you do get a present CompletableFuture it means it has been dispatched and completed. Completed is defined as * {@link java.util.concurrent.CompletableFuture#isDone()} returning true. *

* If caching is disabled there will never be a present Optional returned. *

* NOTE : This will NOT cause a data load to happen. You must called {@link #load(Object)} for that to happen. * * @param key the key to check * * @return an Optional to the future of the value */ public Optional> getIfCompleted(K key) { return helper.getIfCompleted(key); } /** * Requests to load the data with the specified key asynchronously, and returns a future of the resulting value. *

* If batching is enabled (the default), you'll have to call {@link DataLoader#dispatch()} at a later stage to * start batch execution. If you forget this call the future will never be completed (unless already completed, * and returned from cache). *

* The key context object may be useful in the batch loader interfaces such as {@link org.dataloader.BatchLoaderWithContext} or * {@link org.dataloader.MappedBatchLoaderWithContext} to help retrieve data. * * @param key the key to load * @param keyContext a context object that is specific to this key * * @return the future of the value */ public CompletableFuture load(K key, Object keyContext) { return helper.load(key, keyContext); } /** * Requests to load the list of data provided by the specified keys asynchronously, and returns a composite future * of the resulting values. *

* If batching is enabled (the default), you'll have to call {@link DataLoader#dispatch()} at a later stage to * start batch execution. If you forget this call the future will never be completed (unless already completed, * and returned from cache). * * @param keys the list of keys to load * * @return the composite future of the list of values */ public CompletableFuture> loadMany(List keys) { return loadMany(keys, Collections.emptyList()); } /** * Requests to load the list of data provided by the specified keys asynchronously, and returns a composite future * of the resulting values. *

* If batching is enabled (the default), you'll have to call {@link DataLoader#dispatch()} at a later stage to * start batch execution. If you forget this call the future will never be completed (unless already completed, * and returned from cache). *

* The key context object may be useful in the batch loader interfaces such as {@link org.dataloader.BatchLoaderWithContext} or * {@link org.dataloader.MappedBatchLoaderWithContext} to help retrieve data. * * @param keys the list of keys to load * @param keyContexts the list of key calling context objects * * @return the composite future of the list of values */ public CompletableFuture> loadMany(List keys, List keyContexts) { nonNull(keys); nonNull(keyContexts); synchronized (this) { List> collect = new ArrayList<>(); for (int i = 0; i < keys.size(); i++) { K key = keys.get(i); Object keyContext = null; if (i < keyContexts.size()) { keyContext = keyContexts.get(i); } collect.add(load(key, keyContext)); } return CompletableFutureKit.allOf(collect); } } /** * Dispatches the queued load requests to the batch execution function and returns a promise of the result. *

* If batching is disabled, or there are no queued requests, then a succeeded promise is returned. * * @return the promise of the queued load requests */ public CompletableFuture> dispatch() { return helper.dispatch().getPromisedResults(); } /** * Dispatches the queued load requests to the batch execution function and returns both the promise of the result * and the number of entries that were dispatched. *

* If batching is disabled, or there are no queued requests, then a succeeded promise with no entries dispatched is * returned. * * @return the promise of the queued load requests and the number of keys dispatched. */ public DispatchResult dispatchWithCounts() { return helper.dispatch(); } /** * Normally {@link #dispatch()} is an asynchronous operation but this version will 'join' on the * results if dispatch and wait for them to complete. If the {@link CompletableFuture} callbacks make more * calls to this data loader then the {@link #dispatchDepth()} will be > 0 and this method will loop * around and wait for any other extra batch loads to occur. * * @return the list of all results when the {@link #dispatchDepth()} reached 0 */ public List dispatchAndJoin() { List joinedResults = dispatch().join(); List results = new ArrayList<>(joinedResults); while (this.dispatchDepth() > 0) { joinedResults = dispatch().join(); results.addAll(joinedResults); } return results; } /** * @return the depth of the batched key loads that need to be dispatched */ public int dispatchDepth() { return helper.dispatchDepth(); } /** * Clears the future with the specified key from the cache, if caching is enabled, so it will be re-fetched * on the next load request. * * @param key the key to remove * * @return the data loader for fluent coding */ public DataLoader clear(K key) { return clear(key, (v, e) -> { }); } /** * Clears the future with the specified key from the cache remote value store, if caching is enabled * and a remote store is set, so it will be re-fetched and stored on the next load request. * * @param key the key to remove * @param handler a handler that will be called after the async remote clear completes * * @return the data loader for fluent coding */ public DataLoader clear(K key, BiConsumer handler) { Object cacheKey = getCacheKey(key); synchronized (this) { futureCache.delete(cacheKey); valueCache.delete(key).whenComplete(handler); } return this; } /** * Clears the entire cache map of the loader. * * @return the data loader for fluent coding */ public DataLoader clearAll() { return clearAll((v, e) -> { }); } /** * Clears the entire cache map of the loader, and of the cached value store. * * @param handler a handler that will be called after the async remote clear all completes * * @return the data loader for fluent coding */ public DataLoader clearAll(BiConsumer handler) { synchronized (this) { futureCache.clear(); valueCache.clear().whenComplete(handler); } return this; } /** * Primes the cache with the given key and value. Note this will only prime the future cache * and not the value store. Use {@link ValueCache#set(Object, Object)} if you want * to prime it with values before use * * @param key the key * @param value the value * * @return the data loader for fluent coding */ public DataLoader prime(K key, V value) { return prime(key, CompletableFuture.completedFuture(value)); } /** * Primes the cache with the given key and error. * * @param key the key * @param error the exception to prime instead of a value * * @return the data loader for fluent coding */ public DataLoader prime(K key, Exception error) { return prime(key, CompletableFutureKit.failedFuture(error)); } /** * Primes the cache with the given key and value. Note this will only prime the future cache * and not the value store. Use {@link ValueCache#set(Object, Object)} if you want * to prime it with values before use * * @param key the key * @param value the value * * @return the data loader for fluent coding */ public DataLoader prime(K key, CompletableFuture value) { Object cacheKey = getCacheKey(key); synchronized (this) { if (!futureCache.containsKey(cacheKey)) { futureCache.set(cacheKey, value); } } return this; } /** * Gets the object that is used in the internal cache map as key, by applying the cache key function to * the provided key. *

* If no cache key function is present in {@link DataLoaderOptions}, then the returned value equals the input key. * * @param key the input key * * @return the cache key after the input is transformed with the cache key function */ public Object getCacheKey(K key) { return helper.getCacheKey(key); } /** * Gets the statistics associated with this data loader. These will have been gather via * the {@link org.dataloader.stats.StatisticsCollector} passed in via {@link DataLoaderOptions#getStatisticsCollector()} * * @return statistics for this data loader */ public Statistics getStatistics() { return stats.getStatistics(); } /** * Gets the cacheMap associated with this data loader passed in via {@link DataLoaderOptions#cacheMap()} * @return the cacheMap of this data loader */ public CacheMap getCacheMap() { return futureCache; } /** * Gets the valueCache associated with this data loader passed in via {@link DataLoaderOptions#valueCache()} * @return the valueCache of this data loader */ public ValueCache getValueCache() { return valueCache; } }