com.netflix.hollow.api.consumer.HollowConsumer Maven / Gradle / Ivy
/*
* Copyright 2016-2021 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.netflix.hollow.api.consumer;
import static com.netflix.hollow.core.util.Threads.daemonThread;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import com.netflix.hollow.PublicApi;
import com.netflix.hollow.PublicSpi;
import com.netflix.hollow.api.client.FailedTransitionTracker;
import com.netflix.hollow.api.client.HollowAPIFactory;
import com.netflix.hollow.api.client.HollowClientUpdater;
import com.netflix.hollow.api.client.StaleHollowReferenceDetector;
import com.netflix.hollow.api.codegen.HollowAPIClassJavaGenerator;
import com.netflix.hollow.api.consumer.fs.HollowFilesystemBlobRetriever;
import com.netflix.hollow.api.custom.HollowAPI;
import com.netflix.hollow.api.metrics.HollowConsumerMetrics;
import com.netflix.hollow.api.metrics.HollowMetricsCollector;
import com.netflix.hollow.core.HollowConstants;
import com.netflix.hollow.core.memory.MemoryMode;
import com.netflix.hollow.core.read.OptionalBlobPartInput;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.filter.HollowFilterConfig;
import com.netflix.hollow.core.read.filter.TypeFilter;
import com.netflix.hollow.core.util.DefaultHashCodeFinder;
import com.netflix.hollow.core.util.HollowObjectHashCodeFinder;
import com.netflix.hollow.tools.history.HollowHistory;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.UnaryOperator;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* A HollowConsumer is the top-level class used by consumers of Hollow data to initialize and keep up-to-date a local in-memory
* copy of a hollow dataset. The interactions between the "blob" transition store and announcement listener are defined by
* this class, and the implementations of the data retrieval, announcement mechanism are abstracted in the interfaces which
* are provided to this class.
*
* To obtain a HollowConsumer, you should use a builder pattern, for example:
*
{@code
* HollowConsumer consumer = newHollowConsumer().withBlobRetriever(retriever)
* .withAnnouncementWatcher(watcher)
* .withGeneratedAPIClass(MovieAPI.class)
* .build();
* }
*
* The following components are injectable, but only an implementation of the HollowConsumer.BlobRetriever is
* required to be injected, all other components are optional. :
*
* - {@link HollowConsumer.BlobRetriever}
* - Implementations of this class define how to retrieve blob data from the blob store.
*
* - {@link HollowConsumer.AnnouncementWatcher}
* - Implementations of this class define the announcement mechanism, which is used to track the version of the
* currently announced state. It's also expected that implementations will trigger a refresh each time current
* data version is updated.
*
* - a List of {@link HollowConsumer.RefreshListener}s
* - RefreshListener implementations will define what to do when various events happen before, during, and after updating
* local in-memory copies of hollow data sets.
*
* - the Class representing a generated Hollow API
* - Defines how to create a {@link HollowAPI} for the dataset, useful when wrapping a dataset with an api which has
* been generated (via the {@link HollowAPIClassJavaGenerator})
*
* - {@link HollowFilterConfig}
* - Defines what types and fields to load (or not load) into memory from hollow datasets. Generally useful to reduce
* heap footprint on consumers which do not require visibility of an entire dataset.
*
* - {@link HollowConsumer.DoubleSnapshotConfig}
* - Defines whether this consumer may attempt a double snapshot, and how many deltas will be attempted during a single refresh.
* A double snapshot will allow your consumer to update in case of a broken delta chain, but will also result in a doubling of
* the heap footprint while the double snapshot is occurring.
*
* - {@link HollowConsumer.ObjectLongevityConfig}
* - Object longevity is used to guarantee that Hollow objects which are backed by removed records will remain usable and
* consistent until old references are discarded. This behavior is turned off by default. Implementations of this config
* can be used to enable and configure this behavior.
*
* - {@link HollowConsumer.ObjectLongevityDetector}
* - Implementations of this config will be notified when usage of expired Hollow object references is attempted.
*
* - An Executor
* - The Executor which will be used to perform updates when {@link #triggerAsyncRefresh()} is called. This will
* default to a new fixed thread pool with a single refresh thread.
*
*
*/
@SuppressWarnings({"unused", "WeakerAccess"})
@PublicApi
public class HollowConsumer {
private static final Logger LOG = Logger.getLogger(HollowConsumer.class.getName());
protected final AnnouncementWatcher announcementWatcher;
protected final HollowClientUpdater updater;
protected final ReadWriteLock refreshLock;
protected final HollowConsumerMetrics metrics;
private final Executor refreshExecutor;
private final MemoryMode memoryMode;
/**
* @deprecated use {@link HollowConsumer.Builder}
*/
@Deprecated
protected HollowConsumer(BlobRetriever blobRetriever,
AnnouncementWatcher announcementWatcher,
List refreshListeners,
HollowAPIFactory apiFactory,
HollowFilterConfig dataFilter,
ObjectLongevityConfig objectLongevityConfig,
ObjectLongevityDetector objectLongevityDetector,
DoubleSnapshotConfig doubleSnapshotConfig,
HollowObjectHashCodeFinder hashCodeFinder,
Executor refreshExecutor,
MemoryMode memoryMode) {
this(blobRetriever, announcementWatcher, refreshListeners, apiFactory, dataFilter,
objectLongevityConfig, objectLongevityDetector, doubleSnapshotConfig,
hashCodeFinder, refreshExecutor, memoryMode,null);
}
/**
* @deprecated use {@link HollowConsumer.Builder}
*/
@Deprecated
protected HollowConsumer(BlobRetriever blobRetriever,
AnnouncementWatcher announcementWatcher,
List refreshListeners,
HollowAPIFactory apiFactory,
HollowFilterConfig dataFilter,
ObjectLongevityConfig objectLongevityConfig,
ObjectLongevityDetector objectLongevityDetector,
DoubleSnapshotConfig doubleSnapshotConfig,
HollowObjectHashCodeFinder hashCodeFinder,
Executor refreshExecutor,
MemoryMode memoryMode,
HollowMetricsCollector metricsCollector) {
this.metrics = new HollowConsumerMetrics();
this.updater = new HollowClientUpdater(blobRetriever,
refreshListeners,
apiFactory,
doubleSnapshotConfig,
hashCodeFinder,
memoryMode,
objectLongevityConfig,
objectLongevityDetector,
metrics,
metricsCollector);
updater.setFilter(dataFilter);
this.announcementWatcher = announcementWatcher;
this.refreshExecutor = refreshExecutor;
this.refreshLock = new ReentrantReadWriteLock();
if (announcementWatcher != null)
announcementWatcher.subscribeToUpdates(this);
this.memoryMode = memoryMode;
}
protected > HollowConsumer(B builder) {
// duplicated with HollowConsumer(...) constructor above. We cannot chain constructor calls because that
// constructor subscribes to the announcement watcher and we have more setup to do first
this.metrics = new HollowConsumerMetrics();
this.updater = new HollowClientUpdater(builder.blobRetriever,
builder.refreshListeners,
builder.apiFactory,
builder.doubleSnapshotConfig,
builder.hashCodeFinder,
builder.memoryMode,
builder.objectLongevityConfig,
builder.objectLongevityDetector,
metrics,
builder.metricsCollector);
updater.setFilter(builder.typeFilter);
if(builder.skipTypeShardUpdateWithNoAdditions)
updater.setSkipShardUpdateWithNoAdditions(true);
this.announcementWatcher = builder.announcementWatcher;
this.refreshExecutor = builder.refreshExecutor;
this.refreshLock = new ReentrantReadWriteLock();
this.memoryMode = builder.memoryMode;
if (announcementWatcher != null)
announcementWatcher.subscribeToUpdates(this);
}
/**
* Triggers a refresh to the latest version specified by the {@link HollowConsumer.AnnouncementWatcher}.
* If already on the latest version, this operation is a no-op.
*
* If a {@link HollowConsumer.AnnouncementWatcher} is not present, this call trigger a refresh to the
* latest version available in the blob store.
*
* This is a blocking call.
*/
public void triggerRefresh() {
refreshLock.writeLock().lock();
try {
updater.updateTo(announcementWatcher == null ? new VersionInfo(Long.MAX_VALUE)
: announcementWatcher.getLatestVersionInfo());
} catch (Error | RuntimeException e) {
throw e;
} catch (Throwable t) {
throw new RuntimeException(t);
} finally {
refreshLock.writeLock().unlock();
}
}
/**
* Immediately triggers a refresh in a different thread to the latest version
* specified by the {@link HollowConsumer.AnnouncementWatcher}. If already on
* the latest version, this operation is a no-op.
*
* If a {@link HollowConsumer.AnnouncementWatcher} is not present, this call trigger a refresh to the
* latest version available in the blob store.
*
* This is an asynchronous call.
*/
public void triggerAsyncRefresh() {
triggerAsyncRefreshWithDelay(0);
}
/**
* Triggers async refresh after the specified number of milliseconds has passed.
*
* Any subsequent calls for async refresh will not begin until after the specified delay
* has completed.
*
* @param delayMillis the delay, in millseconds, before triggering the refresh
*/
public void triggerAsyncRefreshWithDelay(int delayMillis) {
final long targetBeginTime = System.currentTimeMillis() + delayMillis;
refreshExecutor.execute(() -> {
try {
long delay = targetBeginTime - System.currentTimeMillis();
if (delay > 0)
Thread.sleep(delay);
} catch (InterruptedException e) {
// Interrupting, such as shutting down the executor pool,
// cancels the trigger
LOG.log(Level.INFO, "Async refresh interrupted before trigger, refresh cancelled", e);
return;
}
try {
triggerRefresh();
} catch (Error | RuntimeException e) {
// Ensure exceptions are propagated to the executor
LOG.log(Level.SEVERE, "Async refresh failed", e);
throw e;
}
});
}
/**
* If a {@link HollowConsumer.AnnouncementWatcher} is not specified, then this method will attempt to update
* to the specified version, and if the specified version does not exist then to a different version as specified
* by functionality in the {@code BlobRetriever}.
*
* Otherwise, an UnsupportedOperationException will be thrown.
*
* This is a blocking call.
*
* @param version the version to refresh to
*/
public void triggerRefreshTo(long version) {
triggerRefreshTo(new VersionInfo(version));
}
/**
* Similar to {@link #triggerRefreshTo(long)} but instead of accepting a long version no. it accepts a
* {@link VersionInfo} instance that contains (in addition to version no.) version specific metadata and
* pinning status.
*
* @param versionInfo version no., metadata, and pined status for the desired version
*/
public void triggerRefreshTo(VersionInfo versionInfo) {
if (announcementWatcher != null)
throw new UnsupportedOperationException("Cannot trigger refresh to specified version when a HollowConsumer.AnnouncementWatcher is present");
try {
updater.updateTo(versionInfo);
} catch (Error | RuntimeException e) {
throw e;
} catch (Throwable t) {
throw new RuntimeException(t);
}
}
/**
* @return the {@link HollowReadStateEngine} which is holding the underlying hollow dataset.
*/
public HollowReadStateEngine getStateEngine() {
return updater.getStateEngine();
}
/**
* @return the current version of the dataset. This is the unique identifier of the data's state.
*/
public long getCurrentVersionId() {
return updater.getCurrentVersionId();
}
/**
* Returns a {@code CompletableFuture} that completes after the initial data load succeeds. Also triggers the initial
* load asynchronously, to avoid waiting on a polling interval for the initial load.
*
* Callers can use methods like {@link CompletableFuture#join()} or {@link CompletableFuture#get(long, TimeUnit)}
* to block until the initial load is complete.
*
* A failure during the initial load will not cause the future to complete exceptionally; this allows
* for a subsequent data version to eventually succeed.
*
* In a consumer without published or announced versions – or one that always fails the initial load – the future
* will remain incomplete indefinitely.
*
* @return a future which, when completed, has a value set to the data version that was initially loaded
*/
public CompletableFuture getInitialLoad() {
try {
triggerAsyncRefresh();
} catch (RejectedExecutionException | NullPointerException e) {
LOG.log(Level.INFO, "Refresh triggered by getInitialLoad() failed; future attempts might succeed", e);
}
return updater.getInitialLoad();
}
/**
* @return the api which wraps the underlying dataset.
*/
public HollowAPI getAPI() {
return updater.getAPI();
}
/**
* Equivalent to calling {@link #getAPI()} and casting to the specified API.
*
* @param apiClass the class of the API
* @param the type of the API
* @return the API which wraps the underlying dataset
*/
public T getAPI(Class apiClass) {
return apiClass.cast(updater.getAPI());
}
/**
* Will force a double snapshot refresh on the next update.
*/
public void forceDoubleSnapshotNextUpdate() {
updater.forceDoubleSnapshotNextUpdate();
}
/**
* Clear any failed transitions from the {@link FailedTransitionTracker}, so that they may be reattempted when an update is triggered.
*/
public void clearFailedTransitions() {
updater.clearFailedTransitions();
}
/**
* @return the number of failed snapshot transitions stored in the {@link FailedTransitionTracker}.
*/
public int getNumFailedSnapshotTransitions() {
return updater.getNumFailedSnapshotTransitions();
}
/**
* @return the number of failed delta transitions stored in the {@link FailedTransitionTracker}.
*/
public int getNumFailedDeltaTransitions() {
return updater.getNumFailedDeltaTransitions();
}
/**
* @return a {@link ReadWriteLock#readLock()}, the corresponding writeLock() of which is used to synchronize refreshes.
*
* This is useful if performing long-running operations which require a consistent view of the entire dataset in a
* single data state, to guarantee that updates do not happen while the operation runs.
*/
public Lock getRefreshLock() {
return refreshLock.readLock();
}
/**
* Adds a {@link RefreshListener} to this consumer.
*
* If the listener was previously added to this consumer, as determined by reference equality or {@code Object}
* equality, then this method does nothing.
*
* If a listener is added, concurrently, during the occurrence of a refresh then the listener will not receive
* events until the next refresh. The listener may also be removed concurrently.
*
* If the listener instance implements {@link RefreshRegistrationListener} then before the listener is added
* the {@link RefreshRegistrationListener#onBeforeAddition} method is be invoked. If that method throws an
* exception then that exception will be thrown by this method and the listener will not be added.
*
* @param listener the refresh listener to add
*/
public void addRefreshListener(RefreshListener listener) {
updater.addRefreshListener(listener, this);
}
/**
* Removes a {@link RefreshListener} from this consumer.
*
* If the listener was not previously added to this consumer, as determined by reference equality or {@code Object}
* equality, then this method does nothing.
*
* If a listener is removed, concurrently, during the occurrence of a refresh then the listener will receive all
* events for that refresh but not receive events for subsequent any refreshes.
*
* If the listener instance implements {@link RefreshRegistrationListener} then after the listener is removed
* the {@link RefreshRegistrationListener#onAfterRemoval} method is be invoked. If that method throws an
* exception then that exception will be thrown by this method.
*
* @param listener the refresh listener to remove
*/
public void removeRefreshListener(RefreshListener listener) {
updater.removeRefreshListener(listener, this);
}
/**
* @return the metrics for this consumer
*/
public HollowConsumerMetrics getMetrics() {
return metrics;
}
/**
* An interface which defines the necessary interactions of Hollow with a blob data store.
*
* Implementations will define how to retrieve blob data from a data store.
*/
public interface BlobRetriever {
/**
* Returns the snapshot for the state with the greatest version identifier which is equal to or less than the desired version
* @param desiredVersion the desired version
* @return the blob of the snapshot
*/
HollowConsumer.Blob retrieveSnapshotBlob(long desiredVersion);
/**
* Returns a delta transition which can be applied to the specified version identifier
* @param currentVersion the current version
* @return the blob of the delta
*/
HollowConsumer.Blob retrieveDeltaBlob(long currentVersion);
/**
* Returns a reverse delta transition which can be applied to the specified version identifier
* @param currentVersion the current version
* @return the blob of the reverse delta
*/
HollowConsumer.Blob retrieveReverseDeltaBlob(long currentVersion);
default Set configuredOptionalBlobParts() {
return null;
}
default HollowConsumer.HeaderBlob retrieveHeaderBlob(long currentVersion) {
throw new UnsupportedOperationException();
}
}
protected interface VersionedBlob {
InputStream getInputStream() throws IOException;
default File getFile() throws IOException {
throw new UnsupportedOperationException();
}
}
public static abstract class HeaderBlob implements VersionedBlob{
private final long version;
protected HeaderBlob(long version) {
this.version = version;
}
public long getVersion() {
return this.version;
}
}
/**
* A Blob, which is either a snapshot or a delta, defines three things:
*
* - The "from" version
* - The unique identifier of the state to which a delta transition should be applied. If
* this is a snapshot, then this value is HollowConstants.VERSION_NONE.
*
* - The "to" version
* - The unique identifier of the state at which a dataset will arrive after this blob is applied.
*
* - The actual blob data
* - Implementations will define how to retrieve the actual blob data for this specific blob from a data store as an InputStream.
*
*/
public static abstract class Blob implements VersionedBlob{
protected final long fromVersion;
protected final long toVersion;
private final BlobType blobType;
/**
* Instantiate a snapshot to a specified data state version.
*
* @param toVersion the version
*/
public Blob(long toVersion) {
this(HollowConstants.VERSION_NONE, toVersion);
}
/**
* Instantiate a delta from one data state version to another.
*
* @param fromVersion the version to start the delta from
* @param toVersion the version to end the delta from
*/
public Blob(long fromVersion, long toVersion) {
this.fromVersion = fromVersion;
this.toVersion = toVersion;
if (this.isSnapshot())
this.blobType = BlobType.SNAPSHOT;
else if (this.isReverseDelta())
this.blobType = BlobType.REVERSE_DELTA;
else
this.blobType = BlobType.DELTA;
}
/**
* Implementations will define how to retrieve the actual blob data for this specific transition from a data store.
*
* It is expected that the returned InputStream will not be interrupted. For this reason, it is a good idea to
* retrieve the entire blob (e.g. to disk) from a remote datastore prior to returning this stream.
*
* @return the input stream to the blob
* @throws IOException if the input stream to the blob cannot be obtained
*/
public abstract InputStream getInputStream() throws IOException;
/**
* Implementations may define how to retrieve the optional blob part data for this specific transition from a data store.
*
* It is expected that none of the returned InputStreams will be interrupted. For this reason, it is a good idea to
* retrieve the entire blob part data (e.g. to disk) from a remote datastore prior to returning these streams.
*
* @return OptionalBlobPartInput
* @throws IOException exception in reading from blob or file
*/
public OptionalBlobPartInput getOptionalBlobPartInputs() throws IOException {
return null;
}
/**
* Blobs can be of types {@code SNAPSHOT}, {@code DELTA} or {@code REVERSE_DELTA}.
*/
public enum BlobType {
SNAPSHOT("snapshot"),
DELTA("delta"),
REVERSE_DELTA("reversedelta");
private final String type;
BlobType(String type) {
this.type = type;
}
public String getType() {
return this.type;
}
}
public boolean isSnapshot() {
return fromVersion == HollowConstants.VERSION_NONE;
}
public boolean isReverseDelta() {
return toVersion < fromVersion;
}
public boolean isDelta() {
return !isSnapshot() && !isReverseDelta();
}
public long getFromVersion() {
return fromVersion;
}
public long getToVersion() {
return toVersion;
}
public BlobType getBlobType() {
return blobType;
}
}
/**
* This class holds an announced version, its pinned status and the announcement metadata.
* isPinned and announcementMetadata fields are empty unless they are populated by the AnnouncementWatcher.
* */
public static class VersionInfo {
long version;
Optional isPinned;
Optional> announcementMetadata;
public VersionInfo(long version) {
this(version, Optional.empty(), Optional.empty());
}
public VersionInfo(long version, Optional> announcementMetadata, Optional isPinned) {
this.version = version;
this.announcementMetadata = announcementMetadata;
this.isPinned = isPinned;
}
public long getVersion() {
return version;
}
public Optional> getAnnouncementMetadata() {
return announcementMetadata;
}
public Optional isPinned() {
return isPinned;
}
}
/**
* Implementations of this class are responsible for two things:
*
* 1) Tracking the latest announced data state version.
* 2) Keeping the client up to date by calling triggerAsyncRefresh() on self when the latest version changes.
*
* If an AnnouncementWatcher is provided to a HollowConsumer, then calling HollowConsumer#triggerRefreshTo() is unsupported.
*/
public interface AnnouncementWatcher {
long NO_ANNOUNCEMENT_AVAILABLE = HollowConstants.VERSION_NONE;
/**
* @return the latest announced version.
*/
long getLatestVersion();
/**
* Implementations of this method should subscribe a HollowConsumer to updates to announced versions.
*
* When announcements are received via a push mechanism, or polling reveals a new version, a call should be placed to one
* of the flavors of {@link HollowConsumer#triggerRefresh()} on the provided HollowConsumer.
*
* @param consumer the hollow consumer
*/
void subscribeToUpdates(HollowConsumer consumer);
/***
* @return versionInfo - the latest announced version, its pinned status and announcement metadata.
*/
default VersionInfo getLatestVersionInfo() {
return new VersionInfo(getLatestVersion(), Optional.empty(), Optional.empty());
}
}
public interface DoubleSnapshotConfig {
boolean allowDoubleSnapshot();
int maxDeltasBeforeDoubleSnapshot();
default boolean doubleSnapshotOnSchemaChange() { return false; }
DoubleSnapshotConfig DEFAULT_CONFIG = new DoubleSnapshotConfig() {
@Override
public int maxDeltasBeforeDoubleSnapshot() {
return 32;
}
@Override
public boolean allowDoubleSnapshot() {
return true;
}
};
}
public interface ObjectLongevityConfig {
/**
* @return whether or not long-lived object support is enabled.
*
* Because Hollow reuses pooled memory, if references to Hollow records are held too long, the underlying data may
* be overwritten. When long-lived object support is enabled, Hollow records referenced via a {@link HollowAPI} will,
* after an update, be backed by a reserved copy of the data at the time the reference was created. This guarantees
* that even if a reference is held for a long time, it will continue to return the same data when interrogated.
*
* These reserved copies are backed by the {@link HollowHistory} data structure.
*/
boolean enableLongLivedObjectSupport();
boolean enableExpiredUsageStackTraces();
/**
* @return if long-lived object support is enabled, the number of milliseconds before the {@link StaleHollowReferenceDetector}
* will begin flagging usage of stale objects.
*/
long gracePeriodMillis();
/**
* @return if long-lived object support is enabled, the number of milliseconds, after the grace period, during which
* data is still available in stale references, but usage will be flagged by the {@link StaleHollowReferenceDetector}.
*
* After the grace period + usage detection period have expired, the data from stale references will become inaccessible if
* dropDataAutomatically() is enabled.
*/
long usageDetectionPeriodMillis();
/**
* @return whether or not to drop data behind stale references after the grace period + usage detection period has elapsed, assuming
* that no usage was detected during the usage detection period.
*/
boolean dropDataAutomatically();
/**
* @return whether data is dropped even if flagged during the usage detection period.
*/
boolean forceDropData();
ObjectLongevityConfig DEFAULT_CONFIG = new ObjectLongevityConfig() {
@Override
public boolean enableLongLivedObjectSupport() {
return false;
}
@Override
public boolean dropDataAutomatically() {
return false;
}
@Override
public boolean forceDropData() {
return false;
}
@Override
public boolean enableExpiredUsageStackTraces() {
return false;
}
@Override
public long usageDetectionPeriodMillis() {
return 60 * 60 * 1000;
}
@Override
public long gracePeriodMillis() {
return 60 * 60 * 1000;
}
};
}
/**
* Listens for stale Hollow object usage
*/
public interface ObjectLongevityDetector {
/**
* Stale reference detection hint. This will be called every ~30 seconds.
*
* If a nonzero value is reported, then stale references to Hollow objects may be cached somewhere in your codebase.
*
* This signal can be noisy, and a nonzero value indicates that some reference to stale data exists somewhere.
*
* @param count the count of stale references
*/
void staleReferenceExistenceDetected(int count);
/**
* Stale reference USAGE detection. This will be called every ~30 seconds.
*
* If a nonzero value is reported, then stale references to Hollow objects are being accessed from somewhere in your codebase.
*
* This signal is noiseless, and a nonzero value indicates that some reference to stale data is USED somewhere.
*
* @param count the count of stale references
*/
void staleReferenceUsageDetected(int count);
ObjectLongevityDetector DEFAULT_DETECTOR = new ObjectLongevityDetector() {
@Override
public void staleReferenceUsageDetected(int count) {
}
@Override
public void staleReferenceExistenceDetected(int count) {
}
};
}
/**
* Implementations of this class will define what to do when various events happen before, during, and after updating
* local in-memory copies of hollow data sets.
*/
public interface RefreshListener {
/**
* Called when a new version has been detected in consumer refresh (and just before refreshStarted). Surfaces
* metadata and pinning status pertaining to requested version if available from AnnouncementWatcher. Generally
* useful for logging/metrics.
*
* @param requestedVersionInfo requested version's information comprising version, announcement metadata and its pinned status
* */
default void versionDetected(VersionInfo requestedVersionInfo) {};
/**
* Indicates that a refresh has begun. Generally useful for logging.
*
* A refresh is the process of a consumer getting from a current version to a desired version.
*
* A refresh will consist of one of the following:
*
* - one or more deltas
* - a snapshot load, plus zero or more deltas
*
*
* @param currentVersion the current state version
* @param requestedVersion the version to which the refresh is progressing
*/
void refreshStarted(long currentVersion, long requestedVersion);
/**
* This method is called when either data was initialized for the first time, or an update occurred across a
* discontinuous delta chain (double snapshot).
*
* If this method is called, it means that the current refresh consists of a snapshot load, plus zero or more deltas.
*
* Implementations may initialize (or re-initialize) any indexing which is critical to keep in-sync with the data.
*
* This method will be called a maximum of once per refresh, after the data has reached the final state of the refresh.
*
* @param api the {@link HollowAPI} instance
* @param stateEngine the {@link HollowReadStateEngine}
* @param version the current state version
* @throws Exception thrown if an error occurs in processing
*/
void snapshotUpdateOccurred(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception;
/**
* This method is called whenever a live state engine's data is updated with a delta. This method is not
* called during first time initialization or when an update across a discontinuous delta chain (double snapshot)
* occurs.
*
* Implementations should incrementally update any indexing which is critical to keep in-sync with the data.
*
* If this method is called, it means that the current refresh consists of one or more deltas, and does not include
* a snapshot load.
*
* This method may be called multiple times per refresh, once for each time a delta is applied.
*
* @param api the {@link HollowAPI} instance
* @param stateEngine the {@link HollowReadStateEngine}
* @param version the current state version
* @throws Exception thrown if an error occurs in processing
*/
void deltaUpdateOccurred(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception;
/**
* Called to indicate a blob was loaded (either a snapshot or delta). Generally useful for logging or tracing of applied updates.
*
* @param transition The transition which was applied.
*/
void blobLoaded(HollowConsumer.Blob transition);
/**
* Indicates that a refresh completed successfully.
*
* @param beforeVersion - The version when the refresh started
* @param afterVersion - The version when the refresh completed
* @param requestedVersion - The specific version which was requested
*/
void refreshSuccessful(long beforeVersion, long afterVersion, long requestedVersion);
/**
* Indicates that a refresh failed with an Exception.
*
* @param beforeVersion - The version when the refresh started
* @param afterVersion - The version when the refresh completed
* @param requestedVersion - The specific version which was requested
* @param failureCause - The Exception which caused the failure.
*/
void refreshFailed(long beforeVersion, long afterVersion, long requestedVersion, Throwable failureCause);
}
public interface TransitionAwareRefreshListener extends RefreshListener {
/**
* This method is called whenever a snapshot is processed. In the case of first time initialization or an update
* across a discontinuous delta chain (double snapshot), this method will be called once (as the first transition).
*
* Implementations may initialize (or re-initialize) any indexing which is critical to keep in-sync with the data.
*
* @param api the {@link HollowAPI} instance
* @param stateEngine the {@link HollowReadStateEngine}
* @param version the current state version
* @throws Exception thrown if an error occurs in processing
*/
void snapshotApplied(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception;
/**
* This method is called whenever a delta is processed. In the case of first time initialization or an update
* across a discontinuous delta chain (double snapshot), this method may be called one or more times before arriving
* at the final state (after which {@link #snapshotUpdateOccurred(HollowAPI, HollowReadStateEngine, long)} is called.
*
* Implementations may incrementally update any indexing which is critical to keep in-sync with the data.
*
* @param api the {@link HollowAPI} instance
* @param stateEngine the {@link HollowReadStateEngine}
* @param version the current state version
* @throws Exception thrown if an error occurs in processing
*/
void deltaApplied(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception;
/**
* Called after refresh started and update plan has been initialized, but before the update plan starts executing.
* It is called only once per update plan (and thus only once per consumer refresh). Exposes details of the
* update plan.
* @implSpec The default implementation provided does nothing.
*
* @param beforeVersion The version when refresh started
* @param desiredVersion The version that the consumer refresh tries update to, even though it might not be attainable eg. HollowConstants.VERSION_LATEST
* @param isSnapshotPlan Indicates whether the refresh involves a snapshot transition
* @param transitionSequence List of transitions comprising the refresh
*/
default void transitionsPlanned(long beforeVersion, long desiredVersion, boolean isSnapshotPlan, List transitionSequence) {}
}
/**
* A listener of refresh listener addition and removal.
*
* A {@link RefreshListener} implementation may implement this interface to get notified before
* the listener is added (via a call to {@link #addRefreshListener(RefreshListener)} and after a listener
* is removed (via a call to {@link #removeRefreshListener(RefreshListener)}.
*
* An implementation should not add or remove itself in response to addition or removal. Such actions may result
* in a {@link StackOverflowError} or unspecified behaviour.
*/
public interface RefreshRegistrationListener {
/**
* Called before the refresh listener is added.
* @param c the consumer the associated reference listener is being added to
*/
void onBeforeAddition(HollowConsumer c);
/**
* Called after the refresh listener is removed.
* @param c the consumer the associated reference listener is being removed from
*/
void onAfterRemoval(HollowConsumer c);
}
public static class AbstractRefreshListener implements TransitionAwareRefreshListener {
@Override
public void refreshStarted(long currentVersion, long requestedVersion) {
// no-op
}
@Override
public void transitionsPlanned(long beforeVersion, long desiredVersion, boolean isSnapshotPlan, List transitionSequence) {
// no-op
}
@Override
public void snapshotUpdateOccurred(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception {
// no-op
}
@Override
public void deltaUpdateOccurred(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception {
// no-op
}
@Override
public void blobLoaded(Blob transition) {
// no-op
}
@Override
public void refreshSuccessful(long beforeVersion, long afterVersion, long requestedVersion) {
// no-op
}
@Override
public void refreshFailed(long beforeVersion, long afterVersion, long requestedVersion, Throwable failureCause) {
// no-op
}
@Override
public void snapshotApplied(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception {
// no-op
}
@Override
public void deltaApplied(HollowAPI api, HollowReadStateEngine stateEngine, long version) throws Exception {
// no-op
}
}
public static > HollowConsumer.Builder newHollowConsumer() {
return new Builder<>();
}
/**
* Convenience method for {@code .newHollowConsumer().withBlobRetriever(...)}
*/
public static HollowConsumer.Builder withBlobRetriever(HollowConsumer.BlobRetriever blobRetriever) {
HollowConsumer.Builder builder = new Builder<>();
return builder.withBlobRetriever(blobRetriever);
}
/**
* @deprecated use {@link #newHollowConsumer()}, i.e. {@code newHollowConsumer().withLocalBlobStore(...)}
*/
public static HollowConsumer.Builder withLocalBlobStore(File localBlobStoreDir) {
HollowConsumer.Builder builder = new Builder<>();
return builder.withLocalBlobStore(localBlobStoreDir);
}
@SuppressWarnings("unchecked")
@PublicSpi
public static class Builder> {
protected HollowConsumer.BlobRetriever blobRetriever = null;
protected HollowConsumer.AnnouncementWatcher announcementWatcher = null;
/**
* @deprecated subclasses should use {@code typeFilter}
*/
@Deprecated
protected HollowFilterConfig filterConfig = null; // retained for binary compat
protected TypeFilter typeFilter = null;
protected List refreshListeners = new ArrayList<>();
protected HollowAPIFactory apiFactory = HollowAPIFactory.DEFAULT_FACTORY;
protected HollowObjectHashCodeFinder hashCodeFinder = new DefaultHashCodeFinder();
protected HollowConsumer.DoubleSnapshotConfig doubleSnapshotConfig = DoubleSnapshotConfig.DEFAULT_CONFIG;
protected HollowConsumer.ObjectLongevityConfig objectLongevityConfig = ObjectLongevityConfig.DEFAULT_CONFIG;
protected HollowConsumer.ObjectLongevityDetector objectLongevityDetector = ObjectLongevityDetector.DEFAULT_DETECTOR;
protected File localBlobStoreDir = null;
protected boolean useExistingStaleSnapshot;
protected Executor refreshExecutor = null;
protected MemoryMode memoryMode = MemoryMode.ON_HEAP;
protected HollowMetricsCollector metricsCollector;
protected boolean skipTypeShardUpdateWithNoAdditions = false;
public B withBlobRetriever(HollowConsumer.BlobRetriever blobRetriever) {
this.blobRetriever = blobRetriever;
return (B)this;
}
/**
* This is the same as {@link #withLocalBlobStore(File, boolean)} where the boolean argument
* is set to {@code false}.
* @see #withLocalBlobStore(File, boolean)
*/
public B withLocalBlobStore(File localBlobStoreDir) {
this.localBlobStoreDir = localBlobStoreDir;
return (B)this;
}
/**
* @see #withLocalBlobStore(File)
*/
public B withLocalBlobStore(String localBlobStoreDir) {
return withLocalBlobStore(new File(localBlobStoreDir));
}
/**
* Provide a directory that will be used to cache blobs.
*
* When this is supplied, Hollow will look in this directory for a blob before falling back
* to retrieving using the configured {@link HollowConsumer.BlobRetriever}.
* If it does not find the blob in the configured local blob store directory, it will
* retrieve the blob using the configured blobRetriever and then use it from there.
* Note that cached files are never deleted from {@code localBlobStoreDir}, meaning if you
* use this on a long-running instance you will eventually run out of disk space. As such, a
* local blob store is better suited for local development, testing, or tooling purposes.
*
* @param localBlobStoreDir the directory used to store cached blobs. This will be created
* if it does not already exist.
* @param useExistingStaleSnapshot
* IF this is true
* AND Hollow is trying to retrieve a snapshot
* AND it hasn't already cached a snapshot for that specific version
* AND it has cached a snapshot of an older version
* THEN Hollow will use the older cached snapshot instead of fetching the desired snapshot
*/
public B withLocalBlobStore(File localBlobStoreDir, boolean useExistingStaleSnapshot) {
this.localBlobStoreDir = localBlobStoreDir;
this.useExistingStaleSnapshot = useExistingStaleSnapshot;
return (B)this;
}
/**
* @see #withLocalBlobStore(File, boolean)
*/
public B withLocalBlobStore(String localBlobStoreDir, boolean useExistingStaleSnapshot) {
return withLocalBlobStore(new File(localBlobStoreDir), useExistingStaleSnapshot);
}
public B withAnnouncementWatcher(HollowConsumer.AnnouncementWatcher announcementWatcher) {
this.announcementWatcher = announcementWatcher;
return (B)this;
}
public B withRefreshListener(HollowConsumer.RefreshListener refreshListener) {
refreshListeners.add(refreshListener);
return (B)this;
}
public B withRefreshListeners(HollowConsumer.RefreshListener... refreshListeners) {
Collections.addAll(this.refreshListeners, refreshListeners);
return (B)this;
}
/**
* Provide the code generated API class that extends {@link HollowAPI} with one or more types
* cached for direct field reads.
*
* All hollow record instances are created by one of two factories:
*
*
* - {@link com.netflix.hollow.api.objects.provider.HollowObjectFactoryProvider}
* - creates an instance of the corresponding {@code
DelegateLookupImpl} (e.g.
* {@code StringDelegateLookupImpl} or {@code MovieDelegateLookupImpl} for core
* types or types in a generated client API respectively). Field accesses perform
* a lookup into the underlying high-density cache
*
* - {@link com.netflix.hollow.api.objects.provider.HollowObjectCacheProvider}
* - instantiates and caches the corresponding {@code
DelegateCachedImpl} from
* the generated client API (e.g. {@code MovieDelegateCachedImpl}). For a given ordinal,
* the same {@code HollowRecord} instance is returned assuming the ordinal hasn't been removed.
* All of the type's fields are eagerly looked up from the high-density cache and stored as Java fields,
* making field access in tight loops or the hottest code paths more CPU efficient.
*
*
* Object caching should only be enabled for low cardinality, custom types in your data model.
*
* Use {@link #withGeneratedAPIClass(Class)} to build a consumer with your custom client API and
* using the default high-density cache for all types.
*
* @param generatedAPIClass the code generated API class
* @param cachedType the type to enable cache on to enable object caching on
* @param additionalCachedTypes More types to enable object caching on
*
* @return this builder
*
* @see https://hollow.how/advanced-topics/#caching
*/
public B withGeneratedAPIClass(Class generatedAPIClass,
String cachedType,
String... additionalCachedTypes) {
if (HollowAPI.class.equals(generatedAPIClass))
throw new IllegalArgumentException("must provide a code generated API class");
generatedAPIClass = Objects.requireNonNull(generatedAPIClass, "API class cannot be null");
Objects.requireNonNull(additionalCachedTypes, "null detected for varargs parameter additionalCachedTypes");
String[] cachedTypes = new String[additionalCachedTypes.length + 1];
cachedTypes[0] = cachedType;
System.arraycopy(additionalCachedTypes, 0, cachedTypes, 1, additionalCachedTypes.length);
BitSet nulls = new BitSet(cachedTypes.length);
for (int i = 0; i < cachedTypes.length; ++i) {
if (cachedTypes[i] == null)
nulls.set(i);
}
if (!nulls.isEmpty())
throw new NullPointerException("cached types cannot be null; argsWithNull=" + nulls.toString());
this.apiFactory = new HollowAPIFactory.ForGeneratedAPI<>(generatedAPIClass, cachedTypes);
return (B)this;
}
/**
* Provide the code generated API class that extends {@link HollowAPI}.
*
* The instance returned from {@link HollowConsumer#getAPI()} will be of the provided type and can be cast
* to access generated methods.
*
* @param generatedAPIClass the code generated API class
* @return this builder
* @throws IllegalArgumentException if provided API class is {@code HollowAPI} instead of a subclass
*/
public B withGeneratedAPIClass(Class generatedAPIClass) {
if (HollowAPI.class.equals(generatedAPIClass))
throw new IllegalArgumentException("must provide a code generated API class");
this.apiFactory = new HollowAPIFactory.ForGeneratedAPI<>(generatedAPIClass);
return (B)this;
}
/**
* Specifies a filter config. Clears any type filter specified by {@code withTypeFilter(...)}.
*
* {@link HollowFilterConfig} is deprecated in favor of {@link TypeFilter}.
*
* {@code HollowFilterConfig} has these limitations:
*
*
* - cannot mix inclusions and exclusions in a single filter and cannot compose filters
* - recursive actions requires that callers already have the dataset's schema, leading to
* a chicken-and-egg situation
*
*
* @return this builder
* @see #withTypeFilter(TypeFilter)
* @see #withTypeFilter(UnaryOperator)
* @deprecated use {@link #withTypeFilter(TypeFilter)} or {@link #withTypeFilter(UnaryOperator)}
*/
@Deprecated
public B withFilterConfig(HollowFilterConfig filterConfig) {
this.typeFilter = filterConfig;
return (B)this;
}
/**
* Specifies the type filter. Clears any filter config specified by {@code withFilterConfig()}
*
* {@link TypeFilter} is the replacement API for {@link HollowFilterConfig}. It improves upon its
* limitations, but lacks the ability to serialize/deserialize like {@code HollowFilterConfig} has.
*
* @param typeFilter
* @return this builder
* @see #withTypeFilter(UnaryOperator)
* @see #withFilterConfig(HollowFilterConfig)
*/
public B withTypeFilter(TypeFilter typeFilter) {
this.typeFilter = typeFilter;
return (B)this;
}
/**
* Configures a new type filter on this consumer using a fluent API. Works as if by
* calling {@code withTypeFilter(op.apply(newTypeFilter())}. For example:
*
* {@code
* consumerBuilder
* ...
* .withTypeFilter(filter -> filter
* .excludeAll()
* .includeRecursive("Alpha"))
* .build();
* }
*
* Clears any filter config specified by {@code withFilterConfig(...)}.
*
* {@link TypeFilter} is the replacement API for {@link HollowFilterConfig}. It improves upon its
* limitations, but lacks the ability to serialize/deserialize like {@code HollowFilterConfig} has.
*
* @param op
* @return this builder
* @see #withTypeFilter(TypeFilter)
* @see #withFilterConfig(HollowFilterConfig)
*/
public B withTypeFilter(UnaryOperator op) {
TypeFilter.Builder builder = op.apply(TypeFilter.newTypeFilter());
return withTypeFilter(builder.build());
}
public B withDoubleSnapshotConfig(HollowConsumer.DoubleSnapshotConfig doubleSnapshotConfig) {
this.doubleSnapshotConfig = doubleSnapshotConfig;
return (B)this;
}
public B withObjectLongevityConfig(HollowConsumer.ObjectLongevityConfig objectLongevityConfig) {
this.objectLongevityConfig = objectLongevityConfig;
return (B)this;
}
public B withObjectLongevityDetector(HollowConsumer.ObjectLongevityDetector objectLongevityDetector) {
this.objectLongevityDetector = objectLongevityDetector;
return (B)this;
}
public B withRefreshExecutor(Executor refreshExecutor) {
this.refreshExecutor = refreshExecutor;
return (B)this;
}
/**
* Experimental: By default the Hollow dataset is eagerly loaded to heap, but shared memory mode allows mmap-ing
* the dataset to virtual memory and only loading specific pages to main memory.
*
* This allows faster init and support for datasets larger that available physical memory. However, shared memory
* implementation is not suitable for production use given its limited functionality (no delta refreshes, no
* un-mmap of previous version, feature gaps like indexes continue to live on heap) and limited production hardening.
*/
public B withMemoryMode(MemoryMode memoryMode) {
this.memoryMode = memoryMode;
return (B)this;
}
public B withMetricsCollector(HollowMetricsCollector metricsCollector) {
this.metricsCollector = metricsCollector;
return (B)this;
}
/**
* Experimental: When there are no updates for a type shard in a delta, skip updating that type shard.
*/
public B withSkipTypeShardUpdateWithNoAdditions() {
this.skipTypeShardUpdateWithNoAdditions = true;
return (B)this;
}
@Deprecated
public B withHashCodeFinder(HollowObjectHashCodeFinder hashCodeFinder) {
this.hashCodeFinder = hashCodeFinder;
return (B)this;
}
protected void checkArguments() {
if (filterConfig != null && typeFilter != null) {
// this should only be possible in custom subclasses that override #withFilterConfig(...)
throw new IllegalStateException("Only one of typeFilter and filterConfig can be set");
}
if (blobRetriever == null && localBlobStoreDir == null) {
throw new IllegalArgumentException(
"A HollowBlobRetriever or local blob store directory must be specified when building a HollowClient");
}
BlobRetriever blobRetriever = this.blobRetriever;
if (localBlobStoreDir != null) {
this.blobRetriever = new HollowFilesystemBlobRetriever(
localBlobStoreDir.toPath(), blobRetriever, useExistingStaleSnapshot);
}
if (refreshExecutor == null) {
final Class clazz = getClass(); // to avoid capturing 'this' in lambda
refreshExecutor = newSingleThreadExecutor(r -> daemonThread(r, clazz, "refresh"));
}
if (!memoryMode.consumerSupported()) {
throw new UnsupportedOperationException("Cinder Consumer in " + memoryMode + " mode is not supported");
}
if ((filterConfig != null || typeFilter != null) && !memoryMode.supportsFiltering()) {
throw new UnsupportedOperationException("Filtering is not supported in shared memory mode");
}
}
public HollowConsumer build() {
checkArguments();
if (filterConfig != null) {
// TODO: remove once deprecated #withFilterConfig is removed
typeFilter = filterConfig;
filterConfig = null;
}
return new HollowConsumer((B)this);
}
}
}