nstream.adapter.mongodb.MongoDbIngestingAgent Maven / Gradle / Ivy
Show all versions of nstream-adapter-mongodb Show documentation
// Copyright 2015-2024 Nstream, inc.
//
// Licensed under the Redis Source Available License 2.0 (RSALv2) Agreement;
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://redis.com/legal/rsalv2-agreement/
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nstream.adapter.mongodb;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoCursor;
import nstream.adapter.common.ingress.IngestorMetricsAgent;
import nstream.adapter.common.provision.ProvisionLoader;
import org.bson.Document;
import swim.concurrent.TimerRef;
import swim.structure.Value;
/**
* An abstract Web Agent that can poll a MongoDB database collection.
*
* The {@link MongoDbIngestingAgent} loads agent configuration into a settings
* object on agent start.
* The agent uses the configured {@link MongoClient} provision and timing properties
* to periodically poll the database for documents, deferring processing of
* the ingested {@link Document} to subclasses.
*
* Querying and ingestion are done asynchronously of the agent thread.
*
* @see MongoDbIngressSettings
*/
public abstract class MongoDbIngestingAgent extends IngestorMetricsAgent {
protected MongoClient client;
protected TimerRef pollTimer;
public MongoDbIngestingAgent() {
}
/**
* Assign the {@link MongoClient} to be used for ingestion.
*
* @param client the Mongo client
*/
protected void assignClient(MongoClient client) {
this.client = client;
}
/**
* Loop through the {@link FindIterable} provided by the abstract
* {@link #find()} method, ingesting each document asynchronously.
*/
protected void fetchAndIngest() {
try (MongoCursor cursor = find().cursor()) {
while (cursor.hasNext()) {
final Document document = cursor.next();
ingestOrContinue(document);
}
}
}
/**
* The configurable {@link FindIterable} to be looped through and ingested -
* the find or 'query'.
*
* Can be overridden for bespoke queries.
*
* @return the cursor
*/
protected abstract FindIterable find();
/**
* Cancel the poll timer and therefore ingestion.
* Will cancel any currently active ingestion task.
*/
@Override
protected void cancel() {
if (this.pollTimer != null) {
this.pollTimer.cancel();
}
}
@Override
public void didFailIngest(final Document document, final Exception exception) {
didFail(new RuntimeException(nodeUri() + ": " + document.toJson()
+ " triggered fatal exception; stopping", exception));
}
@Override
protected MongoDbIngressSettings parseIngressSettings(Value prop) {
final MongoDbIngressSettings settings = MongoDbIngressSettings.form().cast(prop);
return settings == null ? MongoDbIngressSettings.defaultSettings() : settings;
}
@Override
protected void stageReception() {
loadSettings("mongoDbIngressConf");
assignClient(ProvisionLoader.getProvision(this.ingressSettings.clientProvisionName())
.value());
this.pollTimer = scheduleWithFixedDelay(() -> this.pollTimer, this.ingressSettings.firstFetchDelayMillis(),
this.ingressSettings.fetchIntervalMillis(), this::fetchAndIngest);
}
}