nstream.adapter.mongodb.MongoDbChangeStreamIngestingAgent Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of nstream-adapter-mongodb Show documentation
Templates for consuming from and producing to MongoDB with Swim
There is a newer version: 4.13.21
Show newest version
// Copyright 2015-2024 Nstream, inc.
//
// Licensed under the Redis Source Available License 2.0 (RSALv2) Agreement;
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://redis.com/legal/rsalv2-agreement/
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nstream.adapter.mongodb;

import com.mongodb.client.ChangeStreamIterable;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.model.changestream.ChangeStreamDocument;
import nstream.adapter.common.ingress.IngestorMetricsAgent;
import nstream.adapter.common.provision.ProvisionLoader;
import org.bson.Document;
import swim.structure.Value;

/**
 * An abstract Web Agent that can subscribe to a MongoDB change stream.
 * 
 * The {@link MongoDbChangeStreamIngestingAgent} loads agent configuration into a settings
 * object on agent start.
 * The agent uses the configured {@link MongoClient} provision to start a change stream
 * receiving any change documents, deferring processing of
 * the ingested {@link ChangeStreamDocument} to subclasses.
 * 

 * Change stream processing is done asynchronously of the agent thread.
 *
 * @see MongoDbIngressSettings
 */
public abstract class MongoDbChangeStreamIngestingAgent extends IngestorMetricsAgent> {

  protected MongoClient client;
  protected volatile MongoCursor> cursor;

  public MongoDbChangeStreamIngestingAgent() {
  }

  /**
   * Assign the {@link MongoClient} to be used for ingestion.
   *
   * @param client the Mongo client
   */
  protected void assignClient(MongoClient client) {
    this.client = client;
  }

  /**
   * Subscribe to the {@link ChangeStreamIterable} provided by the abstract
   * {@link #changeStream()} method and call {@link #ingestOrCancel}
   * on each document.
   *
   * @param didSubscribe did subscribe callback
   */
  protected void subscribe(java.lang.Runnable didSubscribe) {
    try {
      this.cursor = changeStream().cursor();
      didSubscribe.run();
      while (true) {
        ingestOrCancel(this.cursor.next());
      }
    } catch (Exception e) {
      didFail(new RuntimeException(nodeUri() + ": exception reading from cursor; stopping", e));
    }
  }

  /**
   * The configurable {@link ChangeStreamIterable} to be looped through and ingested.
   * 
   * Can be overridden to change the MongoDB object to stream changes for.
   *
   * @return the change stream iterable
   */
  protected abstract ChangeStreamIterable changeStream();

  @Override
  protected void cancel() {
    if (this.cursor != null) {
      this.cursor.close();
      this.cursor = null;
    }
  }

  @Override
  protected void didFailIngest(ChangeStreamDocument document, Exception e) {
    didFail(new RuntimeException(nodeUri() + ": " + document
        + " triggered fatal exception; stopping ", e));
  }

  @Override
  protected MongoDbIngressSettings parseIngressSettings(Value prop) {
    final MongoDbIngressSettings settings = MongoDbIngressSettings.form().cast(prop);
    return settings == null ? MongoDbIngressSettings.defaultSettings() : settings;
  }

  @Override
  protected void didStageReception() {
    super.didStageReception();
    info(nodeUri() + ": successfully staged change stream for reception");
  }

  @Override
  protected void stageReception() {
    loadSettings("mongoDbIngressConf");
    assignClient(ProvisionLoader.getProvision(this.ingressSettings.clientProvisionName())
        .value());
    subscribe(this::didStageReception);
  }

  @Override
  public void didStart() {
    info(nodeUri() + ": didStart");
    execute(this::stageReception);
  }

}