nstream.adapter.dynamodb.DynamoDbIngestingAgent Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of nstream-adapter-dynamodb Show documentation
Templates for consuming from and producing to DynamoDB with Swim
The newest version!
// Copyright 2015-2023 Nstream, inc.
//
// Licensed under the Redis Source Available License 2.0 (RSALv2) Agreement;
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://redis.com/legal/rsalv2-agreement/
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nstream.adapter.dynamodb;

import java.util.Map;
import nstream.adapter.common.ingress.IngestorMetricsAgent;
import nstream.adapter.common.provision.ProvisionLoader;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
import swim.concurrent.TimerRef;
import swim.structure.Value;

/**
 * An abstract Web Agent that can scan a DynamoDB database table.
 * 
 * The {@link DynamoDbIngestingAgent} loads agent configuration into a settings
 * object on agent start.
 * The agent uses the configured {@link software.amazon.awssdk.services.dynamodb.DynamoDbClient}
 * provision and timing properties to periodically poll the database for
 * items, deferring processing of the ingested attribute map to subclasses.
 * 

 * Scanning and ingestion are done asynchronously of the agent thread.
 *
 * @see DynamoDbIngressSettings
 */
public abstract class DynamoDbIngestingAgent extends IngestorMetricsAgent> {

  protected DynamoDbClient client;
  protected TimerRef pollTimer;

  public DynamoDbIngestingAgent() {
  }

  /**
   * Assign the {@link DynamoDbClient} to be used for ingestion.
   *
   * @param client the DynamoDB client
   */
  protected void assignClient(final DynamoDbClient client) {
    this.client = client;
  }

  /**
   * Loop through the {@link ScanRequest} results provided by the abstract
   * {@link #scanRequest()} method, ingesting each document asynchronously.
   */
  protected void fetchAndIngest() {
    final ScanResponse scanResponse = this.client.scan(scanRequest());
    for (Map item : scanResponse.items()) {
      ingestOrContinue(item);
    }
  }

  /**
   * The configurable {@link ScanRequest} to be executed, looped through and ingested.
   * 
   * Can be overridden for bespoke queries.
   *
   * @return the scan request
   */
  protected abstract ScanRequest scanRequest();

  /**
   * Cancel the poll timer and therefore ingestion.
   * Will cancel any currently active ingestion task.
   */
  @Override
  protected void cancel() {
    if (this.pollTimer != null) {
      this.pollTimer.cancel();
    }
  }

  @Override
  public void didFailIngest(final Map item, final Exception exception) {
    didFail(new RuntimeException(nodeUri() + ": " + item.toString()
        + " triggered fatal exception; stopping", exception));
  }

  @Override
  protected DynamoDbIngressSettings parseIngressSettings(Value prop) {
    final DynamoDbIngressSettings settings = DynamoDbIngressSettings.form().cast(prop);
    return settings == null ? DynamoDbIngressSettings.defaultSettings() : settings;
  }

  @Override
  protected void stageReception() {
    loadSettings("dynamoDbIngressConf");
    assignClient(ProvisionLoader.getProvision(this.ingressSettings.clientProvisionName())
        .value());
    this.pollTimer = scheduleWithFixedDelay(() -> this.pollTimer, this.ingressSettings.firstFetchDelayMillis(),
        this.ingressSettings.fetchIntervalMillis(), this::fetchAndIngest);
  }

}