net.snowflake.ingest.streaming.OffsetTokenVerificationFunction Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of snowflake-ingest-sdk Show documentation

Snowflake Ingest SDK

There is a newer version: 3.0.1

/*
 * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved.
 */

package net.snowflake.ingest.streaming;

/**
 * Interface to provide a custom offset verification logic. If specified, verification failures will
 * be logged as warnings and reported to Snowflake. This interface could be used when there are
 * certain assumption about the offset token behavior. Please reach out to Snowflake if you notice
 * any unexpected behaviors.
 *
 * Below is an example that verifies that all offset tokens are monotonically increasing numbers:
 *
 * 
 *     
 *      private static final OffsetTokenVerificationFunction offsetTokenVerificationFunction =
 *       (prevBatchEndOffset, curBatchStartOffset, curBatchEndOffset, rowCount) -> {
 *         boolean isMatch = true;
 *
 *         if (curBatchStartOffset != null) {
 *           try {
 *             long curStart = Long.parseLong(curBatchStartOffset);
 *             long curEnd = Long.parseLong(curBatchEndOffset);
 *
 *             // We verify that the end_offset - start_offset + 1 = row_count
 *             if (curEnd - curStart + 1 != rowCount) {
 *               isMatch = false;
 *             }
 *
 *             // We verify that start_offset_of_current_batch = end_offset_of_previous_batch+1
 *             if (prevBatchEndOffset != null) {
 *               long prevEnd = Long.parseLong(prevBatchEndOffset);
 *               if (curStart != prevEnd + 1) {
 *                 isMatch = false;
 *               }
 *             }
 *           } catch (NumberFormatException ignored) {
 *             // Do nothing since we can't compare the offset, or report a mismatch if number is expected
 *           }
 *         }
 *
 *         return isMatch;
 *       };
 *     
 * 
 */
public interface OffsetTokenVerificationFunction {
  /**
   * @param prevBatchEndOffset end offset token of the previous batch
   * @param curBatchStartOffset start offset token of the current batch
   * @param curBatchEndOffset end offset token of the current batch
   * @param rowCount number of rows in the current batch
   * @return a boolean indicates whether the verification passed or not, if not, we will log a
   *     warning and report it to SF
   */
  boolean verify(
      String prevBatchEndOffset,
      String curBatchStartOffset,
      String curBatchEndOffset,
      long rowCount);
}