fs2.kafka.CommitRecovery.scala Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of fs2-kafka_2.12 Show documentation
fs2-kafka
The newest version!
/*
 * Copyright 2018-2024 OVO Energy Limited
 *
 * SPDX-License-Identifier: Apache-2.0
 */

package fs2.kafka

import scala.concurrent.duration.*

import cats.effect.Temporal
import cats.syntax.applicativeError.*
import cats.syntax.flatMap.*
import cats.syntax.functor.*
import cats.Functor

import org.apache.kafka.clients.consumer.{OffsetAndMetadata, RetriableCommitFailedException}
import org.apache.kafka.common.errors.RebalanceInProgressException
import org.apache.kafka.common.TopicPartition

/**
  * [[CommitRecovery]] describes how to recover from exceptions raised while trying to commit
  * offsets. See [[CommitRecovery#Default]] for the default recovery strategy. If you do not wish to
  * recover from any exceptions, you can use [[CommitRecovery#None]].


  *
  * To create a new [[CommitRecovery]], simply create a new instance and implement the
  * [[recoverCommitWith]] function with the wanted recovery strategy. To use the [[CommitRecovery]],
  * you can simply set it with [[ConsumerSettings#withCommitRecovery]].
  */
abstract class CommitRecovery {

  /**
    * Describes recovery from offset commit exceptions. The `commit` parameter can be used to retry
    * the commit. Note that if more than one retry is desirable, errors from `commit` will need to
    * be handled and recovered.


    *
    * The offsets we are trying to commit are available via the `offsets` parameter. Waiting before
    * retrying again can be done via the provided `Timer` instance, and jitter can be applied using
    * the `Jitter` instance.
    */
  def recoverCommitWith[F[_]](
    offsets: Map[TopicPartition, OffsetAndMetadata],
    commit: F[Unit]
  )(implicit
    F: Temporal[F],
    jitter: Jitter[F]
  ): Throwable => F[Unit]

}

object CommitRecovery {

  /**
    * The default [[CommitRecovery]] used in [[ConsumerSettings]] unless a different one has been
    * specified. The default recovery strategy only retries `RetriableCommitFailedException`s and
    * `RebalanceInProgressException`s. These exceptions are retried with a jittered exponential
    * backoff, where the time in milliseconds before retrying is calculated using:
    *
    * {{{
    * Random.nextDouble() * Math.min(10000, 10 * Math.pow(2, n))
    * }}}
    *
    * where `n` is the retry attempt (first attempt is `n = 1`). This is done for up to 10 attempts,
    * after which we change to retry using a fixed time of 10 seconds, for up to another 5 attempts.
    * If at that point we are still faced with `RetriableCommitFailedException` or
    * `RebalanceInProgressException`, we give up and raise a [[CommitRecoveryException]] with the
    * last such error experienced.


    *
    * The sum of time spent waiting between retries will always be less than 70 220 milliseconds, or
    * ~70 seconds. Note that this does not include the time for attempting to commit offsets. Offset
    * commit times are limited with [[ConsumerSettings.commitTimeout]].
    */
  val Default: CommitRecovery =
    new CommitRecovery {

      private[this] def backoff[F[_]](attempt: Int)(implicit
        F: Functor[F],
        jitter: Jitter[F]
      ): F[FiniteDuration] = {
        val millis = Math.min(10000, 10 * Math.pow(2, attempt.toDouble))
        jitter.withJitter(millis).map(_.millis)
      }

      override def recoverCommitWith[F[_]](
        offsets: Map[TopicPartition, OffsetAndMetadata],
        commit: F[Unit]
      )(implicit
        F: Temporal[F],
        jitter: Jitter[F]
      ): Throwable => F[Unit] = {
        def retry(attempt: Int): Throwable => F[Unit] = {
          case retriable @ (_: RetriableCommitFailedException | _: RebalanceInProgressException) =>
            val commitWithRecovery = commit.handleErrorWith(retry(attempt + 1))
            if (attempt <= 10) backoff(attempt).flatMap(F.sleep) >> commitWithRecovery
            else if (attempt <= 15) F.sleep(10.seconds) >> commitWithRecovery
            else F.raiseError(CommitRecoveryException(attempt - 1, retriable, offsets))

          case nonRetriable: Throwable =>
            F.raiseError(nonRetriable)
        }

        retry(attempt = 1)
      }

      override def toString: String =
        "Default"

    }

  /**
    * A [[CommitRecovery]] that does not retry any exceptions.
    */
  val None: CommitRecovery =
    new CommitRecovery {

      override def recoverCommitWith[F[_]](
        offsets: Map[TopicPartition, OffsetAndMetadata],
        commit: F[Unit]
      )(implicit
        F: Temporal[F],
        jitter: Jitter[F]
      ): Throwable => F[Unit] = F.raiseError

      override def toString: String =
        "None"

    }

}