com.myodov.unicherrygarden.cherrypicker.syncers.AbstractSyncer.scala Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of cherrypicker_2.13 Show documentation
UniCherryGarden: CherryPicker – the subsystem that keeps track of Ethereum blockchain data, stores it in the DB storage and “cherry-picks” the data on the fly (selective currencies, selective addresses to watch)
The newest version!
package com.myodov.unicherrygarden.cherrypicker.syncers

import akka.actor.typed.Behavior
import akka.actor.typed.scaladsl.Behaviors
import com.myodov.unicherrygarden.AbstractEthereumNodeConnector.SingleBlockData
import com.myodov.unicherrygarden.api.DBStorage.Progress
import com.myodov.unicherrygarden.api.GardenMessages.{HeadSyncerMessage, TailSyncerMessage}
import com.myodov.unicherrygarden.api.types.SystemStatus
import com.myodov.unicherrygarden.api.{DBStorage, DBStorageAPI, GardenMessages, dlt}
import com.myodov.unicherrygarden.{AbstractEthereumNodeConnector, CherryGardenComponent, Web3ReadOperations}
import com.typesafe.scalalogging.LazyLogging
import scalikejdbc.DBSession

import scala.language.postfixOps
import scala.util.control.NonFatal

/** Superclass for [[HeadSyncer]] and [[TailSyncer]], containing the common logic.
 *
 * Generic class arguments:
 * 
 * `M` – one of [[HeadSyncerMessage]], [[TailSyncerMessage]].
 * `S` – one of [[HeadSyncerState]], [[TailSyncerState]].
 * `IS` – one of [[IterateHeadSyncer]], [[IterateTailSyncer]].
 * 
 *
 * `state` contains the overall state of the syncer.
 * Unfortunately, we cannot go fully Akka-way having the system state passed through the methods, FSM states
 * and behaviors. If we receive some state-changing message from outside (e.g. the latest state of Ethereum node
 * syncing process; or, for HeadSyncer, the message from TailSyncer), we need to alter the state immediately.
 * But the FSM may be in a 10-second delay after the latest block being processed, and after it a message
 * with the previous state will be posted by the timer. So alas, `state` has to be variable.
 */
abstract private class AbstractSyncer[
  M <: GardenMessages.SyncerMessage,
  S <: AbstractSyncer.SyncerState,
  IS <: GardenMessages.IterateSyncer[M] with M
]
(protected[this] val dbStorage: DBStorageAPI,
 protected[this] val ethereumConnector: AbstractEthereumNodeConnector with Web3ReadOperations,
 protected[this] val state: S)
  extends LazyLogging {

  /** Most important method doing some next iteration of a syncer; must be implemented. */
  def iterate(): Behavior[M]

  /** Construct the specific implementation of `S` generic instance; must be implemented. */
  val iterateMessage: IS

  /** Most basic sanity test for the DB data;
   * fails if we cannot even go further and must wait for the node to continue syncing.
   */
  protected[this] def isNodeReachable(dbProgressData: DBStorage.Progress.ProgressData,
                                      nodeSyncingStatus: SystemStatus.Blockchain): Boolean =
    dbProgressData.blocks.to match {
      case None =>
        // All data is available; but there is no blocks in the DB. This actually is fully okay
        true
      case Some(maxBlocksNum) =>
        // Everything is fine if our latest stored block is not newer than the latest block available
        // to Ethereum node;
        // false/bad otherwise
        maxBlocksNum <= nodeSyncingStatus.syncingData.currentBlock
    }

  /** Validate the syncing progress data/Ethereum node syncing state;
   * and execute the `code` if they are valid, assuming it (maybe) returns some `Behavior`.
   *
   * `RES` is the expected return type from the function; may be `Option[Behavior]`, `[Behavior]` or something similar.
   *
   * @param onError the behavior generator that should be invoked in case of error; the result of invokation
   *                will be returned
   * @param code    the function that will be passed the `ProgressData` and `EthereumNodeStatus` objects.
   *                It can rely upon the fact that `progress.overall.from` is not `None` and contains something.
   */
  protected[this] def withValidatedProgressAndSyncingState[RES](
                                                                 optProgress: Option[Progress.ProgressData],
                                                                 optNodeSyncingStatus: Option[SystemStatus.Blockchain],
                                                                 onError: () => RES
                                                               )
                                                               (
                                                                 code: (Progress.ProgressData, SystemStatus.Blockchain) => RES
                                                               )
                                                               (implicit session: DBSession): RES = {
    (optProgress, optNodeSyncingStatus) match {
      case (None, _) =>
        // we could not even get the DB progress – go to the next round
        logger.error("Some unexpected error when reading the overall progress from the DB")
        onError()
      case (_, None) =>
        // we haven’t received the syncing state from the node
        logger.debug("No syncing status from Ethereum node is available yet, waiting")
        onError()
      case (Some(overallProgress: Progress.ProgressData), Some(nodeSyncStatus: SystemStatus.Blockchain))
        if !isNodeReachable(overallProgress, nodeSyncStatus) =>
        // Sanity test. Does the overall data sanity allows us to proceed?
        // In HeadSyncer, this is Reorg/rewind, phase 1/4: “is node reachable”.
        // In TailSyncer, this is just a sanity test.
        logger.error(s"Ethereum node is probably unavailable: $overallProgress, $nodeSyncStatus")
        onError()
      case (Some(overallProgress: Progress.ProgressData), Some(nodeSyncStatus: SystemStatus.Blockchain)) =>
        // Both CherryPicker syncing progress and Ethereum node status are at least available;
        // but let’s validate them, and only then launch the code.
        if (!overallProgress.isConfigurationValid) {
          onError()
        } else {
          code(overallProgress, nodeSyncStatus)
        }
    }
  }

  def reiterate(): Behavior[M] = {
    logger.debug("FSM: reiterate")
    Behaviors.setup { context =>
      logger.debug("Sending iterate message to self")
      context.self ! iterateMessage
      Behaviors.same
    }
  }

  def pauseThenReiterate(): Behavior[M] = {
    logger.debug("FSM: pauseThenReiterate")
    Behaviors.withTimers[M] { timers =>
      timers.startSingleTimer(
        iterateMessage,
        CherryGardenComponent.BLOCK_ITERATION_PERIOD)
      Behaviors.same
    }
  }

  /** The pause-then-reiterate method that must be implemented in each syncer specifically. */
  def pauseThenReiterateOnError(): Behavior[M]

  /** Perform the regular iteration for a specific block number:
   * read the block from the Ethereum connector, store it into the DB.
   *
   * @return whether syncing of the blocks succeeded.
   */
  protected[this] def syncBlocks(
                                  blocksToSync: dlt.EthereumBlock.BlockNumberRange
                                )(implicit session: DBSession): Boolean = {
    val trackedAddresses: Set[String] = dbStorage.trackedAddresses.getJustAddresses

    logger.debug(s"FSM: syncBlocks - blocks $blocksToSync with tracked addresses $trackedAddresses")

    // Were all of the blocks read and stored well?
    val successes: Seq[Boolean] = ethereumConnector.readBlocks(blocksToSync, trackedAddresses) match {
      case None =>
        logger.error(s"Cannot read blocks $blocksToSync")
        Seq(false)
      case Some(blocks: Seq[SingleBlockData]) => blocks.map { case (block, transactions) =>
        try {
          logger.debug(s"Reading block $block: txes $transactions")

          val thisBlockInDbOpt = dbStorage.blocks.getBlockByNumber(block.number)
          val prevBlockInDbOpt = dbStorage.blocks.getBlockByNumber(block.number - 1)

          logger.debug(s"Storing block: $block; " +
            s"block may be present as $thisBlockInDbOpt, " +
            s"parent may be present as $prevBlockInDbOpt")

          val addingBlockSuccess: Boolean = (thisBlockInDbOpt, prevBlockInDbOpt) match {
            case (None, None) =>
              // This is the simplest case: this is probably the very first block in the DB
              logger.debug(s"Adding first block ${block.number}: " +
                s"neither it nor previous block exist in the DB")
              dbStorage.blocks.addBlock(block.withoutParentHash)
              true
            case (None, Some(prevBlockInDb)) if prevBlockInDb.hash == block.parentHash.get =>
              // Another simplest case: second and further blocks in the DB.
              // Very new block, and its parent matches the existing one
              logger.debug(s"Adding new block ${block.number}; parent block ${block.number - 1} " +
                s"exists already with proper hash")
              dbStorage.blocks.addBlock(block)
              true
            case (Some(thisBlockInDb), _) if thisBlockInDb.hash == block.hash =>
              logger.debug(s"Block ${block.number} exists already in the DB " +
                s"with the same hash ${block.hash}; " +
                "no need to readd the block itself")
              true
            case (Some(thisBlockInDb), _) if thisBlockInDb.hash != block.hash =>
              logger.debug(s"Block ${block.number} exists already in the DB " +
                s"but with ${thisBlockInDb.hash} rather than ${block.hash}; " +
                "need to wipe some blocks maybe!")
              false
            case (None, Some(prevBlockInDb)) if prevBlockInDb.hash != block.parentHash.get =>
              logger.debug(s"Adding new block ${block.number}: " +
                s"expecting parent block to be ${prevBlockInDb.hash} but it is ${block.parentHash.get}; " +
                "need to wipe some blocks maybe!")
              false
            case other =>
              logger.debug(s"No idea what's up with $thisBlockInDbOpt and $prevBlockInDbOpt")
              false
          } // addingBlockSuccess

          if (!addingBlockSuccess) {
            // Bail out early
            false
          } else {
            logger.debug(s"Now trying to store the transactions: $transactions")
            for (tx <- transactions) {
              dbStorage.transactions.addTransaction(tx, block.hash)
              dbStorage.txLogs.addTxLogs(block.number, tx.txhash, tx.txLogs)
            }
            dbStorage.state.advanceProgress(block.number, trackedAddresses)
            true
          }
        } catch {
          case NonFatal(e) =>
            logger.error(s"Unexpected error", e)
            false
        }
      }
    } // successes

    successes.forall(identity)
  }
}

object AbstractSyncer {

  trait SyncerState {
    @volatile var ethereumNodeStatus: Option[SystemStatus.Blockchain]
  }

}