pectator.spectator-ext-spark.1.8.8.source-code.reference.conf Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of spectator-ext-spark Show documentation
spectator-ext-spark
The newest version!

spectator.spark {

  sidecar {
    output-location = "udp://127.0.0.1:1234"

    tags = [
      // {
      //   key = "key"
      //   value = "value"
      // }
    ]
  }

  name-patterns = [

    // EXECUTORS Spark 1.6
    // 97278898-4bd4-49c2-9889-aa5f969a7816-0013.97278898-4bd4-49c2-9889-aa5f969a7816-S1/2.executor.filesystem.file.largeRead_ops
    // 97278898-4bd4-49c2-9889-aa5f969a7816-0013.97278898-4bd4-49c2-9889-aa5f969a7816-S1/2.jvm.heap.committed
    // 97278898-4bd4-49c2-9889-aa5f969a7816-0013.97278898-4bd4-49c2-9889-aa5f969a7816-S1/2.CodeGenerator.compilationTime
    {
      pattern = "^([^.]+)\\.([^.]+)\\/(\\d+)\\.((executor|jvm|CodeGenerator)\\..*)"
      name = 4
      tags = {
        "appId" = 1
        "agentId" = 2
        "executorId" = 3
      }
    },

    // EXECUTORS Spark 2.0/2.1
    // 97278898-4bd4-49c2-9889-aa5f969a7816-0013.2.executor.filesystem.file.largeRead_ops
    // 97278898-4bd4-49c2-9889-aa5f969a7816-0013.2.jvm.heap.committed
    // 97278898-4bd4-49c2-9889-aa5f969a7816-0013.2.CodeGenerator.compilationTime

    // EXECUTORS Spark 2.1
    // 97278898-4bd4-49c2-9889-aa5f969a7816-0013.2.HiveExternalCatalog.fileCacheHits

    {
      pattern = "^([^.]+)\\.(\\d+)\\.((executor|jvm|HiveExternalCatalog|CodeGenerator)\\..*)"
      name = 3
      tags = {
        "appId" = 1
        "executorId" = 2
      }
    },

    // DRIVER 1.6/2.0
    // 97278898-4bd4-49c2-9889-aa5f969a7816-0020.driver.BlockManager.memory.remainingMem_MB
    // 97278898-4bd4-49c2-9889-aa5f969a7816-0020.driver.CodeGenerator.compilationTime
    // 97278898-4bd4-49c2-9889-aa5f969a7816-0020.driver.DAGScheduler.job.activeJobs
    // 97278898-4bd4-49c2-9889-aa5f969a7816-0020.driver.jvm.heap.committed
    // DRIVER 2.1
    // 97278898-4bd4-49c2-9889-aa5f969a7816-0020.driver.HiveExternalCatalog.fileCacheHits
    {
      pattern = "^([^.]+)\\.(driver)\\.((CodeGenerator|DAGScheduler|BlockManager|HiveExternalCatalog|jvm)\\..+[^_MB])(_MB)?"
      name = 3
      tags = {
        "appId" = 1
        "role"  = 2
      }
    },

    // DRIVER STREAMING
    // 52cc96b1-f89a-445d-a6e9-0e4cee70b731-0033.driver.HdfsWordCount.StreamingMetrics.streaming.lastCompletedBatch_processingDelay
    // Note: the .*Delay stats are derived from the .*Time stats. The time stats by themselves
    // aren't that useful sent to the backend.
    {
      pattern = "^([^.]+)\\.(driver)\\.([^.]+)\\.StreamingMetrics\\.(streaming\\..*)"
      name = 4
      tags = {
        "appId" = 1
        "role" = 2
      }
    }

  ]

  value-conversions = [
    {
      pattern = "^.*_MB$"
      factor = 1.0e6
    },
    {
      pattern = "^.*streaming.*_.*Time$"
      factor = 0.001
    },
    {
      pattern = "^.*streaming.*_.*Delay$"
      factor = 0.001
    }
  ]

  // Values that are reported as a gauge, but are monotonically increasing and better mapped into
  // a counter for spectator.
  gauge-counters = "^.*streaming\\.total.*$"
}