• Home
• tech.sourced
• gitbase-spark-connector
• 0.3.0
• source code
• Sources.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

tech.sourced.gitbase.spark.Sources.scala Maven / Gradle / Ivy

package tech.sourced.gitbase.spark

import org.apache.spark.sql.types.{MetadataBuilder, StructField, StructType}

/**
  * Defines the hierarchy between data sources.
  */
object Sources {

  val SourceKey: String = "source"

  /** Sources ordered by their position in the hierarchy. */
  val orderedSources = Array(
    "repositories",
    "refs",
    "ref_commits",
    "commits",
    "commit_trees",
    "commit_blobs",
    "commit_files",
    "tree_entries",
    "blobs",
    "files"
  )

  /**
    * Compares two sources.
    *
    * @param a first source
    * @param b second source
    * @return comparison result
    */
  def compare(a: String, b: String): Int = orderedSources.indexOf(a)
    .compareTo(orderedSources.indexOf(b))

  def addToSchema(schema: StructType, table: String): StructType = {
    val metadata = new MetadataBuilder().putString(Sources.SourceKey, table).build()
    StructType(schema.map(f => {
      StructField(f.name, f.dataType, f.nullable, metadata)
    }))
  }

}