com.sparkutils.quality.impl.util.LookupIdFunctionsImports.scala Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of quality_2.4_2.11 Show documentation

A Spark library for managing in-process data quality rules via Spark SQL

The newest version!

package com.sparkutils.quality.impl.util

import com.sparkutils.quality.RuleSuite
import org.apache.spark.sql.types.StructType

trait LookupIdFunctionsImports {

  /**
   * Retrieves the names (with parent fields following the . notation for any nested fields) from the schema
   *
   * @param schema
   * @return
   */
  def namesFromSchema(schema: StructType): Set[String] = LookupIdFunctions.namesFromSchema(schema)

  /**
   * Use this function to identify which maps / blooms etc. are used by a given rulesuite
   * collects all rules that are using lookup functions but without constant expressions and the list of lookups that are constants.
   *
   */
  def identifyLookups(ruleSuite: RuleSuite): LookupResults = LookupIdFunctions.identifyLookups(ruleSuite)
}