• Home
• io.prophecy
• prophecy-libs_2.12
• 5.2.7-3.0.1
• source code
• BinaryNode23.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

io.prophecy.libs.lineage.parsers.BinaryNode23.scala Maven / Gradle / Ivy

package io.prophecy.libs.lineage.parsers

import io.prophecy.libs.lineage._
import io.prophecy.libs.lineage.LogicalPlanParser._
import io.prophecy.libs.lineage.LogicalRelationParser.{collectTransformations, isPlanPresent}
import org.apache.spark.sql.catalyst.optimizer.OrderedJoin
import org.apache.spark.sql.catalyst.plans.logical.{Except, Intersect, Join, LogicalPlan}
import org.apache.spark.sql.execution.datasources.LogicalRelation

object BinaryNode23 {
  val availableParsers: List[LogicalPlanParser[_]] = List(
    ExceptParser,
    IntersectParser,
    JoinParser,
    OrderedJoinParser
  )
}

object ExceptParser extends LogicalPlanParser[Except] with DefaultPlanExists[Except] {
  def transformations: TransformationCollectionFunction = {
    case ctx: LineageParserContext if ctx.plan.isInstanceOf[Except] ⇒
      collectTransformations(ctx.plan.asInstanceOf[Except], ctx)
  }
  def planExists: FindLogicalPlanFunction = {
    case (plan1: LogicalPlan, plan2: Except) ⇒ isPlanPresent(plan1, plan2)
  }
  override def collectTransformations(plan: Except, ctx: LineageParserContext) = {
    //TODO expression id matcher for sets
    val trans = List.empty
    val next = plan.children.map { child ⇒
      ctx.copy(
        plan = child,
        potentialChildrenPlans = ctx.potentialChildrenPlans
          .find(isLogicalPlanInLogicalPlan(_, child))
          .map(x ⇒ List(x))
          .getOrElse(ctx.potentialChildrenPlans)
      )
    }.toList
    TransformCollectionResult(trans, next)
  }
}

object IntersectParser extends LogicalPlanParser[Intersect] with DefaultPlanExists[Intersect] {
  def transformations: TransformationCollectionFunction = {
    case ctx: LineageParserContext if ctx.plan.isInstanceOf[Intersect] ⇒
      collectTransformations(ctx.plan.asInstanceOf[Intersect], ctx)
  }
  def planExists: FindLogicalPlanFunction = {
    case (plan1: LogicalPlan, plan2: Intersect) ⇒ isPlanPresent(plan1, plan2)
  }
  override def collectTransformations(plan: Intersect, ctx: LineageParserContext) = {
    val outputs = plan.outputSet.map(o ⇒ o.name → o).toMap
    val trans = plan.inputSet.map { i ⇒
      LTransformation(
        sources = ctx.getInputsForExpr(i),
        destination = outputs.get(i.name).flatMap(ctx.getOutputForExpr)
      )
    }.toList
    val next = plan.children.map { child ⇒
      ctx.copy(
        plan = child,
        potentialChildrenPlans = ctx.potentialChildrenPlans
          .find(isLogicalPlanInLogicalPlan(_, child))
          .map(x ⇒ List(x))
          .getOrElse(ctx.potentialChildrenPlans)
      )
    }.toList
    TransformCollectionResult(trans, next)
  }
}

//TODO check if output is correct for all join types - plan.joinType
object JoinParser extends LogicalPlanParser[Join] with DefaultPlanExists[Join] {
  def transformations: TransformationCollectionFunction = {
    case ctx: LineageParserContext if ctx.plan.isInstanceOf[Join] ⇒
      collectTransformations(ctx.plan.asInstanceOf[Join], ctx)
  }
  def planExists: FindLogicalPlanFunction = {
    case (plan1: LogicalPlan, plan2: Join) ⇒ isPlanPresent(plan1, plan2)
  }
  override def collectTransformations(plan: Join, ctx: LineageParserContext) = {
    val (left, right, condition) = (plan.left, plan.right, plan.condition)
    // We are using buildTransformationWithDestination since condition usually contains only a filter in which case it does not make sense to return it
    val trans = condition.flatMap(ctx.buildTransformationWithDestination).map(t ⇒ List(t)).getOrElse(List.empty)
    val nextPotentialPlansForLeft =
      ctx.potentialChildrenPlans.filter(childPlan ⇒ isLogicalPlanInLogicalPlan(childPlan, left)) match {
        case Nil ⇒
          println(s"Potential error: Couldn't find left Join plan: ${left.toString()}")
          ctx.potentialChildrenPlans
        case default ⇒ default
      }

    val leftCtx = ctx.copy(
      plan = left,
      potentialChildrenPlans = nextPotentialPlansForLeft
    )

    val nextPotentialPlansForRight =
      ctx.potentialChildrenPlans.filter(childPlan ⇒ isLogicalPlanInLogicalPlan(childPlan, right)) match {
        case Nil ⇒
          println(s"Potential error: Couldn't find right Join plan: ${right.toString()}")
          ctx.potentialChildrenPlans
        case default ⇒ default
      }
    val rightCtx = ctx.copy(
      plan = right,
      potentialChildrenPlans = nextPotentialPlansForRight
    )
    TransformCollectionResult(trans, leftCtx :: rightCtx :: Nil)
  }
}

// According to docs, same as join
object OrderedJoinParser extends LogicalPlanParser[OrderedJoin] with DefaultPlanExists[OrderedJoin] {
  def transformations: TransformationCollectionFunction = {
    case ctx: LineageParserContext if ctx.plan.isInstanceOf[OrderedJoin] ⇒
      collectTransformations(ctx.plan.asInstanceOf[OrderedJoin], ctx)
  }
  def planExists: FindLogicalPlanFunction = {
    case (plan1: LogicalPlan, plan2: OrderedJoin) ⇒ isPlanPresent(plan1, plan2)
  }
  override def collectTransformations(plan: OrderedJoin, ctx: LineageParserContext) = {
    val (left, right, condition) = (plan.left, plan.right, plan.condition)
    // We are using buildTransformationWithDestination since condition usually contains only a filter in which case it does not make sense to return it
    val trans = condition.flatMap(ctx.buildTransformationWithDestination).map(t ⇒ List(t)).getOrElse(List.empty)
    val nextPotentialPlansForLeft =
      ctx.potentialChildrenPlans.filter(childPlan ⇒ isLogicalPlanInLogicalPlan(childPlan, left)) match {
        case Nil ⇒
          println(s"Potential error: Couldn't find left Join plan: ${left.toString()}")
          ctx.potentialChildrenPlans
        case default ⇒ default
      }
    if (nextPotentialPlansForLeft.isEmpty) {
      println(s"Potential error: Couldn't find left Join plan: ${left.toString()}")
    }
    val leftCtx = ctx.copy(
      plan = left,
      potentialChildrenPlans = nextPotentialPlansForLeft
    )

    val nextPotentialPlansForRight =
      ctx.potentialChildrenPlans.filter(childPlan ⇒ isLogicalPlanInLogicalPlan(childPlan, right)) match {
        case Nil ⇒
          println(s"Potential error: Couldn't find right Join plan: ${right.toString()}")
          ctx.potentialChildrenPlans
        case default ⇒ default
      }
    if (nextPotentialPlansForRight.isEmpty) {
      println(s"Potential error: Couldn't find right Join plan: ${right.toString()}")
    }
    val rightCtx = ctx.copy(
      plan = right,
      potentialChildrenPlans = nextPotentialPlansForRight
    )
    TransformCollectionResult(trans, leftCtx :: rightCtx :: Nil)
  }

}