com.audienceproject.spark.dynamodb.msd.datasource.DynamoDbPartitionReaderFactory.scala Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of mega-spark-diff Show documentation

A diff analysis tool that compares data sets at scale of various types and can be executed in CLOUD or locally

The newest version!

package com.audienceproject.spark.dynamodb.msd.datasource

import com.audienceproject.spark.dynamodb.connector.DynamoConnector
import com.audienceproject.spark.dynamodb.datasource.ScanPartition
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, PartitionReaderFactory}
import org.apache.spark.sql.types.StructType

class DynamoDbPartitionReaderFactory(connector: DynamoConnector, schema: StructType) extends PartitionReaderFactory {
  override def createReader(partition: InputPartition): PartitionReader[InternalRow] =
    new DynamoDbPartitionReader(connector, schema, partition.asInstanceOf[ScanPartition])
}