• Home
• com.twitter
• scalding-parquet-scrooge_2.11
• 0.16.1-RC1
• source code
• ParquetScrooge.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.twitter.scalding.parquet.scrooge.ParquetScrooge.scala Maven / Gradle / Ivy

package com.twitter.scalding.parquet.scrooge

import cascading.scheme.Scheme
import com.twitter.scalding._
import com.twitter.scalding.parquet.thrift.ParquetThriftBase
import com.twitter.scalding.source.{ DailySuffixSource, HourlySuffixSource }
import com.twitter.scrooge.ThriftStruct

trait ParquetScrooge[T <: ThriftStruct] extends ParquetThriftBase[T] {

  override def hdfsScheme = {
    // See docs in Parquet346ScroogeScheme
    val scheme = new Parquet346ScroogeScheme[T](this.config)
    HadoopSchemeInstance(scheme.asInstanceOf[Scheme[_, _, _, _, _]])
  }

}

class DailySuffixParquetScrooge[T <: ThriftStruct](
  path: String,
  dateRange: DateRange)(implicit override val mf: Manifest[T])
  extends DailySuffixSource(path, dateRange) with ParquetScrooge[T]

class HourlySuffixParquetScrooge[T <: ThriftStruct](
  path: String,
  dateRange: DateRange)(implicit override val mf: Manifest[T])
  extends HourlySuffixSource(path, dateRange) with ParquetScrooge[T]

class FixedPathParquetScrooge[T <: ThriftStruct](paths: String*)(implicit override val mf: Manifest[T])
  extends FixedPathSource(paths: _*) with ParquetScrooge[T]