• Home
• io.eels
• eel-hive_2.11
• 1.3.0-a13
• source code
• Compactor.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

io.eels.component.hive.Compactor.scala Maven / Gradle / Ivy

package io.eels.component.hive

import com.sksamuel.exts.Logging
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.hive.metastore.IMetaStoreClient

case class Compactor(dbname: String, tablename: String)
                    (implicit fs: FileSystem, conf: Configuration,client: IMetaStoreClient) extends Logging {

  /**
    * Compacts the files in the given hive datbase.table
    * Returns the original number of files.
    * Automatically handles partitions.
    */
  def compactTo(targetFileCount: Int): Int = {
    val originalPaths = HiveTable(dbname, tablename).paths(false, false)
    HiveSource(dbname, tablename).toDataStream().to(HiveSink(dbname, tablename), targetFileCount)
    originalPaths.foreach(fs.delete(_, false))
    originalPaths.size
  }
}