io.eels.component.hive.partition.DynamicPartitionStrategy.scala Maven / Gradle / Ivy
The newest version!
package io.eels.component.hive.partition
import io.eels.component.hive.HiveOps
import io.eels.schema.Partition
import io.eels.util.HdfsMkdir
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.hive.metastore.IMetaStoreClient
/**
* A PartitionStrategy that creates partitions in the table location using the default
* key=value/key=value path format
*/
class DynamicPartitionStrategy extends PartitionStrategy {
private val cache = scala.collection.mutable.Map.empty[Partition, Path]
def ensurePartition(partition: Partition,
dbName: String,
tableName: String,
inheritPermissions: Boolean,
client: IMetaStoreClient)(implicit fs: FileSystem): Path = {
def createPartition: Path = this.synchronized {
val ops = new HiveOps(client)
ops.partitionMetaData(dbName, tableName, partition) match {
case Some(meta) => meta.location
case _ =>
val tableLocation = ops.tablePath(dbName, tableName)
val partitionPath = new Path(tableLocation, partition.unquoted)
ops.createPartitionIfNotExists(dbName, tableName, partition, partitionPath)
HdfsMkdir(partitionPath, inheritPermissions)
partitionPath
}
}
cache.getOrElseUpdate(partition, createPartition)
}
}