geotrellis.spark.etl.hadoop.HadoopOutput.scala Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of geotrellis-spark-etl_2.11 Show documentation

GeoTrellis is an open source geographic data processing engine for high performance applications.

The newest version!

package geotrellis.spark.etl.hadoop

import geotrellis.spark.etl.OutputPlugin
import geotrellis.spark.io.hadoop._

import org.apache.hadoop.fs.Path
import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil

trait HadoopOutput[K, V, M] extends OutputPlugin[K, V, M] {
  val name = "hadoop"
  val requiredKeys = Array("path")

  //This should be a safe way to get a hadoop configuration that includes all the environment changes from spark
  def attributes(props: Map[String, String]) =
    HadoopAttributeStore(props("path"), SparkHadoopUtil.get.newConfiguration(new SparkConf()))
}