All Downloads are FREE. Search and download functionalities are using the official Maven repository.

au.csiro.variantspark.cli.args.SparkArgs.scala Maven / Gradle / Ivy

The newest version!
package au.csiro.variantspark.cli.args

import org.kohsuke.args4j.Option
import au.csiro.pbdava.ssparkle.spark.SparkApp
import org.apache.spark.rdd.RDD

trait SparkArgs extends SparkApp {

  @Option(name = "-sp", required = false, usage = "Spark parallelism (def=)",
    aliases = Array("--spark-par"))
  val sparkPar: Int = 0

  def textFile(inputFile: String): RDD[String] =
    sc.textFile(inputFile, if (sparkPar > 0) sparkPar else sc.defaultParallelism)

}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy