org.memeticlabs.spark.rdd.trycatch.ErrorHandlingPairRDDFunctions.scala Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of spark-rdd-trycatch Show documentation
Error trapping and handling functionality for Spark's RDD API
The newest version!
/**
	* Copyright 2017 Tristan Nixon
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*
	* Created by Tristan Nixon on 7/9/17.
	*/
package org.memeticlabs.spark.rdd.trycatch

import scala.language.implicitConversions
import scala.reflect._

import org.apache.spark.Partitioner
import org.apache.spark.rdd.{PairRDDFunctions, RDD}
import org.apache.spark.serializer.Serializer

import org.memeticlabs.spark.rdd.trycatch.TryCatchHelpers._

private[memeticlabs] class ErrorHandlingPairRDDFunctions[K, V](val self: ErrorHandlingRDD[(K, V)],
                                                               val errorHandlerBuilder: errorHandlerBuilder[Any] )
                                                              (implicit kt: ClassTag[K], vt: ClassTag[V], ord: Ordering[K] = null)
	extends PairRDDFunctions[K,V]( self )
{
	private def asEHPRDD[U]( tx: (RDD[(K,V)]) => RDD[(K,U)] ) =
		new ErrorHandlingRDD[(K,U)]( tx( self ), errorHandlerBuilder )

	override def combineByKeyWithClassTag[C]( createCombiner: ( V ) => C,
	                                          mergeValue: (C, V) => C,
	                                          mergeCombiners: (C, C) => C,
	                                          partitioner: Partitioner,
	                                          mapSideCombine: Boolean = true,
	                                          serializer: Serializer = null )
	                                        (implicit ct: ClassTag[C]): RDD[(K, C)] =
	{
		val zeroValue: C = null.asInstanceOf[C]
		val errorHandler = errorHandlerBuilder("combineByKeyWithClassTag")

		asEHPRDD[C] { _.combineByKeyWithClassTag( tryCatchAndHandle[V,C]( createCombiner, zeroValue, errorHandler ),
		                                          tryCatchAndHandle[C,V]( mergeValue, zeroValue, errorHandler ),
		                                          tryCatchAndHandle[C]( mergeCombiners, zeroValue, errorHandler ),
		                                          partitioner, mapSideCombine, serializer)(ct) }
	}



	override def mapValues[U](f: ( V ) => U): RDD[(K, U)] =
		asEHPRDD[U] { _.mapValues( tryCatchResult[V,U]( f, errorHandlerBuilder("mapValues") ) )
		               .filter( _._2.isSuccess )
		               .mapValues( _.getResult ) }

	override def flatMapValues[U](f: ( V ) => TraversableOnce[U]): RDD[(K, U)] =
		asEHPRDD[U] { _.mapValues( tryCatchResult[V,TraversableOnce[U]]( f, errorHandlerBuilder("flatMapValues") ) )
		               .filter( _._2.isSuccess )
		               .flatMapValues( _.getResult ) }
}

object ErrorHandlingPairRDDFunctions
{
	implicit def toErrorHandlingPairRDDFunctions[K,V]
		( errorHandlingRDD: ErrorHandlingRDD[(K,V)]  )
		( implicit kt: ClassTag[K], vt: ClassTag[V] ): ErrorHandlingPairRDDFunctions[K, V] =
			new ErrorHandlingPairRDDFunctions[K,V]( errorHandlingRDD, errorHandlingRDD.errorHandlerBuilder )
}