org.memeticlabs.spark.rdd.trycatch.streaming.SourcePropagatingDStream.scala Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of spark-rdd-trycatch Show documentation

Error trapping and handling functionality for Spark's RDD API

The newest version!

/**
	* Copyright 2017 Tristan Nixon
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*
	* Created by Tristan Nixon on 7/12/17.
	*/
package org.memeticlabs.spark.rdd.trycatch.streaming

import scala.reflect.ClassTag

import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.{Duration, Time}
import org.apache.spark.streaming.dstream.DStream

import org.memeticlabs.spark.rdd.trycatch.{SourcePropagatingRDD, ValueWithSource}
import org.memeticlabs.spark.rdd.trycatch.TryCatchHelpers.errorHandlerBuilder

private[memeticlabs] class SourcePropagatingDStream[T,S]
	( parent: DStream[ValueWithSource[T,S]],
	  errorHandlerBuilder: errorHandlerBuilder[ValueWithSource[_,S]] )
	(implicit tt: ClassTag[T], st: ClassTag[S])
	extends DStream[T](parent.context)
{

	override def slideDuration: Duration = parent.slideDuration

	override def dependencies: List[DStream[_]] = List(parent)

	override def compute(validTime: Time): Option[RDD[T]] =
		parent.compute(validTime) match {
			case Some(rdd) => Option( new SourcePropagatingRDD( rdd, errorHandlerBuilder ) )
			case None => None
		}
}

object SourcePropagatingDStream
{
	private[memeticlabs] def asVWS[S](source: DStream[S] ): DStream[ValueWithSource[S,S]] =
		source.map( s => ValueWithSource( s, s ) )

	def fromSource[S: ClassTag]( source: DStream[S],
	                             errorHandlerBuilder: errorHandlerBuilder[ValueWithSource[_,S]] ): DStream[S] =
		new SourcePropagatingDStream[S,S]( asVWS( source ), errorHandlerBuilder )

	def fromSource[S: ClassTag]( source: DStream[S],
	                             errorHandler: ( String, ValueWithSource[_,S], Throwable ) => Unit ): DStream[S] =
	{
		val curriedHandler: errorHandlerBuilder[ValueWithSource[_,S]] = ( op: String ) => errorHandler( op, _, _ )
		fromSource[S]( source, curriedHandler )
	}
}