com.twitter.scalding.TupleUnpacker.scala Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of scalding_2.9.1 Show documentation
scalding
The newest version!
/*
Copyright 2012 Twitter, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package com.twitter.scalding

import cascading.pipe._
import cascading.pipe.joiner._
import cascading.tuple._

import java.lang.reflect.Method

import scala.reflect.Manifest

/** Base class for objects which unpack an object into a tuple.
  * The packer can verify the arity, types, and also the existence
  * of the getter methods at plan time, without having the job
  * blow up in the middle of a run.
  *
  * @author Argyris Zymnis
  * @author Oscar Boykin
  */
object TupleUnpacker extends LowPriorityTupleUnpackers
abstract class TupleUnpacker[T] extends java.io.Serializable {
  def newSetter(fields : Fields) : TupleSetter[T]
}

trait LowPriorityTupleUnpackers extends TupleConversions {
  implicit def genericUnpacker[T : Manifest] = new ReflectionTupleUnpacker[T]
}

class ReflectionTupleUnpacker[T](implicit m : Manifest[T]) extends TupleUnpacker[T] {
  override def newSetter(fields : Fields) = new ReflectionSetter[T](fields)(m)
}

class ReflectionSetter[T](fields : Fields)(implicit m : Manifest[T]) extends TupleSetter[T] {

  validate // Call the validation method at the submitter

  // This is lazy because it is not serializable
  // Contains a list of methods used to set the Tuple from an input of type T
  lazy val setters = makeSetters

  // Methods and Fields are not serializable so we
  // make these defs instead of vals
  // TODO: filter by isAccessible, which somehow seems to fail
  def methodMap = m.erasure
    .getDeclaredMethods
    // Keep only methods with 0 parameter types
    .filter { m => m.getParameterTypes.length == 0 }
    .groupBy { _.getName }
    .mapValues { _.head }

  // TODO: filter by isAccessible, which somehow seems to fail
  def fieldMap = m.erasure
    .getDeclaredFields
    .groupBy { _.getName }
    .mapValues { _.head }

  def makeSetters = {
    (0 until fields.size).map { idx =>
      val fieldName = fields.get(idx).toString
      setterForFieldName(fieldName)
    }
  }

  // This validation makes sure that the setters exist
  // but does not save them in a val (due to serialization issues)
  def validate = makeSetters

  override def apply(input : T) : Tuple = {
    val values = setters.map { setFn => setFn(input) }
    new Tuple(values : _*)
  }

  override def arity = fields.size

  private def setterForFieldName(fieldName : String) : (T => AnyRef) = {
    getValueFromMethod(createGetter(fieldName))
      .orElse(getValueFromMethod(fieldName))
      .orElse(getValueFromField(fieldName))
      .getOrElse(
        throw new TupleUnpackerException("Unrecognized field: " + fieldName + " for class: " + m.erasure.getName)
      )
  }

  private def getValueFromField(fieldName : String) : Option[(T => AnyRef)] = {
    fieldMap.get(fieldName).map { f => (x : T) => f.get(x) }
  }

  private def getValueFromMethod(methodName : String) : Option[(T => AnyRef)] = {
    methodMap.get(methodName).map { m => (x : T) => m.invoke(x) }
  }

  private def upperFirst(s : String) = s.substring(0,1).toUpperCase + s.substring(1)
  private def createGetter(s : String) = "get" + upperFirst(s)
}

class TupleUnpackerException(args : String) extends Exception(args)