• Home
• com.microsoft.ml.spark
• mmlspark_2.11
• 0.18.1
• source code
• NumericFeaturizer.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.microsoft.ml.spark.vw.featurizer.NumericFeaturizer.scala Maven / Gradle / Ivy

// Copyright (C) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in project root for information.

package com.microsoft.ml.spark.vw.featurizer

import org.apache.spark.sql.Row
import org.vowpalwabbit.spark.VowpalWabbitMurmur

import scala.collection.mutable.ArrayBuilder

/**
  * Featurize numeric values into native VW structure. ((hash(column name):value)
  * @param fieldIdx input field index.
  * @param columnName used as feature name prefix.
  * @param namespaceHash pre-hashed namespace.
  * @param mask bit mask applied to final hash.
  * @param getFieldValue lambda to unify the cast/conversion to double.
  */
class NumericFeaturizer(override val fieldIdx: Int, columnName: String, namespaceHash: Int,
                        mask: Int, val getFieldValue: (Row) => Double)
  extends Featurizer(fieldIdx) {

  /**
    * Pre-hashed feature index.
    */
  val featureIdx = mask & VowpalWabbitMurmur.hash(columnName, namespaceHash)

  /**
    * Featurize a single row.
    * @param row input row.
    * @param indices output indices.
    * @param values output values.
    * @note this interface isn't very Scala-esce, but it avoids lots of allocation.
    *       Also due to SparseVector limitations we don't support 64bit indices (e.g. indices are signed 32bit ints)
    */
  override def featurize(row: Row, indices: ArrayBuilder[Int], values: ArrayBuilder[Double]): Unit = {
    val value = getFieldValue(row)

    // Note: 0 valued features are always filtered.
    if (value != 0) {
        indices += featureIdx
        values += value
    }
    ()
  }
}