• Home
• com.intel.analytics.zoo
• analytics-zoo-bigdl_0.12.2-spark_2.2.1
• 0.10.0
• source code
• Utils.scala

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.intel.analytics.zoo.models.recommendation.Utils.scala Maven / Gradle / Ivy

/*
 * Copyright 2018 Analytics Zoo Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.intel.analytics.zoo.models.recommendation

import java.lang

import com.intel.analytics.bigdl.dataset.{Sample, TensorSample}
import com.intel.analytics.bigdl.tensor.Tensor
import com.intel.analytics.bigdl.utils.{RandomGenerator, T}
import org.apache.spark.sql.functions.{max, udf}
import org.apache.spark.sql.{DataFrame, Row}

import scala.collection.mutable
import scala.collection.JavaConverters._

object Utils {

  /**
   * generate negative samples given a dataframe of positive records, label >=2.
   *
   * @param indexed dataframe positive of userId, itemId and label.
   * @return a dataframe of negative samples(label=1) with the same size as indexed dataframe
   */
  def getNegativeSamples(indexed: DataFrame): DataFrame = {
    val schema = indexed.schema
    require(schema.fieldNames.contains("userId"), s"Column userId should exist")
    require(schema.fieldNames.contains("itemId"), s"Column itemId should exist")
    require(schema.fieldNames.contains("label"), s"Column label should exist")

    val indexedDF = indexed.select("userId", "itemId", "label")
    val minMaxRow = indexedDF.agg(max("userId"), max("itemId")).collect()(0)
    val (userCount, itemCount) = (minMaxRow.getInt(0), minMaxRow.getInt(1))
    val sampleDict = indexedDF.rdd.map(row => row(0) + "," + row(1)).collect().toSet

    val dfCount = indexedDF.count.toInt

    import indexed.sqlContext.implicits._

    @transient lazy val ran = RandomGenerator.RNG

    val negative = indexedDF.rdd
      .map(x => {
        val uid = x.getAs[Int](0)
        val iid = ran.uniform(0, itemCount).toInt + 1
        (uid, iid)
      })
      .filter(x => !sampleDict.contains(x._1 + "," + x._2)).distinct()
      .map(x => (x._1, x._2, 1))
      .toDF("userId", "itemId", "label")

    negative
  }

  def buckBucket(bucketSize: Int): (String, String) => Int = {
    val func = (col1: String, col2: String) =>
      (Math.abs((col1 + "_" + col2).hashCode()) % bucketSize + 0)
    func
  }

  def buckBuckets(bucketSize: Int)(col: String*): Int = {
    Math.abs(col.reduce(_ + "_" + _).hashCode()) % bucketSize + 0
  }

  def bucketizedColumn(boundaries: Array[Float]): Float => Int = {
    col1: Float => {
      var index = 0
      while (index < boundaries.length && col1 >= boundaries(index)) {
        index += 1
      }
      index
    }
  }

  // save 0 for uncovered ones
  def categoricalFromVocabList(vocabList: Array[String]): (String) => Int = {
    val func = (sth: String) => {
      val default: Int = 0
      val start: Int = 1
      if (vocabList.contains(sth)) vocabList.indexOf(sth) + start
      else default
    }
    func
  }


  /**
   * convert a row to sample given column information of WideAndDeep model.
   *
   * @param r          Row of userId, itemId, features and label
   * @param columnInfo ColumnFeatureInfo specify information of different features
   * @param modelType  support "wide_n_deep", "wide", "deep" only
   * @return TensorSample as input for WideAndDeep model
   */
  def row2Sample(r: Row, columnInfo: ColumnFeatureInfo, modelType: String): Sample[Float] = {

    val wideTensor: Tensor[Float] = getWideTensor(r, columnInfo)
    val deepTensor: Array[Tensor[Float]] = getDeepTensors(r, columnInfo)
    val l = r.getAs[Int](columnInfo.label)
    val label = Tensor[Float](T(l))
    label.resize(1, 1)

    modelType match {
      case "wide_n_deep" =>
        TensorSample[Float](Array(wideTensor) ++ deepTensor, Array(label))
      case "wide" =>
        TensorSample[Float](Array(wideTensor), Array(label))
      case "deep" =>
        TensorSample[Float](deepTensor, Array(label))
      case _ =>
        throw new IllegalArgumentException("unknown type")
    }
  }

  /**
   * convert a row to sample given column information of WideAndDeep Sequential model.
   *
   * @param r Row of userId, itemId, features and label
   * @param columnInfo ColumnFeatureInfo specify information of different features
   * @param modelType support "wide_n_deep", "wide", "deep" only
   * @return TensorSample as input for WideAndDeep Sequential model
   */
  def row2SampleSequential(r: Row, columnInfo: ColumnFeatureInfo, modelType: String): Sample[Float]
  = {
    val wideTensor: Tensor[Float] = getWideTensorSequential(r, columnInfo)
    val deepTensor: Tensor[Float] = getDeepTensor(r, columnInfo)
    val l = r.getAs[Int](columnInfo.label)

    val label = Tensor[Float](T(l))
    label.resize(1, 1)

    modelType match {
      case "wide_n_deep" =>
        TensorSample[Float](Array(wideTensor, deepTensor), Array(label))
      case "wide" =>
        TensorSample[Float](Array(wideTensor), Array(label))
      case "deep" =>
        TensorSample[Float](Array(deepTensor), Array(label))
      case _ =>
        throw new IllegalArgumentException("unknown type")
    }
  }

  /**
   * prepare tensor for wide part of WideAndDeep model based on SparseDense.
   *
   * @param r          Row of userId, itemId, features and label
   * @param columnInfo ColumnFeatureInfo specify information of different features
   * @return a tensor as input for wide part of a WideAndDeep model
   */
  def getWideTensor(r: Row, columnInfo: ColumnFeatureInfo): Tensor[Float] = {
    val wideColumns = columnInfo.wideBaseCols ++ columnInfo.wideCrossCols
    val wideDims = columnInfo.wideBaseDims ++ columnInfo.wideCrossDims
    val wideLength = wideColumns.length
    var acc = 0
    val indices: Array[Int] = (0 to wideLength - 1).map(i => {
      val index = r.getAs[Int](wideColumns(i))
      if (i == 0) {index}
      else {
        acc = acc + wideDims(i - 1)
        acc + index
      }
    }).toArray
    val values = indices.map(_ => 1.0f)
    val shape = Array(wideDims.sum)

    Tensor.sparse(Array(indices), values, shape)
  }

  /**
   * prepare tensor for wide part of WideAndDeep based on sequential api and LookupTableSparse.
   *
   * @param r          Row of userId, itemId, features and label
   * @param columnInfo ColumnFeatureInfo specify information of different features
   * @return a tensor as input for wide part of a WideAndDeep model
   */
  def getWideTensorSequential(r: Row, columnInfo: ColumnFeatureInfo): Tensor[Float] = {
    val wideColumns = columnInfo.wideBaseCols ++ columnInfo.wideCrossCols
    val wideDims = columnInfo.wideBaseDims ++ columnInfo.wideCrossDims
    val wideLength = wideColumns.length
    var acc = 0
    val indices: Array[Int] = (0 to wideLength - 1).map(i => {
      val index = r.getAs[Int](wideColumns(i))
      if (i == 0) index
      else {
        acc = acc + wideDims(i - 1)
        acc + index
      }
    }).toArray
    val values = indices.map(_ + 1.0f)
    val shape = Array(wideDims.sum)

    Tensor.sparse(Array(indices), values, shape)
  }

  /**
   * convert a row to tensors given column feature information of WideAndDeep model.
   *
   * @param r          Row of userId, itemId, features and label
   * @param columnInfo ColumnFeatureInfo specify information of different features
   * @return an array of tensors as input for deep part of a WideAndDeep model
   */
  def getDeepTensors(r: Row, columnInfo: ColumnFeatureInfo): Array[Tensor[Float]] = {

    val indCol = columnInfo.indicatorCols.length
    val embCol = columnInfo.embedCols.length
    val contCol = columnInfo.continuousCols.length

    val indTensor = Tensor[Float](columnInfo.indicatorDims.sum).fill(0)

    // setup indicators
    var acc = 0
    (0 to indCol - 1).map {
      i =>
        val index = r.getAs[Int](columnInfo.indicatorCols(i))
        val accIndex = if (i == 0) {
          index
        }
        else {
          acc = acc + columnInfo.indicatorDims(i - 1)
          acc + index
        }
        indTensor.setValue(accIndex + 1, 1)
    }

    val embTensor = Tensor[Float](embCol).fill(0)
    (0 to embCol - 1).map(i =>
      embTensor.setValue(i + 1, r.getAs[Int](columnInfo.embedCols(i)).toFloat))


    val contTensor = Tensor[Float](contCol).fill(0)
    (0 to contCol - 1).map(i => {
      val data = r.getAs[Any](columnInfo.continuousCols(i))
      val td = data match {
        case n: Int => n.toFloat
        case n: Long => n.toFloat
        case n: Double => n.toFloat
        case n: Float => n
        case _ => throw new Exception("wrong data type")
      }
      contTensor.setValue(i + 1, td)
    })

    (indCol > 0, embCol > 0, contCol > 0) match {

      case (true, true, true) =>
        Array(indTensor, embTensor, contTensor)
      case (false, true, true) =>
        Array(embTensor, contTensor)
      case (true, false, true) =>
        Array(indTensor, contTensor)
      case (true, true, false) =>
        Array(indTensor, embTensor)
      case (false, true, false) =>
        Array(embTensor)
      case (false, false, true) =>
        Array(contTensor)
      case (true, false, false) =>
        Array(indTensor)
      case _ =>
        Array[Tensor[Float]]()
    }

  }

  // setup deep tensor
  def getDeepTensor(r: Row, columnInfo: ColumnFeatureInfo): Tensor[Float] = {
    val deepColumns1 = columnInfo.indicatorCols
    val deepColumns2 = columnInfo.embedCols ++ columnInfo.continuousCols
    val deepLength = columnInfo.indicatorDims.sum + deepColumns2.length
    val deepTensor = Tensor[Float](deepLength).fill(0)

    // setup indicators
    var acc = 0
    (0 to deepColumns1.length - 1).map {
      i =>
        val index = r.getAs[Int](columnInfo.indicatorCols(i))
        val accIndex = if (i == 0) index
        else {
          acc = acc + columnInfo.indicatorDims(i - 1)
          acc + index
        }
        deepTensor.setValue(accIndex + 1, 1)
    }

    // setup embedding and continuous
    (0 to deepColumns2.length - 1).map {
      i =>
        deepTensor.setValue(i + 1 + columnInfo.indicatorDims.sum,
          r.getAs[Int](deepColumns2(i)).toFloat)
    }
    deepTensor
  }

  def row2sampleSession(r: Row,
                        sessionLength: Int,
                        includeHistory: Boolean,
                        historyLength: Int): Sample[Float] = {
    val label = Tensor[Float](T(r.getAs[Float]("label")))
    val rnnFeature: Array[Float] = r
      .getAs[mutable.WrappedArray[java.lang.Float]]("session").array.map(_.toFloat)
    val rnnTensor = Tensor(rnnFeature, Array(sessionLength))

    val sample = if (includeHistory) {
      val mlpFeature: Array[Float] = r
        .getAs[mutable.WrappedArray[java.lang.Float]]("purchase_history").array.map(_.toFloat)
      val mlpTensor = Tensor(mlpFeature, Array(historyLength))
      Sample[Float](Array(rnnTensor, mlpTensor), Array(label))
    }
    else {
      Sample[Float](Array(rnnTensor), Array(label))
    }
    sample
  }

  def prePadding(maxLength: Int): mutable.WrappedArray[java.lang.Float] => Array[Float] = {

    (seq: mutable.WrappedArray[java.lang.Float]) => {
      if (seq.array.size < maxLength) {
        seq.array.map(_.toFloat).reverse.padTo(maxLength, 0f).reverse
      }
      else {
        seq.array.map(_.toFloat).takeRight(maxLength)
      }
    }
  }

  def slideSession(df: DataFrame, sessionLength: Int): DataFrame = {
    val sqlContext = df.sqlContext
    import sqlContext.implicits._

    val dfSlided = df.rdd.flatMap(x => {
      val session: Array[Float] = x.getAs[mutable.WrappedArray[java.lang.Float]]("session")
        .array.map(_.toFloat)
      val feature2 = x.getAs[mutable.WrappedArray[java.lang.Float]]("purchase_history")
        .array.map(_.toFloat)
      val featureLabel = for (label <- session.slice(1, session.size)) yield {
        val endIdx = session.indexOf(label)
        val beginIdx = if (session.size <= sessionLength) 0 else endIdx - sessionLength
        val feature1 = session.slice(beginIdx, endIdx)
        (feature1, feature2, label)
      }
      featureLabel
    }).toDF("session", "purchase_history", "label").na.drop()
    dfSlided
  }

}