All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.flink.ml.recommendation.ALS.scala Maven / Gradle / Ivy

There is a newer version: 0.9.1
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.ml.recommendation

import java.{util, lang}

import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint
import org.apache.flink.api.scala._
import org.apache.flink.api.common.operators.Order
import org.apache.flink.core.memory.{DataOutputView, DataInputView}
import org.apache.flink.ml.common._
import org.apache.flink.ml.pipeline.{FitOperation, PredictDataSetOperation, Predictor}
import org.apache.flink.types.Value
import org.apache.flink.util.Collector
import org.apache.flink.api.common.functions.{Partitioner => FlinkPartitioner, GroupReduceFunction, CoGroupFunction}

import com.github.fommil.netlib.BLAS.{ getInstance => blas }
import com.github.fommil.netlib.LAPACK.{ getInstance => lapack }
import org.netlib.util.intW

import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import scala.util.Random

/** Alternating least squares algorithm to calculate a matrix factorization.
  *
  * Given a matrix `R`, ALS calculates two matricess `U` and `V` such that `R ~~ U^TV`. The
  * unknown row dimension is given by the number of latent factors. Since matrix factorization
  * is often used in the context of recommendation, we'll call the first matrix the user and the
  * second matrix the item matrix. The `i`th column of the user matrix is `u_i` and the `i`th
  * column of the item matrix is `v_i`. The matrix `R` is called the ratings matrix and
  * `(R)_{i,j} = r_{i,j}`.
  *
  * In order to find the user and item matrix, the following problem is solved:
  *
  * `argmin_{U,V} sum_(i,j\ with\ r_{i,j} != 0) (r_{i,j} - u_{i}^Tv_{j})^2 +
  * lambda (sum_(i) n_{u_i} ||u_i||^2 + sum_(j) n_{v_j} ||v_j||^2)`
  *
  * with `\lambda` being the regularization factor, `n_{u_i}` being the number of items the user `i`
  * has rated and `n_{v_j}` being the number of times the item `j` has been rated. This
  * regularization scheme to avoid overfitting is called weighted-lambda-regularization. Details
  * can be found in the work of [[http://dx.doi.org/10.1007/978-3-540-68880-8_32 Zhou et al.]].
  *
  * By fixing one of the matrices `U` or `V` one obtains a quadratic form which can be solved. The
  * solution of the modified problem is guaranteed to decrease the overall cost function. By
  * applying this step alternately to the matrices `U` and `V`, we can iteratively improve the
  * matrix factorization.
  *
  * The matrix `R` is given in its sparse representation as a tuple of `(i, j, r)` where `i` is the
  * row index, `j` is the column index and `r` is the matrix value at position `(i,j)`.
  *
  * @example
  *          {{{
  *             val inputDS: DataSet[(Int, Int, Double)] = env.readCsvFile[(Int, Int, Double)](
  *               pathToTrainingFile)
  *
  *             val als = ALS()
  *               .setIterations(10)
  *               .setNumFactors(10)
  *
  *             als.fit(inputDS)
  *
  *             val data2Predict: DataSet[(Int, Int)] = env.readCsvFile[(Int, Int)](pathToData)
  *
  *             als.predict(data2Predict)
  *          }}}
  *
  * =Parameters=
  *
  *  - [[org.apache.flink.ml.recommendation.ALS.NumFactors]]:
  *  The number of latent factors. It is the dimension of the calculated user and item vectors.
  *  (Default value: '''10''')
  *
  *  - [[org.apache.flink.ml.recommendation.ALS.Lambda]]:
  *  Regularization factor. Tune this value in order to avoid overfitting/generalization.
  *  (Default value: '''1''')
  *
  *  - [[org.apache.flink.ml.regression.MultipleLinearRegression.Iterations]]:
  *  The number of iterations to perform. (Default value: '''10''')
  *
  *  - [[org.apache.flink.ml.recommendation.ALS.Blocks]]:
  *  The number of blocks into which the user and item matrix a grouped. The fewer
  *  blocks one uses, the less data is sent redundantly. However, bigger blocks entail bigger
  *  update messages which have to be stored on the Heap. If the algorithm fails because of
  *  an OutOfMemoryException, then try to increase the number of blocks. (Default value: '''None''')
  *
  *  - [[org.apache.flink.ml.recommendation.ALS.Seed]]:
  *  Random seed used to generate the initial item matrix for the algorithm.
  *  (Default value: '''0''')
  *
  *  - [[org.apache.flink.ml.recommendation.ALS.TemporaryPath]]:
  *  Path to a temporary directory into which intermediate results are stored. If
  *  this value is set, then the algorithm is split into two preprocessing steps, the ALS iteration
  *  and a post-processing step which calculates a last ALS half-step. The preprocessing steps
  *  calculate the [[org.apache.flink.ml.recommendation.ALS.OutBlockInformation]] and
  *  [[org.apache.flink.ml.recommendation.ALS.InBlockInformation]] for the given rating matrix.
  *  The result of the individual steps are stored in the specified directory. By splitting the
  *  algorithm into multiple smaller steps, Flink does not have to split the available memory
  *  amongst too many operators. This allows the system to process bigger individual messasges and
  *  improves the overall performance. (Default value: '''None''')
  *
  * The ALS implementation is based on Spark's MLLib implementation of ALS which you can find
  * [[https://github.com/apache/spark/blob/master/mllib/src/main/scala/org/apache/spark/mllib/
  * recommendation/ALS.scala here]].
  */
class ALS extends Predictor[ALS] {

  import ALS._

  // Stores the matrix factorization after the fitting phase
  var factorsOption: Option[(DataSet[Factors], DataSet[Factors])] = None

  /** Sets the number of latent factors/row dimension of the latent model
    *
    * @param numFactors
    * @return
    */
  def setNumFactors(numFactors: Int): ALS = {
    parameters.add(NumFactors, numFactors)
    this
  }

  /** Sets the regularization coefficient lambda
    *
    * @param lambda
    * @return
    */
  def setLambda(lambda: Double): ALS = {
    parameters.add(Lambda, lambda)
    this
  }

  /** Sets the number of iterations of the ALS algorithm
    * 
    * @param iterations
    * @return
    */
  def setIterations(iterations: Int): ALS = {
    parameters.add(Iterations, iterations)
    this
  }

  /** Sets the number of blocks into which the user and item matrix shall be partitioned
    * 
    * @param blocks
    * @return
    */
  def setBlocks(blocks: Int): ALS = {
    parameters.add(Blocks, blocks)
    this
  }

  /** Sets the random seed for the initial item matrix initialization
    * 
    * @param seed
    * @return
    */
  def setSeed(seed: Long): ALS = {
    parameters.add(Seed, seed)
    this
  }

  /** Sets the temporary path into which intermediate results are written in order to increase
    * performance.
    * 
    * @param temporaryPath
    * @return
    */
  def setTemporaryPath(temporaryPath: String): ALS = {
    parameters.add(TemporaryPath, temporaryPath)
    this
  }

  /** Empirical risk of the trained model (matrix factorization).
    *
    * @param labeledData Reference data
    * @param riskParameters Additional parameters for the empirical risk calculation
    * @return
    */
  def empiricalRisk(
      labeledData: DataSet[(Int, Int, Double)],
      riskParameters: ParameterMap = ParameterMap.Empty)
    : DataSet[Double] = {
    val resultingParameters = parameters ++ riskParameters

    val lambda = resultingParameters(Lambda)

    val data = labeledData map {
      x => (x._1, x._2)
    }

    factorsOption match {
      case Some((userFactors, itemFactors)) => {
        val predictions = data.join(userFactors, JoinHint.REPARTITION_HASH_SECOND).where(0)
          .equalTo(0).join(itemFactors, JoinHint.REPARTITION_HASH_SECOND).where("_1._2")
          .equalTo(0).map {
          triple => {
            val (((uID, iID), uFactors), iFactors) = triple

            val uFactorsVector = uFactors.factors
            val iFactorsVector = iFactors.factors

            val squaredUNorm2 = blas.ddot(
              uFactorsVector.length,
              uFactorsVector,
              1,
              uFactorsVector,
              1)
            val squaredINorm2 = blas.ddot(
              iFactorsVector.length,
              iFactorsVector,
              1,
              iFactorsVector,
              1)

            val prediction = blas.ddot(uFactorsVector.length, uFactorsVector, 1, iFactorsVector, 1)

            (uID, iID, prediction, squaredUNorm2, squaredINorm2)
          }
        }

        labeledData.join(predictions).where(0,1).equalTo(0,1) {
          (left, right) => {
            val (_, _, expected) = left
            val (_, _, predicted, squaredUNorm2, squaredINorm2) = right

            val residual = expected - predicted

            residual * residual + lambda * (squaredUNorm2 + squaredINorm2)
          }
        } reduce {
          _ + _
        }
      }

      case None => throw new RuntimeException("The ALS model has not been fitted to data. " +
        "Prior to predicting values, it has to be trained on data.")
    }
  }
}

object ALS {
  val USER_FACTORS_FILE = "userFactorsFile"
  val ITEM_FACTORS_FILE = "itemFactorsFile"

  // ========================================= Parameters ==========================================

  case object NumFactors extends Parameter[Int] {
    val defaultValue: Option[Int] = Some(10)
  }

  case object Lambda extends Parameter[Double] {
    val defaultValue: Option[Double] = Some(1.0)
  }

  case object Iterations extends Parameter[Int] {
    val defaultValue: Option[Int] = Some(10)
  }

  case object Blocks extends Parameter[Int] {
    val defaultValue: Option[Int] = None
  }

  case object Seed extends Parameter[Long] {
    val defaultValue: Option[Long] = Some(0L)
  }

  case object TemporaryPath extends Parameter[String] {
    val defaultValue: Option[String] = None
  }

  // ==================================== ALS type definitions =====================================

  /** Representation of a user-item rating
    *
    * @param user User ID of the rating user
    * @param item Item iD of the rated item
    * @param rating Rating value
    */
  case class Rating(user: Int, item: Int, rating: Double)

  /** Latent factor model vector
    *
    * @param id
    * @param factors
    */
  case class Factors(id: Int, factors: Array[Double]) {
    override def toString = s"($id, ${factors.mkString(",")})"
  }

  case class Factorization(userFactors: DataSet[Factors], itemFactors: DataSet[Factors])

  case class OutBlockInformation(elementIDs: Array[Int], outLinks: OutLinks) {
    override def toString: String = {
      s"OutBlockInformation:((${elementIDs.mkString(",")}), ($outLinks))"
    }
  }

  class OutLinks(var links: Array[scala.collection.mutable.BitSet]) extends Value {
    def this() = this(null)

    override def toString: String = {
      s"${links.mkString("\n")}"
    }

    override def write(out: DataOutputView): Unit = {
      out.writeInt(links.length)
      links foreach {
        link => {
          val bitMask = link.toBitMask
          out.writeInt(bitMask.length)
          for (element <- bitMask) {
            out.writeLong(element)
          }
        }
      }
    }

    override def read(in: DataInputView): Unit = {
      val length = in.readInt()
      links = new Array[scala.collection.mutable.BitSet](length)

      for (i <- 0 until length) {
        val bitMaskLength = in.readInt()
        val bitMask = new Array[Long](bitMaskLength)
        for (j <- 0 until bitMaskLength) {
          bitMask(j) = in.readLong()
        }
        links(i) = mutable.BitSet.fromBitMask(bitMask)
      }
    }

    def apply(idx: Int) = links(idx)
  }

  case class InBlockInformation(elementIDs: Array[Int], ratingsForBlock: Array[BlockRating]) {

    override def toString: String = {
      s"InBlockInformation:((${elementIDs.mkString(",")}), (${ratingsForBlock.mkString("\n")}))"
    }
  }

  case class BlockRating(var ratings: Array[(Array[Int], Array[Double])]) {
    def apply(idx: Int) = ratings(idx)

    override def toString: String = {
      ratings.map {
        case (left, right) => s"((${left.mkString(",")}),(${right.mkString(",")}))"
      }.mkString(",")
    }
  }

  case class BlockedFactorization(userFactors: DataSet[(Int, Array[Array[Double]])],
                                  itemFactors: DataSet[(Int, Array[Array[Double]])])

  class BlockIDPartitioner extends FlinkPartitioner[Int] {
    override def partition(blockID: Int, numberOfPartitions: Int): Int = {
      blockID % numberOfPartitions
    }
  }

  class BlockIDGenerator(blocks: Int) extends Serializable {
    def apply(id: Int): Int = {
      id % blocks
    }
  }

  // ================================= Factory methods =============================================

  def apply(): ALS = {
    new ALS()
  }

  // ===================================== Operations ==============================================

  /** Predict operation which calculates the matrix entry for the given indices  */
  implicit val predictRating = new PredictDataSetOperation[ALS, (Int, Int), (Int ,Int, Double)] {
    override def predictDataSet(
        instance: ALS,
        predictParameters: ParameterMap,
        input: DataSet[(Int, Int)])
      : DataSet[(Int, Int, Double)] = {

      instance.factorsOption match {
        case Some((userFactors, itemFactors)) => {
          input.join(userFactors, JoinHint.REPARTITION_HASH_SECOND).where(0).equalTo(0)
            .join(itemFactors, JoinHint.REPARTITION_HASH_SECOND).where("_1._2").equalTo(0).map {
            triple => {
              val (((uID, iID), uFactors), iFactors) = triple

              val uFactorsVector = uFactors.factors
              val iFactorsVector = iFactors.factors

              val prediction = blas.ddot(
                uFactorsVector.length,
                uFactorsVector,
                1,
                iFactorsVector,
                1)

              (uID, iID, prediction)
            }
          }
        }

        case None => throw new RuntimeException("The ALS model has not been fitted to data. " +
          "Prior to predicting values, it has to be trained on data.")
      }
    }
  }

  /** Calculates the matrix factorization for the given ratings. A rating is defined as
    * a tuple of user ID, item ID and the corresponding rating.
    *
    * @return Factorization containing the user and item matrix
    */
  implicit val fitALS =  new FitOperation[ALS, (Int, Int, Double)] {
    override def fit(
        instance: ALS,
        fitParameters: ParameterMap,
        input: DataSet[(Int, Int, Double)])
      : Unit = {
      val resultParameters = instance.parameters ++ fitParameters

      val userBlocks = resultParameters.get(Blocks).getOrElse(input.count.toInt)
      val itemBlocks = userBlocks
      val persistencePath = resultParameters.get(TemporaryPath)
      val seed = resultParameters(Seed)
      val factors = resultParameters(NumFactors)
      val iterations = resultParameters(Iterations)
      val lambda = resultParameters(Lambda)

      val ratings = input.map {
        entry => {
          val (userID, itemID, rating) = entry
          Rating(userID, itemID, rating)
        }
      }

      val blockIDPartitioner = new BlockIDPartitioner()

      val ratingsByUserBlock = ratings.map{
        rating =>
          val blockID = rating.user % userBlocks
          (blockID, rating)
      } partitionCustom(blockIDPartitioner, 0)

      val ratingsByItemBlock = ratings map {
        rating =>
          val blockID = rating.item % itemBlocks
          (blockID, new Rating(rating.item, rating.user, rating.rating))
      } partitionCustom(blockIDPartitioner, 0)

      val (uIn, uOut) = createBlockInformation(userBlocks, itemBlocks, ratingsByUserBlock,
        blockIDPartitioner)
      val (iIn, iOut) = createBlockInformation(itemBlocks, userBlocks, ratingsByItemBlock,
        blockIDPartitioner)

      val (userIn, userOut) = persistencePath match {
        case Some(path) => FlinkMLTools.persist(uIn, uOut, path + "userIn", path + "userOut")
        case None => (uIn, uOut)
      }

      val (itemIn, itemOut) = persistencePath match {
        case Some(path) => FlinkMLTools.persist(iIn, iOut, path + "itemIn", path + "itemOut")
        case None => (iIn, iOut)
      }

      val initialItems = itemOut.partitionCustom(blockIDPartitioner, 0).map{
        outInfos =>
          val blockID = outInfos._1
          val infos = outInfos._2

          (blockID, infos.elementIDs.map{
            id =>
              val random = new Random(id ^ seed)
              randomFactors(factors, random)
          })
      }.withForwardedFields("0")

      // iteration to calculate the item matrix
      val items = initialItems.iterate(iterations) {
        items => {
          val users = updateFactors(userBlocks, items, itemOut, userIn, factors, lambda,
            blockIDPartitioner)
          updateFactors(itemBlocks, users, userOut, itemIn, factors, lambda, blockIDPartitioner)
        }
      }

      val pItems = persistencePath match {
        case Some(path) => FlinkMLTools.persist(items, path + "items")
        case None => items
      }

      // perform last half-step to calculate the user matrix
      val users = updateFactors(userBlocks, pItems, itemOut, userIn, factors, lambda,
        blockIDPartitioner)

      instance.factorsOption = Some((
        unblock(users, userOut, blockIDPartitioner),
        unblock(pItems, itemOut, blockIDPartitioner)))
    }
  }

  /** Calculates a single half step of the ALS optimization. The result is the new value for
    * either the user or item matrix, depending with which matrix the method was called.
    *
    * @param numUserBlocks Number of blocks in the respective dimension
    * @param items Fixed matrix value for the half step
    * @param itemOut Out information to know where to send the vectors
    * @param userIn In information for the cogroup step
    * @param factors Number of latent factors
    * @param lambda Regularization constant
    * @param blockIDPartitioner Custom Flink partitioner
    * @return New value for the optimized matrix (either user or item)
    */
  def updateFactors(numUserBlocks: Int,
    items: DataSet[(Int, Array[Array[Double]])],
    itemOut: DataSet[(Int, OutBlockInformation)],
    userIn: DataSet[(Int, InBlockInformation)],
    factors: Int,
    lambda: Double, blockIDPartitioner: FlinkPartitioner[Int]):
  DataSet[(Int, Array[Array[Double]])] = {
    // send the item vectors to the blocks whose users have rated the items
    val partialBlockMsgs = itemOut.join(items).where(0).equalTo(0).
      withPartitioner(blockIDPartitioner).apply {
      (left, right, col: Collector[(Int, Int, Array[Array[Double]])]) => {
        val blockID = left._1
        val outInfo = left._2
        val factors = right._2
        var userBlock = 0
        var itemIdx = 0

        while(userBlock < numUserBlocks){
          itemIdx = 0
          val buffer = new ArrayBuffer[Array[Double]]
          while(itemIdx < outInfo.elementIDs.length){
            if(outInfo.outLinks(userBlock)(itemIdx)){
              buffer += factors(itemIdx)
            }
            itemIdx += 1
          }

          if(buffer.nonEmpty){
            // send update message to userBlock
            col.collect(userBlock, blockID, buffer.toArray)
          }

          userBlock += 1
        }
      }
    }

    // collect the partial update messages and calculate for each user block the new user vectors
    partialBlockMsgs.coGroup(userIn).where(0).equalTo(0).sortFirstGroup(1, Order.ASCENDING).
      withPartitioner(blockIDPartitioner).apply{
      new CoGroupFunction[(Int, Int, Array[Array[Double]]), (Int,
        InBlockInformation), (Int, Array[Array[Double]])](){

        // in order to save space, store only the upper triangle of the XtX matrix
        val triangleSize = (factors*factors - factors)/2 + factors
        val matrix = Array.fill(triangleSize)(0.0)
        val fullMatrix = Array.fill(factors * factors)(0.0)
        val userXtX = new ArrayBuffer[Array[Double]]()
        val userXy = new ArrayBuffer[Array[Double]]()
        val numRatings = new ArrayBuffer[Int]()

        override def coGroup(left: lang.Iterable[(Int, Int, Array[Array[Double]])],
          right: lang.Iterable[(Int, InBlockInformation)],
          collector: Collector[(Int, Array[Array[Double]])]): Unit = {
          // there is only one InBlockInformation per user block
          val inInfo = right.iterator().next()._2
          val updates = left.iterator()

          val numUsers = inInfo.elementIDs.length
          var blockID = -1

          var i = 0

          // clear old matrices and allocate new ones
          val matricesToClear = if (numUsers > userXtX.length) {
            val oldLength = userXtX.length

            while(i < (numUsers - oldLength)) {
              userXtX += Array.fill(triangleSize)(0.0)
              userXy += Array.fill(factors)(0.0)
              numRatings.+=(0)

              i += 1
            }

            oldLength
          } else {
            numUsers
          }

          i = 0
          while(i  < matricesToClear){
            numRatings(i) = 0

            util.Arrays.fill(userXtX(i), 0.0)
            util.Arrays.fill(userXy(i), 0.0)

            i += 1
          }

          var itemBlock = 0

          // build XtX matrices and Xy vector
          while(updates.hasNext){
            val update = updates.next()
            val blockFactors = update._3
            blockID = update._1

            var p = 0
            while(p < blockFactors.length){
              val vector = blockFactors(p)

              outerProduct(vector, matrix, factors)

              val (users, ratings) = inInfo.ratingsForBlock(itemBlock)(p)

              var i = 0
              while (i < users.length) {
                numRatings(users(i)) += 1
                blas.daxpy(matrix.length, 1, matrix, 1, userXtX(users(i)), 1)
                blas.daxpy(vector.length, ratings(i), vector, 1, userXy(users(i)), 1)

                i += 1
              }
              p += 1
            }

            itemBlock += 1
          }

          val array = new Array[Array[Double]](numUsers)

          i = 0
          while(i < numUsers){
            generateFullMatrix(userXtX(i), fullMatrix, factors)

            var f = 0

            // add regularization constant
            while(f < factors){
              fullMatrix(f*factors + f) += lambda * numRatings(i)
              f += 1
            }

            // calculate new user vector
            val result = new intW(0)
            lapack.dposv("U", factors, 1, fullMatrix, factors , userXy(i), factors, result)
            array(i) = userXy(i)

            i += 1
          }

          collector.collect((blockID, array))
        }
      }
    }.withForwardedFieldsFirst("0").withForwardedFieldsSecond("0")
  }

  /** Creates the meta information needed to route the item and user vectors to the respective user
    * and item blocks.
    * * @param userBlocks
    * @param itemBlocks
    * @param ratings
    * @param blockIDPartitioner
    * @return
    */
  def createBlockInformation(userBlocks: Int, itemBlocks: Int, ratings: DataSet[(Int, Rating)],
    blockIDPartitioner: BlockIDPartitioner):
  (DataSet[(Int, InBlockInformation)], DataSet[(Int, OutBlockInformation)]) = {
    val blockIDGenerator = new BlockIDGenerator(itemBlocks)

    val usersPerBlock = createUsersPerBlock(ratings)

    val outBlockInfos = createOutBlockInformation(ratings, usersPerBlock, itemBlocks,
      blockIDGenerator)

    val inBlockInfos = createInBlockInformation(ratings, usersPerBlock, blockIDGenerator)

    (inBlockInfos, outBlockInfos)
  }

  /** Calculates the userIDs in ascending order of each user block
    *
    * @param ratings
    * @return
    */
  def createUsersPerBlock(ratings: DataSet[(Int, Rating)]): DataSet[(Int, Array[Int])] = {
    ratings.map{ x => (x._1, x._2.user)}.withForwardedFields("0").groupBy(0).
      sortGroup(1, Order.ASCENDING).reduceGroup {
      users => {
        val result = ArrayBuffer[Int]()
        var id = -1
        var oldUser = -1

        while(users.hasNext) {
          val user = users.next()

          id = user._1

          if (user._2 != oldUser) {
            result.+=(user._2)
            oldUser = user._2
          }
        }

        val userIDs = result.toArray
        (id, userIDs)
      }
    }.withForwardedFields("0")
  }

  /** Creates the outgoing block information
    *
    * Creates for every user block the outgoing block information. The out block information
    * contains for every item block a [[scala.collection.mutable.BitSet]] which indicates which
    * user vector has to be sent to this block. If a vector v has to be sent to a block b, then
    * bitsets(b)'s bit v is set to 1, otherwise 0. Additionally the user IDataSet are replaced by
    * the user vector's index value.
    *
    * @param ratings
    * @param usersPerBlock
    * @param itemBlocks
    * @param blockIDGenerator
    * @return
    */
  def createOutBlockInformation(ratings: DataSet[(Int, Rating)],
    usersPerBlock: DataSet[(Int, Array[Int])],
    itemBlocks: Int, blockIDGenerator: BlockIDGenerator):
  DataSet[(Int, OutBlockInformation)] = {
    ratings.coGroup(usersPerBlock).where(0).equalTo(0).apply {
      (ratings, users) =>
        val userIDs = users.next()._2
        val numUsers = userIDs.length

        val userIDToPos = userIDs.zipWithIndex.toMap

        val shouldDataSend = Array.fill(itemBlocks)(new scala.collection.mutable.BitSet(numUsers))
        var blockID = -1
        while (ratings.hasNext) {
          val r = ratings.next()

          val pos =
            try {
              userIDToPos(r._2.user)
            }catch{
              case e: NoSuchElementException =>
                throw new RuntimeException(s"Key ${r._2.user} not  found. BlockID $blockID. " +
                  s"Elements in block ${userIDs.take(5).mkString(", ")}. " +
                  s"UserIDList contains ${userIDs.contains(r._2.user)}.", e)
            }

          blockID = r._1
          shouldDataSend(blockIDGenerator(r._2.item))(pos) = true
        }

        (blockID, OutBlockInformation(userIDs, new OutLinks(shouldDataSend)))
    }.withForwardedFieldsFirst("0").withForwardedFieldsSecond("0")
  }

  /** Creates the incoming block information
    *
    * Creates for every user block the incoming block information. The incoming block information
    * contains the userIDs of the users in the respective block and for every item block a
    * BlockRating instance. The BlockRating instance describes for every incoming set of item
    * vectors of an item block, which user rated these items and what the rating was. For that
    * purpose it contains for every incoming item vector a tuple of an id array us and a rating
    * array rs. The array us contains the indices of the users having rated the respective
    * item vector with the ratings in rs.
    *
    * @param ratings
    * @param usersPerBlock
    * @param blockIDGenerator
    * @return
    */
  def createInBlockInformation(ratings: DataSet[(Int, Rating)],
    usersPerBlock: DataSet[(Int, Array[Int])],
    blockIDGenerator: BlockIDGenerator):
  DataSet[(Int, InBlockInformation)] = {
    // Group for every user block the users which have rated the same item and collect their ratings
    val partialInInfos = ratings.map { x => (x._1, x._2.item, x._2.user, x._2.rating)}
      .withForwardedFields("0").groupBy(0, 1).reduceGroup {
      x =>
        var userBlockID = -1
        var itemID = -1
        val userIDs = ArrayBuffer[Int]()
        val ratings = ArrayBuffer[Double]()

        while (x.hasNext) {
          val (uBlockID, item, user, rating) = x.next
          userBlockID = uBlockID
          itemID = item

          userIDs += user
          ratings += rating
        }

        (userBlockID, blockIDGenerator(itemID), itemID, (userIDs.toArray, ratings.toArray))
    }.withForwardedFields("0")

    // Aggregate all ratings for items belonging to the same item block. Sort ascending with
    // respect to the itemID, because later the item vectors of the update message are sorted
    // accordingly.
    val collectedPartialInfos = partialInInfos.groupBy(0, 1).sortGroup(2, Order.ASCENDING).
      reduceGroup {
      new GroupReduceFunction[(Int, Int, Int, (Array[Int], Array[Double])), (Int,
        Int, Array[(Array[Int], Array[Double])])](){
        val buffer = new ArrayBuffer[(Array[Int], Array[Double])]

        override def reduce(iterable: lang.Iterable[(Int, Int, Int, (Array[Int],
          Array[Double]))], collector: Collector[(Int, Int, Array[(Array[Int],
          Array[Double])])]): Unit = {

          val infos = iterable.iterator()
          var counter = 0

          var blockID = -1
          var itemBlockID = -1

          while (infos.hasNext && counter < buffer.length) {
            val info = infos.next()
            blockID = info._1
            itemBlockID = info._2

            buffer(counter) = info._4

            counter += 1
          }

          while (infos.hasNext) {
            val info = infos.next()
            blockID = info._1
            itemBlockID = info._2

            buffer += info._4

            counter += 1
          }

          val array = new Array[(Array[Int], Array[Double])](counter)

          buffer.copyToArray(array)

          collector.collect((blockID, itemBlockID, array))
        }
      }
    }.withForwardedFields("0", "1")

    // Aggregate all item block ratings with respect to their user block ID. Sort the blocks with
    // respect to their itemBlockID, because the block update messages are sorted the same way
    collectedPartialInfos.coGroup(usersPerBlock).where(0).equalTo(0).
      sortFirstGroup(1, Order.ASCENDING).apply{
      new CoGroupFunction[(Int, Int, Array[(Array[Int], Array[Double])]),
        (Int, Array[Int]), (Int, InBlockInformation)] {
        val buffer = ArrayBuffer[BlockRating]()

        override def coGroup(partialInfosIterable:
        lang.Iterable[(Int, Int,  Array[(Array[Int], Array[Double])])],
          userIterable: lang.Iterable[(Int, Array[Int])],
          collector: Collector[(Int, InBlockInformation)]): Unit = {

          val users = userIterable.iterator()
          val partialInfos = partialInfosIterable.iterator()

          val userWrapper = users.next()
          val id = userWrapper._1
          val userIDs = userWrapper._2
          val userIDToPos = userIDs.zipWithIndex.toMap

          var counter = 0

          while (partialInfos.hasNext && counter < buffer.length) {
            val partialInfo = partialInfos.next()
            // entry contains the ratings and userIDs of a complete item block
            val entry = partialInfo._3

            // transform userIDs to positional indices
            for (row <- 0 until entry.length; col <- 0 until entry(row)._1.length) {
              entry(row)._1(col) = userIDToPos(entry(row)._1(col))
            }

            buffer(counter).ratings = entry

            counter += 1
          }

          while (partialInfos.hasNext) {
            val partialInfo = partialInfos.next()
            // entry contains the ratings and userIDs of a complete item block
            val entry = partialInfo._3

            // transform userIDs to positional indices
            for (row <- 0 until entry.length; col <- 0 until entry(row)._1.length) {
              entry(row)._1(col) = userIDToPos(entry(row)._1(col))
            }

            buffer += new BlockRating(entry)

            counter += 1
          }

          val array = new Array[BlockRating](counter)

          buffer.copyToArray(array)

          collector.collect((id, InBlockInformation(userIDs, array)))
        }
      }
    }.withForwardedFieldsFirst("0").withForwardedFieldsSecond("0")
  }

  /** Unblocks the blocked user and item matrix representation so that it is at DataSet of
    * column vectors.
    *
    * @param users
    * @param outInfo
    * @param blockIDPartitioner
    * @return
    */
  def unblock(users: DataSet[(Int, Array[Array[Double]])],
    outInfo: DataSet[(Int, OutBlockInformation)],
    blockIDPartitioner: BlockIDPartitioner): DataSet[Factors] = {
    users.join(outInfo).where(0).equalTo(0).withPartitioner(blockIDPartitioner).apply {
      (left, right, col: Collector[Factors]) => {
        val outInfo = right._2
        val factors = left._2

        for(i <- 0 until outInfo.elementIDs.length){
          val id = outInfo.elementIDs(i)
          val factorVector = factors(i)
          col.collect(Factors(id, factorVector))
        }
      }
    }
  }

  // ================================ Math helper functions ========================================

  def outerProduct(vector: Array[Double], matrix: Array[Double], factors: Int): Unit = {
    var row = 0
    var pos = 0
    while(row < factors){
      var col = 0
      while(col <= row){
        matrix(pos) = vector(row) * vector(col)
        col += 1
        pos += 1
      }

      row += 1
    }
  }

  def generateFullMatrix(triangularMatrix: Array[Double], fullMatrix: Array[Double],
    factors: Int): Unit = {
    var row = 0
    var pos = 0

    while(row < factors){
      var col = 0
      while(col < row){
        fullMatrix(row*factors + col) = triangularMatrix(pos)
        fullMatrix(col*factors + row) = triangularMatrix(pos)

        pos += 1
        col += 1
      }

      fullMatrix(row*factors + row) = triangularMatrix(pos)

      pos += 1
      row += 1
    }
  }

  def generateRandomMatrix(users: DataSet[Int], factors: Int, seed: Long): DataSet[Factors] = {
    users map {
      id =>{
        val random = new Random(id ^ seed)
        Factors(id, randomFactors(factors, random))
      }
    }
  }

  def randomFactors(factors: Int, random: Random): Array[Double] = {
    Array.fill(factors)(random.nextDouble())
  }
}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy