com.intel.analytics.bigdl.nn.Highway.scala Maven / Gradle / Ivy

/*
 * Copyright 2016 The BigDL Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.intel.analytics.bigdl.nn


import com.intel.analytics.bigdl.nn.abstractnn.TensorModule
import com.intel.analytics.bigdl.optim.Regularizer
import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric

import scala.reflect.ClassTag


object Highway {
  /**
   * Densely connected highway network.
   * Highway layers are a natural extension of LSTMs to feedforward networks.
   *
   * @param size input size
   * @param withBias whether to include a bias
   * @param activation activation function
   * @param wRegularizer: instance of [[Regularizer]]
   *                    (eg. L1 or L2 regularization), applied to the input weights matrices.
   * @param bRegularizer: instance of [[Regularizer]]
   *                    applied to the bias.
   */
  def apply[@specialized(Float, Double) T: ClassTag](size: Int, withBias: Boolean = true,
    activation: TensorModule[T] = null,
    wRegularizer: Regularizer[T] = null,
    bRegularizer: Regularizer[T] = null)
    (implicit ev: TensorNumeric[T]): Graph[T] = {
    val input = Input()
    val l1 = Linear(size, size, withBias = withBias, wRegularizer = wRegularizer,
      bRegularizer = bRegularizer).inputs(input)
    val transformWeight = Sigmoid().inputs(l1)
    val negatedGate = AddConstant(1).inputs(Negative().inputs(transformWeight))
    val l2 = Linear(size, size, withBias = withBias, wRegularizer = wRegularizer,
      bRegularizer = bRegularizer).inputs(input)
    val transformed = if (null != activation) activation.inputs(l2) else l2
    val transformedGated = CMulTable().inputs(transformWeight, transformed)
    val identityGate = CMulTable().inputs(negatedGate, input)
    val value = CAddTable().inputs(transformedGated, identityGate)
    Graph(Array(input), Array(value))
  }
}