Merge pull request #3454 from mrdaybird/adapt_hard_tanh

Updated hard_tanh and added to layer_types

HISTORY.md

@@ -1,6 +1,8 @@
 ### mlpack ?.?.?
 ###### ????-??-??
 
+  * Adapt HardTanH layer (#3454).
+
   * Adapt Softmin layer for new neural network API (#3437).
 
   * Adapt PReLU layer for new neural network API (#3420).

src/mlpack/methods/ann/layer/not_adapted/hard_tanh.hpp → src/mlpack/methods/ann/layer/hard_tanh.hpp

@@ -1,6 +1,7 @@
 /**
  * @file methods/ann/layer/hard_tanh.hpp
  * @author Dhawal Arora
+ * @author Vaibhav Pathak
  *
  * Definition and implementation of the HardTanH layer.
  *
@@ -44,8 +45,8 @@ namespace mlpack {
  *    to also be in this type. The type also allows the computation and weight
  *    type to differ from the input type (Default: arma::mat).
  */
-template <typename InputType = arma::mat, typename OutputType = arma::mat>
-class HardTanHType : public Layer<InputType, OutputType>
+template <typename MatType = arma::mat>
+class HardTanHType : public Layer<MatType>
 {
  public:
   /**
@@ -57,6 +58,20 @@ class HardTanHType : public Layer<InputType, OutputType>
    * @param minValue Range of the linear region minimum value.
    */
   HardTanHType(const double maxValue = 1, const double minValue = -1);
+
+  virtual ~HardTanHType() { }
+
+  //! Copy the other HardTanH layer
+  HardTanHType(const HardTanHType& layer);
+
+  //! Take ownership of the members of the other HardTanH Layer
+  HardTanHType(HardTanHType&& layer);
+
+  //! Copy the other HardTanH layer
+  HardTanHType& operator=(const HardTanHType& layer);
+
+  //! Take ownership of the members of the other HardTanH Layer
+  HardTanHType& operator=(HardTanHType&& layer);
 
   //! Clone the HardTanHType object. This handles polymorphism correctly.
   HardTanHType* Clone() const { return new HardTanHType(*this); }
@@ -68,7 +83,7 @@ class HardTanHType : public Layer<InputType, OutputType>
    * @param input Input data used for evaluating the specified function.
    * @param output Resulting output activation.
    */
-  void Forward(const InputType& input, OutputType& output);
+  void Forward(const MatType& input, MatType& output);
 
   /**
    * Ordinary feed backward pass of a neural network, calculating the function
@@ -79,7 +94,7 @@ class HardTanHType : public Layer<InputType, OutputType>
    * @param gy The backpropagated error.
    * @param g The calculated gradient.
    */
-  void Backward(const InputType& input, const OutputType& gy, OutputType& g);
+  void Backward(const MatType& input, const MatType& gy, MatType& g);
 
   //! Get the maximum value.
   double const& MaxValue() const { return maxValue; }
@@ -108,7 +123,7 @@ class HardTanHType : public Layer<InputType, OutputType>
 // Convenience typedefs.
 
 // Standard HardTanH layer.
-typedef HardTanHType<arma::mat, arma::mat> HardTanH;
+typedef HardTanHType<arma::mat> HardTanH;
 
 } // namespace mlpack
 

src/mlpack/methods/ann/layer/hard_tanh_impl.hpp

@@ -0,0 +1,119 @@
+/**
+ * @file methods/ann/layer/hard_tanh_impl.hpp
+ * @author Dhawal Arora
+ * @author Vaibhav Pathak
+ *
+ * Implementation and implementation of the HardTanH layer.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_HARD_TANH_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_HARD_TANH_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "hard_tanh.hpp"
+
+namespace mlpack {
+
+template<typename MatType>
+HardTanHType<MatType>::HardTanHType(
+    const double maxValue,
+    const double minValue) :
+    Layer<MatType>(),
+    maxValue(maxValue),
+    minValue(minValue)
+{
+  // Nothing to do here.
+}
+
+template<typename MatType>
+HardTanHType<MatType>::HardTanHType(const HardTanHType& layer) :
+    Layer<MatType>(layer),
+    maxValue(layer.maxValue),
+    minValue(layer.minValue)
+{
+  // Nothing to do here.
+}
+
+template<typename MatType>
+HardTanHType<MatType>::HardTanHType(HardTanHType&& layer) :
+    Layer<MatType>(std::move(layer)),
+    maxValue(std::move(layer.maxValue)),
+    minValue(std::move(layer.minValue))
+{
+  // Nothing to do here.
+}
+
+template<typename MatType>
+HardTanHType<MatType>& HardTanHType<MatType>::operator=(const HardTanHType& layer)
+{
+  if (&layer != this)
+  {
+    Layer<MatType>::operator=(layer);
+    maxValue = layer.maxValue;
+    minValue = layer.minValue;
+  }
+
+  return *this;
+} 
+
+template<typename MatType>
+HardTanHType<MatType>& HardTanHType<MatType>::operator=(HardTanHType&& layer)
+{
+  if (&layer != this)
+  {
+    Layer<MatType>::operator=(std::move(layer));
+    maxValue = std::move(layer.maxValue);
+    minValue = std::move(layer.minValue);
+  }
+
+  return *this;
+}
+template<typename MatType>
+void HardTanHType<MatType>::Forward(
+    const MatType& input, MatType& output)
+{
+  #pragma omp parallel for
+  for (size_t i = 0; i < input.n_elem; ++i)
+  {
+    output(i) = (input(i) > maxValue ? maxValue :
+        (input(i) < minValue ? minValue : input(i)));
+  }
+}
+
+template<typename MatType>
+void HardTanHType<MatType>::Backward(
+    const MatType& input, const MatType& gy, MatType& g)
+{
+  g = gy;
+
+  #pragma omp parallel for
+  for (size_t i = 0; i < input.n_elem; ++i)
+  {
+    // input should not have any values greater than maxValue
+    // and lesser than minValue
+    if (input(i) <= minValue || input(i) >= maxValue)
+    {
+      g(i) = 0;
+    }
+  }
+}
+
+template<typename MatType>
+template<typename Archive>
+void HardTanHType<MatType>::serialize(
+    Archive& ar,
+    const uint32_t /* version */)
+{
+  ar(cereal::base_class<Layer<MatType>>(this));
+
+  ar(CEREAL_NVP(maxValue));
+  ar(CEREAL_NVP(minValue));
+}
+
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/layer_types.hpp

@@ -32,6 +32,7 @@
 #include <mlpack/methods/ann/layer/dropout.hpp>
 #include <mlpack/methods/ann/layer/elu.hpp>
 #include <mlpack/methods/ann/layer/grouped_convolution.hpp>
+#include <mlpack/methods/ann/layer/hard_tanh.hpp>
 #include <mlpack/methods/ann/layer/identity.hpp>
 #include <mlpack/methods/ann/layer/leaky_relu.hpp>
 #include <mlpack/methods/ann/layer/linear.hpp>

src/mlpack/methods/ann/layer/serialization.hpp

@@ -69,6 +69,7 @@
     CEREAL_REGISTER_TYPE(mlpack::RBFType<__VA_ARGS__>); \
     CEREAL_REGISTER_TYPE(mlpack::SoftmaxType<__VA_ARGS__>); \
     CEREAL_REGISTER_TYPE(mlpack::SoftminType<__VA_ARGS__>); \
+    CEREAL_REGISTER_TYPE(mlpack::HardTanHType<__VA_ARGS__>); \
 
 CEREAL_REGISTER_MLPACK_LAYERS(arma::mat);
 

src/mlpack/tests/ann/layer/hard_tanh.cpp

@@ -0,0 +1,58 @@
+/*
+ * @file tests/ann/layer/hard_tanh.cpp
+ * @author Vaibhav Pathak
+ *
+ * Tests the hard_tanh layer
+ *
+ */
+
+#include <mlpack/core.hpp>
+#include <mlpack/methods/ann.hpp>
+
+#include "../../test_catch_tools.hpp"
+#include "../../catch.hpp"
+#include "../../serialization.hpp"
+#include "../ann_test_tools.hpp"
+
+using namespace mlpack;
+
+/**
+ * Simple HardTanH module test
+ */
+
+TEST_CASE("SimpleHardTanHTest", "[ANNLayerTest]")
+{
+  arma::mat output, gy, g;
+  arma::mat input = {{-1.3743, -0.5565,  0.2742, -0.0151,  -1.4871},
+    {1.5797, -4.2711, -2.2505, -1.7105, -1.2544},
+    {0.4023,  0.5676, 2.3100, 1.6658, -0.1907},
+    {0.1897,  0.9097, 0.1418, -1.5349, 0.1225},
+    {-0.1101, -3.3656, -5.4033, -2.2240, -3.3235}};
+  arma::mat actualOutput = {{-1.0000, -0.5565, 0.2742, -0.0151, -1.0000},
+    {1.0000,  -1.0000,  -1.0000,  -1.0000,  -1.0000},
+    {0.4023,   0.5676,   1.0000,   1.0000,  -0.1907},
+    {0.1897,   0.9097,   0.1418,  -1.0000,   0.1225},
+    {-0.1101,  -1.0000,  -1.0000,  -1.0000,  -1.0000}};
+
+  HardTanH module;
+
+  output.set_size(5,5);
+  // Test the Forward function
+  module.Forward(input, output);
+  REQUIRE(arma::accu(output - actualOutput) == Approx(0).epsilon(1e-4));
+
+  arma::mat delta = {{0  , 1.0, 1.0, 1.0, 0.0},
+    {0  , 0  , 0  , 0.0, 0.0},
+    {1.0, 1.0, 0  , 0.0, 1.0},
+    {1.0, 1.0, 1.0, 0.0, 1.0},
+    {1.0, 0  , 0.0, 0.0, 0.0}};
+
+  gy.set_size(5,5);
+  gy.fill(1);
+  g.set_size(5,5);
+
+  //Test the Backward function
+  module.Backward(output, gy, g);
+  REQUIRE(arma::accu(g - delta) == Approx(0).epsilon(1e-4));
+}
+

src/mlpack/tests/ann/layer_test.cpp

@@ -26,6 +26,7 @@
 #include "layer/concatenate.cpp"
 #include "layer/dropout.cpp"
 #include "layer/grouped_convolution.cpp"
+#include "layer/hard_tanh.cpp"
 #include "layer/identity.cpp"
 #include "layer/linear3d.cpp"
 #include "layer/linear_no_bias.cpp"