temp: Full RNN fixes

tmva/tmva/inc/TMVA/DNN/Architectures/Cpu.h

@@ -386,6 +386,9 @@ class TCpu
     *  tensor \p B. */
    static void Deflatten(std::vector<TCpuMatrix<AReal>> &A, const TCpuMatrix<AReal> &B, size_t index, size_t nRows,
                          size_t nCols);
+   /** Rearrage data accoring to time fill B x T x D out with T x B x D matrix in*/
+   static void Rearrange(std::vector<TCpuMatrix<AReal>> &out, const std::vector<TCpuMatrix<AReal>> &in); 
+
 
    ///@}
 

tmva/tmva/inc/TMVA/DNN/Architectures/Cuda.h

@@ -392,6 +392,8 @@ class TCuda
    /** Transforms each row of \p B to a matrix and stores it in the tensor \p B. */
    static void Deflatten(std::vector<TCudaMatrix<AFloat>> &A, const TCudaMatrix<AFloat> &B, size_t index, size_t nRows,
                          size_t nCols);
+   /** Rearrage data accoring to time fill B x T x D out with T x B x D matrix in*/
+   static void Rearrange(std::vector<TCudaMatrix<AReal>> &out, const std::vector<TCudaMatrix<AReal>> &in); 
 
    ///@}
 

tmva/tmva/inc/TMVA/DNN/Architectures/Reference.h

@@ -388,6 +388,8 @@ class TReference
    /** Transforms each row of \p B to a matrix and stores it in the tensor \p B. */
    static void Deflatten(std::vector<TMatrixT<AReal>> &A, const TMatrixT<Scalar_t> &B, size_t index, size_t nRows,
                          size_t nCols);
+   /** Rearrage data accoring to time fill B x T x D out with T x B x D matrix in*/
+   static void Rearrange(std::vector<TMatrixT<AReal>> &out, const std::vector<TMatrixT<AReal>> &in); 
 
    ///@}
 

tmva/tmva/inc/TMVA/DNN/DeepNet.h

@@ -136,7 +136,7 @@ class TDeepNet {
 
    /*! Function for adding Recurrent Layer in the Deep Neural Network,
     * with given parameters */
-   TBasicRNNLayer<Architecture_t> *AddBasicRNNLayer(size_t batchSize, size_t stateSize, size_t inputSize,
+   TBasicRNNLayer<Architecture_t> *AddBasicRNNLayer(size_t stateSize, size_t inputSize,
                                                     size_t timeSteps, bool rememberState = false);
 
    /*! Function for adding Vanilla RNN when the layer is already created
@@ -491,12 +491,12 @@ void TDeepNet<Architecture_t, Layer_t>::AddMaxPoolLayer(TMaxPoolLayer<Architectu
 
 //______________________________________________________________________________
 template <typename Architecture_t, typename Layer_t>
-TBasicRNNLayer<Architecture_t> *TDeepNet<Architecture_t, Layer_t>::AddBasicRNNLayer(size_t batchSize, size_t stateSize,
+TBasicRNNLayer<Architecture_t> *TDeepNet<Architecture_t, Layer_t>::AddBasicRNNLayer(size_t stateSize,
                                                                                     size_t inputSize, size_t timeSteps,
                                                                                     bool rememberState)
 {
    TBasicRNNLayer<Architecture_t> *basicRNNLayer = new TBasicRNNLayer<Architecture_t>(
-      batchSize, stateSize, inputSize, timeSteps, rememberState, DNN::EActivationFunction::kTanh, fIsTraining);
+      this->GetBatchSize(), stateSize, inputSize, timeSteps, rememberState, DNN::EActivationFunction::kTanh, fIsTraining, this->GetInitialization());
    fLayers.push_back(basicRNNLayer);
    return basicRNNLayer;
 }
@@ -684,6 +684,22 @@ auto TDeepNet<Architecture_t, Layer_t>::Initialize() -> void
    }
 }
 
+template <typename Architecture>
+auto debugTensor(const std::vector<typename Architecture::Matrix_t> &A, const std::string name = "tensor")
+-> void
+{
+  std::cout << name << "\n";
+  for (size_t l = 0; l < A.size(); ++l) {
+      for (size_t i = 0; i < A[l].GetNrows(); ++i) {
+        for (size_t j = 0; j < A[l].GetNcols(); ++j) {
+            std::cout << A[l](i, j) << " ";
+        }
+        std::cout << "\n";
+      }
+      std::cout << "********\n";
+  } 
+}
+
 //______________________________________________________________________________
 template <typename Architecture_t, typename Layer_t>
 auto TDeepNet<Architecture_t, Layer_t>::Forward(std::vector<Matrix_t> input, bool applyDropout) -> void
@@ -715,6 +731,7 @@ auto TDeepNet<Architecture_t, Layer_t>::ParallelForward(std::vector<TDeepNet<Arc
       }
    }
 }
+
 //_____________________________________________________________________________
 template <typename Architecture_t, typename Layer_t>
 auto TDeepNet<Architecture_t, Layer_t>::PreTrain(std::vector<Matrix_t> &input,
@@ -804,6 +821,7 @@ auto TDeepNet<Architecture_t, Layer_t>::PreTrain(std::vector<Matrix_t> &input,
       fLayers.back()->Print();
    }
 }
+
 //______________________________________________________________________________
 template <typename Architecture_t, typename Layer_t>
 auto TDeepNet<Architecture_t, Layer_t>::FineTune(std::vector<Matrix_t> &input, std::vector<Matrix_t> &testInput,
@@ -846,14 +864,18 @@ auto TDeepNet<Architecture_t, Layer_t>::Backward(std::vector<Matrix_t> input, co
    evaluateGradients<Architecture_t>(fLayers.back()->GetActivationGradientsAt(0), this->GetLossFunction(), groundTruth,
                                      fLayers.back()->GetOutputAt(0), weights);
    for (size_t i = fLayers.size() - 1; i > 0; i--) {
-      std::vector<Matrix_t> activation_gradient_backward = fLayers[i - 1]->GetActivationGradients();
-      std::vector<Matrix_t> activations_backward = fLayers[i - 1]->GetOutput();
+      std::vector<Matrix_t> &activation_gradient_backward = fLayers[i - 1]->GetActivationGradients();
+      std::vector<Matrix_t> &activations_backward = fLayers[i - 1]->GetOutput();
       fLayers[i]->Backward(activation_gradient_backward, activations_backward, inp1, inp2);
+      //debugTensor<Architecture_t>(activation_gradient_backward, "act grad backward after back of dense");
    }
 
-   std::vector<Matrix_t> dummy;
+   std::vector<Matrix_t> gradient_input;
+   for (size_t i = 0; i < input.size(); i++) {
+      gradient_input.emplace_back(input[i].GetNrows(), input[i].GetNcols());
+   }
 
-   fLayers[0]->Backward(dummy, input, inp1, inp2);
+   fLayers[0]->Backward(gradient_input, input, inp1, inp2);
 }
 
 //______________________________________________________________________________

tmva/tmva/inc/TMVA/DNN/RNN/RNNLayer.h

@@ -63,16 +63,6 @@ template<typename Architecture_t>
 
 private:
 
-   /* from GeneralLayer:
-    * fBatchSize
-    * fInputDepth = 1
-    * fInputHeight = 1
-    * fInputWidth = inputSize
-    * fOutputDepth = 1
-    * fOutputHeight = 1
-    * fOutputWidth = stateSize
-    * fOutput = timeSteps x batchSize x stateSize */
-
    size_t fTimeSteps;              ///< Timesteps for RNN
    size_t fStateSize;              ///< Hidden state size of RNN
    bool   fRememberState;          ///< Remember state in next pass
@@ -95,7 +85,7 @@ template<typename Architecture_t>
    TBasicRNNLayer(size_t batchSize, size_t stateSize, size_t inputSize,
                   size_t timeSteps, bool rememberState = false,
                   DNN::EActivationFunction f = DNN::EActivationFunction::kTanh,
-                  bool training = true);
+                  bool training = true, DNN::EInitialization fA = DNN::EInitialization::kZero);
 
    /** Copy Constructor */
    TBasicRNNLayer(const TBasicRNNLayer &);
@@ -131,17 +121,9 @@ template<typename Architecture_t>
                               const Matrix_t & precStateActivations, const Matrix_t & currStateActivations,
                               const Matrix_t & input, Matrix_t & input_gradient);
 
-   /*! Return a vector of all learnable weights */
-   //std::vector<Matrix_t*> GetWeights() const;
-
-   ///*! Return a vector of all learnable weights' gradients */
-   //std::vector<Matrix_t*> GetWeightGradients() const;
-
-   /*! Return a vector of all learnable biases */
-   //std::vector<Matrix_t*> GetBiases();
-
-   /*! Return a vector of all learnable bias' gradients */
-   //std::vector<Matrix_t*> GetBiasGradients();
+   /*! Rearrage data accoring to time
+    * fill B x T x D out with T x B x D matrix in*/
+   //void Rearrange(Tensor_t &out, const Tensor_t &in); 
 
    /** Prints the info about the layer */
    void Print() const;
@@ -158,10 +140,10 @@ template<typename Architecture_t>
    const Matrix_t & GetWeightsInput()   const {return fWeightsInput;}
    Matrix_t        & GetWeightsState()        {return fWeightsState;}
    const Matrix_t & GetWeightsState()   const {return fWeightsState;}
-   Matrix_t        & GetBiases()              {return fBiases;}
-   const Matrix_t & GetBiases()         const {return fBiases;}
-   Matrix_t        & GetBiasGradients()            {return fBiasGradients;}
-   const Matrix_t & GetBiasGradients() const {return fBiasGradients;}
+   //Matrix_t        & GetBiases()              {return fBiases;}
+   //const Matrix_t & GetBiases()         const {return fBiases;}
+   //Matrix_t        & GetBiasGradients()            {return fBiasGradients;}
+   //const Matrix_t & GetBiasGradients() const {return fBiasGradients;}
    Matrix_t        & GetWeightInputGradients()         {return fWeightInputGradients;}
    const Matrix_t & GetWeightInputGradients()    const {return fWeightInputGradients;}
    Matrix_t        & GetWeightStateGradients()         {return fWeightStateGradients;}
@@ -177,9 +159,9 @@ template<typename Architecture_t>
 TBasicRNNLayer<Architecture_t>::TBasicRNNLayer(size_t batchSize, size_t stateSize, size_t inputSize,
                                               size_t timeSteps, bool rememberState,
                                               DNN::EActivationFunction f,
-                                              bool training)
+                                              bool training, DNN::EInitialization fA)
    : VGeneralLayer<Architecture_t>(batchSize, 1, 1, inputSize, 1, 1, stateSize, 2, {stateSize, stateSize}, {inputSize, stateSize},
-   1, {stateSize}, {1}, timeSteps, batchSize, stateSize, DNN::EInitialization::kZero),
+   1, {stateSize}, {1}, timeSteps, batchSize, stateSize, fA),
    fTimeSteps(timeSteps), fStateSize(stateSize), fRememberState(rememberState), fWeightsInput(this->GetWeightsAt(0)), fF(f),
    fState(batchSize, stateSize), fWeightsState(this->GetWeightsAt(1)), fBiases(this->GetBiasesAt(0)), fDerivatives(batchSize, stateSize),
    fWeightInputGradients(this->GetWeightGradientsAt(0)), fWeightStateGradients(this->GetWeightGradientsAt(1)), fBiasGradients(this->GetBiasGradientsAt(0))
@@ -230,47 +212,46 @@ auto TBasicRNNLayer<Architecture_t>::Print() const
              << "Hidden State Size: " << this->GetStateSize() << "\n";
 }
 
-////______________________________________________________________________________
-//template<typename Architecture_t>
-//auto TBasicRNNLayer<Architecture_t>::GetWeights() const
-//->   std::vector<Matrix_t*>
-//{
-//  std::vector<Matrix_t*> weights;
-//  weights.emplace_back(&fWeightsInput);
-//  weights.emplace_back(&fWeightsState);
-//  return weights;
-//}
-//
-////______________________________________________________________________________
-//template<typename Architecture_t>
-//auto TBasicRNNLayer<Architecture_t>::GetWeightGradients() const
-//->   std::vector<Matrix_t*>
-//{
-//  std::vector<Matrix_t*> weightGradients;
-//  weightGradients.emplace_back(&fWeightInputGradients);
-//  weightGradients.emplace_back(&fWeightStateGradients);
-//  return weightGradients;
-//}
-//
 //______________________________________________________________________________
 //template<typename Architecture_t>
-//auto TBasicRNNLayer<Architecture_t>::GetBiases() const
-//->   std::vector<Matrix_t*>
+//auto TBasicRNNLayer<Architecture_t>::Rearrange(Tensor_t &out, const Tensor_t &in)
+//-> void
 //{
-//  std::vector<Matrix_t*> biases;
-//  biases.emplace_back(&fBiases);
-//  return biases;
+//   // B x T x D out --- T x B x D in*/
+//   size_t B = out.size();
+//   size_t T = out[0].GetNrows();
+//   size_t D = out[0].GetNcols();
+//   if ((T != in.size()) || (B != in[0].GetNrows()) 
+//       || (D != in[0].GetNcols())) {
+//      std::cout << "Incompatible Dimensions\n"
+//         << in.size() << "x" << in[0].GetNrows() << "x" << in[0].GetNcols() 
+//         << " --> " << B << "x" << T << "x" << D << "\n";
+//      return;
+//   }
+//   for (size_t i = 0; i < B; ++i) {
+//      for (size_t j = 0; j < T; ++j) {
+//         for (size_t k = 0; k < D; ++k) {
+//            out[i](j, k) = in[j](i, k);
+//         }
+//      }
+//   }
+//   return;
 //}
 
-//______________________________________________________________________________
-//template<typename Architecture_t>
-//auto TBasicRNNLayer<Architecture_t>::GetBiasGradients() const
-//->   std::vector<Matrix_t*>
-//{
-//  std::vector<Matrix_t*> biasGradients;
-//  biasGradients.emplace_back(&fBiasGradients);
-//  return biasGradients;
-//}
+template <typename Architecture>
+auto debugMatrix(const typename Architecture::Matrix_t &A, const std::string name = "matrix")
+-> void
+{
+  std::cout << name << "\n";
+  for (size_t i = 0; i < A.GetNrows(); ++i) {
+    for (size_t j = 0; j < A.GetNcols(); ++j) {
+        std::cout << A(i, j) << " ";
+    }
+    std::cout << "\n";
+  }
+  std::cout << "********\n";
+}
+
 
 //______________________________________________________________________________
 template <typename Architecture_t>
@@ -291,9 +272,12 @@ auto inline TBasicRNNLayer<Architecture_t>::CellForward(Matrix_t &input)
 {
    // State = act(W_input . input + W_state . state + bias)
    const DNN::EActivationFunction fF = this->GetActivationFunction();
+   //debugMatrix<Architecture_t>(input, "input");
    Matrix_t tmpState(fState.GetNrows(), fState.GetNcols());
    Architecture_t::MultiplyTranspose(tmpState, fState, fWeightsState);
    Architecture_t::MultiplyTranspose(fState, input, fWeightsInput);
+   //debugMatrix<Architecture_t>(fWeightsInput, "weights input");
+   //debugMatrix<Architecture_t>(fState, "fState");
    Architecture_t::ScaleAdd(fState, tmpState);
    Architecture_t::AddRowWise(fState, fBiases);
    DNN::evaluate<Architecture_t>(fState, fF);
@@ -310,7 +294,7 @@ auto inline TBasicRNNLayer<Architecture_t>::Backward(Tensor_t &gradients_backwar
    // activations backward is input
    // gradients_backward is activationGradients of layer before it, which is input layer
    // currently gradient_backward is for input(x) and not for state
-   // we also need the one for state as
+   // TODO use this to change initial state??
    Matrix_t state_gradients_backward(this->GetBatchSize(), fStateSize);  // B x H
    DNN::initialize<Architecture_t>(state_gradients_backward,  DNN::EInitialization::kZero);
 
@@ -340,9 +324,18 @@ auto inline TBasicRNNLayer<Architecture_t>::CellBackward(Matrix_t & state_gradie
 -> Matrix_t &
 {
    DNN::evaluateDerivative<Architecture_t>(fDerivatives, this->GetActivationFunction(), currStateActivations);
-   return Architecture_t::RecurrentLayerBackward(state_gradients_backward, fWeightInputGradients, fWeightStateGradients,
+   //debugMatrix<Architecture_t>(state_gradients_backward, "0 state grad");
+   //debugMatrix<Architecture_t>(fWeightInputGradients, "0 wx grad");
+   //debugMatrix<Architecture_t>(fWeightStateGradients, "0 wh grad");
+   //debugMatrix<Architecture_t>(fDerivatives, "bef df");
+   auto &lol =  Architecture_t::RecurrentLayerBackward(state_gradients_backward, fWeightInputGradients, fWeightStateGradients,
                                                  fBiasGradients, fDerivatives, precStateActivations, fWeightsInput,
                                                  fWeightsState, input, input_gradient);
+   //debugMatrix<Architecture_t>(state_gradients_backward, "state grad");
+   //debugMatrix<Architecture_t>(fWeightInputGradients, "wx grad");
+   //debugMatrix<Architecture_t>(fWeightStateGradients, "wh grad");
+   //debugMatrix<Architecture_t>(fDerivatives, "df");
+   return lol;
 }
 
 } // namespace RNN

tmva/tmva/src/DNN/Architectures/Cpu/Propagation.cxx

@@ -394,5 +394,30 @@ void TCpu<AFloat>::Deflatten(std::vector<TCpuMatrix<AFloat>> &A, const TCpuMatri
    }
 }
 
+//______________________________________________________________________________
+template <typename AReal>
+void TCpu<AReal>::Rearrange(std::vector<TCpuMatrix<AReal>> &out, const std::vector<TCpuMatrix<AReal>> &in)
+{
+   // B x T x D out --- T x B x D in*/
+   size_t B = out.size();
+   size_t T = out[0].GetNrows();
+   size_t D = out[0].GetNcols();
+   if ((T != in.size()) || (B != in[0].GetNrows()) 
+       || (D != in[0].GetNcols())) {
+      std::cout << "Incompatible Dimensions\n"
+         << in.size() << "x" << in[0].GetNrows() << "x" << in[0].GetNcols() 
+         << " --> " << B << "x" << T << "x" << D << "\n";
+      return;
+   }
+   for (size_t i = 0; i < B; ++i) {
+      for (size_t j = 0; j < T; ++j) {
+         for (size_t k = 0; k < D; ++k) {
+            out[i](j, k) = in[j](i, k);
+         }
+      }
+   }
+   return;
+}
+
 } // namespace DNN
 } // namespace TMVA

tmva/tmva/src/DNN/Architectures/Cuda/Propagation.cu

@@ -280,7 +280,32 @@ void TCuda<AFloat>::MaxPoolLayerBackward(std::vector<TCudaMatrix<AFloat>> & acti
 template<typename AFloat>
 void TCuda<AFloat>::Reshape(TCudaMatrix<AFloat> &A, const TCudaMatrix<AFloat> &B)
 {
-
+   //TODO    
+}
+
+//______________________________________________________________________________
+template <typename AReal>
+void TCuda<AReal>::Rearrange(std::vector<TCudaMatrix<AReal>> &out, const std::vector<TCudaMatrix<AReal>> &in)
+{
+   // B x T x D out --- T x B x D in*/
+   size_t B = out.size();
+   size_t T = out[0].GetNrows();
+   size_t D = out[0].GetNcols();
+   if ((T != in.size()) || (B != in[0].GetNrows()) 
+       || (D != in[0].GetNcols())) {
+      std::cout << "Incompatible Dimensions\n"
+         << in.size() << "x" << in[0].GetNrows() << "x" << in[0].GetNcols() 
+         << " --> " << B << "x" << T << "x" << D << "\n";
+      return;
+   }
+   for (size_t i = 0; i < B; ++i) {
+      for (size_t j = 0; j < T; ++j) {
+         for (size_t k = 0; k < D; ++k) {
+            out[i](j, k) = in[j](i, k);
+         }
+      }
+   }
+   return;
 }
 
 //____________________________________________________________________________