diff --git a/.gitignore b/.gitignore index 2936cd7982e31..944960f839e50 100644 --- a/.gitignore +++ b/.gitignore @@ -500,3 +500,8 @@ CMakeLists.txt.user /tutorials/tree/cernstaff.root /tutorials/tree/staff.root /tutorials/tree/basic.root + +# TMVA datasets +/tmva/tmva/test/DNN/CNN/dataset/SingleElectronPt50_FEVTDEBUG_n250k_IMG_CROPS32.root +/tmva/tmva/test/DNN/CNN/dataset/SinglePhotonPt50_FEVTDEBUG_n250k_IMG_CROPS32.root +/tmva/tmva/test/DNN/CNN/dataset/tmva_class_example.root diff --git a/tmva/tmva/CMakeLists.txt b/tmva/tmva/CMakeLists.txt index 3d0e167e0329a..b19534b9fd2e3 100644 --- a/tmva/tmva/CMakeLists.txt +++ b/tmva/tmva/CMakeLists.txt @@ -16,7 +16,7 @@ set(headers1 Configurable.h Factory.h MethodBase.h MethodCompositeBase.h MethodKNN.h MethodCFMlpANN.h MethodCFMlpANN_Utils.h MethodLikelihood.h MethodHMatrix.h MethodPDERS.h MethodBDT.h MethodDT.h MethodSVM.h MethodBayesClassifier.h MethodFDA.h MethodMLP.h MethodBoost.h - MethodPDEFoam.h MethodLD.h MethodCategory.h MethodDNN.h) + MethodPDEFoam.h MethodLD.h MethodCategory.h MethodDNN.h MethodDL.h) set(headers2 TSpline2.h TSpline1.h PDF.h BinaryTree.h BinarySearchTreeNode.h BinarySearchTree.h Timer.h RootFinder.h CrossEntropy.h DecisionTree.h DecisionTreeNode.h MisClassificationError.h Node.h SdivSqrtSplusB.h SeparationBase.h RegressionVariance.h Tools.h Reader.h @@ -47,7 +47,8 @@ endforeach() #---Assign source files to the implementations ----------------- SET(DNN_FILES src/DNN/Architectures/Reference.cxx - src/DNN/Architectures/Reference/DataLoader.cxx) + src/DNN/Architectures/Reference/DataLoader.cxx + src/DNN/Architectures/Reference/TensorDataLoader.cxx) SET(DNN_CUDA_FILES src/DNN/Architectures/Cuda.cu src/DNN/Architectures/Cuda/CudaBuffers.cxx src/DNN/Architectures/Cuda/CudaMatrix.cu) @@ -95,6 +96,9 @@ if(NOT gnuinstall) endif() ROOT_ADD_TEST_SUBDIRECTORY(test/DNN) +ROOT_ADD_TEST_SUBDIRECTORY(test/DNN/CNN) +ROOT_ADD_TEST_SUBDIRECTORY(test/DNN/DAE) +ROOT_ADD_TEST_SUBDIRECTORY(test/DNN/RNN) diff --git a/tmva/tmva/inc/LinkDef1.h b/tmva/tmva/inc/LinkDef1.h index 3f099cb409fd8..7824fb837d0de 100644 --- a/tmva/tmva/inc/LinkDef1.h +++ b/tmva/tmva/inc/LinkDef1.h @@ -62,5 +62,6 @@ #pragma link C++ class TMVA::MethodLD+; #pragma link C++ class TMVA::MethodCategory+; #pragma link C++ class TMVA::MethodDNN+; +#pragma link C++ class TMVA::MethodDL+; #endif diff --git a/tmva/tmva/inc/TMVA/Config.h b/tmva/tmva/inc/TMVA/Config.h index 48734c8a156c5..b2965d81d10ba 100644 --- a/tmva/tmva/inc/TMVA/Config.h +++ b/tmva/tmva/inc/TMVA/Config.h @@ -41,13 +41,14 @@ #endif #include "Rtypes.h" #include "TString.h" - +#include namespace TMVA { class MsgLogger; class Config { - + ROOT::TThreadExecutor fPool; + public: static Config& Instance(); @@ -64,6 +65,7 @@ namespace TMVA { Bool_t DrawProgressBar() const { return fDrawProgressBar; } void SetDrawProgressBar( Bool_t d ) { fDrawProgressBar = d; } + ROOT::TThreadExecutor &GetThreadExecutor() { return fPool; } public: diff --git a/tmva/tmva/inc/TMVA/DNN/Architectures/Cpu.h b/tmva/tmva/inc/TMVA/DNN/Architectures/Cpu.h index 56e3b952b2b3e..0107608a772f6 100644 --- a/tmva/tmva/inc/TMVA/DNN/Architectures/Cpu.h +++ b/tmva/tmva/inc/TMVA/DNN/Architectures/Cpu.h @@ -20,6 +20,7 @@ #include "Cpu/CpuBuffer.h" #include "Cpu/CpuMatrix.h" +#include namespace TMVA { @@ -83,6 +84,17 @@ class TCpu const TCpuMatrix & activationGradients, const TCpuMatrix & weights, const TCpuMatrix & activationBackward); + /** Backward pass for Recurrent Networks */ + static Matrix_t & RecurrentLayerBackward(TCpuMatrix & state_gradients_backward, // BxH + TCpuMatrix & input_weight_gradients, + TCpuMatrix & state_weight_gradients, + TCpuMatrix & bias_gradients, + TCpuMatrix & df, //DxH + const TCpuMatrix & state, // BxH + const TCpuMatrix & weights_input, // HxD + const TCpuMatrix & weights_state, // HxH + const TCpuMatrix & input, // BxD + TCpuMatrix & input_gradient); /** Adds a the elements in matrix B scaled by c to the elements in * the matrix A. This is required for the weight update in the gradient * descent step.*/ @@ -92,6 +104,15 @@ class TCpu static void Copy(TCpuMatrix & B, const TCpuMatrix & A); + + /** Above functions extended to vectors */ + static void ScaleAdd(std::vector> & A, + const std::vector> & B, + Scalar_t beta = 1.0); + + static void Copy(std::vector> & A, + const std::vector> & B); + ///@} //____________________________________________________________________________ @@ -245,6 +266,130 @@ class TCpu * matrix \p A and scale the result by reciprocal of \p p. */ static void Dropout(TCpuMatrix & A, Scalar_t p); + ///@} + + //____________________________________________________________________________ + // + // Convolutional Layer Propagation + //____________________________________________________________________________ + + /** @name Forward Propagation in Convolutional Layer + */ + ///@{ + + /** Transform the matrix B in local view format, suitable for + * convolution, and store it in matrix A */ + static void Im2col(TCpuMatrix &A, TCpuMatrix &B, size_t imgHeight, size_t imgWidth, size_t fltHeight, + size_t fltWidth, size_t strideRows, size_t strideCols, size_t zeroPaddingHeight, + size_t zeroPaddingWidth); + + /** Rotates the matrix \p B, which is representing a weights, + * and stores them in the matrix \p A. */ + static void RotateWeights(TCpuMatrix &A, const TCpuMatrix &B, size_t filterDepth, size_t filterHeight, + size_t filterWidth, size_t numFilters); + + /** Add the biases in the Convolutional Layer. */ + static void AddConvBiases(TCpuMatrix &output, const TCpuMatrix &biases); + ///@} + + /** @name Backward Propagation in Convolutional Layer + */ + ///@{ + + /** Perform the complete backward propagation step in a Convolutional Layer. + * If the provided \p activationGradientsBackward matrix is not empty, compute the + * gradients of the objective function with respect to the activations + * of the previous layer (backward direction). + * Also compute the weight and the bias gradients. Modifies the values + * in \p df and thus produces only a valid result, if it is applied the + * first time after the corresponding forward propagation has been per- + * formed. */ + static void ConvLayerBackward(std::vector> &activationGradientsBackward, + TCpuMatrix &weightGradients, TCpuMatrix &biasGradients, + std::vector> &df, + const std::vector> &activationGradients, + const TCpuMatrix &weights, + const std::vector> &activationBackward, size_t batchSize, + size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, + size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t nLocalViews); + + /** Utility function for calculating the activation gradients of the layer + * before the convolutional layer. */ + static void CalculateConvActivationGradients(std::vector> &activationGradientsBackward, + std::vector> &df, + const TCpuMatrix &weights, size_t batchSize, + size_t inputHeight, size_t inputWidth, size_t depth, size_t height, + size_t width, size_t filterDepth, size_t filterHeight, + size_t filterWidth); + + /** Utility function for calculating the weight gradients of the convolutional + * layer. */ + static void CalculateConvWeightGradients(TCpuMatrix &weightGradients, + std::vector> &df, + const std::vector> &activations_backward, + size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, + size_t height, size_t width, size_t filterDepth, size_t filterHeight, + size_t filterWidth, size_t nLocalViews); + + /** Utility function for calculating the bias gradients of the convolutional + * layer */ + static void CalculateConvBiasGradients(TCpuMatrix &biasGradients, std::vector> &df, + size_t batchSize, size_t depth, size_t nLocalViews); + ///@} + + //____________________________________________________________________________ + // + // Max Pooling Layer Propagation + //____________________________________________________________________________ + /** @name Forward Propagation in Max Pooling Layer + */ + ///@{ + + /** Downsample the matrix \p C to the matrix \p A, using max + * operation, such that the winning indices are stored in matrix + * \p B. */ + static void Downsample(TCpuMatrix &A, TCpuMatrix &B, const TCpuMatrix &C, size_t imgHeight, + size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols); + + ///@} + + /** @name Backward Propagation in Max Pooling Layer + */ + ///@{ + /** Perform the complete backward propagation step in a Pooling Layer. Based on the + * winning idices stored in the index matrix, it just forwards the actiovation + * gradients to the previous layer. */ + static void MaxPoolLayerBackward(std::vector> &activationGradientsBackward, + const std::vector> &activationGradients, + const std::vector> &indexMatrix, size_t batchSize, size_t depth, + size_t nLocalViews); + + ///@} + + //____________________________________________________________________________ + // + // Reshape Layer Propagation + //____________________________________________________________________________ + /** @name Forward and Backward Propagation in Reshape Layer + */ + ///@{ + + /** Transform the matrix \p B to a matrix with different dimensions \p A */ + static void Reshape(TCpuMatrix &A, const TCpuMatrix &B); + + /** Flattens the tensor \p B, such that each matrix, is stretched in + * one row, resulting with a matrix \p A. */ + static void Flatten(TCpuMatrix &A, const std::vector> &B, size_t size, size_t nRows, + size_t nCols); + + /** Transforms each row of \p B to a matrix and stores it in the + * tensor \p B. */ + static void Deflatten(std::vector> &A, const TCpuMatrix &B, size_t index, size_t nRows, + size_t nCols); + /** Rearrage data accoring to time fill B x T x D out with T x B x D matrix in*/ + static void Rearrange(std::vector> &out, const std::vector> &in); + + ///@} //____________________________________________________________________________ diff --git a/tmva/tmva/inc/TMVA/DNN/Architectures/Cpu/CpuMatrix.h b/tmva/tmva/inc/TMVA/DNN/Architectures/Cpu/CpuMatrix.h index 63916e1841b05..23eee3fd061a5 100644 --- a/tmva/tmva/inc/TMVA/DNN/Architectures/Cpu/CpuMatrix.h +++ b/tmva/tmva/inc/TMVA/DNN/Architectures/Cpu/CpuMatrix.h @@ -21,9 +21,34 @@ #include #include "TMatrix.h" -#include "ROOT/TThreadExecutor.hxx" +#include "TMVA/Config.h" #include "CpuBuffer.h" +//#define DEBUG_TMVA_TCPUMATRIX +#if defined(DEBUG_TMVA_TCPUMATRIX) +#define PrintMatrix(mat, text) \ + { \ + auto _dpointer = mat.GetRawDataPointer(); \ + if (_dpointer == NULL) { \ + std::cout << #mat << " is null pointer" << std::endl; \ + exit(1); \ + } \ + auto _nrows = mat.GetNrows(); \ + auto _ncols = mat.GetNcols(); \ + std::cout << "---------------------" << text << " " << #mat << "(" << _nrows << "," << _ncols << ")" \ + << "--------------------" << std::endl; \ + for (size_t _i = 0; _i < _nrows; _i++) { \ + for (size_t _j = 0; _j < _ncols; _j++) { \ + std::cout << mat(_i, _j); \ + if (_j < _ncols - 1) std::cout << ","; \ + } \ + std::cout << std::endl; \ + } \ + } +#else +#define PrintMatrix(mat, text) +#endif + namespace TMVA { namespace DNN @@ -46,8 +71,6 @@ template class TCpuMatrix { private: - - static ROOT::TThreadExecutor fPool; static std::vector fOnes; ///< Vector filled with ones used for BLAS calls. TCpuBuffer fBuffer; ///< The buffer holding the matrix elements @@ -61,6 +84,8 @@ class TCpuMatrix * of the number of columns of every instantiated CpuMatrix object. */ static const AFloat * GetOnePointer() {return fOnes.data();} + static size_t GetOnePointerSize() { return fOnes.size(); } + /** Construct matrix and allocate space for its elements. */ TCpuMatrix(size_t nRows, size_t nCols); /** Construct a TCpuMatrix object by (deeply) copying from a @@ -103,7 +128,7 @@ class TCpuMatrix AFloat * GetRawDataPointer() {return fBuffer;} const AFloat * GetRawDataPointer() const {return fBuffer;} - ROOT::TThreadExecutor & GetThreadExecutor() const {return fPool;} + ROOT::TThreadExecutor &GetThreadExecutor() const { return TMVA::Config::Instance().GetThreadExecutor(); } private: @@ -111,9 +136,6 @@ class TCpuMatrix }; -template -ROOT::TThreadExecutor TCpuMatrix::fPool {}; - template std::vector TCpuMatrix::fOnes {}; @@ -132,7 +154,7 @@ inline void TCpuMatrix::Map(Function_t &f) return 0; }; - fPool.Map(ff, ROOT::TSeqI(fNCols * fNRows)); + TMVA::Config::Instance().GetThreadExecutor().Map(ff, ROOT::TSeqI(fNCols * fNRows)); } template @@ -148,7 +170,7 @@ inline void TCpuMatrix::MapFrom(Function_t &f, const TCpuMatrix &A) return 0; }; - fPool.Map(ff, ROOT::TSeqI(fNCols * fNRows)); + TMVA::Config::Instance().GetThreadExecutor().Map(ff, ROOT::TSeqI(fNCols * fNRows)); } } // namespace DNN diff --git a/tmva/tmva/inc/TMVA/DNN/Architectures/Cuda.h b/tmva/tmva/inc/TMVA/DNN/Architectures/Cuda.h index 9d0b37505ce64..5b7bc22f4ce76 100644 --- a/tmva/tmva/inc/TMVA/DNN/Architectures/Cuda.h +++ b/tmva/tmva/inc/TMVA/DNN/Architectures/Cuda.h @@ -23,6 +23,7 @@ #include "Cuda/CudaMatrix.h" #include "TMVA/DNN/DataLoader.h" #include +#include namespace TMVA { @@ -87,6 +88,17 @@ class TCuda const TCudaMatrix & activationGradients, const TCudaMatrix & weights, const TCudaMatrix & activationBackward); + /** Backward pass for Recurrent Networks */ + static Matrix_t & RecurrentLayerBackward(TCudaMatrix & state_gradients_backward, // BxH + TCudaMatrix & input_weight_gradients, + TCudaMatrix & state_weight_gradients, + TCudaMatrix & bias_gradients, + TCudaMatrix & df, //DxH + const TCudaMatrix & state, // BxH + const TCudaMatrix & weights_input, // HxD + const TCudaMatrix & weights_state, // HxH + const TCudaMatrix & input, // BxD + TCudaMatrix & input_gradient); /** Adds a the elements in matrix B scaled by c to the elements in * the matrix A. This is required for the weight update in the gradient * descent step.*/ @@ -96,6 +108,15 @@ class TCuda /** Copy the elements of matrix A into matrix B. */ static void Copy(TCudaMatrix & B, const TCudaMatrix & A); + + /** Above functions extended to vectors */ + static void ScaleAdd(std::vector> & A, + const std::vector> & B, + Scalar_t beta = 1.0); + + static void Copy(std::vector> & A, + const std::vector> & B); + ///@} //____________________________________________________________________________ @@ -252,6 +273,130 @@ class TCuda ///@} + //____________________________________________________________________________ + // + // Convolutional Layer Propagation + //____________________________________________________________________________ + + /** @name Forward Propagation in Convolutional Layer + */ + ///@{ + + /** Transform the matrix \p B in local view format, suitable for + * convolution, and store it in matrix \p A. */ + static void Im2col(TCudaMatrix &A, const TCudaMatrix &B, size_t imgHeight, size_t imgWidth, + size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, size_t zeroPaddingHeight, + size_t zeroPaddingWidth); + + /** Rotates the matrix \p B, which is representing a weights, + * and stores them in the matrix \p A. */ + static void RotateWeights(TCudaMatrix &A, const TCudaMatrix &B, size_t filterDepth, + size_t filterHeight, size_t filterWidth, size_t numFilters); + + /** Add the biases in the Convolutional Layer. */ + static void AddConvBiases(TCudaMatrix &output, const TCudaMatrix &biases); + + ///@} + + /** @name Backward Propagation in Convolutional Layer + */ + ///@{ + + /** Perform the complete backward propagation step in a Convolutional Layer. + * If the provided \p activationGradientsBackward matrix is not empty, compute the + * gradients of the objective function with respect to the activations + * of the previous layer (backward direction). + * Also compute the weight and the bias gradients. Modifies the values + * in \p df and thus produces only a valid result, if it is applied the + * first time after the corresponding forward propagation has been per- + * formed. */ + static void ConvLayerBackward(std::vector> &activationGradientsBackward, + TCudaMatrix &weightGradients, TCudaMatrix &biasGradients, + std::vector> &df, + const std::vector> &activationGradients, + const TCudaMatrix &weights, + const std::vector> &activationBackward, size_t batchSize, + size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, + size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t nLocalViews); + + /** Utility function for calculating the activation gradients of the layer + * before the convolutional layer. */ + static void CalculateConvActivationGradients(std::vector> &activationGradientsBackward, + std::vector> &df, + const TCudaMatrix &weights, size_t batchSize, + size_t inputHeight, size_t inputWidth, size_t depth, size_t height, + size_t width, size_t filterDepth, size_t filterHeight, + size_t filterWidth); + + /** Utility function for calculating the weight gradients of the convolutional + * layer. */ + static void CalculateConvWeightGradients(TCudaMatrix &weightGradients, std::vector> &df, + const std::vector> &activations_backward, + size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, + size_t height, size_t width, size_t filterDepth, size_t filterHeight, + size_t filterWidth, size_t nLocalViews); + + /** Utility function for calculating the bias gradients of the convolutional + * layer */ + static void CalculateConvBiasGradients(TCudaMatrix &biasGradients, std::vector> &df, + size_t batchSize, size_t depth, size_t nLocalViews); + + ///@} + + //____________________________________________________________________________ + // + // Max Pooling Layer Propagation + //____________________________________________________________________________ + /** @name Forward Propagation in Max Pooling Layer + */ + ///@{ + + /** Downsample the matrix \p C to the matrix \p A, using max + * operation, such that the winning indices are stored in matrix + * \p B. */ + static void Downsample(TCudaMatrix &A, TCudaMatrix &B, const TCudaMatrix &C, + const int imgHeight, const int imgWidth, const int fltHeight, const int fltWidth, + const int strideRows, const int strideCols); + ///@} + + /** @name Backward Propagation in Max Pooling Layer + */ + ///@{ + + /** Perform the complete backward propagation step in a Pooling Layer. Based on the + * winning idices stored in the index matrix, it just forwards the actiovation + * gradients to the previous layer. */ + static void MaxPoolLayerBackward(std::vector> &activationGradientsBackward, + const std::vector> &activationGradients, + const std::vector> &indexMatrix, size_t batchSize, size_t depth, + size_t nLocalViews); + + ///@} + + //____________________________________________________________________________ + // + // Reshape Layer Propagation + //____________________________________________________________________________ + /** @name Forward and Backward Propagation in Reshape Layer + */ + ///@{ + + /** Transform the matrix \p B to a matrix with different dimensions \p A */ + static void Reshape(TCudaMatrix &A, const TCudaMatrix &B); + + /** Flattens the tensor \p B, such that each matrix, is stretched in + * one row, resulting with a matrix \p A. */ + static void Flatten(TCudaMatrix &A, const std::vector> &B, size_t size, size_t nRows, + size_t nCols); + + /** Transforms each row of \p B to a matrix and stores it in the tensor \p B. */ + static void Deflatten(std::vector> &A, const TCudaMatrix &B, size_t index, size_t nRows, + size_t nCols); + /** Rearrage data accoring to time fill B x T x D out with T x B x D matrix in*/ + static void Rearrange(std::vector> &out, const std::vector> &in); + + ///@} + //____________________________________________________________________________ // // Additional Arithmetic Functions diff --git a/tmva/tmva/inc/TMVA/DNN/Architectures/Reference.h b/tmva/tmva/inc/TMVA/DNN/Architectures/Reference.h index b72437e40b57e..f6d98aa58fdfa 100644 --- a/tmva/tmva/inc/TMVA/DNN/Architectures/Reference.h +++ b/tmva/tmva/inc/TMVA/DNN/Architectures/Reference.h @@ -20,6 +20,8 @@ #include "TMatrix.h" #include "TMVA/DNN/Architectures/Reference/DataLoader.h" +#include "TMVA/DNN/Architectures/Reference/TensorDataLoader.h" +#include namespace TMVA { @@ -82,6 +84,17 @@ class TReference const TMatrixT & activationGradients, const TMatrixT & weights, const TMatrixT & activationBackward); + /** Backpropagation step for a Recurrent Neural Network */ + static Matrix_t & RecurrentLayerBackward(TMatrixT & state_gradients_backward, // BxH + TMatrixT & input_weight_gradients, + TMatrixT & state_weight_gradients, + TMatrixT & bias_gradients, + TMatrixT & df, //DxH + const TMatrixT & state, // BxH + const TMatrixT & weights_input, // HxD + const TMatrixT & weights_state, // HxH + const TMatrixT & input, // BxD + TMatrixT & input_gradient); /** Adds a the elements in matrix B scaled by c to the elements in * the matrix A. This is required for the weight update in the gradient * descent step.*/ @@ -91,6 +104,15 @@ class TReference static void Copy(TMatrixT & A, const TMatrixT & B); + + /** Above functions extended to vectors */ + static void ScaleAdd(std::vector> & A, + const std::vector> & B, + Scalar_t beta = 1.0); + + static void Copy(std::vector> & A, + const std::vector> & B); + ///@} //____________________________________________________________________________ @@ -250,6 +272,127 @@ class TReference ///@} + + //____________________________________________________________________________ + // + // Convolutional Layer Propagation + //____________________________________________________________________________ + + /** @name Forward Propagation in Convolutional Layer + */ + ///@{ + + /** Transform the matrix \p B in local view format, suitable for + * convolution, and store it in matrix \p A. */ + static void Im2col(TMatrixT &A, TMatrixT &B, size_t imgHeight, size_t imgWidth, size_t fltHeight, + size_t fltWidth, size_t strideRows, size_t strideCols, size_t zeroPaddingHeight, + size_t zeroPaddingWidth); + + /** Rotates the matrix \p B, which is representing a weights, + * and stores them in the matrix \p A. */ + static void RotateWeights(TMatrixT &A, const TMatrixT &B, size_t filterDepth, size_t filterHeight, + size_t filterWidth, size_t numFilters); + + /** Add the biases in the Convolutional Layer. */ + static void AddConvBiases(TMatrixT &output, const TMatrixT &biases); + ///@} + + /** @name Backward Propagation in Convolutional Layer + */ + ///@{ + + /** Perform the complete backward propagation step in a Convolutional Layer. + * If the provided \p activationGradientsBackward matrix is not empty, compute the + * gradients of the objective function with respect to the activations + * of the previous layer (backward direction). + * Also compute the weight and the bias gradients. Modifies the values + * in \p df and thus produces only a valid result, if it is applied the + * first time after the corresponding forward propagation has been per- + * formed. */ + static void ConvLayerBackward(std::vector> &activationGradientsBackward, + TMatrixT &weightGradients, TMatrixT &biasGradients, + std::vector> &df, + const std::vector> &activationGradients, + const TMatrixT &weights, const std::vector> &activationBackward, + size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, + size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth, + size_t nLocalViews); + + /** Utility function for calculating the activation gradients of the layer + * before the convolutional layer. */ + static void CalculateConvActivationGradients(std::vector> &activationGradientsBackward, + std::vector> &df, const TMatrixT &weights, + size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, + size_t height, size_t width, size_t filterDepth, size_t filterHeight, + size_t filterWidth); + + /** Utility function for calculating the weight gradients of the convolutional + * layer. */ + static void CalculateConvWeightGradients(TMatrixT &weightGradients, std::vector> &df, + const std::vector> &activationBackward, size_t batchSize, + size_t inputHeight, size_t inputWidth, size_t depth, size_t height, + size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth, + size_t nLocalViews); + + /** Utility function for calculating the bias gradients of the convolutional + * layer. */ + static void CalculateConvBiasGradients(TMatrixT &biasGradients, std::vector> &df, + size_t batchSize, size_t depth, size_t nLocalViews); + ///@} + + + //____________________________________________________________________________ + // + // Max Pooling Layer Propagation + //____________________________________________________________________________ + /** @name Forward Propagation in Max Pooling Layer + */ + ///@{ + + /** Downsample the matrix \p C to the matrix \p A, using max + * operation, such that the winning indices are stored in matrix + * \p B. */ + static void Downsample(TMatrixT &A, TMatrixT &B, const TMatrixT &C, size_t imgHeight, + size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols); + + ///@} + + /** @name Backward Propagation in Max Pooling Layer + */ + ///@{ + + /** Perform the complete backward propagation step in a Max Pooling Layer. Based on the + * winning idices stored in the index matrix, it just forwards the actiovation + * gradients to the previous layer. */ + static void MaxPoolLayerBackward(std::vector> &activationGradientsBackward, + const std::vector> &activationGradients, + const std::vector> &indexMatrix, size_t batchSize, size_t depth, + size_t nLocalViews); + ///@} + + //____________________________________________________________________________ + // + // Reshape Layer Propagation + //____________________________________________________________________________ + /** @name Forward and Backward Propagation in Reshape Layer + */ + ///@{ + + /** Transform the matrix \p B to a matrix with different dimensions \p A */ + static void Reshape(TMatrixT &A, const TMatrixT &B); + + /** Flattens the tensor \p B, such that each matrix, is stretched in one row, resulting with a matrix \p A. */ + static void Flatten(TMatrixT &A, const std::vector> &B, size_t size, size_t nRows, + size_t nCols); + + /** Transforms each row of \p B to a matrix and stores it in the tensor \p B. */ + static void Deflatten(std::vector> &A, const TMatrixT &B, size_t index, size_t nRows, + size_t nCols); + /** Rearrage data accoring to time fill B x T x D out with T x B x D matrix in*/ + static void Rearrange(std::vector> &out, const std::vector> &in); + + ///@} + //____________________________________________________________________________ // // Additional Arithmetic Functions @@ -259,6 +402,60 @@ class TReference * m elements in \p A. */ static void SumColumns(TMatrixT &B, const TMatrixT &A); + + //____________________________________________________________________________ + // + // AutoEncoder Propagation + //____________________________________________________________________________ + + // Add Biases to the output + static void AddBiases(TMatrixT &A, + const TMatrixT &biases); + + // Updating parameters after every backward pass. Weights and biases are + // updated. + static void + UpdateParams(TMatrixT &x, TMatrixT &tildeX, TMatrixT &y, + TMatrixT &z, TMatrixT &fVBiases, + TMatrixT &fHBiases, TMatrixT &fWeights, + TMatrixT &VBiasError, TMatrixT &HBiasError, + AReal learningRate, size_t fBatchSize); + + // Softmax functions redifined + static void SoftmaxAE(TMatrixT & A); + + + // Corrupt the input values randomly on corruption Level. + //Basically inputs are masked currently. + static void CorruptInput(TMatrixT & input, + TMatrixT & corruptedInput, + AReal corruptionLevel); + + //Encodes the input Values in the compressed form. + static void EncodeInput(TMatrixT &input, + TMatrixT &compressedInput, + TMatrixT &Weights); + + // reconstructs the input. The reconstructed Input has same dimensions as that + // of the input. + static void ReconstructInput(TMatrixT & compressedInput, + TMatrixT & reconstructedInput, + TMatrixT &fWeights); + + + static void ForwardLogReg(TMatrixT &input, + TMatrixT &p, + TMatrixT &fWeights); + + static void UpdateParamsLogReg(TMatrixT &input, + TMatrixT &output, + TMatrixT &difference, + TMatrixT &p, + TMatrixT &fWeights, + TMatrixT &fBiases, + AReal learningRate, + size_t fBatchSize); + }; } // namespace DNN diff --git a/tmva/tmva/inc/TMVA/DNN/Architectures/Reference/TensorDataLoader.h b/tmva/tmva/inc/TMVA/DNN/Architectures/Reference/TensorDataLoader.h new file mode 100644 index 0000000000000..e6e5468649f93 --- /dev/null +++ b/tmva/tmva/inc/TMVA/DNN/Architectures/Reference/TensorDataLoader.h @@ -0,0 +1,149 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : TTensorDataLoader * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Specialization of the Tensor Data Loader Class * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +////////////////////////////////////////////////////////////////////////// +// Partial specialization of the TTensorDataLoader class to adapt // +// it to the TMatrix class. Also the data transfer is kept simple, // +// since this implementation (being intended as reference and fallback) // +// is not optimized for performance. // +////////////////////////////////////////////////////////////////////////// + +#ifndef TMVA_DNN_ARCHITECTURES_REFERENCE_TENSORDATALOADER +#define TMVA_DNN_ARCHITECTURES_REFERENCE_TENSORDATALOADER + +#include "TMVA/DNN/TensorDataLoader.h" +#include + +namespace TMVA { +namespace DNN { + +template +class TReference; + +template +class TTensorDataLoader> { +private: + using BatchIterator_t = TTensorBatchIterator>; + + const AData &fData; ///< The data that should be loaded in the batches. + + size_t fNSamples; ///< The total number of samples in the dataset. + size_t fBatchSize; ///< The size of a batch. + size_t fBatchDepth; ///< The number of matrices in the tensor. + size_t fBatchHeight; ///< The number od rows in each matrix. + size_t fBatchWidth; ///< The number of columns in each matrix. + size_t fNOutputFeatures; ///< The number of outputs from the classifier/regressor. + size_t fBatchIndex; ///< The index of the batch when there are multiple batches in parallel. + + std::vector> inputTensor; ///< The 3D tensor used to keep the input data. + TMatrixT outputMatrix; ///< The matrix used to keep the output. + TMatrixT weightMatrix; ///< The matrix used to keep the batch weights. + + std::vector fSampleIndices; ///< Ordering of the samples in the epoch. + +public: + /*! Constructor. */ + TTensorDataLoader(const AData &data, size_t nSamples, size_t batchSize, size_t batchDepth, size_t batchHeight, + size_t batchWidth, size_t nOutputFeatures, size_t nStreams = 1); + + TTensorDataLoader(const TTensorDataLoader &) = default; + TTensorDataLoader(TTensorDataLoader &&) = default; + TTensorDataLoader &operator=(const TTensorDataLoader &) = default; + TTensorDataLoader &operator=(TTensorDataLoader &&) = default; + + /** Copy input tensor into the given host buffer. Function to be specialized by + * the architecture-specific backend. */ + void CopyTensorInput(std::vector> &tensor, IndexIterator_t sampleIterator); + /** Copy output matrix into the given host buffer. Function to be specialized + * by the architecture-spcific backend. */ + void CopyTensorOutput(TMatrixT &matrix, IndexIterator_t sampleIterator); + /** Copy weight matrix into the given host buffer. Function to be specialized + * by the architecture-spcific backend. */ + void CopyTensorWeights(TMatrixT &matrix, IndexIterator_t sampleIterator); + + BatchIterator_t begin() { return BatchIterator_t(*this); } + BatchIterator_t end() { return BatchIterator_t(*this, fNSamples / fBatchSize); } + + /** Shuffle the order of the samples in the batch. The shuffling is indirect, + * i.e. only the indices are shuffled. No input data is moved by this + * routine. */ + void Shuffle(); + + /** Return the next batch from the training set. The TTensorDataLoader object + * keeps an internal counter that cycles over the batches in the training + * set. */ + TTensorBatch> GetTensorBatch(); +}; + +// +// TTensorDataLoader Class. +//______________________________________________________________________________ +template +TTensorDataLoader>::TTensorDataLoader(const AData &data, size_t nSamples, size_t batchSize, + size_t batchDepth, size_t batchHeight, size_t batchWidth, + size_t nOutputFeatures, size_t /* nStreams */) + : fData(data), fNSamples(nSamples), fBatchSize(batchSize), fBatchDepth(batchDepth), fBatchHeight(batchHeight), + fBatchWidth(batchWidth), fNOutputFeatures(nOutputFeatures), fBatchIndex(0), inputTensor(), + outputMatrix(batchSize, nOutputFeatures), weightMatrix(batchSize, 1), fSampleIndices() +{ + + inputTensor.reserve(batchDepth); + for (size_t i = 0; i < batchDepth; i++) { + inputTensor.emplace_back(batchHeight, batchWidth); + } + + fSampleIndices.reserve(fNSamples); + for (size_t i = 0; i < fNSamples; i++) { + fSampleIndices.push_back(i); + } +} + +template +void TTensorDataLoader>::Shuffle() +{ + std::random_shuffle(fSampleIndices.begin(), fSampleIndices.end()); +} + +template +auto TTensorDataLoader>::GetTensorBatch() -> TTensorBatch> +{ + fBatchIndex %= (fNSamples / fBatchSize); // Cycle through samples. + + size_t sampleIndex = fBatchIndex * fBatchSize; + IndexIterator_t sampleIndexIterator = fSampleIndices.begin() + sampleIndex; + + CopyTensorInput(inputTensor, sampleIndexIterator); + CopyTensorOutput(outputMatrix, sampleIndexIterator); + CopyTensorWeights(weightMatrix, sampleIndexIterator); + + fBatchIndex++; + return TTensorBatch>(inputTensor, outputMatrix, weightMatrix); +} + +} // namespace DNN +} // namespace TMVA + +#endif \ No newline at end of file diff --git a/tmva/tmva/inc/TMVA/DNN/CNN/ConvLayer.h b/tmva/tmva/inc/TMVA/DNN/CNN/ConvLayer.h new file mode 100644 index 0000000000000..6c98e08697f1c --- /dev/null +++ b/tmva/tmva/inc/TMVA/DNN/CNN/ConvLayer.h @@ -0,0 +1,265 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : TConvLayer * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Convolutional Deep Neural Network Layer * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_CNN_CONVLAYER +#define TMVA_CNN_CONVLAYER + +#include "TMatrix.h" + +#include "TMVA/DNN/GeneralLayer.h" +#include "TMVA/DNN/Functions.h" + +#include +#include + +namespace TMVA { +namespace DNN { +namespace CNN { + +template +class TConvLayer : public VGeneralLayer { +public: + using Matrix_t = typename Architecture_t::Matrix_t; + using Scalar_t = typename Architecture_t::Scalar_t; + +private: + size_t fFilterDepth; ///< The depth of the filter. + size_t fFilterHeight; ///< The height of the filter. + size_t fFilterWidth; ///< The width of the filter. + + size_t fStrideRows; ///< The number of row pixels to slid the filter each step. + size_t fStrideCols; ///< The number of column pixels to slid the filter each step. + + size_t fPaddingHeight; ///< The number of zero layers added top and bottom of the input. + size_t fPaddingWidth; ///< The number of zero layers left and right of the input. + + size_t fNLocalViewPixels; ///< The number of pixels in one local image view. + size_t fNLocalViews; ///< The number of local views in one image. + + Scalar_t fDropoutProbability; ///< Probability that an input is active. + + std::vector fDerivatives; ///< First fDerivatives of the activations of this layer. + + EActivationFunction fF; ///< Activation function of the layer. + ERegularization fReg; ///< The regularization method. + Scalar_t fWeightDecay; ///< The weight decay. + +public: + /*! Constructor. */ + TConvLayer(size_t BatchSize, size_t InputDepth, size_t InputHeight, size_t InputWidth, size_t Depth, size_t Height, + size_t Width, size_t WeightsNRows, size_t WeightsNCols, size_t BiasesNRows, size_t BiasesNCols, + size_t OutputNSlices, size_t OutputNRows, size_t OutputNCols, EInitialization Init, size_t FilterDepth, + size_t FilterHeight, size_t FilterWidth, size_t StrideRows, size_t StrideCols, size_t PaddingHeight, + size_t PaddingWidth, Scalar_t DropoutProbability, EActivationFunction f, ERegularization Reg, + Scalar_t WeightDecay); + + /*! Copy the conv layer provided as a pointer */ + TConvLayer(TConvLayer *layer); + + /*! Copy constructor. */ + TConvLayer(const TConvLayer &); + + /*! Destructor. */ + ~TConvLayer(); + + /*! Computes activation of the layer for the given input. The input + * must be in 3D tensor form with the different matrices corresponding to + * different events in the batch. Computes activations as well as + * the first partial derivative of the activation function at those + * activations. */ + void Forward(std::vector &input, bool applyDropout = false); + + /*! Compute weight, bias and activation gradients. Uses the precomputed + * first partial derviatives of the activation function computed during + * forward propagation and modifies them. Must only be called directly + * at the corresponding call to Forward(...). */ + void Backward(std::vector &gradients_backward, const std::vector &activations_backward, + std::vector &inp1, std::vector &inp2); + + /*! Prints the info about the layer. */ + void Print() const; + + /*! Getters */ + size_t GetFilterDepth() const { return fFilterDepth; } + size_t GetFilterHeight() const { return fFilterHeight; } + size_t GetFilterWidth() const { return fFilterWidth; } + + size_t GetStrideRows() const { return fStrideRows; } + size_t GetStrideCols() const { return fStrideCols; } + + size_t GetPaddingHeight() const { return fPaddingHeight; } + size_t GetPaddingWidth() const { return fPaddingWidth; } + + size_t GetNLocalViewPixels() const { return fNLocalViewPixels; } + size_t GetNLocalViews() const { return fNLocalViews; } + + Scalar_t GetDropoutProbability() const { return fDropoutProbability; } + + const std::vector &GetDerivatives() const { return fDerivatives; } + std::vector &GetDerivatives() { return fDerivatives; } + + Matrix_t &GetDerivativesAt(size_t i) { return fDerivatives[i]; } + const Matrix_t &GetDerivativesAt(size_t i) const { return fDerivatives[i]; } + + EActivationFunction GetActivationFunction() const { return fF; } + ERegularization GetRegularization() const { return fReg; } + Scalar_t GetWeightDecay() const { return fWeightDecay; } +}; + +// +// +// Conv Layer Class - Implementation +//______________________________________________________________________________ +template +TConvLayer::TConvLayer(size_t batchSize, size_t inputDepth, size_t inputHeight, size_t inputWidth, + size_t depth, size_t height, size_t width, size_t weightsNRows, + size_t weightsNCols, size_t biasesNRows, size_t biasesNCols, + size_t outputNSlices, size_t outputNRows, size_t outputNCols, + EInitialization init, size_t filterDepth, size_t filterHeight, + size_t filterWidth, size_t strideRows, size_t strideCols, size_t paddingHeight, + size_t paddingWidth, Scalar_t dropoutProbability, EActivationFunction f, + ERegularization reg, Scalar_t weightDecay) + : VGeneralLayer(batchSize, inputDepth, inputHeight, inputWidth, depth, height, width, 1, + weightsNRows, weightsNCols, 1, biasesNRows, biasesNCols, outputNSlices, outputNRows, + outputNCols, init), + fFilterDepth(filterDepth), fFilterHeight(filterHeight), fFilterWidth(filterWidth), fStrideRows(strideRows), + fStrideCols(strideCols), fPaddingHeight(paddingHeight), fPaddingWidth(paddingWidth), + fNLocalViewPixels(filterDepth * filterHeight * filterWidth), fNLocalViews(height * width), + fDropoutProbability(dropoutProbability), fDerivatives(), fF(f), fReg(reg), fWeightDecay(weightDecay) +{ + for (size_t i = 0; i < outputNSlices; i++) { + fDerivatives.emplace_back(outputNRows, outputNCols); + } +} + +//______________________________________________________________________________ +template +TConvLayer::TConvLayer(TConvLayer *layer) + : VGeneralLayer(layer), fFilterDepth(layer->GetFilterDepth()), + fFilterHeight(layer->GetFilterHeight()), fFilterWidth(layer->GetFilterWidth()), + fStrideRows(layer->GetStrideRows()), fStrideCols(layer->GetStrideCols()), + fPaddingHeight(layer->GetPaddingHeight()), fPaddingWidth(layer->GetPaddingWidth()), + fNLocalViewPixels(layer->GetNLocalViewPixels()), fNLocalViews(layer->GetNLocalViews()), + fDropoutProbability(layer->GetDropoutProbability()), fF(layer->GetActivationFunction()), + fReg(layer->GetRegularization()), fWeightDecay(layer->GetWeightDecay()) +{ + size_t outputNSlices = (layer->GetDerivatives()).size(); + size_t outputNRows = 0; + size_t outputNCols = 0; + + for (size_t i = 0; i < outputNSlices; i++) { + outputNRows = (layer->GetDerivativesAt(i)).GetNrows(); + outputNCols = (layer->GetDerivativesAt(i)).GetNcols(); + + fDerivatives.emplace_back(outputNRows, outputNCols); + } +} + +//______________________________________________________________________________ +template +TConvLayer::TConvLayer(const TConvLayer &convLayer) + : VGeneralLayer(convLayer), fFilterDepth(convLayer.fFilterDepth), + fFilterHeight(convLayer.fFilterHeight), fFilterWidth(convLayer.fFilterWidth), fStrideRows(convLayer.fStrideRows), + fStrideCols(convLayer.fStrideCols), fPaddingHeight(convLayer.fPaddingHeight), + fPaddingWidth(convLayer.fPaddingWidth), fNLocalViewPixels(convLayer.fNLocalViewPixels), + fNLocalViews(convLayer.fNLocalViews), fDropoutProbability(convLayer.fDropoutProbability), fF(convLayer.fF), + fReg(convLayer.fReg), fWeightDecay(convLayer.fWeightDecay) +{ + size_t outputNSlices = convLayer.fDerivatives.size(); + size_t outputNRows = convLayer.GetDerivativesAt(0).GetNrows(); + size_t outputNCols = convLayer.GetDerivativesAt(0).GetNcols(); + + for (size_t i = 0; i < outputNSlices; i++) { + fDerivatives.emplace_back(outputNRows, outputNCols); + } +} + +//______________________________________________________________________________ +template +TConvLayer::~TConvLayer() +{ +} + +//______________________________________________________________________________ +template +auto TConvLayer::Forward(std::vector &input, bool applyDropout) -> void +{ + for (size_t i = 0; i < this->GetBatchSize(); i++) { + + if (applyDropout && (this->GetDropoutProbability() != 1.0)) { + Architecture_t::Dropout(input[i], this->GetDropoutProbability()); + } + + Matrix_t inputTr(this->GetNLocalViews(), this->GetNLocalViewPixels()); + Architecture_t::Im2col(inputTr, input[i], this->GetInputHeight(), this->GetInputWidth(), this->GetFilterHeight(), + this->GetFilterWidth(), this->GetStrideRows(), this->GetStrideCols(), + this->GetPaddingHeight(), this->GetPaddingWidth()); + + Architecture_t::MultiplyTranspose(this->GetOutputAt(i), this->GetWeightsAt(0), inputTr); + Architecture_t::AddConvBiases(this->GetOutputAt(i), this->GetBiasesAt(0)); + + evaluateDerivative(this->GetDerivativesAt(i), fF, this->GetOutputAt(i)); + evaluate(this->GetOutputAt(i), fF); + } +} + +//______________________________________________________________________________ +template +auto TConvLayer::Backward(std::vector &gradients_backward, + const std::vector &activations_backward, + std::vector & /*inp1*/, std::vector & + /*inp2*/) -> void +{ + Architecture_t::ConvLayerBackward( + gradients_backward, this->GetWeightGradientsAt(0), this->GetBiasGradientsAt(0), this->GetDerivatives(), + this->GetActivationGradients(), this->GetWeightsAt(0), activations_backward, this->GetBatchSize(), + this->GetInputHeight(), this->GetInputWidth(), this->GetDepth(), this->GetHeight(), this->GetWidth(), + this->GetFilterDepth(), this->GetFilterHeight(), this->GetFilterWidth(), this->GetNLocalViews()); + + addRegularizationGradients(this->GetWeightGradientsAt(0), this->GetWeightsAt(0), + this->GetWeightDecay(), this->GetRegularization()); +} + +//______________________________________________________________________________ +template +auto TConvLayer::Print() const -> void +{ + std::cout << "\t\t CONV LAYER: " << std::endl; + std::cout << "\t\t\t Width = " << this->GetWidth() << std::endl; + std::cout << "\t\t\t Height = " << this->GetHeight() << std::endl; + std::cout << "\t\t\t Depth = " << this->GetDepth() << std::endl; + + std::cout << "\t\t\t Filter Width = " << this->GetFilterWidth() << std::endl; + std::cout << "\t\t\t Filter Height = " << this->GetFilterHeight() << std::endl; + std::cout << "\t\t\t Local Views = " << this->GetNLocalViews() << std::endl; + std::cout << "\t\t\t Activation Function = " << static_cast(fF) << std::endl; +} + +} // namespace CNN +} // namespace DNN +} // namespace TMVA + +#endif diff --git a/tmva/tmva/inc/TMVA/DNN/CNN/MaxPoolLayer.h b/tmva/tmva/inc/TMVA/DNN/CNN/MaxPoolLayer.h new file mode 100644 index 0000000000000..609f29f8776a5 --- /dev/null +++ b/tmva/tmva/inc/TMVA/DNN/CNN/MaxPoolLayer.h @@ -0,0 +1,213 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : TMaxPoolLayer * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Max Pool Deep Neural Network Layer * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef MAXPOOLLAYER_H_ +#define MAXPOOLLAYER_H_ + +#include "TMatrix.h" + +#include "TMVA/DNN/GeneralLayer.h" +#include "TMVA/DNN/Functions.h" + +#include + +namespace TMVA { +namespace DNN { +namespace CNN { + +/** \class TMaxPoolLayer + + Generic Max Pooling Layer class. + + This generic Max Pooling Layer Class represents a pooling layer of + a CNN. It inherits all of the properties of the generic virtual base class + VGeneralLayer. In addition to that, it contains a matrix of winning units. + + The height and width of the weights and biases is set to 0, since this + layer does not contain any weights. + + */ +template +class TMaxPoolLayer : public VGeneralLayer { +public: + using Matrix_t = typename Architecture_t::Matrix_t; + using Scalar_t = typename Architecture_t::Scalar_t; + +private: + std::vector indexMatrix; ///< Matrix of indices for the backward pass. + + size_t fFrameHeight; ///< The height of the frame. + size_t fFrameWidth; ///< The width of the frame. + + size_t fStrideRows; ///< The number of row pixels to slid the filter each step. + size_t fStrideCols; ///< The number of column pixels to slid the filter each step. + + size_t fNLocalViewPixels; ///< The number of pixels in one local image view. + size_t fNLocalViews; ///< The number of local views in one image. + + Scalar_t fDropoutProbability; ///< Probability that an input is active. + +public: + /*! Constructor. */ + TMaxPoolLayer(size_t BatchSize, size_t InputDepth, size_t InputHeight, size_t InputWidth, size_t Height, + size_t Width, size_t OutputNSlices, size_t OutputNRows, size_t OutputNCols, size_t FrameHeight, + size_t FrameWidth, size_t StrideRows, size_t StrideCols, Scalar_t DropoutProbability); + + /*! Copy the max pooling layer provided as a pointer */ + TMaxPoolLayer(TMaxPoolLayer *layer); + + /*! Copy constructor. */ + TMaxPoolLayer(const TMaxPoolLayer &); + + /*! Destructor. */ + ~TMaxPoolLayer(); + + /*! Computes activation of the layer for the given input. The input + * must be in 3D tensor form with the different matrices corresponding to + * different events in the batch. It spatially downsamples the input + * matrices. */ + void Forward(std::vector &input, bool applyDropout = false); + + /*! Depending on the winning units determined during the Forward pass, + * it only forwards the derivatives to the right units in the previous + * layer. Must only be called directly at the corresponding call + * to Forward(...). */ + void Backward(std::vector &gradients_backward, const std::vector &activations_backward, + std::vector &inp1, std::vector &inp2); + + /*! Prints the info about the layer. */ + void Print() const; + + /*! Getters */ + const std::vector &GetIndexMatrix() const { return indexMatrix; } + std::vector &GetIndexMatrix() { return indexMatrix; } + + size_t GetFrameHeight() const { return fFrameHeight; } + size_t GetFrameWidth() const { return fFrameWidth; } + + size_t GetStrideRows() const { return fStrideRows; } + size_t GetStrideCols() const { return fStrideCols; } + + size_t GetNLocalViewPixels() const { return fNLocalViewPixels; } + size_t GetNLocalViews() const { return fNLocalViews; } + + Scalar_t GetDropoutProbability() const { return fDropoutProbability; } +}; + +//______________________________________________________________________________ +template +TMaxPoolLayer::TMaxPoolLayer(size_t batchSize, size_t inputDepth, size_t inputHeight, size_t inputWidth, + size_t height, size_t width, size_t outputNSlices, size_t outputNRows, + size_t outputNCols, size_t frameHeight, size_t frameWidth, + size_t strideRows, size_t strideCols, Scalar_t dropoutProbability) + : VGeneralLayer(batchSize, inputDepth, inputHeight, inputWidth, inputDepth, height, width, 0, 0, 0, + 0, 0, 0, outputNSlices, outputNRows, outputNCols, EInitialization::kZero), + indexMatrix(), fFrameHeight(frameHeight), fFrameWidth(frameWidth), fStrideRows(strideRows), + fStrideCols(strideCols), fNLocalViewPixels(inputDepth * frameHeight * frameWidth), fNLocalViews(height * width), + fDropoutProbability(dropoutProbability) +{ + for (size_t i = 0; i < this->GetBatchSize(); i++) { + indexMatrix.emplace_back(this->GetDepth(), this->GetNLocalViews()); + } +} + +//______________________________________________________________________________ +template +TMaxPoolLayer::TMaxPoolLayer(TMaxPoolLayer *layer) + : VGeneralLayer(layer), indexMatrix(), fFrameHeight(layer->GetFrameHeight()), + fFrameWidth(layer->GetFrameWidth()), fStrideRows(layer->GetStrideRows()), fStrideCols(layer->GetStrideCols()), + fNLocalViewPixels(layer->GetNLocalViewPixels()), fNLocalViews(layer->GetNLocalViews()), + fDropoutProbability(layer->GetDropoutProbability()) +{ + for (size_t i = 0; i < layer->GetBatchSize(); i++) { + indexMatrix.emplace_back(layer->GetDepth(), layer->GetNLocalViews()); + } +} + +//______________________________________________________________________________ +template +TMaxPoolLayer::TMaxPoolLayer(const TMaxPoolLayer &layer) + : VGeneralLayer(layer), indexMatrix(), fFrameHeight(layer.fFrameHeight), + fFrameWidth(layer.fFrameWidth), fStrideRows(layer.fStrideRows), fStrideCols(layer.fStrideCols), + fNLocalViewPixels(layer.fNLocalViewPixels), fNLocalViews(layer.fNLocalViews), + fDropoutProbability(layer.fDropoutProbability) +{ + for (size_t i = 0; i < layer.fBatchSize; i++) { + indexMatrix.emplace_back(layer.fDepth, layer.fNLocalViews); + } +} + +//______________________________________________________________________________ +template +TMaxPoolLayer::~TMaxPoolLayer() +{ +} + +//______________________________________________________________________________ +template +auto TMaxPoolLayer::Forward(std::vector &input, bool applyDropout) -> void +{ + for (size_t i = 0; i < this->GetBatchSize(); i++) { + + if (applyDropout && (this->GetDropoutProbability() != 1.0)) { + Architecture_t::Dropout(input[i], this->GetDropoutProbability()); + } + + Architecture_t::Downsample(this->GetOutputAt(i), indexMatrix[i], input[i], this->GetInputHeight(), + this->GetInputWidth(), this->GetFrameHeight(), this->GetFrameWidth(), + this->GetStrideRows(), this->GetStrideCols()); + } +} + +//______________________________________________________________________________ +template +auto TMaxPoolLayer::Backward(std::vector &gradients_backward, + const std::vector & /*activations_backward*/, + std::vector & /*inp1*/, std::vector & + /*inp2*/) -> void +{ + Architecture_t::MaxPoolLayerBackward(gradients_backward, this->GetActivationGradients(), indexMatrix, + this->GetBatchSize(), this->GetDepth(), this->GetNLocalViews()); +} + +//______________________________________________________________________________ +template +auto TMaxPoolLayer::Print() const -> void +{ + std::cout << "\t\t POOL LAYER: " << std::endl; + std::cout << "\t\t\t Width = " << this->GetWidth() << std::endl; + std::cout << "\t\t\t Height = " << this->GetHeight() << std::endl; + std::cout << "\t\t\t Depth = " << this->GetDepth() << std::endl; + + std::cout << "\t\t\t Frame Width = " << this->GetFrameWidth() << std::endl; + std::cout << "\t\t\t Frame Height = " << this->GetFrameHeight() << std::endl; +} + +} // namespace CNN +} // namespace DNN +} // namespace TMVA + +#endif diff --git a/tmva/tmva/inc/TMVA/DNN/DAE/CompressionLayer.h b/tmva/tmva/inc/TMVA/DNN/DAE/CompressionLayer.h new file mode 100644 index 0000000000000..dea7bc919b65e --- /dev/null +++ b/tmva/tmva/inc/TMVA/DNN/DAE/CompressionLayer.h @@ -0,0 +1,184 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Akshay Vashistha + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : TCompressionLayer * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Compressed Layer for DeepAutoEncoders * + * * + * Authors (alphabetical): * + * Akshay Vashistha - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + + +#ifndef TMVA_DAE_COMPRESSION_LAYER +#define TMVA_DAE_COMPRESSION_LAYER + +#include "TMatrix.h" + +#include "TMVA/DNN/GeneralLayer.h" +#include "TMVA/DNN/Functions.h" + +#include +#include + +namespace TMVA { +namespace DNN { +namespace DAE { + + /** \class TCompressionLayer + * Used to Compress the input values. + * Here input is stored into compressed number of hidden units. + * These are basically the features after every layer. + */ + +template +class TCompressionLayer : public VGeneralLayer { +public: + using Matrix_t = typename Architecture_t::Matrix_t; + using Scalar_t = typename Architecture_t::Scalar_t; + + + size_t fVisibleUnits; ///< total number of visible units + + size_t fHiddenUnits; ///< tital number of hidden units. + + Scalar_t fDropoutProbability; ///< Probability that an input is active. + + size_t fType; ///< Type of layer + + EActivationFunction fF; ///< Activation function of the layer. + + /*! Constructor. */ + TCompressionLayer(size_t BatchSize, size_t VisibleUnits, size_t HiddenUnits, + Scalar_t DropoutProbability, EActivationFunction f, + std::vector Weights, std::vector Biases); + + /*! Copy the compression layer provided as a pointer */ + TCompressionLayer(TCompressionLayer *layer); + + /*! Copy constructor. */ + TCompressionLayer(const TCompressionLayer &); + + /* Destructor */ + ~TCompressionLayer(); + + /* This forward pass compresses the input. Updated weights and biases are used from previous layers. */ + void Forward(std::vector &input, bool applyDropout = false); + + /* Not required in this layer */ + void Backward(std::vector &gradients_backward, + const std::vector &activations_backward, + std::vector &inp1, + std::vector &inp2); + + void Print() const; + + size_t GetVisibleUnits() const { return fVisibleUnits; } + size_t GetHiddenUnits() const {return fHiddenUnits;} + size_t GetType() const {return fType;} + Scalar_t GetDropoutProbability() const { return fDropoutProbability; } + EActivationFunction GetActivationFunction() const { return fF; } + +}; + +//______________________________________________________________________________ +template +TCompressionLayer::TCompressionLayer(size_t batchSize, size_t visibleUnits, size_t hiddenUnits, + Scalar_t dropoutProbability, EActivationFunction f, + std::vector weights, std::vector biases) + : VGeneralLayer(batchSize, 1, 1, 0, 0, 0, 0, 1, {hiddenUnits}, {visibleUnits}, 2, + {hiddenUnits, visibleUnits}, {1, 1}, batchSize, hiddenUnits, 1, + EInitialization::kZero), + fVisibleUnits(visibleUnits), fHiddenUnits(hiddenUnits), fDropoutProbability(dropoutProbability), fType(2), fF(f) + +{ + Architecture_t::Copy(this->GetWeightsAt(0),weights[0]); + Architecture_t::Copy(this->GetBiasesAt(0),biases[0]); + } +//______________________________________________________________________________ + template + TCompressionLayer::TCompressionLayer(TCompressionLayer *layer) + : VGeneralLayer(layer), fVisibleUnits(layer->GetVisibleUnits()), + fHiddenUnits(layer->GetHiddenUnits()), fDropoutProbability(layer->GetDropoutProbability()), fType(2), + fF(layer->GetActivationFunction()) + { + Architecture_t::Copy(this->GetWeightsAt(0), layer->weights[0]); + Architecture_t::Copy(this->GetBiasesAt(0), layer->biases[0]); + // Output Tensor will be created in General Layer +} +//______________________________________________________________________________ +template +TCompressionLayer::TCompressionLayer(const TCompressionLayer &compress) + : VGeneralLayer(compress), fVisibleUnits(compress.fVisibleUnits), + fHiddenUnits(compress.fHiddenUnits), fDropoutProbability(compress.fDropoutProbability), fType(2), fF(compress.fF) + +{ + Architecture_t::Copy(this->GetWeightsAt(0), compress.weights[0]); + Architecture_t::Copy(this->GetBiasesAt(0), compress.biases[0]); + // Output Tensor will be created in General Layer +} +//______________________________________________________________________________ + +template TCompressionLayer::~TCompressionLayer() {} + +//______________________________________________________________________________ +template +auto TCompressionLayer::Forward(std::vector &input, bool /*applyDropout*/) -> void +{ + + for (size_t i = 0; i < this->GetBatchSize(); i++) { + + Architecture_t::EncodeInput(input[i], this->GetOutputAt(i), this->GetWeightsAt(0)); + Architecture_t::AddBiases(this->GetOutputAt(i), this->GetBiasesAt(0)); + evaluate(this->GetOutputAt(i), fF); + } + +} +//______________________________________________________________________________ +template +auto inline TCompressionLayer::Backward(std::vector & /*gradients_backward*/, + const std::vector & /*activations_backward*/, + std::vector & /*inp1*/, std::vector & + /*inp2*/) -> void + +{ +} +//______________________________________________________________________________ +template +auto TCompressionLayer::Print() const +-> void +{ + std::cout << "Batch Size: " << this->GetBatchSize() << "\n" + << "Input Units: " << this->GetVisibleUnits() << "\n" + << "Hidden units " << this->GetHiddenUnits() << "\n"; + + std::cout<<"Compressed Input: "<GetWeightsAt(0).GetNrows(); j++) { + for (Int_t k = 0; k < this->GetWeightsAt(0).GetNcols(); k++) { + std::cout<GetWeightsAt(0)(j,k)<<"\t"; + } + std::cout< - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + + +#ifndef TMVA_DAE_CORRUPTION_LAYER +#define TMVA_DAE_CORRUPTION_LAYER + +#include "TMatrix.h" + +#include "TMVA/DNN/GeneralLayer.h" +#include "TMVA/DNN/Functions.h" + +#include +#include + +namespace TMVA { +namespace DNN { +namespace DAE { + + /** \class TCorruptionLayer + * Used to corrupt the input values according to defined Corruption level. + * Here we mask the values of input according to corruption level. + */ + +template +class TCorruptionLayer : public VGeneralLayer { +public: + using Matrix_t = typename Architecture_t::Matrix_t; + using Scalar_t = typename Architecture_t::Scalar_t; + + + size_t fVisibleUnits; ///< total number of visible units + + size_t fHiddenUnits; ///< total number of hidden units in a denoise layer + + Scalar_t fDropoutProbability; ///< Probability that an input is active. + + size_t fType; ///< Type of layer + + Scalar_t fCorruptionLevel; /// *layer); + + /*! Copy constructor. */ + TCorruptionLayer(const TCorruptionLayer &); + + /* Destructor */ + ~TCorruptionLayer(); + + /* The corruption function. All the inputs are corrupted in one forward pass */ + void Forward(std::vector &input, bool applyDropout = false); + + /* Not required for this layer */ + void Backward(std::vector &gradients_backward, + const std::vector &activations_backward, + std::vector &inp1, + std::vector &inp2); + + void Print() const; + + /* Getters */ + size_t GetVisibleUnits() const { return fVisibleUnits; } + size_t GetHiddenUnits() const {return fHiddenUnits;} + size_t GetType() const {return fType;} + Scalar_t GetDropoutProbability() const { return fDropoutProbability; } + Scalar_t GetCorruptionLevel() const {return fCorruptionLevel;} + +}; + +//______________________________________________________________________________ +template +TCorruptionLayer::TCorruptionLayer(size_t batchSize, size_t visibleUnits, size_t hiddenUnits, + Scalar_t dropoutProbability, Scalar_t corruptionLevel) + : VGeneralLayer(batchSize, 1, 1, 0, 0, 0, 0, 1, {hiddenUnits}, {visibleUnits}, 2, + {hiddenUnits, visibleUnits}, {1, 1}, batchSize, visibleUnits, 1, + EInitialization::kUniform), + fVisibleUnits(visibleUnits), fHiddenUnits(hiddenUnits), fDropoutProbability(dropoutProbability), fType(1), + fCorruptionLevel(corruptionLevel) + +{ +} +//______________________________________________________________________________ +template +TCorruptionLayer::TCorruptionLayer(TCorruptionLayer *layer) + : VGeneralLayer(layer), fVisibleUnits(layer->GetVisibleUnits()), + fHiddenUnits(layer->GetHiddenUnits()), fDropoutProbability(layer->GetDropoutProbability()), fType(1), + fCorruptionLevel(layer->GetCorruptionLevel()) +{ + // Output Tensor will be created in General Layer +} +//______________________________________________________________________________ +template +TCorruptionLayer::TCorruptionLayer(const TCorruptionLayer &corrupt) + : VGeneralLayer(corrupt), + fVisibleUnits(corrupt.fVisibleUnits),fType(1), + fHiddenUnits(corrupt.fHiddenUnits), + fDropoutProbability(corrupt.fDropoutProbability), + fCorruptionLevel(corrupt.fCorruptionLevel) + + +{ + // Output Tensor will be created in General Layer + +} +//______________________________________________________________________________ + +template TCorruptionLayer::~TCorruptionLayer() {} + +//______________________________________________________________________________ +template +auto TCorruptionLayer::Forward(std::vector &input, bool applyDropout) +-> void +{ + for (size_t i = 0; i < this->GetBatchSize(); i++) + { + if (applyDropout && (this->GetDropoutProbability() != 1.0)) + { + Architecture_t::Dropout(input[i], this->GetDropoutProbability()); + } + Architecture_t::CorruptInput(input[i], this->GetOutputAt(i), this->GetCorruptionLevel()); + } +} +//______________________________________________________________________________ +template +auto inline TCorruptionLayer::Backward(std::vector & /*gradients_backward*/, + const std::vector & /*activations_backward*/, + std::vector & /*inp1*/, std::vector & + /*inp2*/) -> void +{ +} + +//______________________________________________________________________________ +template +auto TCorruptionLayer::Print() const +-> void +{ + std::cout << "Batch Size: " << this->GetBatchSize() << "\n" + << "Input Units: " << this->GetVisibleUnits() << "\n" + << "Hidden units: "<< this->GetHiddenUnits() <<"\n"; + std::cout<<"Corrupted Output: "<GetBatchSize(); i++) + { + for (Int_t j = 0; j < this->GetOutputAt(i).GetNrows(); j++) { + for (Int_t k = 0; k < this->GetOutputAt(i).GetNcols(); k++) { + std::cout<GetOutputAt(i)(j,k)<<"\t"; + } + std::cout< - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + + + #ifndef TMVA_DAE_LOGISTIC_REGRESSION_LAYER + #define TMVA_DAE_LOGISTIC_REGRESSION_LAYER + + #include "TMatrix.h" + + #include "TMVA/DNN/GeneralLayer.h" + #include "TMVA/DNN/Functions.h" + + #include + #include + + namespace TMVA { + namespace DNN { + namespace DAE { + + /** \class TLogisticRegressionLayer + LogisticRegression Layer is used in the finetune step of training for AutoEncoders. + This is the supervised learning step to classify output. + */ + +template +class TLogisticRegressionLayer : public VGeneralLayer { +public: + using Matrix_t = typename Architecture_t::Matrix_t; + using Scalar_t = typename Architecture_t::Scalar_t; + + size_t fInputUnits; ///< Number of input units + + size_t fOutputUnits; ///< Number of output Units + + size_t fTestDataBatchSize; ///< Number of testing units in testing btch + + Scalar_t fLearningRate; ///< Learning Rate + + + /* constructor */ + TLogisticRegressionLayer(size_t BatchSize, size_t InputUnits, size_t OutputUnits, size_t TestDataBatchSize, + Scalar_t LearningRate); + + /*! Copy the denoise layer provided as a pointer */ + TLogisticRegressionLayer(TLogisticRegressionLayer *layer); + + /* copy constructor */ + TLogisticRegressionLayer(const TLogisticRegressionLayer &); + + // This is basically the prediction step. Can be used in DeepNet to Predict output. + void Forward(std::vector &input, bool applyDropout = false); + + void Backward(std::vector &gradients_backward, + const std::vector &activations_backward, + std::vector &inp1, + std::vector &inp2); + + /* Getters */ + size_t GetInputUnits() const {return fInputUnits;} + size_t GetOutputUnits() const {return fOutputUnits;} + size_t GetTestDataBatchSize() const {return fTestDataBatchSize;} + Scalar_t GetLearningRate() const {return fLearningRate;} + + + /* Train the Logistic Regression Layer */ + + /* Predict output of Logistic Regression Layer, should be used as a + successive call after TrainLogReg() */ + //std::vector PredictLogReg(std::vector &input); + + void Print() const; + +}; +//______________________________________________________________________________ + +template +TLogisticRegressionLayer::TLogisticRegressionLayer(size_t batchSize, size_t inputUnits, + size_t outputUnits, size_t testDataBatchSize, + Scalar_t learningRate) + : VGeneralLayer(batchSize, 1, 1, 0, 0, 0, 0, 1, {outputUnits}, {inputUnits}, 1, {outputUnits}, + {1}, testDataBatchSize, outputUnits, 1, EInitialization::kUniform), + fInputUnits(inputUnits), fOutputUnits(outputUnits), + fTestDataBatchSize(testDataBatchSize), fLearningRate(learningRate) + +{ + // Output Tensor will be created in General Layer +} + + + +//______________________________________________________________________________ +template +TLogisticRegressionLayer::TLogisticRegressionLayer(TLogisticRegressionLayer *layer) + : VGeneralLayer(layer), + fInputUnits(layer->GetInputUnits()), fOutputUnits(layer->GetOutputUnits()), + fTestDataBatchSize(layer->GetTestDataBatchSize()), fLearningRate(layer->GetLearningRate()) +{ + // Output Tensor will be created in General Layer +} + +//______________________________________________________________________________ +template +TLogisticRegressionLayer::TLogisticRegressionLayer(const TLogisticRegressionLayer &logistic) + : VGeneralLayer(logistic), + fInputUnits(logistic.fInputUnits), fOutputUnits(logistic.fOutputUnits), + fTestDataBatchSize(logistic.fTestDataBatchSize), fLearningRate(logistic.fLearningRate) + +{ + // Output Tensor will be created in General Layer + +} +//______________________________________________________________________________ +//______________________________________________________________________________ +template +auto inline TLogisticRegressionLayer::Backward(std::vector &inputLabel, + const std::vector & /*inp1*/, + std::vector &input, std::vector & + /*inp2*/) -> void +{ + for(size_t i=0; iGetBatchSize(); i++) + { + Matrix_t p(this->GetOutputUnits(), 1); + Matrix_t difference(this->GetOutputUnits(), 1); + for(size_t j=0; j<(size_t)p.GetNrows(); j++) + { + for(size_t k=0; k<(size_t)p.GetNcols(); k++) + { + p(j,k)=0; + difference(j,k)=0; + } + } + Architecture_t::ForwardLogReg(input[i], p, this->GetWeightsAt(0)); + Architecture_t::AddBiases(p, this->GetBiasesAt(0)); + Architecture_t::SoftmaxAE(p); + Architecture_t::UpdateParamsLogReg(input[i], inputLabel[i], difference, p, + this->GetWeightsAt(0), this->GetBiasesAt(0), + this->GetLearningRate(), this->GetBatchSize()); + + } + +} + +//______________________________________________________________________________ +template +auto TLogisticRegressionLayer::Forward(std::vector &input, bool /*applyDropout*/) -> void +{ + for(size_t i=0; iGetTestDataBatchSize(); i++) + { + Architecture_t::ForwardLogReg(input[i], this->GetOutputAt(i), this->GetWeightsAt(0)); + Architecture_t::AddBiases(this->GetOutputAt(i), this->GetBiasesAt(0)); + Architecture_t::SoftmaxAE(this->GetOutputAt(i)); + + } +} + +//______________________________________________________________________________ +template +auto TLogisticRegressionLayer::Print() const +-> void +{ + std::cout << "Input Batch Size: " << this->GetBatchSize() << "\n" + << "Output Batch size: " << this->GetTestDataBatchSize() << "\n" + << "Input Units: " << this->GetInputUnits() << "\n" + << "Output Units: " << this->GetOutputUnits() << "\n"; + + std::cout<<"output: "<GetOutput().size(); i++) + { + for (Int_t j = 0; j < this->GetOutputAt(i).GetNrows(); j++) { + for (Int_t k = 0; k < this->GetOutputAt(i).GetNcols(); k++) { + std::cout << this->GetOutputAt(i)(j, k) << "\t"; + } + std::cout << std::endl; + } + } +} + +//______________________________________________________________________________ + + +//______________________________________________________________________________ + +//______________________________________________________________________________ + +}// namespace DAE +}//namespace DNN +}//namespace TMVA +#endif /* TMVA_DAE_LOGISTIC_REGRESSION_LAYER */ diff --git a/tmva/tmva/inc/TMVA/DNN/DAE/ReconstructionLayer.h b/tmva/tmva/inc/TMVA/DNN/DAE/ReconstructionLayer.h new file mode 100644 index 0000000000000..860ef5ec9176b --- /dev/null +++ b/tmva/tmva/inc/TMVA/DNN/DAE/ReconstructionLayer.h @@ -0,0 +1,255 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Akshay Vashistha + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : TReconstructionLayer * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Reconstruction Layer for DeepAutoEncoders * + * * + * Authors (alphabetical): * + * Akshay Vashistha - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + + +#ifndef TMVA_DAE_RECONSTRUCTION_LAYER +#define TMVA_DAE_RECONSTRUCTION_LAYER + +#include "TMatrix.h" + +#include "TMVA/DNN/GeneralLayer.h" +#include "TMVA/DNN/Functions.h" + +#include +#include + +namespace TMVA { +namespace DNN { +namespace DAE { + +/** \class TReconstructionLayer + * Reconstruction Layer for AutoEncoders. + * This reconstructs input based on the difference between Actual and Corrupted values. + * It takes weights and biases from previous layers. And used concept of tied weights + * to update parameters. +*/ + +template +class TReconstructionLayer : public VGeneralLayer { +public: + using Matrix_t = typename Architecture_t::Matrix_t; + using Scalar_t = typename Architecture_t::Scalar_t; + + + size_t fVisibleUnits; ///< total number of visible units + + size_t fHiddenUnits; ///< Number of compressed inputs + + Matrix_t fVBiasError; ///< Errors associated with visible Units + + Matrix_t fHBiasError; ///< Errors associated with Hidden Units + + Scalar_t fLearningRate; + + size_t fType; ///< Type of layer + + EActivationFunction fF; + + Scalar_t fCorruptionLevel; + + Scalar_t fDropoutProbability; + + + + /*! Constructor. */ + TReconstructionLayer(size_t BatchSize, size_t VisibleUnits, + size_t HiddenUnits, Scalar_t learningRate, EActivationFunction f, + std::vector weights, std::vector biases, + Scalar_t CorruptionLevel, Scalar_t dropoutProbability); + + /*! Copy the denoise layer provided as a pointer */ + TReconstructionLayer(TReconstructionLayer *layer); + + /*! Copy constructor. */ + TReconstructionLayer(const TReconstructionLayer &); + + /*! Destructor. */ + ~TReconstructionLayer(); + + + + void Forward(std::vector &input, bool applyDropout = false); + + void Backward(std::vector &compressedInput, + const std::vector &input, + std::vector &inp1, + std::vector &inp2); + + + + void Print() const; + + /* Getters */ + //size_t GetBatchSize() const { return fBatchSize;} + size_t GetVisibleUnits() const { return fVisibleUnits; } + size_t GetHiddenUnits() const { return fHiddenUnits; } + size_t GetType() const {return fType;} + Scalar_t GetCorruptionLevel() const {return fCorruptionLevel;} + Scalar_t GetDropoutProbability() const { return fDropoutProbability; } + Scalar_t GetLearningRate() const {return fLearningRate; } + + EActivationFunction GetActivationFunction() const { return fF; } + + const Matrix_t &GetVBiasError() const { return fVBiasError; } + Matrix_t &GetVBiasError() { return fVBiasError; } + + const Matrix_t &GetHBiasError() const { return fHBiasError; } + Matrix_t &GetHBiasError() { return fHBiasError; } + +}; + +// +// +// Denoise Layer Class - Implementation +//______________________________________________________________________________ + +template +TReconstructionLayer::TReconstructionLayer(size_t batchSize, size_t visibleUnits, size_t hiddenUnits, + Scalar_t learningRate, EActivationFunction f, + std::vector weights, std::vector biases, + Scalar_t corruptionLevel, Scalar_t dropoutProbability) + : VGeneralLayer(batchSize, 1, 1, 0, 0, 0, 0, 1, {hiddenUnits}, {visibleUnits}, 2, + {hiddenUnits, visibleUnits}, {1, 1}, batchSize, visibleUnits, 1, + EInitialization::kZero), + fVisibleUnits(visibleUnits), fHiddenUnits(hiddenUnits), fVBiasError(visibleUnits, 1), fHBiasError(hiddenUnits, 1), + fLearningRate(learningRate), fType(3), fF(f), fCorruptionLevel(corruptionLevel), + fDropoutProbability(dropoutProbability) + +{ + Architecture_t::Copy(this->GetWeightsAt(0),weights[0]); + Architecture_t::Copy(this->GetBiasesAt(0),biases[0]); + +} + +//______________________________________________________________________________ +template +TReconstructionLayer::TReconstructionLayer(TReconstructionLayer *layer) + : VGeneralLayer(layer), fVisibleUnits(layer->GetVisibleUnits()), + fHiddenUnits(layer->GetHiddenUnits()), fHBiasError(layer->GetHiddenUnits(), 1), + fVBiasError(layer->GetVisibleUnits(), 1), fType(3), fF(layer->GetActivationFunction()), + fCorruptionLevel(layer->GetCorruptionLevel()), fDropoutProbability(layer->GetDropoutProbability()) + +{ + Architecture_t::Copy(this->GetWeightsAt(0),layer->weights[0]); + Architecture_t::Copy(this->GetBiasesAt(0),layer->biases[0]); +} + +//______________________________________________________________________________ + +template +TReconstructionLayer::TReconstructionLayer(const TReconstructionLayer &dae) + : VGeneralLayer(dae), + fVisibleUnits(dae.fVisibleUnits), + fHiddenUnits(dae.fHiddenUnits), + fHBiasError(dae.fHiddenUnits, 1), + fVBiasError(dae.fVisibleUnits,1),fType(3), + fF(dae.fActivationFunction), + fCorruptionLevel(dae.fCorruptionLevel), + fDropoutProbability(dae.fDropoutProbability), + fLearningRate(dae.fLearningRate) + +{ + Architecture_t::Copy(this->GetWeightsAt(0),dae.weights[0]); + Architecture_t::Copy(this->GetBiasesAt(0),dae.biases[0]); + +} + +//______________________________________________________________________________ + +template TReconstructionLayer::~TReconstructionLayer() {} + +//______________________________________________________________________________ +// +// using concept of tied weights, i.e. using same weights as associated with +// previous layer for reconstruction. +//______________________________________________________________________________ +// Input here should be compressedInput +// reconstruction step +template +auto TReconstructionLayer::Forward(std::vector &input, bool /*applyDropout*/) -> void +{ + std::cout<<"Reconstruction Forward starts "<GetBatchSize(); i++) + { + Architecture_t::ReconstructInput(input[i],this->GetOutputAt(i),this->GetWeightsAt(0)); + + Architecture_t::AddBiases(this->GetOutputAt(i), this->GetBiasesAt(1)); + + evaluate(this->GetOutputAt(i), fF); + + } + std::cout<<"Reconstruction Forward ends "< +auto inline TReconstructionLayer::Backward(std::vector &compressedInput, + const std::vector & /*gradient*/, + std::vector &corruptedInput, + std::vector &input) -> void +{ + for (size_t i = 0; i < this->GetBatchSize(); i++) + { + Architecture_t::UpdateParams(input[i], corruptedInput[i], + compressedInput[i], + this->GetOutputAt(i), + this->GetBiasesAt(1), + this->GetBiasesAt(0), this->GetWeightsAt(0), + this->GetVBiasError(), this->GetHBiasError(), + this->GetLearningRate(), this->GetBatchSize()); + } + +} +//______________________________________________________________________________ +template +auto TReconstructionLayer::Print() const +-> void +{ + std::cout << "Batch Size: " << this->GetBatchSize() << "\n" + << "Input Units: " << this->GetVisibleUnits() << "\n" + << "Hidden Units: " << this->GetHiddenUnits() << "\n"; + std::cout<<"Reconstructed Input: "<GetBatchSize(); i++) + { + for (Int_t j = 0; j < this->GetOutputAt(i).GetNrows(); j++) { + for (Int_t k = 0; k < this->GetOutputAt(i).GetNcols(); k++) { + std::cout << this->GetOutputAt(i)(j, k) << "\t"; + } + std::cout << std::endl; + } + std::cout< - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_DNN_DLMINIMIZERS +#define TMVA_DNN_DLMINIMIZERS + +#include "TMVA/DNN/TensorDataLoader.h" +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DeepNet.h" + +#include +#include + +namespace TMVA { +namespace DNN { + +/*** \class TDLGradientDescent + * + * Generic implementation of gradient descent minimization for the + * deep learning neural nets. + * + * The TDLGradientDescent class implements an architecture, input data and + * deep learning neural network type independent implementation of the gradient + * descent minimization algorithm. + * +* This is provided by the Step(...), StepMomentum(...) and + * StepNesterov(...) functions that perform a single minimization step. + * + * The main training characteristics are defined by the provided learning rate, + * the test interval, and the convergence steps required for convergence. The + * test interval defines how often the error on the validation set is computed, + * and the values with which the step counter is increased each time the + * HasConverged() member function is called. A convergence step is defined as + * a step in which the test error is NOT less than 0.999 times the current + * minimal test error that has been reached. If between two subsequent calls + * to HasConverged(Double_t) the test error has not been sufficiently reduced + * it is assumed that a number of convergence steps equal to the test interval + * has been performed. + */ + +template +class TDLGradientDescent { +public: + using DeepNet_t = TDeepNet; + using Scalar_t = typename Architecture_t::Scalar_t; + using Matrix_t = typename Architecture_t::Matrix_t; + +private: + size_t fBatchSize; ///< Batch size to use for the training. + size_t fStepCount; ///< Number of steps performed in the current training session + size_t fConvergenceSteps; ///< Number of training epochs without considerable + ///< decrease in the test error for convergence. + size_t fConvergenceCount; ///< Current number of training epochs without + ///< considerable decrease in the test error. + size_t fTestInterval; ///< Interval for the computation of the test error. + Scalar_t fTrainingError; ///< Holds the most recently computed training loss. + Scalar_t fTestError; ///< Holds the most recently computed test loss. + Scalar_t fLearningRate; ///< Learning rate \f$\alpha\f$ + Scalar_t fMinimumError; ///< The minimum loss achieved on the training set + ///< during the current traning session. + +public: + TDLGradientDescent(); + TDLGradientDescent(Scalar_t learningRate, size_t convergenceSteps, size_t testInterval); + + /** Reset minimizer object to default state. */ + void Reset() + { + fMinimumError = std::numeric_limits::infinity(); + fConvergenceCount = 0; + fStepCount = 0; + }; + + /** Perform a single optimization step on a given batch. Propagates the input + matrix foward through the net, evaluates the loss and propagates the gradients + backward through the net. The computed gradients are scaled by the learning + rate \f$\alpha\f$ and subtracted from the weights and bias values of each + layer. */ + void Step(DeepNet_t &deepNet, std::vector &input, const Matrix_t &output, const Matrix_t &weights); + + /** Does not evaluate the loss and therefore not trigger a possible synchronization + * with the device. Trains the weights of each layer, but only the bias terms of + * the first layer for compatibility with the previous implementation. */ + void StepReducedWeights(DeepNet_t &deepNet, std::vector &input, const Matrix_t &output, + const Matrix_t &weights); + + /** Same as Step(...) but also evaluate the loss on the given training data. + * Note that this requires synchronization between host and device. */ + Scalar_t StepLoss(DeepNet_t &deepNet, std::vector &input, const Matrix_t &output, const Matrix_t &weights); + + /** Similar to StepReducedWeights(...) but also evaluates the loss. May trigger + * synchronization with the device. */ + Scalar_t StepReducedWeightsLoss(DeepNet_t &deepNet, std::vector &input, const Matrix_t &output, + const Matrix_t &weights); + + /** Perform multiple optimization steps simultaneously. Performs the + * backprop algorithm on the input batches given in \p batches on + * the neural networks given in \p nets. The forward and backward propagation + * steps are executed in an interleaving manner in order to exploit potential + * batch-level parallelism for asynchronous device calls. + */ + void Step(DeepNet_t &master, std::vector &nets, std::vector> &batches); + + /** Same as the Step(...) method for multiple batches but uses momentum. */ + void StepMomentum(DeepNet_t &master, std::vector &nets, + std::vector> &batches, Scalar_t momentum); + + /** Same as the Step(...) method for multiple batches but uses Nesterov + * momentum. */ + void StepNesterov(DeepNet_t &master, std::vector &nets, + std::vector> &batches, Scalar_t momentum); + + /** Increases the minimization step counter by the test error evaluation + * period and uses the current internal value of the test error to + * determine if the minimization has converged. */ + bool HasConverged(); + + /** Increases the minimization step counter by the test error evaluation + * period and uses the provided test error value to determine if the + * minimization has converged. */ + bool HasConverged(Scalar_t testError); + + /** Getters */ + size_t GetConvergenceCount() const { return fConvergenceCount; } + size_t GetConvergenceSteps() const { return fConvergenceSteps; } + Scalar_t GetTrainingError() const { return fTrainingError; } + Scalar_t GetTestError() const { return fTestError; } + size_t GetTestInterval() const { return fTestInterval; } + + /** Setters */ + void SetConvergenceSteps(size_t steps) { fConvergenceSteps = steps; } + void SetTestInterval(size_t interval) { fTestInterval = interval; } + void SetLearningRate(Scalar_t rate) { fLearningRate = rate; } + void SetBatchSize(Scalar_t rate) { fBatchSize = rate; } +}; + +// +// Implementation +//______________________________________________________________________________ +template +TDLGradientDescent::TDLGradientDescent() + : fBatchSize(0), fStepCount(0), fConvergenceSteps(0), fConvergenceCount(0), fTestInterval(0), fLearningRate(0), + fMinimumError(std::numeric_limits::infinity()) +{ + // Nothing to do here. +} + +//______________________________________________________________________________ +template +TDLGradientDescent::TDLGradientDescent(Scalar_t learningRate, size_t convergenceSteps, + size_t testInterval) + : fBatchSize(0), fStepCount(0), fConvergenceSteps(convergenceSteps), fConvergenceCount(0), + fTestInterval(testInterval), fLearningRate(learningRate), fMinimumError(std::numeric_limits::infinity()) +{ + // Nothing to do here. +} + +//______________________________________________________________________________ +template +void TDLGradientDescent::Step(DeepNet_t &deepNet, std::vector &input, const Matrix_t &output, + const Matrix_t &weights) +{ + // Make forward and backward pass and update the net afterwards + deepNet.Forward(input, true); + deepNet.Backward(input, output, weights); + deepNet.Update(fLearningRate); +} + +//______________________________________________________________________________ +template +void TDLGradientDescent::StepReducedWeights(DeepNet_t &deepNet, std::vector &input, + const Matrix_t &output, const Matrix_t &weights) +{ + // Make forward and backward pass and update the net afterwards + deepNet.Forward(input, true); + deepNet.Backward(input, output, weights); + + for (size_t i = 0; i < deepNet.GetDepth(); i++) { + auto *layer = deepNet.GetLayerAt(i); + + layer->UpdateWeights(layer->GetWeightGradients(), fLearningRate); + if (i == 0) { + layer->UpdateBiases(layer->GetBiasGradients(), fLearningRate); + } + } +} + +//______________________________________________________________________________ +template +auto TDLGradientDescent::StepLoss(DeepNet_t &deepNet, std::vector &input, + const Matrix_t &output, const Matrix_t &weights) -> Scalar_t +{ + Scalar_t loss = deepNet.Loss(input, output); + deepNet.Backward(input, output, weights); + deepNet.Update(fLearningRate); + + return loss; +} + +//______________________________________________________________________________ +template +auto TDLGradientDescent::StepReducedWeightsLoss(DeepNet_t &deepNet, std::vector &input, + const Matrix_t &output, const Matrix_t &weights) + -> Scalar_t +{ + Scalar_t loss = deepNet.Loss(input, output); + fTrainingError = loss; + deepNet.Backward(input, output, weights); + + for (size_t i = 0; i < deepNet.GetDepth(); i++) { + auto *layer = deepNet.GetLayerAt(i); + + layer->UpdateWeights(layer->GetWeightGradients(), fLearningRate); + if (i == 0) { + layer->UpdateBiases(layer->GetBiasGradients(), fLearningRate); + } + } + + return loss; +} + +//______________________________________________________________________________ +template +void TDLGradientDescent::Step(DeepNet_t &master, std::vector &nets, + std::vector> &batches) +{ + + master.ParallelForward(nets, batches); + master.ParallelBackward(nets, batches, fLearningRate); +} + +//______________________________________________________________________________ +template +void TDLGradientDescent::StepMomentum(DeepNet_t &master, std::vector &nets, + std::vector> &batches, + Scalar_t momentum) +{ + master.ParallelForward(nets, batches); + master.ParallelBackwardMomentum(nets, batches, fLearningRate, momentum); +} + +//______________________________________________________________________________ +template +void TDLGradientDescent::StepNesterov(DeepNet_t &master, std::vector &nets, + std::vector> &batches, + Scalar_t momentum) +{ + master.ParallelForward(nets, batches); + master.ParallelBackwardNestorov(nets, batches, fLearningRate, momentum); +} + +//______________________________________________________________________________ +template +bool TDLGradientDescent::HasConverged() +{ + if (fTestError < fMinimumError * 0.999) { + fConvergenceCount = 0; + fMinimumError = fTestError; + } else { + fConvergenceCount++; + } + + return (fConvergenceCount >= fConvergenceSteps); +} + +//______________________________________________________________________________ +template +bool TDLGradientDescent::HasConverged(Scalar_t testError) +{ + fTestError = testError; + if (fTestError < fMinimumError * 0.999) { + fConvergenceCount = 0; + fMinimumError = fTestError; + } else { + fConvergenceCount += fTestInterval; + } + return (fConvergenceCount >= fConvergenceSteps); +} + +} // namespace DNN +} // namespace TMVA + +#endif diff --git a/tmva/tmva/inc/TMVA/DNN/DeepNet.h b/tmva/tmva/inc/TMVA/DNN/DeepNet.h new file mode 100644 index 0000000000000..ff937a3f52a5e --- /dev/null +++ b/tmva/tmva/inc/TMVA/DNN/DeepNet.h @@ -0,0 +1,1134 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : TDeepNet * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Deep Neural Network * + * * + * Authors (alphabetical): * + * Akshay Vashistha - CERN, Switzerland * + * Vladimir Ilievski - CERN, Switzerland * + * Saurav Shekhar - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_DNN_DEEPNET +#define TMVA_DNN_DEEPNET + +#include "TString.h" + +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/TensorDataLoader.h" + +#include "TMVA/DNN/GeneralLayer.h" +#include "TMVA/DNN/DenseLayer.h" +#include "TMVA/DNN/ReshapeLayer.h" + +#include "TMVA/DNN/CNN/ConvLayer.h" +#include "TMVA/DNN/CNN/MaxPoolLayer.h" + +#include "TMVA/DNN/RNN/RNNLayer.h" + +#include "TMVA/DNN/DAE/CompressionLayer.h" +#include "TMVA/DNN/DAE/CorruptionLayer.h" +#include "TMVA/DNN/DAE/ReconstructionLayer.h" +#include "TMVA/DNN/DAE/LogisticRegressionLayer.h" + +#include +#include + +using namespace TMVA::DNN::CNN; +using namespace TMVA::DNN::RNN; +using namespace TMVA::DNN::DAE; + +namespace TMVA { +namespace DNN { + +/** \class TDeepNet + + Generic Deep Neural Network class. + + This classs encapsulates the information for all types of Deep Neural Networks. + + \tparam Architecture The Architecture type that holds the + architecture-specific data types. + */ +template > +class TDeepNet { +public: + using Matrix_t = typename Architecture_t::Matrix_t; + using Scalar_t = typename Architecture_t::Scalar_t; + +private: + bool inline isInteger(Scalar_t x) const { return x == floor(x); } + size_t calculateDimension(int imgDim, int fltDim, int padding, int stride); + +private: + std::vector fLayers; ///< The layers consisting the DeepNet + + size_t fBatchSize; ///< Batch size used for training and evaluation. + size_t fInputDepth; ///< The depth of the input. + size_t fInputHeight; ///< The height of the input. + size_t fInputWidth; ///< The width of the input. + + size_t fBatchDepth; ///< The depth of the batch used for training/testing. + size_t fBatchHeight; ///< The height of the batch used for training/testing. + size_t fBatchWidth; ///< The width of the batch used for training/testing. + + bool fIsTraining; ///< Is the network training? + + ELossFunction fJ; ///< The loss function of the network. + EInitialization fI; ///< The initialization method of the network. + ERegularization fR; ///< The regularization used for the network. + Scalar_t fWeightDecay; ///< The weight decay factor. + +public: + /*! Default Constructor */ + TDeepNet(); + + /*! Constructor */ + TDeepNet(size_t BatchSize, size_t InputDepth, size_t InputHeight, size_t InputWidth, size_t BatchDepth, + size_t BatchHeight, size_t BatchWidth, ELossFunction fJ, EInitialization fI = EInitialization::kZero, + ERegularization fR = ERegularization::kNone, Scalar_t fWeightDecay = 0.0, bool isTraining = false); + + /*! Copy-constructor */ + TDeepNet(const TDeepNet &); + + /*! Destructor */ + ~TDeepNet(); + + /*! Function for adding Convolution layer in the Deep Neural Network, + * with a given depth, filter height and width, striding in rows and columns, + * the zero paddings, as well as the activation function and the dropout + * probability. Based on these parameters, it calculates the width and height + * of the convolutional layer. */ + TConvLayer *AddConvLayer(size_t depth, size_t filterHeight, size_t filterWidth, size_t strideRows, + size_t strideCols, size_t paddingHeight, size_t paddingWidth, + EActivationFunction f, Scalar_t dropoutProbability = 1.0); + + /*! Function for adding Convolution Layer in the Deep Neural Network, + * when the layer is already created. */ + void AddConvLayer(TConvLayer *convLayer); + + /*! Function for adding Pooling layer in the Deep Neural Network, + * with a given filter height and width, striding in rows and columns as + * well as the dropout probability. The depth is same as the previous + * layer depth. Based on these parameters, it calculates the width and + * height of the pooling layer. */ + TMaxPoolLayer *AddMaxPoolLayer(size_t frameHeight, size_t frameWidth, size_t strideRows, + size_t strideCols, Scalar_t dropoutProbability = 1.0); + /*! Function for adding Max Pooling layer in the Deep Neural Network, + * when the layer is already created. */ + void AddMaxPoolLayer(TMaxPoolLayer *maxPoolLayer); + + /*! Function for adding Recurrent Layer in the Deep Neural Network, + * with given parameters */ + TBasicRNNLayer *AddBasicRNNLayer(size_t stateSize, size_t inputSize, size_t timeSteps, + bool rememberState = false); + + /*! Function for adding Vanilla RNN when the layer is already created + */ + void AddBasicRNNLayer(TBasicRNNLayer *basicRNNLayer); + + /*! Function for adding Dense Connected Layer in the Deep Neural Network, + * with a given width, activation function and dropout probability. + * Based on the previous layer dimensions, it calculates the input width + * of the fully connected layer. */ + TDenseLayer *AddDenseLayer(size_t width, EActivationFunction f, Scalar_t dropoutProbability = 1.0); + + /*! Function for adding Dense Layer in the Deep Neural Network, when + * the layer is already created. */ + void AddDenseLayer(TDenseLayer *denseLayer); + + /*! Function for adding Reshape Layer in the Deep Neural Network, with a given + * height and width. It will take every matrix from the previous layer and + * reshape it to a matrix with new dimensions. */ + TReshapeLayer *AddReshapeLayer(size_t depth, size_t height, size_t width, bool flattening); + + /*! Function for adding Reshape Layer in the Deep Neural Network, when + * the layer is already created. */ + void AddReshapeLayer(TReshapeLayer *reshapeLayer); + + /*! Function for adding Corruption layer in the Deep Neural Network, + * with given number of visibleUnits and hiddenUnits. It corrupts input + * according to given corruptionLevel and dropoutProbability. */ + TCorruptionLayer *AddCorruptionLayer(size_t visibleUnits, size_t hiddenUnits, + Scalar_t dropoutProbability, Scalar_t corruptionLevel); + + /*! Function for adding Corruption Layer in the Deep Neural Network, + * when the layer is already created. */ + void AddCorruptionLayer(TCorruptionLayer *corruptionLayer); + + /*! Function for adding Compression layer in the Deep Neural Network, + * with given number of visibleUnits and hiddenUnits. It compresses the input units + * taking weights and biases from prev layers. */ + TCompressionLayer *AddCompressionLayer(size_t visibleUnits, size_t hiddenUnits, + Scalar_t dropoutProbability, EActivationFunction f, + std::vector weights, std::vector biases); + + /*! Function for adding Compression Layer in the Deep Neural Network, when + * the layer is already created. */ + void AddCompressionLayer(TCompressionLayer *compressionLayer); + + /*! Function for adding Reconstruction layer in the Deep Neural Network, + * with given number of visibleUnits and hiddenUnits. It reconstructs the input units + * taking weights and biases from prev layers. Same corruptionLevel and dropoutProbability + * must be passed as in corruptionLayer. */ + TReconstructionLayer *AddReconstructionLayer(size_t visibleUnits, size_t hiddenUnits, + Scalar_t learningRate, EActivationFunction f, + std::vector weights, + std::vector biases, Scalar_t corruptionLevel, + Scalar_t dropoutProbability); + + /*! Function for adding Reconstruction Layer in the Deep Neural Network, when + * the layer is already created. */ + void AddReconstructionLayer(TReconstructionLayer *reconstructionLayer); + + /*! Function for adding logisticRegressionLayer in the Deep Neural Network, + * with given number of inputUnits and outputUnits. It classifies the outputUnits. */ + TLogisticRegressionLayer *AddLogisticRegressionLayer(size_t inputUnits, size_t outputUnits, + size_t testDataBatchSize, + Scalar_t learningRate); + + /*! Function for adding logisticRegressionLayer in the Deep Neural Network, when + * the layer is already created. */ + void AddLogisticRegressionLayer(TLogisticRegressionLayer *logisticRegressionLayer); + + /*! Function for initialization of the Neural Net. */ + void Initialize(); + + /*! Function that executes the entire forward pass in the network. */ + void Forward(std::vector &input, bool applyDropout = false); + + /*! Function for parallel forward in the vector of deep nets, where the master + * net is the net calling this function. There is one batch for one deep net.*/ + void ParallelForward(std::vector> &nets, + std::vector> &batches, bool applyDropout = false); + + /*! Function that executes the entire backward pass in the network. */ + void Backward(std::vector &input, const Matrix_t &groundTruth, const Matrix_t &weights); + + /* To train the Deep AutoEncoder network with required number of Corruption, Compression and Reconstruction + * layers. */ + void PreTrain(std::vector &input, std::vector numHiddenUnitsPerLayer, Scalar_t learningRate, + Scalar_t corruptionLevel, Scalar_t dropoutProbability, size_t epochs, EActivationFunction f, + bool applyDropout = false); + + /* To classify outputLabel in Deep AutoEncoder. Should be used after PreTrain if required. + * Currently, it used Logistic Regression Layer. Otherwise we can use any other classification layer also. + */ + void FineTune(std::vector &input, std::vector &testInput, std::vector &outputLabel, + size_t outputUnits, size_t testDataBatchSize, Scalar_t learningRate, size_t epochs); + + /*! Function for parallel backward in the vector of deep nets, where the master + * net is the net calling this function and getting the updates from the other nets. + * There is one batch for one deep net.*/ + void ParallelBackward(std::vector> &nets, + std::vector> &batches, Scalar_t learningRate); + + /*! Function for parallel backward in the vector of deep nets, where the master + * net is the net calling this function and getting the updates from the other nets, + * following the momentum strategy. There is one batch for one deep net.*/ + void ParallelBackwardMomentum(std::vector> &nets, + std::vector> &batches, Scalar_t learningRate, + Scalar_t momentum); + + /*! Function for parallel backward in the vector of deep nets, where the master + * net is the net calling this function and getting the updates from the other nets, + * following the Nestorov momentum strategy. There is one batch for one deep net.*/ + void ParallelBackwardNestorov(std::vector> &nets, + std::vector> &batches, Scalar_t learningRate, + Scalar_t momentum); + + /*! Function that will update the weights and biases in the layers that + * contain weights and biases. */ + void Update(Scalar_t learningRate); + + /*! Function for evaluating the loss, based on the activations stored + * in the last layer. */ + Scalar_t Loss(const Matrix_t &groundTruth, const Matrix_t &weights, bool includeRegularization = true) const; + + /*! Function for evaluating the loss, based on the propagation of the given input. */ + Scalar_t Loss(std::vector input, const Matrix_t &groundTruth, const Matrix_t &weights, + bool applyDropout = false, bool includeRegularization = true); + + /*! Prediction based on activations stored in the last layer. */ + void Prediction(Matrix_t &predictions, EOutputFunction f) const; + + /*! Prediction for the given inputs, based on what network learned. */ + void Prediction(Matrix_t &predictions, std::vector input, EOutputFunction f); + + /*! Print the Deep Net Info */ + void Print(); + + /*! Get the layer in the vector of layers at poistion i */ + inline Layer_t *GetLayerAt(size_t i) { return fLayers[i]; } + inline const Layer_t *GetLayerAt(size_t i) const { return fLayers[i]; } + + /* Depth and the output width of the network. */ + inline size_t GetDepth() { return fLayers.size(); } + inline size_t GetOutputWidth() { return fLayers.back()->GetWidth(); } + + /* Return a reference to the layers. */ + inline std::vector &GetLayers() { return fLayers; } + inline const std::vector &GetLayers() const { return fLayers; } + + /*! Remove all layers from the network. */ + inline void Clear() { fLayers.clear(); } + + /*! Getters */ + inline size_t GetBatchSize() const { return fBatchSize; } + inline size_t GetInputDepth() const { return fInputDepth; } + inline size_t GetInputHeight() const { return fInputHeight; } + inline size_t GetInputWidth() const { return fInputWidth; } + + inline size_t GetBatchDepth() const { return fBatchDepth; } + inline size_t GetBatchHeight() const { return fBatchHeight; } + inline size_t GetBatchWidth() const { return fBatchWidth; } + + inline bool IsTraining() const { return fIsTraining; } + + inline ELossFunction GetLossFunction() const { return fJ; } + inline EInitialization GetInitialization() const { return fI; } + inline ERegularization GetRegularization() const { return fR; } + inline Scalar_t GetWeightDecay() const { return fWeightDecay; } + + /*! Setters */ + // FIXME many of these won't work as the data structure storing activations + // and gradients have not changed in all the layers, also params in layers + // have not changed either + inline void SetBatchSize(size_t batchSize) { fBatchSize = batchSize; } + inline void SetInputDepth(size_t inputDepth) { fInputDepth = inputDepth; } + inline void SetInputHeight(size_t inputHeight) { fInputHeight = inputHeight; } + inline void SetInputWidth(size_t inputWidth) { fInputWidth = inputWidth; } + inline void SetBatchDepth(size_t batchDepth) { fBatchDepth = batchDepth; } + inline void SetBatchHeight(size_t batchHeight) { fBatchHeight = batchHeight; } + inline void SetBatchWidth(size_t batchWidth) { fBatchWidth = batchWidth; } + inline void SetLossFunction(ELossFunction J) { fJ = J; } + inline void SetInitialization(EInitialization I) { fI = I; } + inline void SetRegularization(ERegularization R) { fR = R; } + inline void SetWeightDecay(Scalar_t weightDecay) { fWeightDecay = weightDecay; } +}; + +// +// Deep Net Class - Implementation +// +//______________________________________________________________________________ +template +TDeepNet::TDeepNet() + : fLayers(), fBatchSize(0), fInputDepth(0), fInputHeight(0), fInputWidth(0), fBatchDepth(0), fBatchHeight(0), + fBatchWidth(0), fJ(ELossFunction::kMeanSquaredError), fI(EInitialization::kZero), fR(ERegularization::kNone), + fIsTraining(true), fWeightDecay(0.0) +{ + // Nothing to do here. +} + +//______________________________________________________________________________ +template +TDeepNet::TDeepNet(size_t batchSize, size_t inputDepth, size_t inputHeight, size_t inputWidth, + size_t batchDepth, size_t batchHeight, size_t batchWidth, ELossFunction J, + EInitialization I, ERegularization R, Scalar_t weightDecay, bool isTraining) + : fLayers(), fBatchSize(batchSize), fInputDepth(inputDepth), fInputHeight(inputHeight), fInputWidth(inputWidth), + fBatchDepth(batchDepth), fBatchHeight(batchHeight), fBatchWidth(batchWidth), fIsTraining(isTraining), fJ(J), fI(I), + fR(R), fWeightDecay(weightDecay) +{ + // Nothing to do here. +} + +//______________________________________________________________________________ +template +TDeepNet::TDeepNet(const TDeepNet &deepNet) + : fLayers(), fBatchSize(deepNet.fBatchSize), fInputDepth(deepNet.fInputDepth), fInputHeight(deepNet.fInputHeight), + fInputWidth(deepNet.fInputWidth), fBatchDepth(deepNet.fBatchDepth), fBatchHeight(deepNet.fBatchHeight), + fBatchWidth(deepNet.fBatchWidth), fIsTraining(deepNet.fIsTraining), fJ(deepNet.fJ), fI(deepNet.fI), fR(deepNet.fR), + fWeightDecay(deepNet.fWeightDecay) +{ + // Nothing to do here. +} + +//______________________________________________________________________________ +template +TDeepNet::~TDeepNet() +{ + // Relese the layers memory +} + +//______________________________________________________________________________ +template +auto TDeepNet::calculateDimension(int imgDim, int fltDim, int padding, int stride) -> size_t +{ + Scalar_t dimension = ((imgDim - fltDim + 2 * padding) / stride) + 1; + if (!isInteger(dimension) || dimension <= 0) { + std::cout << "calculateDimension - Not compatible hyper parameters (imgDim, fltDim, padding, stride)" + << imgDim << " , " << fltDim << " , " << padding << " , " << stride<< " resulting dim is " << dimension << std::endl; + std::exit(EXIT_FAILURE); + } + + return (size_t)dimension; +} + +//______________________________________________________________________________ +template +TConvLayer *TDeepNet::AddConvLayer(size_t depth, size_t filterHeight, + size_t filterWidth, size_t strideRows, + size_t strideCols, size_t paddingHeight, + size_t paddingWidth, EActivationFunction f, + Scalar_t dropoutProbability) +{ + // All variables defining a convolutional layer + size_t batchSize = this->GetBatchSize(); + size_t inputDepth; + size_t inputHeight; + size_t inputWidth; + size_t height; + size_t width; + size_t filterDepth; + size_t weightsNRows = depth; + size_t weightsNCols; + size_t biasesNRows = depth; + size_t biasesNCols = 1; + size_t outputNSlices = this->GetBatchSize(); + size_t outputNRows = depth; + size_t outputNCols; + EInitialization init = this->GetInitialization(); + ERegularization reg = this->GetRegularization(); + Scalar_t decay = this->GetWeightDecay(); + + if (fLayers.size() == 0) { + inputDepth = this->GetInputDepth(); + inputHeight = this->GetInputHeight(); + inputWidth = this->GetInputWidth(); + } else { + Layer_t *lastLayer = fLayers.back(); + inputDepth = lastLayer->GetDepth(); + inputHeight = lastLayer->GetHeight(); + inputWidth = lastLayer->GetWidth(); + } + + height = calculateDimension(inputHeight, filterHeight, paddingHeight, strideRows); + width = calculateDimension(inputWidth, filterWidth, paddingWidth, strideCols); + + filterDepth = inputDepth; + + weightsNCols = filterDepth * filterHeight * filterWidth; + outputNCols = height * width; + + // Create the conv layer + TConvLayer *convLayer = new TConvLayer( + batchSize, inputDepth, inputHeight, inputWidth, depth, height, width, weightsNRows, weightsNCols, biasesNRows, + biasesNCols, outputNSlices, outputNRows, outputNCols, init, filterDepth, filterHeight, filterWidth, strideRows, + strideCols, paddingHeight, paddingWidth, dropoutProbability, f, reg, decay); + + fLayers.push_back(convLayer); + return convLayer; +} + +//______________________________________________________________________________ +template +void TDeepNet::AddConvLayer(TConvLayer *convLayer) +{ + fLayers.push_back(convLayer); +} + +//______________________________________________________________________________ +template +TMaxPoolLayer *TDeepNet::AddMaxPoolLayer(size_t frameHeight, size_t frameWidth, + size_t strideRows, size_t strideCols, + Scalar_t dropoutProbability) +{ + size_t batchSize = this->GetBatchSize(); + size_t inputDepth; + size_t inputHeight; + size_t inputWidth; + size_t height; + size_t width; + size_t outputNSlices = this->GetBatchSize(); + size_t outputNRows; + size_t outputNCols; + + if (fLayers.size() == 0) { + inputDepth = this->GetInputDepth(); + inputHeight = this->GetInputHeight(); + inputWidth = this->GetInputWidth(); + } else { + Layer_t *lastLayer = fLayers.back(); + inputDepth = lastLayer->GetDepth(); + inputHeight = lastLayer->GetHeight(); + inputWidth = lastLayer->GetWidth(); + } + + height = calculateDimension(inputHeight, frameHeight, 0, strideRows); + width = calculateDimension(inputWidth, frameWidth, 0, strideCols); + + outputNRows = inputDepth; + outputNCols = height * width; + + TMaxPoolLayer *maxPoolLayer = new TMaxPoolLayer( + batchSize, inputDepth, inputHeight, inputWidth, height, width, outputNSlices, outputNRows, outputNCols, + frameHeight, frameWidth, strideRows, strideCols, dropoutProbability); + + // But this creates a copy or what? + fLayers.push_back(maxPoolLayer); + + return maxPoolLayer; +} + +//______________________________________________________________________________ +template +void TDeepNet::AddMaxPoolLayer(TMaxPoolLayer *maxPoolLayer) +{ + fLayers.push_back(maxPoolLayer); +} + +//______________________________________________________________________________ +template +TBasicRNNLayer *TDeepNet::AddBasicRNNLayer(size_t stateSize, size_t inputSize, + size_t timeSteps, + bool rememberState) +{ + TBasicRNNLayer *basicRNNLayer = + new TBasicRNNLayer(this->GetBatchSize(), stateSize, inputSize, timeSteps, rememberState, + DNN::EActivationFunction::kTanh, fIsTraining, this->GetInitialization()); + fLayers.push_back(basicRNNLayer); + return basicRNNLayer; +} + +//______________________________________________________________________________ +template +void TDeepNet::AddBasicRNNLayer(TBasicRNNLayer *basicRNNLayer) +{ + fLayers.push_back(basicRNNLayer); +} +//______________________________________________________________________________ +template +TCorruptionLayer *TDeepNet::AddCorruptionLayer(size_t visibleUnits, + size_t hiddenUnits, + Scalar_t dropoutProbability, + Scalar_t corruptionLevel) +{ + size_t batchSize = this->GetBatchSize(); + + TCorruptionLayer *corruptionLayer = + new TCorruptionLayer(batchSize, visibleUnits, hiddenUnits, dropoutProbability, corruptionLevel); + fLayers.push_back(corruptionLayer); + return corruptionLayer; +} +//______________________________________________________________________________ + +template +void TDeepNet::AddCorruptionLayer(TCorruptionLayer *corruptionLayer) +{ + fLayers.push_back(corruptionLayer); +} + +//______________________________________________________________________________ +template +TCompressionLayer *TDeepNet::AddCompressionLayer( + size_t visibleUnits, size_t hiddenUnits, Scalar_t dropoutProbability, EActivationFunction f, + std::vector weights, std::vector biases) +{ + size_t batchSize = this->GetBatchSize(); + + TCompressionLayer *compressionLayer = new TCompressionLayer( + batchSize, visibleUnits, hiddenUnits, dropoutProbability, f, weights, biases); + fLayers.push_back(compressionLayer); + return compressionLayer; +} +//______________________________________________________________________________ + +template +void TDeepNet::AddCompressionLayer(TCompressionLayer *compressionLayer) +{ + fLayers.push_back(compressionLayer); +} + +//______________________________________________________________________________ +template +TReconstructionLayer *TDeepNet::AddReconstructionLayer( + size_t visibleUnits, size_t hiddenUnits, Scalar_t learningRate, EActivationFunction f, std::vector weights, + std::vector biases, Scalar_t corruptionLevel, Scalar_t dropoutProbability) +{ + size_t batchSize = this->GetBatchSize(); + + TReconstructionLayer *reconstructionLayer = new TReconstructionLayer( + batchSize, visibleUnits, hiddenUnits, learningRate, f, weights, biases, corruptionLevel, dropoutProbability); + fLayers.push_back(reconstructionLayer); + return reconstructionLayer; +} +//______________________________________________________________________________ + +template +void TDeepNet::AddReconstructionLayer( + TReconstructionLayer *reconstructionLayer) +{ + fLayers.push_back(reconstructionLayer); +} + +//______________________________________________________________________________ +template +TLogisticRegressionLayer *TDeepNet::AddLogisticRegressionLayer( + size_t inputUnits, size_t outputUnits, size_t testDataBatchSize, Scalar_t learningRate) +{ + size_t batchSize = this->GetBatchSize(); + + TLogisticRegressionLayer *logisticRegressionLayer = + new TLogisticRegressionLayer(batchSize, inputUnits, outputUnits, testDataBatchSize, learningRate); + fLayers.push_back(logisticRegressionLayer); + return logisticRegressionLayer; +} +//______________________________________________________________________________ +template +void TDeepNet::AddLogisticRegressionLayer( + TLogisticRegressionLayer *logisticRegressionLayer) +{ + fLayers.push_back(logisticRegressionLayer); +} + +//______________________________________________________________________________ +template +TDenseLayer *TDeepNet::AddDenseLayer(size_t width, EActivationFunction f, + Scalar_t dropoutProbability) +{ + size_t batchSize = this->GetBatchSize(); + size_t inputWidth; + EInitialization init = this->GetInitialization(); + ERegularization reg = this->GetRegularization(); + Scalar_t decay = this->GetWeightDecay(); + + if (fLayers.size() == 0) { + inputWidth = this->GetInputWidth(); + } else { + Layer_t *lastLayer = fLayers.back(); + inputWidth = lastLayer->GetWidth(); + } + + TDenseLayer *denseLayer = + new TDenseLayer(batchSize, inputWidth, width, init, dropoutProbability, f, reg, decay); + + fLayers.push_back(denseLayer); + + return denseLayer; +} + +//______________________________________________________________________________ +template +void TDeepNet::AddDenseLayer(TDenseLayer *denseLayer) +{ + fLayers.push_back(denseLayer); +} + +//______________________________________________________________________________ +template +TReshapeLayer *TDeepNet::AddReshapeLayer(size_t depth, size_t height, + size_t width, bool flattening) +{ + size_t batchSize = this->GetBatchSize(); + size_t inputDepth; + size_t inputHeight; + size_t inputWidth; + size_t outputNSlices; + size_t outputNRows; + size_t outputNCols; + + if (fLayers.size() == 0) { + inputDepth = this->GetInputDepth(); + inputHeight = this->GetInputHeight(); + inputWidth = this->GetInputWidth(); + } else { + Layer_t *lastLayer = fLayers.back(); + inputDepth = lastLayer->GetDepth(); + inputHeight = lastLayer->GetHeight(); + inputWidth = lastLayer->GetWidth(); + } + + if (flattening) { + outputNSlices = 1; + outputNRows = this->GetBatchSize(); + outputNCols = depth * height * width; + } else { + outputNSlices = this->GetBatchSize(); + outputNRows = depth; + outputNCols = height * width; + } + + TReshapeLayer *reshapeLayer = + new TReshapeLayer(batchSize, inputDepth, inputHeight, inputWidth, depth, height, width, + outputNSlices, outputNRows, outputNCols, flattening); + + fLayers.push_back(reshapeLayer); + + return reshapeLayer; +} + +//______________________________________________________________________________ +template +void TDeepNet::AddReshapeLayer(TReshapeLayer *reshapeLayer) +{ + fLayers.push_back(reshapeLayer); +} + +//______________________________________________________________________________ +template +auto TDeepNet::Initialize() -> void +{ + for (size_t i = 0; i < fLayers.size(); i++) { + fLayers[i]->Initialize(); + } +} + +template +auto debugTensor(const std::vector &A, const std::string name = "tensor") -> void +{ + std::cout << name << "\n"; + for (size_t l = 0; l < A.size(); ++l) { + for (size_t i = 0; i < A[l].GetNrows(); ++i) { + for (size_t j = 0; j < A[l].GetNcols(); ++j) { + std::cout << A[l](i, j) << " "; + } + std::cout << "\n"; + } + std::cout << "********\n"; + } +} + +//______________________________________________________________________________ +template +auto TDeepNet::Forward(std::vector &input, bool applyDropout) -> void +{ + fLayers.front()->Forward(input, applyDropout); + + for (size_t i = 1; i < fLayers.size(); i++) { + fLayers[i]->Forward(fLayers[i - 1]->GetOutput(), applyDropout); + } +} + +//______________________________________________________________________________ +template +auto TDeepNet::ParallelForward(std::vector> &nets, + std::vector> &batches, + bool applyDropout) -> void +{ + size_t depth = this->GetDepth(); + + // The first layer of each deep net + for (size_t i = 0; i < nets.size(); i++) { + nets[i].GetLayerAt(0)->Forward(batches[i].GetInput(), applyDropout); + } + + // The i'th layer of each deep net + for (size_t i = 1; i < depth; i++) { + for (size_t j = 0; j < nets.size(); j++) { + nets[j].GetLayerAt(i)->Forward(nets[j].GetLayerAt(i - 1)->GetOutput(), applyDropout); + } + } +} + +//_____________________________________________________________________________ +template +auto TDeepNet::PreTrain(std::vector &input, + std::vector numHiddenUnitsPerLayer, Scalar_t learningRate, + Scalar_t corruptionLevel, Scalar_t dropoutProbability, size_t epochs, + EActivationFunction f, bool applyDropout) -> void +{ + std::vector inp1; + std::vector inp2; + size_t numOfHiddenLayers = sizeof(numHiddenUnitsPerLayer) / sizeof(numHiddenUnitsPerLayer[0]); + // size_t batchSize = this->GetBatchSize(); + size_t visibleUnits = (size_t)input[0].GetNrows(); + + AddCorruptionLayer(visibleUnits, numHiddenUnitsPerLayer[0], dropoutProbability, corruptionLevel); + fLayers.back()->Initialize(); + fLayers.back()->Forward(input, applyDropout); + // fLayers.back()->Print(); + + AddCompressionLayer(visibleUnits, numHiddenUnitsPerLayer[0], dropoutProbability, f, fLayers.back()->GetWeights(), + fLayers.back()->GetBiases()); + fLayers.back()->Initialize(); + fLayers.back()->Forward(fLayers[fLayers.size() - 2]->GetOutput(), applyDropout); // as we have to pass corrupt input + + AddReconstructionLayer(visibleUnits, numHiddenUnitsPerLayer[0], learningRate, f, fLayers.back()->GetWeights(), + fLayers.back()->GetBiases(), corruptionLevel, dropoutProbability); + fLayers.back()->Initialize(); + fLayers.back()->Forward(fLayers[fLayers.size() - 2]->GetOutput(), + applyDropout); // as we have to pass compressed Input + fLayers.back()->Backward(fLayers[fLayers.size() - 2]->GetOutput(), inp1, fLayers[fLayers.size() - 3]->GetOutput(), + input); + // three layers are added, now pointer is on third layer + size_t weightsSize = fLayers.back()->GetWeights().size(); + size_t biasesSize = fLayers.back()->GetBiases().size(); + for (size_t epoch = 0; epoch < epochs - 1; epoch++) { + // fLayers[fLayers.size() - 3]->Forward(input,applyDropout); + for (size_t j = 0; j < weightsSize; j++) { + Architecture_t::Copy(fLayers[fLayers.size() - 2]->GetWeightsAt(j), fLayers.back()->GetWeightsAt(j)); + } + for (size_t j = 0; j < biasesSize; j++) { + Architecture_t::Copy(fLayers[fLayers.size() - 2]->GetBiasesAt(j), fLayers.back()->GetBiasesAt(j)); + } + fLayers[fLayers.size() - 2]->Forward(fLayers[fLayers.size() - 3]->GetOutput(), applyDropout); + fLayers[fLayers.size() - 1]->Forward(fLayers[fLayers.size() - 2]->GetOutput(), applyDropout); + fLayers[fLayers.size() - 1]->Backward(fLayers[fLayers.size() - 2]->GetOutput(), inp1, + fLayers[fLayers.size() - 3]->GetOutput(), input); + } + fLayers.back()->Print(); + + for (size_t i = 1; i < numOfHiddenLayers; i++) { + + AddCorruptionLayer(numHiddenUnitsPerLayer[i - 1], numHiddenUnitsPerLayer[i], dropoutProbability, corruptionLevel); + fLayers.back()->Initialize(); + fLayers.back()->Forward(fLayers[fLayers.size() - 3]->GetOutput(), + applyDropout); // as we have to pass compressed Input + + AddCompressionLayer(numHiddenUnitsPerLayer[i - 1], numHiddenUnitsPerLayer[i], dropoutProbability, f, + fLayers.back()->GetWeights(), fLayers.back()->GetBiases()); + fLayers.back()->Initialize(); + fLayers.back()->Forward(fLayers[fLayers.size() - 2]->GetOutput(), applyDropout); + + AddReconstructionLayer(numHiddenUnitsPerLayer[i - 1], numHiddenUnitsPerLayer[i], learningRate, f, + fLayers.back()->GetWeights(), fLayers.back()->GetBiases(), corruptionLevel, + dropoutProbability); + fLayers.back()->Initialize(); + fLayers.back()->Forward(fLayers[fLayers.size() - 2]->GetOutput(), + applyDropout); // as we have to pass compressed Input + fLayers.back()->Backward(fLayers[fLayers.size() - 2]->GetOutput(), inp1, fLayers[fLayers.size() - 3]->GetOutput(), + fLayers[fLayers.size() - 5]->GetOutput()); + + // three layers are added, now pointer is on third layer + size_t _weightsSize = fLayers.back()->GetWeights().size(); + size_t _biasesSize = fLayers.back()->GetBiases().size(); + for (size_t epoch = 0; epoch < epochs - 1; epoch++) { + // fLayers[fLayers.size() - 3]->Forward(input,applyDropout); + for (size_t j = 0; j < _weightsSize; j++) { + Architecture_t::Copy(fLayers[fLayers.size() - 2]->GetWeightsAt(j), fLayers.back()->GetWeightsAt(j)); + } + for (size_t j = 0; j < _biasesSize; j++) { + Architecture_t::Copy(fLayers[fLayers.size() - 2]->GetBiasesAt(j), fLayers.back()->GetBiasesAt(j)); + } + fLayers[fLayers.size() - 2]->Forward(fLayers[fLayers.size() - 3]->GetOutput(), applyDropout); + fLayers[fLayers.size() - 1]->Forward(fLayers[fLayers.size() - 2]->GetOutput(), applyDropout); + fLayers[fLayers.size() - 1]->Backward(fLayers[fLayers.size() - 2]->GetOutput(), inp1, + fLayers[fLayers.size() - 3]->GetOutput(), + fLayers[fLayers.size() - 5]->GetOutput()); + } + fLayers.back()->Print(); + } +} + +//______________________________________________________________________________ +template +auto TDeepNet::FineTune(std::vector &input, std::vector &testInput, + std::vector &inputLabel, size_t outputUnits, + size_t testDataBatchSize, Scalar_t learningRate, size_t epochs) -> void +{ + std::vector inp1; + std::vector inp2; + if (fLayers.size() == 0) // only Logistic Regression Layer + { + size_t inputUnits = input[0].GetNrows(); + + AddLogisticRegressionLayer(inputUnits, outputUnits, testDataBatchSize, learningRate); + fLayers.back()->Initialize(); + for (size_t i = 0; i < epochs; i++) { + fLayers.back()->Backward(inputLabel, inp1, input, inp2); + } + fLayers.back()->Forward(input, false); + fLayers.back()->Print(); + } else { // if used after any other layer + size_t inputUnits = fLayers.back()->GetOutputAt(0).GetNrows(); + AddLogisticRegressionLayer(inputUnits, outputUnits, testDataBatchSize, learningRate); + fLayers.back()->Initialize(); + for (size_t i = 0; i < epochs; i++) { + fLayers.back()->Backward(inputLabel, inp1, fLayers[fLayers.size() - 2]->GetOutput(), inp2); + } + fLayers.back()->Forward(testInput, false); + fLayers.back()->Print(); + } +} + +//______________________________________________________________________________ +template +auto TDeepNet::Backward(std::vector &input, const Matrix_t &groundTruth, + const Matrix_t &weights) -> void +{ + std::vector inp1; + std::vector inp2; + // Last layer should be dense layer + evaluateGradients(fLayers.back()->GetActivationGradientsAt(0), this->GetLossFunction(), groundTruth, + fLayers.back()->GetOutputAt(0), weights); + for (size_t i = fLayers.size() - 1; i > 0; i--) { + std::vector &activation_gradient_backward = fLayers[i - 1]->GetActivationGradients(); + std::vector &activations_backward = fLayers[i - 1]->GetOutput(); + fLayers[i]->Backward(activation_gradient_backward, activations_backward, inp1, inp2); + } + + // need to have a dummy tensor (size=0) to pass for activation gradient backward which + // are not computed for the first layer + std::vector dummy; + fLayers[0]->Backward(dummy, input, inp1, inp2); +} + +//______________________________________________________________________________ +template +auto TDeepNet::ParallelBackward(std::vector> &nets, + std::vector> &batches, + Scalar_t learningRate) -> void +{ + std::vector inp1; + std::vector inp2; + size_t depth = this->GetDepth(); + + // Evaluate the gradients of the last layers in each deep net + for (size_t i = 0; i < nets.size(); i++) { + evaluateGradients(nets[i].GetLayerAt(depth - 1)->GetActivationGradientsAt(0), + nets[i].GetLossFunction(), batches[i].GetOutput(), + nets[i].GetLayerAt(depth - 1)->GetOutputAt(0), batches[i].GetWeights()); + } + + // Backpropagate the error in i'th layer of each deep net + for (size_t i = depth - 1; i > 0; i--) { + for (size_t j = 0; j < nets.size(); j++) { + nets[j].GetLayerAt(i)->Backward(nets[j].GetLayerAt(i - 1)->GetActivationGradients(), + nets[j].GetLayerAt(i - 1)->GetOutput(), inp1, inp2); + } + } + + std::vector dummy; + + // First layer of each deep net + for (size_t i = 0; i < nets.size(); i++) { + nets[i].GetLayerAt(0)->Backward(dummy, batches[i].GetInput(), inp1, inp2); + } + + // Update and copy + for (size_t i = 0; i < nets.size(); i++) { + for (size_t j = 0; j < depth; j++) { + Layer_t *masterLayer = this->GetLayerAt(j); + Layer_t *layer = nets[i].GetLayerAt(j); + + masterLayer->UpdateWeights(layer->GetWeightGradients(), learningRate); + layer->CopyWeights(masterLayer->GetWeights()); + + masterLayer->UpdateBiases(layer->GetBiasGradients(), learningRate); + layer->CopyBiases(masterLayer->GetBiases()); + } + } +} + +//______________________________________________________________________________ +template +auto TDeepNet::ParallelBackwardMomentum(std::vector> &nets, + std::vector> &batches, + Scalar_t learningRate, Scalar_t momentum) -> void +{ + std::vector inp1; + std::vector inp2; + size_t depth = this->GetDepth(); + + // Evaluate the gradients of the last layers in each deep net + for (size_t i = 0; i < nets.size(); i++) { + evaluateGradients(nets[i].GetLayerAt(depth - 1)->GetActivationGradientsAt(0), + nets[i].GetLossFunction(), batches[i].GetOutput(), + nets[i].GetLayerAt(depth - 1)->GetOutputAt(0), batches[i].GetWeights()); + } + + // Backpropagate the error in i'th layer of each deep net + for (size_t i = depth - 1; i > 0; i--) { + Layer_t *masterLayer = this->GetLayerAt(i); + + for (size_t j = 0; j < nets.size(); j++) { + Layer_t *layer = nets[j].GetLayerAt(i); + + layer->Backward(nets[j].GetLayerAt(i - 1)->GetActivationGradients(), nets[j].GetLayerAt(i - 1)->GetOutput(), + inp1, inp2); + masterLayer->UpdateWeightGradients(layer->GetWeightGradients(), learningRate / momentum); + masterLayer->UpdateBiasGradients(layer->GetBiasGradients(), learningRate / momentum); + } + + masterLayer->UpdateWeightGradients(masterLayer->GetWeightGradients(), 1.0 - momentum); + masterLayer->UpdateBiasGradients(masterLayer->GetBiasGradients(), 1.0 - momentum); + } + + std::vector dummy; + + // First layer of each deep net + Layer_t *masterFirstLayer = this->GetLayerAt(0); + for (size_t i = 0; i < nets.size(); i++) { + Layer_t *layer = nets[i].GetLayerAt(0); + + layer->Backward(dummy, batches[i].GetInput(), inp1, inp2); + + masterFirstLayer->UpdateWeightGradients(layer->GetWeightGradients(), learningRate / momentum); + masterFirstLayer->UpdateBiasGradients(layer->GetBiasGradients(), learningRate / momentum); + } + + masterFirstLayer->UpdateWeightGradients(masterFirstLayer->GetWeightGradients(), 1.0 - momentum); + masterFirstLayer->UpdateBiasGradients(masterFirstLayer->GetBiasGradients(), 1.0 - momentum); + + for (size_t i = 0; i < depth; i++) { + Layer_t *masterLayer = this->GetLayerAt(i); + masterLayer->Update(1.0); + + for (size_t j = 0; j < nets.size(); j++) { + Layer_t *layer = nets[j].GetLayerAt(i); + + layer->CopyWeights(masterLayer->GetWeights()); + layer->CopyBiases(masterLayer->GetBiases()); + } + } +} + +//______________________________________________________________________________ +template +auto TDeepNet::ParallelBackwardNestorov(std::vector> &nets, + std::vector> &batches, + Scalar_t learningRate, Scalar_t momentum) -> void +{ + std::cout << "Parallel Backward Nestorov" << std::endl; + std::vector inp1; + std::vector inp2; + size_t depth = this->GetDepth(); + + // Evaluate the gradients of the last layers in each deep net + for (size_t i = 0; i < nets.size(); i++) { + evaluateGradients(nets[i].GetLayerAt(depth - 1)->GetActivationGradientsAt(0), + nets[i].GetLossFunction(), batches[i].GetOutput(), + nets[i].GetLayerAt(depth - 1)->GetOutputAt(0), batches[i].GetWeights()); + } + + // Backpropagate the error in i'th layer of each deep net + for (size_t i = depth - 1; i > 0; i--) { + for (size_t j = 0; j < nets.size(); j++) { + Layer_t *layer = nets[j].GetLayerAt(i); + + layer->Backward(nets[j].GetLayerAt(i - 1)->GetActivationGradients(), nets[j].GetLayerAt(i - 1)->GetOutput(), + inp1, inp2); + } + } + + std::vector dummy; + + // First layer of each deep net + for (size_t i = 0; i < nets.size(); i++) { + Layer_t *layer = nets[i].GetLayerAt(0); + layer->Backward(dummy, batches[i].GetInput(), inp1, inp2); + } + + for (size_t i = 0; i < depth; i++) { + Layer_t *masterLayer = this->GetLayerAt(i); + for (size_t j = 0; j < nets.size(); j++) { + Layer_t *layer = nets[j].GetLayerAt(i); + + layer->CopyWeights(masterLayer->GetWeights()); + layer->CopyBiases(masterLayer->GetBiases()); + + layer->UpdateWeights(masterLayer->GetWeightGradients(), 1.0); + layer->UpdateBiases(masterLayer->GetBiasGradients(), 1.0); + } + + for (size_t j = 0; j < nets.size(); j++) { + Layer_t *layer = nets[j].GetLayerAt(i); + + masterLayer->UpdateWeightGradients(layer->GetWeightGradients(), learningRate / momentum); + masterLayer->UpdateBiasGradients(layer->GetBiasGradients(), learningRate / momentum); + } + + masterLayer->UpdateWeightGradients(masterLayer->GetWeightGradients(), 1.0 - momentum); + masterLayer->UpdateBiasGradients(masterLayer->GetBiasGradients(), 1.0 - momentum); + + masterLayer->Update(1.0); + } +} + +//______________________________________________________________________________ +template +auto TDeepNet::Update(Scalar_t learningRate) -> void +{ + for (size_t i = 0; i < fLayers.size(); i++) { + fLayers[i]->Update(learningRate); + } +} + +//______________________________________________________________________________ +template +auto TDeepNet::Loss(const Matrix_t &groundTruth, const Matrix_t &weights, + bool includeRegularization) const -> Scalar_t +{ + // Last layer should not be deep + auto loss = evaluate(this->GetLossFunction(), groundTruth, fLayers.back()->GetOutputAt(0), weights); + includeRegularization &= (this->GetRegularization() != ERegularization::kNone); + + if (includeRegularization) { + for (size_t i = 0; i < fLayers.size(); i++) { + for (size_t j = 0; j < (fLayers[i]->GetWeights()).size(); j++) { + loss += this->GetWeightDecay() * + regularization(fLayers[i]->GetWeightsAt(j), this->GetRegularization()); + } + } + } + + return loss; +} + +//______________________________________________________________________________ +template +auto TDeepNet::Loss(std::vector input, const Matrix_t &groundTruth, + const Matrix_t &weights, bool applyDropout, bool includeRegularization) + -> Scalar_t +{ + Forward(input, applyDropout); + return Loss(groundTruth, weights, includeRegularization); +} + +//______________________________________________________________________________ +template +auto TDeepNet::Prediction(Matrix_t &predictions, EOutputFunction f) const -> void +{ + // Last layer should not be deep + evaluate(predictions, f, fLayers.back()->GetOutputAt(0)); +} + +//______________________________________________________________________________ +template +auto TDeepNet::Prediction(Matrix_t &predictions, std::vector input, + EOutputFunction f) -> void +{ + Forward(input, false); + // Last layer should not be deep + evaluate(predictions, f, fLayers.back()->GetOutputAt(0)); +} + +//______________________________________________________________________________ +template +auto TDeepNet::Print() -> void +{ + std::cout << "DEEP NEURAL NETWORK:" << std::endl; + std::cout << "\t Loss function = " << static_cast(this->GetLossFunction()) << std::endl; + std::cout << "\t Network Depth = " << this->GetDepth() << std::endl; + std::cout << "\t Input depth = " << this->GetInputDepth() << std::endl; + std::cout << "\t Input height = " << this->GetInputHeight() << std::endl; + std::cout << "\t Input width = " << this->GetInputWidth() << std::endl; + std::cout << "\t Batch size = " << this->GetBatchSize() << std::endl; + + std::cout << "\t Layers: " << std::endl; + + for (size_t i = 0; i < fLayers.size(); i++) { + fLayers[i]->Print(); + } +} +} // namespace DNN +} // namespace TMVA + +#endif diff --git a/tmva/tmva/inc/TMVA/DNN/DenseLayer.h b/tmva/tmva/inc/TMVA/DNN/DenseLayer.h new file mode 100644 index 0000000000000..c244d11a4e3e6 --- /dev/null +++ b/tmva/tmva/inc/TMVA/DNN/DenseLayer.h @@ -0,0 +1,203 @@ + +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : TDenseLayer * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Dense Layer Class * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_DNN_DENSELAYER +#define TMVA_DNN_DENSELAYER + +#include "TMatrix.h" + +#include "TMVA/DNN/GeneralLayer.h" +#include "TMVA/DNN/Functions.h" + +#include + +namespace TMVA { +namespace DNN { +/** \class TDenseLayer + +Generic layer class. + +This generic layer class represents a dense layer of a neural network with +a given width n and activation function f. The activation function of each +layer is given by \f$\mathbf{u} = \mathbf{W}\mathbf{x} + \boldsymbol{\theta}\f$. + +In addition to the weight and bias matrices, each layer allocates memory +for its activations and the corresponding first partial fDerivatives of +the activation function as well as the gradients of the weights and biases. + +The layer provides member functions for the forward propagation of +activations through the given layer. +*/ +template +class TDenseLayer : public VGeneralLayer { +public: + using Scalar_t = typename Architecture_t::Scalar_t; + using Matrix_t = typename Architecture_t::Matrix_t; + +private: + std::vector fDerivatives; ///< First fDerivatives of the activations of this layer. + + Scalar_t fDropoutProbability; ///< Probability that an input is active. + + EActivationFunction fF; ///< Activation function of the layer. + ERegularization fReg; ///< The regularization method. + Scalar_t fWeightDecay; ///< The weight decay. + +public: + /*! Constructor */ + TDenseLayer(size_t BatchSize, size_t InputWidth, size_t Width, EInitialization init, Scalar_t DropoutProbability, + EActivationFunction f, ERegularization reg, Scalar_t weightDecay); + + /*! Copy the dense layer provided as a pointer */ + TDenseLayer(TDenseLayer *layer); + + /*! Copy Constructor */ + TDenseLayer(const TDenseLayer &); + + /*! Destructor */ + ~TDenseLayer(); + + /*! Compute activation of the layer for the given input. The input + * must be in 3D tensor form with the different matrices corresponding to + * different events in the batch. Computes activations as well as + * the first partial derivative of the activation function at those + * activations. */ + void Forward(std::vector &input, bool applyDropout = false); + + /*! Compute weight, bias and activation gradients. Uses the precomputed + * first partial derviatives of the activation function computed during + * forward propagation and modifies them. Must only be called directly + * a the corresponding call to Forward(...). */ + void Backward(std::vector &gradients_backward, const std::vector &activations_backward, + std::vector &inp1, std::vector &inp2); + + /*! Printing the layer info. */ + void Print() const; + + /*! Getters */ + Scalar_t GetDropoutProbability() const { return fDropoutProbability; } + + const std::vector &GetDerivatives() const { return fDerivatives; } + std::vector &GetDerivatives() { return fDerivatives; } + + Matrix_t &GetDerivativesAt(size_t i) { return fDerivatives[i]; } + const Matrix_t &GetDerivativesAt(size_t i) const { return fDerivatives[i]; } + + EActivationFunction GetActivationFunction() const { return fF; } + ERegularization GetRegularization() const { return fReg; } + Scalar_t GetWeightDecay() const { return fWeightDecay; } +}; + +// +// +// The Dense Layer Class - Implementation +//______________________________________________________________________________ +template +TDenseLayer::TDenseLayer(size_t batchSize, size_t inputWidth, size_t width, EInitialization init, + Scalar_t dropoutProbability, EActivationFunction f, ERegularization reg, + Scalar_t weightDecay) + : VGeneralLayer(batchSize, 1, 1, inputWidth, 1, 1, width, 1, width, inputWidth, 1, width, 1, 1, + batchSize, width, init), + fDerivatives(), fDropoutProbability(dropoutProbability), fF(f), fReg(reg), fWeightDecay(weightDecay) +{ + fDerivatives.emplace_back(batchSize, width); +} + +//______________________________________________________________________________ +template +TDenseLayer::TDenseLayer(TDenseLayer *layer) + : VGeneralLayer(layer), fDerivatives(), fDropoutProbability(layer->GetDropoutProbability()), + fF(layer->GetActivationFunction()), fReg(layer->GetRegularization()), fWeightDecay(layer->GetWeightDecay()) +{ + fDerivatives.emplace_back(layer->GetBatchSize(), layer->GetWidth()); +} + +//______________________________________________________________________________ +template +TDenseLayer::TDenseLayer(const TDenseLayer &layer) + : VGeneralLayer(layer), fDerivatives(), fDropoutProbability(layer.fDropoutProbability), fF(layer.fF), + fReg(layer.fReg), fWeightDecay(layer.fWeightDecay) +{ + fDerivatives.emplace_back(layer.fBatchSize, layer.fWidth); +} + +//______________________________________________________________________________ +template +TDenseLayer::~TDenseLayer() +{ + // Nothing to do here. +} + +//______________________________________________________________________________ +template +auto TDenseLayer::Forward(std::vector &input, bool applyDropout) -> void +{ + if (applyDropout && (this->GetDropoutProbability() != 1.0)) { + Architecture_t::Dropout(input[0], this->GetDropoutProbability()); + } + + Architecture_t::MultiplyTranspose(this->GetOutputAt(0), input[0], this->GetWeightsAt(0)); + Architecture_t::AddRowWise(this->GetOutputAt(0), this->GetBiasesAt(0)); + evaluateDerivative(this->GetDerivativesAt(0), this->GetActivationFunction(), this->GetOutputAt(0)); + evaluate(this->GetOutputAt(0), this->GetActivationFunction()); +} + +//______________________________________________________________________________ +template +auto TDenseLayer::Backward(std::vector &gradients_backward, + const std::vector &activations_backward, + std::vector & /*inp1*/, std::vector & + /*inp2*/) -> void +{ + if (gradients_backward.size() == 0) { + Matrix_t dummy(0, 0); + Architecture_t::Backward(dummy, this->GetWeightGradientsAt(0), this->GetBiasGradientsAt(0), + this->GetDerivativesAt(0), this->GetActivationGradientsAt(0), this->GetWeightsAt(0), + activations_backward[0]); + + } else { + Architecture_t::Backward(gradients_backward[0], this->GetWeightGradientsAt(0), this->GetBiasGradientsAt(0), + this->GetDerivativesAt(0), this->GetActivationGradientsAt(0), this->GetWeightsAt(0), + activations_backward[0]); + } + + addRegularizationGradients(this->GetWeightGradientsAt(0), this->GetWeightsAt(0), + this->GetWeightDecay(), this->GetRegularization()); +} + +//______________________________________________________________________________ +template +void TDenseLayer::Print() const +{ + std::cout << "Rows = " << this->GetWeightsAt(0).GetNrows(); + std::cout << "Cols = " << this->GetWeightsAt(0).GetNcols(); + std::cout << ", Activation Function = "; + std::cout << static_cast(fF) << std::endl; +} +} // namespace DNN +} // namespace TMVA + +#endif diff --git a/tmva/tmva/inc/TMVA/DNN/GeneralLayer.h b/tmva/tmva/inc/TMVA/DNN/GeneralLayer.h new file mode 100644 index 0000000000000..bdd7d0d54781a --- /dev/null +++ b/tmva/tmva/inc/TMVA/DNN/GeneralLayer.h @@ -0,0 +1,439 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : TGeneralLayer * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * General Deep Neural Network Layer * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_DNN_GENERALLAYER +#define TMVA_DNN_GENERALLAYER + +#include + +namespace TMVA { +namespace DNN { + +/** \class VGeneralLayer + Generic General Layer class. + + This class represents the general class for all layers in the Deep Learning + Module. + */ +template +class VGeneralLayer { + using Matrix_t = typename Architecture_t::Matrix_t; + using Scalar_t = typename Architecture_t::Scalar_t; + +protected: + size_t fBatchSize; ///< Batch size used for training and evaluation + + size_t fInputDepth; ///< The depth of the previous layer or input. + size_t fInputHeight; ///< The height of the previous layer or input. + size_t fInputWidth; ///< The width of the previous layer or input. + + size_t fDepth; ///< The depth of the layer. + size_t fHeight; ///< The height of the layer. + size_t fWidth; ///< The width of this layer. + + bool fIsTraining; ///< Flag indicatig the mode + + std::vector fWeights; ///< The weights associated to the layer. + std::vector fBiases; ///< The biases associated to the layer. + + std::vector fWeightGradients; ///< Gradients w.r.t. the weights of the layer. + std::vector fBiasGradients; ///< Gradients w.r.t. the bias values of the layer. + + std::vector fOutput; ///< Activations of this layer. + std::vector fActivationGradients; ///< Gradients w.r.t. the activations of this layer. + + EInitialization fInit; ///< The initialization method. + +public: + /*! Constructor */ + VGeneralLayer(size_t BatchSize, size_t InputDepth, size_t InputHeight, size_t InputWidth, size_t Depth, + size_t Height, size_t Width, size_t WeightsNSlices, size_t WeightsNRows, size_t WeightsNCols, + size_t BiasesNSlices, size_t BiasesNRows, size_t BiasesNCols, size_t OutputNSlices, size_t OutputNRows, + size_t OutputNCols, EInitialization Init); + + /*! General Constructor with different weights dimension */ + VGeneralLayer(size_t BatchSize, size_t InputDepth, size_t InputHeight, size_t InputWidth, size_t Depth, + size_t Height, size_t Width, size_t WeightsNSlices, std::vector WeightsNRows, + std::vector WeightsNCols, size_t BiasesNSlices, std::vector BiasesNRows, + std::vector BiasesNCols, size_t OutputNSlices, size_t OutputNRows, size_t OutputNCols, + EInitialization Init); + + /*! Copy the layer provided as a pointer */ + VGeneralLayer(VGeneralLayer *layer); + + /*! Copy Constructor */ + VGeneralLayer(const VGeneralLayer &); + + /*! Virtual Destructor. */ + virtual ~VGeneralLayer(); + + /*! Initialize the weights and biases according to the given initialization method. */ + void Initialize(); + + /*! Computes activation of the layer for the given input. The input + * must be in 3D tensor form with the different matrices corresponding to + * different events in the batch. */ + virtual void Forward(std::vector &input, bool applyDropout = false) = 0; + + /*! Backpropagates the error. Must only be called directly at the corresponding + * call to Forward(...). */ + virtual void Backward(std::vector &gradients_backward, const std::vector &activations_backward, + std::vector &inp1, std::vector &inp2) = 0; + + /*! Updates the weights and biases, given the learning rate */ + void Update(const Scalar_t learningRate); + + /*! Updates the weights, given the gradients and the learning rate, */ + void UpdateWeights(const std::vector &weightGradients, const Scalar_t learningRate); + + /*! Updates the biases, given the gradients and the learning rate. */ + void UpdateBiases(const std::vector &biasGradients, const Scalar_t learningRate); + + /*! Updates the weight gradients, given some other weight gradients and learning rate. */ + void UpdateWeightGradients(const std::vector &weightGradients, const Scalar_t learningRate); + + /*! Updates the bias gradients, given some other weight gradients and learning rate. */ + void UpdateBiasGradients(const std::vector &biasGradients, const Scalar_t learningRate); + + /*! Copies the weights provided as an input. */ + void CopyWeights(const std::vector &otherWeights); + + /*! Copies the biases provided as an input. */ + void CopyBiases(const std::vector &otherBiases); + + /*! Prints the info about the layer. */ + virtual void Print() const = 0; + + /*! Getters */ + size_t GetBatchSize() const { return fBatchSize; } + size_t GetInputDepth() const { return fInputDepth; } + size_t GetInputHeight() const { return fInputHeight; } + size_t GetInputWidth() const { return fInputWidth; } + size_t GetDepth() const { return fDepth; } + size_t GetHeight() const { return fHeight; } + size_t GetWidth() const { return fWidth; } + bool IsTraining() const { return fIsTraining; } + + const std::vector &GetWeights() const { return fWeights; } + std::vector &GetWeights() { return fWeights; } + + const Matrix_t &GetWeightsAt(size_t i) const { return fWeights[i]; } + Matrix_t &GetWeightsAt(size_t i) { return fWeights[i]; } + + const std::vector &GetBiases() const { return fBiases; } + std::vector &GetBiases() { return fBiases; } + + const Matrix_t &GetBiasesAt(size_t i) const { return fBiases[i]; } + Matrix_t &GetBiasesAt(size_t i) { return fBiases[i]; } + + const std::vector &GetWeightGradients() const { return fWeightGradients; } + std::vector &GetWeightGradients() { return fWeightGradients; } + + const Matrix_t &GetWeightGradientsAt(size_t i) const { return fWeightGradients[i]; } + Matrix_t &GetWeightGradientsAt(size_t i) { return fWeightGradients[i]; } + + const std::vector &GetBiasGradients() const { return fBiasGradients; } + std::vector &GetBiasGradients() { return fBiasGradients; } + + const Matrix_t &GetBiasGradientsAt(size_t i) const { return fBiasGradients[i]; } + Matrix_t &GetBiasGradientsAt(size_t i) { return fBiasGradients[i]; } + + const std::vector &GetOutput() const { return fOutput; } + std::vector &GetOutput() { return fOutput; } + + const std::vector &GetActivationGradients() const { return fActivationGradients; } + std::vector &GetActivationGradients() { return fActivationGradients; } + + Matrix_t &GetOutputAt(size_t i) { return fOutput[i]; } + const Matrix_t &GetOutputAt(size_t i) const { return fOutput[i]; } + + Matrix_t &GetActivationGradientsAt(size_t i) { return fActivationGradients[i]; } + const Matrix_t &GetActivationGradientsAt(size_t i) const { return fActivationGradients[i]; } + + EInitialization GetInitialization() const { return fInit; } + + /*! Setters */ + void SetBatchSize(size_t batchSize) { fBatchSize = batchSize; } + void SetInputDepth(size_t inputDepth) { fInputDepth = inputDepth; } + void SetInputHeight(size_t inputHeight) { fInputHeight = inputHeight; } + void SetInputWidth(size_t inputWidth) { fInputWidth = inputWidth; } + void SetDepth(size_t depth) { fDepth = depth; } + void SetHeight(size_t height) { fHeight = height; } + void SetWidth(size_t width) { fWidth = width; } + void SetIsTraining(bool isTraining) { fIsTraining = isTraining; } +}; + +// +// +// The General Layer Class - Implementation +//_________________________________________________________________________________________________ +template +VGeneralLayer::VGeneralLayer(size_t batchSize, size_t inputDepth, size_t inputHeight, size_t inputWidth, + size_t depth, size_t height, size_t width, size_t weightsNSlices, + size_t weightsNRows, size_t weightsNCols, size_t biasesNSlices, + size_t biasesNRows, size_t biasesNCols, size_t outputNSlices, + size_t outputNRows, size_t outputNCols, EInitialization init) + : fBatchSize(batchSize), fInputDepth(inputDepth), fInputHeight(inputHeight), fInputWidth(inputWidth), fDepth(depth), + fHeight(height), fWidth(width), fIsTraining(true), fWeights(), fBiases(), fWeightGradients(), fBiasGradients(), + fOutput(), fActivationGradients(), fInit(init) +{ + + for (size_t i = 0; i < weightsNSlices; i++) { + fWeights.emplace_back(weightsNRows, weightsNCols); + fWeightGradients.emplace_back(weightsNRows, weightsNCols); + } + + for (size_t i = 0; i < biasesNSlices; i++) { + fBiases.emplace_back(biasesNRows, biasesNCols); + fBiasGradients.emplace_back(biasesNRows, biasesNCols); + } + + for (size_t i = 0; i < outputNSlices; i++) { + fOutput.emplace_back(outputNRows, outputNCols); + fActivationGradients.emplace_back(outputNRows, outputNCols); + } +} + +//_________________________________________________________________________________________________ +template +VGeneralLayer::VGeneralLayer(size_t batchSize, size_t inputDepth, size_t inputHeight, size_t inputWidth, + size_t depth, size_t height, size_t width, size_t weightsNSlices, + std::vector weightsNRows, std::vector weightsNCols, + size_t biasesNSlices, std::vector biasesNRows, + std::vector biasesNCols, size_t outputNSlices, size_t outputNRows, + size_t outputNCols, EInitialization init) + : fBatchSize(batchSize), fInputDepth(inputDepth), fInputHeight(inputHeight), fInputWidth(inputWidth), fDepth(depth), + fHeight(height), fWidth(width), fIsTraining(true), fWeights(), fBiases(), fWeightGradients(), fBiasGradients(), + fOutput(), fActivationGradients(), fInit(init) +{ + + for (size_t i = 0; i < weightsNSlices; i++) { + fWeights.emplace_back(weightsNRows[i], weightsNCols[i]); + fWeightGradients.emplace_back(weightsNRows[i], weightsNCols[i]); + } + + for (size_t i = 0; i < biasesNSlices; i++) { + fBiases.emplace_back(biasesNRows[i], biasesNCols[i]); + fBiasGradients.emplace_back(biasesNRows[i], biasesNCols[i]); + } + + for (size_t i = 0; i < outputNSlices; i++) { + fOutput.emplace_back(outputNRows, outputNCols); + fActivationGradients.emplace_back(outputNRows, outputNCols); + } +} + +//_________________________________________________________________________________________________ +template +VGeneralLayer::VGeneralLayer(VGeneralLayer *layer) + : fBatchSize(layer->GetBatchSize()), fInputDepth(layer->GetInputDepth()), fInputHeight(layer->GetInputHeight()), + fInputWidth(layer->GetInputWidth()), fDepth(layer->GetDepth()), fHeight(layer->GetHeight()), + fWidth(layer->GetWidth()), fIsTraining(layer->IsTraining()), fWeights(), fBiases(), fWeightGradients(), + fBiasGradients(), fOutput(), fActivationGradients(), fInit(layer->GetInitialization()) +{ + size_t weightsNSlices = (layer->GetWeights()).size(); + size_t weightsNRows = 0; + size_t weightsNCols = 0; + + for (size_t i = 0; i < weightsNSlices; i++) { + weightsNRows = (layer->GetWeightsAt(i)).GetNrows(); + weightsNCols = (layer->GetWeightsAt(i)).GetNcols(); + + fWeights.emplace_back(weightsNRows, weightsNCols); + fWeightGradients.emplace_back(weightsNRows, weightsNCols); + + Architecture_t::Copy(fWeights[i], layer->GetWeightsAt(i)); + } + + size_t biasesNSlices = (layer->GetBiases()).size(); + size_t biasesNRows = 0; + size_t biasesNCols = 0; + + for (size_t i = 0; i < biasesNSlices; i++) { + biasesNRows = (layer->GetBiasesAt(i)).GetNrows(); + biasesNCols = (layer->GetBiasesAt(i)).GetNcols(); + + fBiases.emplace_back(biasesNRows, biasesNCols); + fBiasGradients.emplace_back(biasesNRows, biasesNCols); + + Architecture_t::Copy(fBiases[i], layer->GetBiasesAt(i)); + } + + size_t outputNSlices = (layer->GetOutput()).size(); + size_t outputNRows = 0; + size_t outputNCols = 0; + + for (size_t i = 0; i < outputNSlices; i++) { + outputNRows = (layer->GetOutputAt(i)).GetNrows(); + outputNCols = (layer->GetOutputAt(i)).GetNcols(); + + fOutput.emplace_back(outputNRows, outputNCols); + fActivationGradients.emplace_back(outputNRows, outputNCols); + } +} + +//_________________________________________________________________________________________________ +template +VGeneralLayer::VGeneralLayer(const VGeneralLayer &layer) + : fBatchSize(layer.fBatchSize), fInputDepth(layer.fInputDepth), fInputHeight(layer.fInputHeight), + fInputWidth(layer.fInputWidth), fDepth(layer.fDepth), fHeight(layer.fHeight), fWidth(layer.fWidth), + fIsTraining(layer.fIsTraining), fWeights(), fBiases(), fWeightGradients(), fBiasGradients(), fOutput(), + fActivationGradients(), fInit(layer.fInit) +{ + size_t weightsNSlices = layer.fWeights.size(); + size_t weightsNRows = 0; + size_t weightsNCols = 0; + + for (size_t i = 0; i < weightsNSlices; i++) { + weightsNRows = (layer.fWeights[i]).GetNrows(); + weightsNCols = (layer.fWeights[i]).GetNcols(); + + fWeights.emplace_back(weightsNRows, weightsNCols); + fWeightGradients.emplace_back(weightsNRows, weightsNCols); + + Architecture_t::Copy(fWeights[i], layer.fWeights[i]); + } + + size_t biasesNSlices = layer.fBiases.size(); + size_t biasesNRows = 0; + size_t biasesNCols = 0; + + for (size_t i = 0; i < biasesNSlices; i++) { + biasesNRows = (layer.fBiases[i]).GetNrows(); + biasesNCols = (layer.fBiases[i]).GetNcols(); + + fBiases.emplace_back(biasesNRows, biasesNCols); + fBiasGradients.emplace_back(biasesNRows, biasesNCols); + + Architecture_t::Copy(fBiases[i], layer.fBiases[i]); + } + + size_t outputNSlices = layer.fOutput.size(); + size_t outputNRows = 0; + size_t outputNCols = 0; + + for (size_t i = 0; i < outputNSlices; i++) { + outputNRows = (layer.fOutput[i]).GetNrows(); + outputNCols = (layer.fOutput[i]).GetNcols(); + + fOutput.emplace_back(outputNRows, outputNCols); + fActivationGradients.emplace_back(outputNRows, outputNCols); + } +} + +//_________________________________________________________________________________________________ +template +VGeneralLayer::~VGeneralLayer() +{ + // Nothing to do here. +} + +//_________________________________________________________________________________________________ +template +auto VGeneralLayer::Initialize() -> void +{ + for (size_t i = 0; i < fWeights.size(); i++) { + initialize(fWeights[i], this->GetInitialization()); + initialize(fWeightGradients[i], EInitialization::kZero); + } + + for (size_t i = 0; i < fBiases.size(); i++) { + initialize(fBiases[i], EInitialization::kZero); + initialize(fBiasGradients[i], EInitialization::kZero); + } +} + +//_________________________________________________________________________________________________ +template +auto VGeneralLayer::Update(const Scalar_t learningRate) -> void +{ + this->UpdateWeights(fWeightGradients, learningRate); + this->UpdateBiases(fBiasGradients, learningRate); +} + +//_________________________________________________________________________________________________ +template +auto VGeneralLayer::UpdateWeights(const std::vector &weightGradients, + const Scalar_t learningRate) -> void +{ + for (size_t i = 0; i < fWeights.size(); i++) { + Architecture_t::ScaleAdd(fWeights[i], weightGradients[i], -learningRate); + } +} + +//_________________________________________________________________________________________________ +template +auto VGeneralLayer::UpdateBiases(const std::vector &biasGradients, + const Scalar_t learningRate) -> void +{ + for (size_t i = 0; i < fBiases.size(); i++) { + Architecture_t::ScaleAdd(fBiases[i], biasGradients[i], -learningRate); + } +} + +//_________________________________________________________________________________________________ +template +auto VGeneralLayer::UpdateWeightGradients(const std::vector &weightGradients, + const Scalar_t learningRate) -> void +{ + for (size_t i = 0; i < fWeightGradients.size(); i++) { + Architecture_t::ScaleAdd(fWeightGradients[i], weightGradients[i], -learningRate); + } +} + +//_________________________________________________________________________________________________ +template +auto VGeneralLayer::UpdateBiasGradients(const std::vector &biasGradients, + const Scalar_t learningRate) -> void +{ + for (size_t i = 0; i < fBiasGradients.size(); i++) { + Architecture_t::ScaleAdd(fBiasGradients[i], biasGradients[i], -learningRate); + } +} + +//_________________________________________________________________________________________________ +template +auto VGeneralLayer::CopyWeights(const std::vector &otherWeights) -> void +{ + + for (size_t i = 0; i < fWeights.size(); i++) { + Architecture_t::Copy(fWeights[i], otherWeights[i]); + } +} + +//_________________________________________________________________________________________________ +template +auto VGeneralLayer::CopyBiases(const std::vector &otherBiases) -> void +{ + for (size_t i = 0; i < fBiases.size(); i++) { + Architecture_t::Copy(fBiases[i], otherBiases[i]); + } +} + +} // namespace DNN +} // namespace TMVA + +#endif diff --git a/tmva/tmva/inc/TMVA/DNN/RNN/RNNLayer.h b/tmva/tmva/inc/TMVA/DNN/RNN/RNNLayer.h new file mode 100644 index 0000000000000..759ebbe0d5e18 --- /dev/null +++ b/tmva/tmva/inc/TMVA/DNN/RNN/RNNLayer.h @@ -0,0 +1,347 @@ +// @(#)root/tmva/tmva/dnn/rnn:$Id$ +// Author: Saurav Shekhar 19/07/17 + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : BasicRNNLayer * + * * + * Description: * + * NeuralNetwork * + * * + * Authors (alphabetical): * + * Saurav Shekhar - ETH Zurich, Switzerland * + * * + * Copyright (c) 2005-2015: * + * All rights reserved. * + * CERN, Switzerland * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + **********************************************************************************/ + +//#pragma once + +////////////////////////////////////////////////////////////////////// +// // +////////////////////////////////////////////////////////////////////// + +#ifndef TMVA_DNN_RNN_LAYER +#define TMVA_DNN_RNN_LAYER + +#include +#include +#include + +#include "TMatrix.h" +#include "TMVA/DNN/Functions.h" + +namespace TMVA +{ +namespace DNN +{ +namespace RNN +{ + +//______________________________________________________________________________ +// +// Basic RNN Layer +//______________________________________________________________________________ + +/** \class BasicRNNLayer + Generic implementation +*/ +template + class TBasicRNNLayer : public VGeneralLayer +{ + +public: + + using Matrix_t = typename Architecture_t::Matrix_t; + using Scalar_t = typename Architecture_t::Scalar_t; + using Tensor_t = std::vector; + +private: + + size_t fTimeSteps; ///< Timesteps for RNN + size_t fStateSize; ///< Hidden state size of RNN + bool fRememberState; ///< Remember state in next pass + + DNN::EActivationFunction fF; ///< Activation function of the hidden state + + Matrix_t fState; ///< Hidden State + Matrix_t &fWeightsInput; ///< Input weights, fWeights[0] + Matrix_t &fWeightsState; ///< Prev state weights, fWeights[1] + Matrix_t &fBiases; ///< Biases + + Matrix_t fDerivatives; ///< First fDerivatives of the activations + Matrix_t &fWeightInputGradients; ///< Gradients w.r.t. the input weights + Matrix_t &fWeightStateGradients; ///< Gradients w.r.t. the recurring weights + Matrix_t &fBiasGradients; ///< Gradients w.r.t. the bias values + +public: + + /** Constructor */ + TBasicRNNLayer(size_t batchSize, size_t stateSize, size_t inputSize, + size_t timeSteps, bool rememberState = false, + DNN::EActivationFunction f = DNN::EActivationFunction::kTanh, + bool training = true, DNN::EInitialization fA = DNN::EInitialization::kZero); + + /** Copy Constructor */ + TBasicRNNLayer(const TBasicRNNLayer &); + + /*! Initialize the weights according to the given initialization + ** method. */ + //void Initialize(DNN::EInitialization m); + + /*! Initialize the state + ** method. */ + void InitState(DNN::EInitialization m = DNN::EInitialization::kZero); + + /*! Compute and return the next state with given input + * matrix */ + void Forward(Tensor_t &input, bool isTraining = true); + + /*! Forward for a single cell (time unit) */ + void CellForward(Matrix_t &input); + + /*! Backpropagates the error. Must only be called directly at the corresponding + * call to Forward(...). */ + void Backward(Tensor_t &gradients_backward, + const Tensor_t &activations_backward, + std::vector &inp1, + std::vector &inp2); + + /* Updates weights and biases, given the learning rate */ + void Update(const Scalar_t learningRate); + + /*! Backward for a single time unit + * a the corresponding call to Forward(...). */ + inline Matrix_t & CellBackward(Matrix_t & state_gradients_backward, + const Matrix_t & precStateActivations, const Matrix_t & currStateActivations, + const Matrix_t & input, Matrix_t & input_gradient); + + /** Prints the info about the layer */ + void Print() const; + + /** Getters */ + size_t GetTimeSteps() const { return fTimeSteps; } + size_t GetStateSize() const { return fStateSize; } + size_t GetInputSize() const { return this->GetInputWidth(); } + inline bool IsRememberState() const {return fRememberState;} + inline DNN::EActivationFunction GetActivationFunction() const {return fF;} + Matrix_t & GetState() {return fState;} + const Matrix_t & GetState() const {return fState;} + Matrix_t & GetWeightsInput() {return fWeightsInput;} + const Matrix_t & GetWeightsInput() const {return fWeightsInput;} + Matrix_t & GetWeightsState() {return fWeightsState;} + const Matrix_t & GetWeightsState() const {return fWeightsState;} + Matrix_t & GetDerivatives() {return fDerivatives;} + const Matrix_t & GetDerivatives() const {return fDerivatives;} + //Matrix_t & GetBiases() {return fBiases;} + //const Matrix_t & GetBiases() const {return fBiases;} + //Matrix_t & GetBiasGradients() {return fBiasGradients;} + //const Matrix_t & GetBiasGradients() const {return fBiasGradients;} + Matrix_t & GetWeightInputGradients() {return fWeightInputGradients;} + const Matrix_t & GetWeightInputGradients() const {return fWeightInputGradients;} + Matrix_t & GetWeightStateGradients() {return fWeightStateGradients;} + const Matrix_t & GetWeightStateGradients() const {return fWeightStateGradients;} +}; + +//______________________________________________________________________________ +// +// BasicRNNLayer Implementation +//______________________________________________________________________________ +template +TBasicRNNLayer::TBasicRNNLayer(size_t batchSize, size_t stateSize, size_t inputSize, size_t timeSteps, + bool rememberState, DNN::EActivationFunction f, bool /*training*/, + DNN::EInitialization fA) + // TODO inputDepth and outputDepth changed to batchSize?? + : VGeneralLayer(batchSize, 1, timeSteps, inputSize, 1, timeSteps, stateSize, 2, + {stateSize, stateSize}, {inputSize, stateSize}, 1, {stateSize}, {1}, batchSize, + timeSteps, stateSize, fA), + fTimeSteps(timeSteps), + fStateSize(stateSize), + fRememberState(rememberState), + fF(f), + fState(batchSize, stateSize), + fWeightsInput(this->GetWeightsAt(0)), + fWeightsState(this->GetWeightsAt(1)), + fBiases(this->GetBiasesAt(0)), + fDerivatives(batchSize, stateSize), + fWeightInputGradients(this->GetWeightGradientsAt(0)), + fWeightStateGradients(this->GetWeightGradientsAt(1)), + fBiasGradients(this->GetBiasGradientsAt(0)) +{ + // Nothing +} + +//______________________________________________________________________________ +template +TBasicRNNLayer::TBasicRNNLayer(const TBasicRNNLayer &layer) + : VGeneralLayer(layer), fTimeSteps(layer.fTimeSteps), fStateSize(layer.fStateSize), + fRememberState(layer.fRememberState), fF(layer.GetActivationFunction()), + fState(layer.GetBatchSize(), layer.GetStateSize()), fWeightsInput(this->GetWeightsAt(0)), + fWeightsState(this->GetWeightsAt(1)), fBiases(this->GetBiasesAt(0)), + fDerivatives(layer.GetBatchSize(), layer.GetStateSize()), fWeightInputGradients(this->GetWeightGradientsAt(0)), + fWeightStateGradients(this->GetWeightGradientsAt(1)), fBiasGradients(this->GetBiasGradientsAt(0)) +{ + // Gradient matrices not copied + Architecture_t::Copy(fState, layer.GetState()); + Architecture_t::Copy(fDerivatives, layer.GetDerivatives()); +} + +//______________________________________________________________________________ +//template +//auto TBasicRNNLayer::Initialize(DNN::EInitialization m) +//-> void +//{ +// DNN::initialize(fWeightsInput, m); +// DNN::initialize(fWeightsState, m); +// DNN::initialize(fBiases, DNN::EInitialization::kZero); +//} + +//______________________________________________________________________________ +template +auto TBasicRNNLayer::InitState(DNN::EInitialization /*m*/) -> void +{ + DNN::initialize(this->GetState(), DNN::EInitialization::kZero); +} + +//______________________________________________________________________________ +template +auto TBasicRNNLayer::Print() const +-> void +{ + std::cout << "Batch Size: " << this->GetBatchSize() << "\n" + << "Input Size: " << this->GetInputSize() << "\n" + << "Hidden State Size: " << this->GetStateSize() << "\n"; +} + +template +auto debugMatrix(const typename Architecture_t::Matrix_t &A, const std::string name = "matrix") +-> void +{ + std::cout << name << "\n"; + for (size_t i = 0; i < A.GetNrows(); ++i) { + for (size_t j = 0; j < A.GetNcols(); ++j) { + std::cout << A(i, j) << " "; + } + std::cout << "\n"; + } + std::cout << "********\n"; +} + + +//______________________________________________________________________________ +template +auto inline TBasicRNNLayer::Forward(Tensor_t &input, bool /*isTraining*/) // B x T x D + -> void +{ + Tensor_t arrInput; + for (size_t t = 0; t < fTimeSteps; ++t) arrInput.emplace_back(this->GetBatchSize(), this->GetInputWidth()); // T x B x D + Architecture_t::Rearrange(arrInput, input); + Tensor_t arrOutput; + for (size_t t = 0; t < fTimeSteps;++t) arrOutput.emplace_back(this->GetBatchSize(), fStateSize); // T x B x H + + if (!this->fRememberState) InitState(DNN::EInitialization::kZero); + for (size_t t = 0; t < fTimeSteps; ++t) { + CellForward(arrInput[t]); + Architecture_t::Copy(arrOutput[t], fState); + } + Architecture_t::Rearrange(this->GetOutput(), arrOutput); // B x T x D +} + +//______________________________________________________________________________ +template +auto inline TBasicRNNLayer::CellForward(Matrix_t &input) +-> void +{ + // State = act(W_input . input + W_state . state + bias) + const DNN::EActivationFunction fAF = this->GetActivationFunction(); + Matrix_t tmpState(fState.GetNrows(), fState.GetNcols()); + Architecture_t::MultiplyTranspose(tmpState, fState, fWeightsState); + Architecture_t::MultiplyTranspose(fState, input, fWeightsInput); + Architecture_t::ScaleAdd(fState, tmpState); + Architecture_t::AddRowWise(fState, fBiases); + DNN::evaluate(fState, fAF); +} + +//____________________________________________________________________________ +template +auto inline TBasicRNNLayer::Backward(Tensor_t &gradients_backward, // B x T x D + const Tensor_t &activations_backward, // B x T x D + std::vector & /*inp1*/, std::vector & + /*inp2*/) -> void +{ + // activations backward is input + // gradients_backward is activationGradients of layer before it, which is input layer + // currently gradient_backward is for input(x) and not for state + // TODO use this to change initial state?? + + bool dummy = false; + if (gradients_backward.size() == 0 || gradients_backward[0].GetNrows() == 0 || gradients_backward[0].GetNcols() == 0) { + dummy = true; + } + Tensor_t arr_gradients_backward; + for (size_t t = 0; t < fTimeSteps; ++t) arr_gradients_backward.emplace_back(this->GetBatchSize(), this->GetInputSize()); // T x B x D + + if (!dummy) { + // TODO gradients_backward will be written back on the matrix + //Architecture_t::Rearrange(arr_gradients_backward, gradients_backward); + } + Tensor_t arr_activations_backward; + for (size_t t = 0; t < fTimeSteps; ++t) arr_activations_backward.emplace_back(this->GetBatchSize(), this->GetInputSize()); // T x B x D + Architecture_t::Rearrange(arr_activations_backward, activations_backward); + + Matrix_t state_gradients_backward(this->GetBatchSize(), fStateSize); // B x H + DNN::initialize(state_gradients_backward, DNN::EInitialization::kZero); + + Matrix_t initState(this->GetBatchSize(), fStateSize); // B x H + DNN::initialize(initState, DNN::EInitialization::kZero); + + Tensor_t arr_output; + for (size_t t = 0; t < fTimeSteps; ++t) arr_output.emplace_back(this->GetBatchSize(), fStateSize); + Architecture_t::Rearrange(arr_output, this->GetOutput()); + + Tensor_t arr_actgradients; + for (size_t t = 0; t < fTimeSteps; ++t) arr_actgradients.emplace_back(this->GetBatchSize(), fStateSize); + Architecture_t::Rearrange(arr_actgradients, this->GetActivationGradients()); + + for (size_t t = fTimeSteps; t > 0; t--) { + const Matrix_t & currStateActivations = arr_output[t - 1]; + Architecture_t::ScaleAdd(state_gradients_backward, arr_actgradients[t - 1]); + if (t > 1) { + const Matrix_t & precStateActivations = arr_output[t - 2]; + CellBackward(state_gradients_backward, precStateActivations, currStateActivations, arr_activations_backward[t - 1], + arr_gradients_backward[t - 1]); + } else { + const Matrix_t & precStateActivations = initState; + CellBackward(state_gradients_backward, precStateActivations, currStateActivations, arr_activations_backward[t - 1], + arr_gradients_backward[t - 1]); + } + } + if (!dummy) { + Architecture_t::Rearrange(arr_gradients_backward, gradients_backward); + } + //Architecture_t::Rearrange(arr_activations_backward, activations_backward); +} + +//______________________________________________________________________________ +template +auto inline TBasicRNNLayer::CellBackward(Matrix_t & state_gradients_backward, + const Matrix_t & precStateActivations, const Matrix_t & currStateActivations, + const Matrix_t & input, Matrix_t & input_gradient) +-> Matrix_t & +{ + DNN::evaluateDerivative(fDerivatives, this->GetActivationFunction(), currStateActivations); + return Architecture_t::RecurrentLayerBackward(state_gradients_backward, fWeightInputGradients, fWeightStateGradients, + fBiasGradients, fDerivatives, precStateActivations, fWeightsInput, + fWeightsState, input, input_gradient); +} + +} // namespace RNN +} // namespace DNN +} // namespace TMVA + +#endif diff --git a/tmva/tmva/inc/TMVA/DNN/ReshapeLayer.h b/tmva/tmva/inc/TMVA/DNN/ReshapeLayer.h new file mode 100644 index 0000000000000..c53a2e7bdb4ef --- /dev/null +++ b/tmva/tmva/inc/TMVA/DNN/ReshapeLayer.h @@ -0,0 +1,168 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : TReshapeLayer * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Reshape Deep Neural Network Layer * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_DNN_RESHAPELAYER +#define TMVA_DNN_RESHAPELAYER + +#include "TMatrix.h" + +#include "TMVA/DNN/GeneralLayer.h" +#include "TMVA/DNN/Functions.h" + +#include + +namespace TMVA { +namespace DNN { + +template +class TReshapeLayer : public VGeneralLayer { +public: + using Matrix_t = typename Architecture_t::Matrix_t; + using Scalar_t = typename Architecture_t::Scalar_t; + +private: + bool fFlattening; ///< Whather the layer is doing flattening + +public: + /*! Constructor */ + TReshapeLayer(size_t BatchSize, size_t InputDepth, size_t InputHeight, size_t InputWidth, size_t Depth, + size_t Height, size_t Width, size_t OutputNSlices, size_t OutputNRows, size_t OutputNCols, + bool Flattening); + + /*! Copy the reshape layer provided as a pointer */ + TReshapeLayer(TReshapeLayer *layer); + + /*! Copy Constructor */ + TReshapeLayer(const TReshapeLayer &); + + /*! Destructor. */ + ~TReshapeLayer(); + + /*! The input must be in 3D tensor form with the different matrices + * corresponding to different events in the batch. It transforms the + * input matrices. */ + void Forward(std::vector &input, bool applyDropout = false); + + void Backward(std::vector &gradients_backward, const std::vector &activations_backward, + std::vector &inp1, std::vector &inp2); + + /*! Prints the info about the layer. */ + void Print() const; + + /*! TODO Add documentation + * Does this layer flatten? (necessary for DenseLayer) + * B x D1 x D2 --> 1 x B x (D1 * D2) */ + bool isFlattening() const { return fFlattening; } +}; + +// +// +// The Reshape Layer Class - Implementation +//_________________________________________________________________________________________________ +template +TReshapeLayer::TReshapeLayer(size_t batchSize, size_t inputDepth, size_t inputHeight, size_t inputWidth, + size_t depth, size_t height, size_t width, size_t outputNSlices, + size_t outputNRows, size_t outputNCols, bool flattening) + : VGeneralLayer(batchSize, inputDepth, inputHeight, inputWidth, depth, height, width, 0, 0, 0, 0, 0, + 0, outputNSlices, outputNRows, outputNCols, EInitialization::kZero), + fFlattening(flattening) +{ + if (this->GetInputDepth() * this->GetInputHeight() * this->GetInputWidth() != + this->GetDepth() * this->GetHeight() * this->GetWidth()) { + std::cout << "Reshape Dimensions not compatible \n" + << this->GetInputDepth() << " x " << this->GetInputHeight() << " x " << this->GetInputWidth() << " --> " + << this->GetDepth() << " x " << this->GetHeight() << " x " << this->GetWidth() << std::endl; + return; + } +} + +//_________________________________________________________________________________________________ +template +TReshapeLayer::TReshapeLayer(TReshapeLayer *layer) + : VGeneralLayer(layer), fFlattening(layer->isFlattening()) +{ +} + +//_________________________________________________________________________________________________ +template +TReshapeLayer::TReshapeLayer(const TReshapeLayer &layer) + : VGeneralLayer(layer), fFlattening(layer.fFlattening) +{ + // Nothing to do here. +} + +//_________________________________________________________________________________________________ +template +TReshapeLayer::~TReshapeLayer() +{ + // Nothing to do here. +} + +//_________________________________________________________________________________________________ +template +auto TReshapeLayer::Forward(std::vector &input, bool /*applyDropout*/) -> void +{ + if (fFlattening) { + size_t size = input.size(); + size_t nRows = input[0].GetNrows(); + size_t nCols = input[0].GetNcols(); + Architecture_t::Flatten(this->GetOutputAt(0), input, size, nRows, nCols); + } else { + for (size_t i = 0; i < this->GetBatchSize(); i++) { + Architecture_t::Reshape(this->GetOutputAt(i), input[i]); + } + } +} + +//_________________________________________________________________________________________________ +template +auto TReshapeLayer::Backward(std::vector &gradients_backward, + const std::vector & /*activations_backward*/, + std::vector & /*inp1*/, std::vector & + /*inp2*/) -> void +{ + if (fFlattening) { + size_t size = gradients_backward.size(); + size_t nRows = gradients_backward[0].GetNrows(); + size_t nCols = gradients_backward[0].GetNcols(); + Architecture_t::Deflatten(gradients_backward, this->GetActivationGradientsAt(0), size, nRows, nCols); + } else { + for (size_t i = 0; i < this->GetBatchSize(); i++) { + Architecture_t::Reshape(gradients_backward[i], this->GetActivationGradientsAt(i)); + } + } +} + +//_________________________________________________________________________________________________ +template +auto TReshapeLayer::Print() const -> void +{ +} + +} // namespace DNN +} // namespace TMVA + +#endif diff --git a/tmva/tmva/inc/TMVA/DNN/TensorDataLoader.h b/tmva/tmva/inc/TMVA/DNN/TensorDataLoader.h new file mode 100644 index 0000000000000..471bc7ba5f0f5 --- /dev/null +++ b/tmva/tmva/inc/TMVA/DNN/TensorDataLoader.h @@ -0,0 +1,276 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : TTensorDataLoader * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Tensor Data Loader Class * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_DNN_TENSORDATALOADER +#define TMVA_DNN_TENSORDATALOADER + +#include "TMatrix.h" +#include "TMVA/Event.h" + +namespace TMVA { +namespace DNN { + +// +// Input Data Types +//______________________________________________________________________________ +using TensorInput = + std::tuple> &, const TMatrixT &, const TMatrixT &>; + +using TMVAInput_t = std::vector; +using IndexIterator_t = typename std::vector::iterator; + +/** TTensorBatch + * + * Class representing training batches consisting of a vector of matrices as input data + * and a matrix of output data. The input and output data can be accessed using + * the GetInput() and GetOutput() member functions. + * + * \tparam Architecture_t The underlying architecture. + */ + +template +class TTensorBatch { +public: + using Matrix_t = typename Architecture_t::Matrix_t; + +private: + std::vector fInputTensor; ///< The input tensor batch, one matrix one input. + Matrix_t fOutputMatrix; ///< The output matrix representing the ground truth. + Matrix_t fWeightMatrix; + +public: + TTensorBatch(std::vector &, Matrix_t &, Matrix_t &); + TTensorBatch(const TTensorBatch &) = default; + TTensorBatch(TTensorBatch &&) = default; + TTensorBatch &operator=(const TTensorBatch &) = default; + TTensorBatch &operator=(TTensorBatch &&) = default; + + /** Return the tensor representing the input data */ + std::vector &GetInput() { return fInputTensor; } + /** Return the matrix representing the output data. */ + Matrix_t &GetOutput() { return fOutputMatrix; } + /** Return the matrix holding the event weights. */ + Matrix_t &GetWeights() { return fWeightMatrix; } +}; + +template +class TTensorDataLoader; + +/** TTensorBatchIterator + * + * Simple iterator class for the iterations over the training batches in + * a given data set represented by a TTensorDataLoader object. + * + * \tparam Data_t The input data type. + * \tparam Architecture_t The underlying architecture type. + */ +template +class TTensorBatchIterator { +private: + TTensorDataLoader &fTensorDataLoader; + size_t fBatchIndex; + +public: + TTensorBatchIterator(TTensorDataLoader &tensorDataLoader, size_t index = 0) + : fTensorDataLoader(tensorDataLoader), fBatchIndex(index) + { + // Nothing to do here. + } + + TTensorBatch operator*() { return fTensorDataLoader.GetTensorBatch(); } + TTensorBatchIterator operator++() + { + fBatchIndex++; + return *this; + } + bool operator!=(const TTensorBatchIterator &other) { return fBatchIndex != other.fBatchIndex; } +}; + +/** TTensorDataLoader + * + * Service class managing the streaming of the training data from the input data + * type to the accelerator device or the CPU. A TTensorDataLoader object manages + * a number of host and device buffer pairs that are used in a round-robin manner + * for the transfer of batches to the device. + * + * Each TTensorDataLoader object has an associated batch size and a number of total + * samples in the dataset. One epoch is the number of buffers required to transfer + * the complete training set. Using the begin() and end() member functions allows + * the user to iterate over the batches in one epoch. + * + * \tparam Data_t The input data type. + * \tparam Architecture_t The achitecture class of the underlying architecture. + */ +template +class TTensorDataLoader { +private: + using HostBuffer_t = typename Architecture_t::HostBuffer_t; + using DeviceBuffer_t = typename Architecture_t::DeviceBuffer_t; + using Matrix_t = typename Architecture_t::Matrix_t; + using BatchIterator_t = TTensorBatchIterator; + + const Data_t &fData; ///< The data that should be loaded in the batches. + + size_t fNSamples; ///< The total number of samples in the dataset. + size_t fBatchSize; ///< The size of a batch. + size_t fBatchDepth; ///< The number of matrices in the tensor. + size_t fBatchHeight; ///< The number od rows in each matrix. + size_t fBatchWidth; ///< The number of columns in each matrix. + size_t fNOutputFeatures; ///< The number of outputs from the classifier/regressor. + size_t fBatchIndex; ///< The index of the batch when there are multiple batches in parallel + + size_t fNStreams; ///< Number of buffer pairs. + std::vector fDeviceBuffers; ///< The device buffers used to keep the input, output and weight data. + std::vector fHostBuffers; ///< The host buffers used to load the input, output and weight data. + + std::vector fSampleIndices; ///< Ordering of the samples in the epoch. + +public: + /*! Constructor. */ + TTensorDataLoader(const Data_t &data, size_t nSamples, size_t batchSize, size_t batchDepth, size_t batchHeight, + size_t batchWidth, size_t nOutputFeatures, size_t nStreams = 1); + + TTensorDataLoader(const TTensorDataLoader &) = default; + TTensorDataLoader(TTensorDataLoader &&) = default; + TTensorDataLoader &operator=(const TTensorDataLoader &) = default; + TTensorDataLoader &operator=(TTensorDataLoader &&) = default; + + /** Copy input tensor into the given host buffer. Function to be specialized by + * the architecture-specific backend. */ + void CopyTensorInput(HostBuffer_t &buffer, IndexIterator_t begin); + /** Copy output matrix into the given host buffer. Function to be specialized + * by the architecture-spcific backend. */ + void CopyTensorOutput(HostBuffer_t &buffer, IndexIterator_t begin); + /** Copy weight matrix into the given host buffer. Function to be specialized + * by the architecture-spcific backend. */ + void CopyTensorWeights(HostBuffer_t &buffer, IndexIterator_t begin); + + BatchIterator_t begin() { return TTensorBatchIterator(*this); } + BatchIterator_t end() { return TTensorBatchIterator(*this, fNSamples / fBatchSize); } + + /** Shuffle the order of the samples in the batch. The shuffling is indirect, + * i.e. only the indices are shuffled. No input data is moved by this + * routine. */ + void Shuffle(); + + /** Return the next batch from the training set. The TTensorDataLoader object + * keeps an internal counter that cycles over the batches in the training + * set. */ + TTensorBatch GetTensorBatch(); +}; + +// +// TTensorBatch Class. +//______________________________________________________________________________ +template +TTensorBatch::TTensorBatch(std::vector &inputTensor, Matrix_t &outputMatrix, + Matrix_t &weightMatrix) + : fInputTensor(inputTensor), fOutputMatrix(outputMatrix), fWeightMatrix(weightMatrix) +{ + // Nothing to do here. +} + +// +// TTensorDataLoader Class. +//______________________________________________________________________________ +template +TTensorDataLoader::TTensorDataLoader(const Data_t &data, size_t nSamples, size_t batchSize, + size_t batchDepth, size_t batchHeight, size_t batchWidth, + size_t nOutputFeatures, size_t nStreams) + : fData(data), fNSamples(nSamples), fBatchSize(batchSize), fBatchDepth(batchDepth), fBatchHeight(batchHeight), + fBatchWidth(batchWidth), fNOutputFeatures(nOutputFeatures), fBatchIndex(0), fNStreams(nStreams), fDeviceBuffers(), + fHostBuffers(), fSampleIndices() +{ + size_t inputTensorSize = fBatchDepth * fBatchHeight * fBatchWidth; + size_t outputMatrixSize = fBatchSize * fNOutputFeatures; + size_t weightMatrixSize = fBatchSize; + + for (size_t i = 0; i < fNStreams; i++) { + fHostBuffers.push_back(HostBuffer_t(inputTensorSize + outputMatrixSize + weightMatrixSize)); + fDeviceBuffers.push_back(DeviceBuffer_t(inputTensorSize + outputMatrixSize + weightMatrixSize)); + } + + fSampleIndices.reserve(fNSamples); + for (size_t i = 0; i < fNSamples; i++) { + fSampleIndices.push_back(i); + } +} + +//______________________________________________________________________________ +template +TTensorBatch TTensorDataLoader::GetTensorBatch() +{ + fBatchIndex %= (fNSamples / fBatchSize); // Cycle through samples. + + size_t inputTensorSize = fBatchDepth * fBatchHeight * fBatchWidth; + size_t outputMatrixSize = fBatchSize * fNOutputFeatures; + size_t weightMatrixSize = fBatchSize; + + size_t streamIndex = fBatchIndex % fNStreams; + HostBuffer_t &hostBuffer = fHostBuffers[streamIndex]; + DeviceBuffer_t &deviceBuffer = fDeviceBuffers[streamIndex]; + + HostBuffer_t inputHostBuffer = hostBuffer.GetSubBuffer(0, inputTensorSize); + HostBuffer_t outputHostBuffer = hostBuffer.GetSubBuffer(inputTensorSize, outputMatrixSize); + HostBuffer_t weightHostBuffer = hostBuffer.GetSubBuffer(inputTensorSize + outputMatrixSize, weightMatrixSize); + + DeviceBuffer_t inputDeviceBuffer = deviceBuffer.GetSubBuffer(0, inputTensorSize); + DeviceBuffer_t outputDeviceBuffer = deviceBuffer.GetSubBuffer(inputTensorSize, outputMatrixSize); + DeviceBuffer_t weightDeviceBuffer = deviceBuffer.GetSubBuffer(inputTensorSize + outputMatrixSize, weightMatrixSize); + + size_t sampleIndex = fBatchIndex * fBatchSize; + IndexIterator_t sampleIndexIterator = fSampleIndices.begin() + sampleIndex; + + CopyTensorInput(inputHostBuffer, sampleIndexIterator); + CopyTensorOutput(outputHostBuffer, sampleIndexIterator); + CopyTensorWeights(weightHostBuffer, sampleIndexIterator); + + deviceBuffer.CopyFrom(hostBuffer); + + std::vector inputTensor; + size_t jump = fBatchHeight * fBatchWidth; + for (size_t i = 0; i < fBatchDepth; i++) { + DeviceBuffer_t subInputDeviceBuffer = inputDeviceBuffer.GetSubBuffer(i * jump, jump); + inputTensor.emplace_back(subInputDeviceBuffer, fBatchHeight, fBatchWidth); + } + Matrix_t outputMatrix(outputDeviceBuffer, fBatchSize, fNOutputFeatures); + Matrix_t weightMatrix(weightDeviceBuffer, fBatchSize, fNOutputFeatures); + + fBatchIndex++; + return TTensorBatch(inputTensor, outputMatrix, weightMatrix); +} + +//______________________________________________________________________________ +template +void TTensorDataLoader::Shuffle() +{ + std::random_shuffle(fSampleIndices.begin(), fSampleIndices.end()); +} + +} // namespace DNN +} // namespace TMVA + +#endif \ No newline at end of file diff --git a/tmva/tmva/inc/TMVA/MethodDL.h b/tmva/tmva/inc/TMVA/MethodDL.h new file mode 100644 index 0000000000000..813f17d3f5724 --- /dev/null +++ b/tmva/tmva/inc/TMVA/MethodDL.h @@ -0,0 +1,243 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Vladimir Ilievski, Saurav Shekhar + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : MethodDL * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Deep Neural Network Method * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * Saurav Shekhar - ETH Zurich, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef ROOT_TMVA_MethodDL +#define ROOT_TMVA_MethodDL + +////////////////////////////////////////////////////////////////////////// +// // +// MethodDL // +// // +// Method class for all Deep Learning Networks // +// // +////////////////////////////////////////////////////////////////////////// + +#include "TString.h" + +#include "TMVA/MethodBase.h" +#include "TMVA/Types.h" + +#include "TMVA/DNN/Architectures/Reference.h" + +#ifdef DNNCPU +#include "TMVA/DNN/Architectures/Cpu.h" +#endif + +#ifdef DNNCUDA +#include "TMVA/DNN/Architectures/Cuda.h" +#endif + +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DeepNet.h" + +#include + +namespace TMVA { + +/*! All of the options that can be specified in the training string */ +struct TTrainingSettings { + size_t batchSize; + size_t testInterval; + size_t convergenceSteps; + DNN::ERegularization regularization; + Double_t learningRate; + Double_t momentum; + Double_t weightDecay; + std::vector dropoutProbabilities; + bool multithreading; +}; + +class MethodDL : public MethodBase { + +private: + // Key-Value vector type, contining the values for the training options + using KeyValueVector_t = std::vector>; +#ifdef DNNCPU + using ArchitectureCpu_t = TMVA::DNN::TCpu; + using DeepNetCpu_t = TMVA::DNN::TDeepNet; + std::unique_ptr fNet; +#endif + /*! The option handling methods */ + void DeclareOptions(); + void ProcessOptions(); + + void Init(); + + // Function to parse the layout of the input + void ParseInputLayout(); + void ParseBatchLayout(); + + /*! After calling the ProcesOptions(), all of the options are parsed, + * so using the parsed options, and given the architecture and the + * type of the layers, we build the Deep Network passed as + * a reference in the function. */ + template + void CreateDeepNet(DNN::TDeepNet &deepNet, + std::vector> &nets); + + template + void ParseDenseLayer(DNN::TDeepNet &deepNet, + std::vector> &nets, TString layerString, TString delim); + + template + void ParseConvLayer(DNN::TDeepNet &deepNet, + std::vector> &nets, TString layerString, TString delim); + + template + void ParseMaxPoolLayer(DNN::TDeepNet &deepNet, + std::vector> &nets, TString layerString, + TString delim); + + template + void ParseReshapeLayer(DNN::TDeepNet &deepNet, + std::vector> &nets, TString layerString, + TString delim); + + template + void ParseRnnLayer(DNN::TDeepNet &deepNet, + std::vector> &nets, TString layerString, TString delim); + + template + void ParseLstmLayer(DNN::TDeepNet &deepNet, + std::vector> &nets, TString layerString, TString delim); + + size_t fInputDepth; ///< The depth of the input. + size_t fInputHeight; ///< The height of the input. + size_t fInputWidth; ///< The width of the input. + + size_t fBatchDepth; ///< The depth of the batch used to train the deep net. + size_t fBatchHeight; ///< The height of the batch used to train the deep net. + size_t fBatchWidth; ///< The width of the batch used to train the deep net. + + DNN::EInitialization fWeightInitialization; ///< The initialization method + DNN::EOutputFunction fOutputFunction; ///< The output function for making the predictions + DNN::ELossFunction fLossFunction; ///< The loss function + + TString fInputLayoutString; ///< The string defining the layout of the input + TString fBatchLayoutString; ///< The string defining the layout of the batch + TString fLayoutString; ///< The string defining the layout of the deep net + TString fErrorStrategy; ///< The string defining the error strategy for training + TString fTrainingStrategyString; ///< The string defining the training strategy + TString fWeightInitializationString; ///< The string defining the weight initialization method + TString fArchitectureString; ///< The string defining the architecure: CPU or GPU + bool fResume; + + KeyValueVector_t fSettings; ///< Map for the training strategy + std::vector fTrainingSettings; ///< The vector defining each training strategy + + ClassDef(MethodDL, 0); + +protected: + // provide a help message + void GetHelpMessage() const; + +public: + /*! Constructor */ + MethodDL(const TString &jobName, const TString &methodTitle, DataSetInfo &theData, const TString &theOption); + + /*! Constructor */ + MethodDL(DataSetInfo &theData, const TString &theWeightFile); + + /*! Virtual Destructor */ + virtual ~MethodDL(); + + /*! Function for parsing the training settings, provided as a string + * in a key-value form. */ + KeyValueVector_t ParseKeyValueString(TString parseString, TString blockDelim, TString tokenDelim); + + /*! Check the type of analysis the deep learning network can do */ + Bool_t HasAnalysisType(Types::EAnalysisType type, UInt_t numberClasses, UInt_t numberTargets); + + /*! Methods for training the deep learning network */ + void Train(); + void TrainGpu(); + void TrainCpu(); + + Double_t GetMvaValue(Double_t *err = 0, Double_t *errUpper = 0); + + /*! Methods for writing and reading weights */ + using MethodBase::ReadWeightsFromStream; + void AddWeightsXMLTo(void *parent) const; + void ReadWeightsFromXML(void *wghtnode); + void ReadWeightsFromStream(std::istream &); + + /* Create ranking */ + const Ranking *CreateRanking(); + + /* Getters */ + size_t GetInputDepth() { return fInputDepth; } + size_t GetInputHeight() { return fInputHeight; } + size_t GetInputWidth() { return fInputWidth; } + + size_t GetBatchDepth() { return fBatchDepth; } + size_t GetBatchHeight() { return fBatchHeight; } + size_t GetBatchWidth() { return fBatchWidth; } + + DNN::EInitialization GetWeightInitialization() { return fWeightInitialization; } + DNN::EOutputFunction GetOutputFunction() { return fOutputFunction; } + DNN::ELossFunction GetLossFunction() { return fLossFunction; } + + TString GetInputLayoutString() { return fInputLayoutString; } + TString GetBatchLayoutString() { return fBatchLayoutString; } + TString GetLayoutString() { return fLayoutString; } + TString GetErrorStrategyString() { return fErrorStrategy; } + TString GetTrainingStrategyString() { return fTrainingStrategyString; } + TString GetWeightInitializationString() { return fWeightInitializationString; } + TString GetArchitectureString() { return fArchitectureString; } + + const std::vector &GetTrainingSettings() const { return fTrainingSettings; } + std::vector &GetTrainingSettings() { return fTrainingSettings; } + const KeyValueVector_t &GetKeyValueSettings() const { return fSettings; } + KeyValueVector_t &GetKeyValueSettings() { return fSettings; } + + /** Setters */ + void SetInputDepth(size_t inputDepth) { fInputDepth = inputDepth; } + void SetInputHeight(size_t inputHeight) { fInputHeight = inputHeight; } + void SetInputWidth(size_t inputWidth) { fInputWidth = inputWidth; } + + void SetBatchDepth(size_t batchDepth) { fBatchDepth = batchDepth; } + void SetBatchHeight(size_t batchHeight) { fBatchHeight = batchHeight; } + void SetBatchWidth(size_t batchWidth) { fBatchWidth = batchWidth; } + + void SetWeightInitialization(DNN::EInitialization weightInitialization) + { + fWeightInitialization = weightInitialization; + } + void SetOutputFunction(DNN::EOutputFunction outputFunction) { fOutputFunction = outputFunction; } + void SetErrorStrategyString(TString errorStrategy) { fErrorStrategy = errorStrategy; } + void SetTrainingStrategyString(TString trainingStrategyString) { fTrainingStrategyString = trainingStrategyString; } + void SetWeightInitializationString(TString weightInitializationString) + { + fWeightInitializationString = weightInitializationString; + } + void SetArchitectureString(TString architectureString) { fArchitectureString = architectureString; } + void SetLayoutString(TString layoutString) { fLayoutString = layoutString; } +}; + +} // namespace TMVA + +#endif diff --git a/tmva/tmva/inc/TMVA/Types.h b/tmva/tmva/inc/TMVA/Types.h index cc0f9f81d399f..d8881731fbb88 100644 --- a/tmva/tmva/inc/TMVA/Types.h +++ b/tmva/tmva/inc/TMVA/Types.h @@ -98,6 +98,7 @@ namespace TMVA { kPlugins , kCategory , kDNN , + kDL , kPyRandomForest , kPyAdaBoost , kPyGTB , diff --git a/tmva/tmva/src/DNN/Architectures/Cpu.cxx b/tmva/tmva/src/DNN/Architectures/Cpu.cxx index 42d947d009a64..4c25fc50a6eda 100644 --- a/tmva/tmva/src/DNN/Architectures/Cpu.cxx +++ b/tmva/tmva/src/DNN/Architectures/Cpu.cxx @@ -23,6 +23,7 @@ #include "Cpu/OutputFunctions.cxx" #include "Cpu/Propagation.cxx" #include "Cpu/Regularization.cxx" +#include "Cpu/RecurrentPropagation.cxx" namespace TMVA { namespace DNN { diff --git a/tmva/tmva/src/DNN/Architectures/Cpu/Arithmetic.cxx b/tmva/tmva/src/DNN/Architectures/Cpu/Arithmetic.cxx index 8974524fa6fe4..51b9c8211b794 100644 --- a/tmva/tmva/src/DNN/Architectures/Cpu/Arithmetic.cxx +++ b/tmva/tmva/src/DNN/Architectures/Cpu/Arithmetic.cxx @@ -33,6 +33,10 @@ void TCpu::Multiply(TCpuMatrix &C, int k = (int) A.GetNcols(); int n = (int) B.GetNcols(); + R__ASSERT((int) C.GetNrows() == m); + R__ASSERT((int) C.GetNcols() == n); + R__ASSERT((int) B.GetNrows() == k); + char transa = 'N'; char transb = 'N'; @@ -50,13 +54,17 @@ void TCpu::Multiply(TCpuMatrix &C, //____________________________________________________________________________ template void TCpu::TransposeMultiply(TCpuMatrix &C, - const TCpuMatrix &A, - const TCpuMatrix &B) + const TCpuMatrix &A, + const TCpuMatrix &B) { int m = (int) A.GetNcols(); int k = (int) A.GetNrows(); int n = (int) B.GetNcols(); + R__ASSERT((int) C.GetNrows() == m); + R__ASSERT((int) C.GetNcols() == n); + R__ASSERT((int) B.GetNrows() == k); + char transa = 'T'; char transb = 'N'; @@ -133,5 +141,28 @@ void TCpu::Copy(TCpuMatrix &B, B.MapFrom(f, A); } + +//____________________________________________________________________________ +template +void TCpu::ScaleAdd(std::vector> &B, + const std::vector> &A, + Real_t alpha) +{ + for (size_t i = 0; i < B.size(); ++i) { + ScaleAdd(B[i], A[i], alpha); + } +} + +//____________________________________________________________________________ +template +void TCpu::Copy(std::vector> &B, + const std::vector> &A) +{ + for (size_t i = 0; i < B.size(); ++i) { + Copy(B[i], A[i]); + } +} + + } // DNN } // TMVA diff --git a/tmva/tmva/src/DNN/Architectures/Cpu/CpuBuffer.cxx b/tmva/tmva/src/DNN/Architectures/Cpu/CpuBuffer.cxx index 813aba1e03c55..463072e3e4dcd 100644 --- a/tmva/tmva/src/DNN/Architectures/Cpu/CpuBuffer.cxx +++ b/tmva/tmva/src/DNN/Architectures/Cpu/CpuBuffer.cxx @@ -16,65 +16,61 @@ #include #include #include "TMVA/DNN/DataLoader.h" +#include "TMVA/DNN/TensorDataLoader.h" #include "TMVA/DNN/Architectures/Cpu.h" #include "Rtypes.h" #include -namespace TMVA -{ -namespace DNN -{ +namespace TMVA { +namespace DNN { //______________________________________________________________________________ -template -void TCpuBuffer::TDestructor::operator()(AReal ** pointer) +template +void TCpuBuffer::TDestructor::operator()(AReal **pointer) { delete[] * pointer; delete[] pointer; } //______________________________________________________________________________ -template -TCpuBuffer::TCpuBuffer(size_t size) - : fSize(size), fOffset(0) +template +TCpuBuffer::TCpuBuffer(size_t size) : fSize(size), fOffset(0) { - AReal ** pointer = new AReal * [1]; - * pointer = new AReal[size]; - fBuffer = std::shared_ptr(pointer, fDestructor); + AReal **pointer = new AReal *[1]; + *pointer = new AReal[size]; + fBuffer = std::shared_ptr(pointer, fDestructor); } //______________________________________________________________________________ -template +template TCpuBuffer TCpuBuffer::GetSubBuffer(size_t offset, size_t size) { TCpuBuffer buffer = *this; buffer.fOffset = offset; - buffer.fSize = size; + buffer.fSize = size; return buffer; } //______________________________________________________________________________ -template -void TCpuBuffer::CopyFrom(TCpuBuffer & other) +template +void TCpuBuffer::CopyFrom(TCpuBuffer &other) { std::swap(*this->fBuffer, *other.fBuffer); } //______________________________________________________________________________ -template -void TCpuBuffer::CopyTo(TCpuBuffer & other) +template +void TCpuBuffer::CopyTo(TCpuBuffer &other) { std::swap(*this->fBuffer, *other.fBuffer); } //______________________________________________________________________________ -template<> -void TDataLoader>::CopyInput( - TCpuBuffer & buffer, - IndexIterator_t sampleIterator, - size_t batchSize) +template <> +void TDataLoader>::CopyInput(TCpuBuffer &buffer, IndexIterator_t sampleIterator, + size_t batchSize) { - const TMatrixT &inputMatrix = std::get<0>(fData); + const TMatrixT &inputMatrix = std::get<0>(fData); size_t n = inputMatrix.GetNcols(); for (size_t i = 0; i < batchSize; i++) { @@ -88,13 +84,11 @@ void TDataLoader>::CopyInput( } //______________________________________________________________________________ -template<> -void TDataLoader>::CopyOutput( - TCpuBuffer & buffer, - IndexIterator_t sampleIterator, - size_t batchSize) +template <> +void TDataLoader>::CopyOutput(TCpuBuffer &buffer, IndexIterator_t sampleIterator, + size_t batchSize) { - const TMatrixT &outputMatrix = std::get<1>(fData); + const TMatrixT &outputMatrix = std::get<1>(fData); size_t n = outputMatrix.GetNcols(); for (size_t i = 0; i < batchSize; i++) { @@ -126,7 +120,7 @@ template <> void TDataLoader>::CopyInput(TCpuBuffer &buffer, IndexIterator_t sampleIterator, size_t batchSize) { - const TMatrixT &inputMatrix = std::get<0>(fData); + const TMatrixT &inputMatrix = std::get<0>(fData); size_t n = inputMatrix.GetNcols(); for (size_t i = 0; i < batchSize; i++) { @@ -140,13 +134,11 @@ void TDataLoader>::CopyInput(TCpuBuffer } //______________________________________________________________________________ -template<> -void TDataLoader>::CopyOutput( - TCpuBuffer & buffer, - IndexIterator_t sampleIterator, - size_t batchSize) +template <> +void TDataLoader>::CopyOutput(TCpuBuffer &buffer, + IndexIterator_t sampleIterator, size_t batchSize) { - const TMatrixT &outputMatrix = std::get<1>(fData); + const TMatrixT &outputMatrix = std::get<1>(fData); size_t n = outputMatrix.GetNcols(); for (size_t i = 0; i < batchSize; i++) { @@ -178,13 +170,13 @@ template <> void TDataLoader>::CopyInput(TCpuBuffer &buffer, IndexIterator_t sampleIterator, size_t batchSize) { - Event * event = fData.front(); - size_t n = event->GetNVariables(); + Event *event = fData.front(); + size_t n = event->GetNVariables(); // Copy input variables. for (size_t i = 0; i < batchSize; i++) { - size_t sampleIndex = * sampleIterator++; + size_t sampleIndex = *sampleIterator++; event = fData[sampleIndex]; for (size_t j = 0; j < n; j++) { size_t bufferIndex = j * batchSize + i; @@ -194,19 +186,17 @@ void TDataLoader>::CopyInput(TCpuBuffer &b } //______________________________________________________________________________ -template<> -void TDataLoader>::CopyOutput( - TCpuBuffer & buffer, - IndexIterator_t sampleIterator, - size_t batchSize) +template <> +void TDataLoader>::CopyOutput(TCpuBuffer &buffer, IndexIterator_t sampleIterator, + size_t batchSize) { - Event * event = fData.front(); - size_t n = buffer.GetSize() / batchSize; + Event *event = fData.front(); + size_t n = buffer.GetSize() / batchSize; // Copy target(s). for (size_t i = 0; i < batchSize; i++) { - size_t sampleIndex = * sampleIterator++; + size_t sampleIndex = *sampleIterator++; event = fData[sampleIndex]; for (size_t j = 0; j < n; j++) { // Copy output matrices. @@ -249,13 +239,13 @@ template <> void TDataLoader>::CopyInput(TCpuBuffer &buffer, IndexIterator_t sampleIterator, size_t batchSize) { - Event * event = fData.front(); - size_t n = event->GetNVariables(); + Event *event = fData.front(); + size_t n = event->GetNVariables(); // Copy input variables. for (size_t i = 0; i < batchSize; i++) { - size_t sampleIndex = * sampleIterator++; + size_t sampleIndex = *sampleIterator++; event = fData[sampleIndex]; for (size_t j = 0; j < n; j++) { size_t bufferIndex = j * batchSize + i; @@ -265,19 +255,17 @@ void TDataLoader>::CopyInput(TCpuBuffer &buffe } //______________________________________________________________________________ -template<> -void TDataLoader>::CopyOutput( - TCpuBuffer & buffer, - IndexIterator_t sampleIterator, - size_t batchSize) +template <> +void TDataLoader>::CopyOutput(TCpuBuffer &buffer, IndexIterator_t sampleIterator, + size_t batchSize) { - Event * event = fData.front(); - size_t n = buffer.GetSize() / batchSize; + Event *event = fData.front(); + size_t n = buffer.GetSize() / batchSize; // Copy target(s). for (size_t i = 0; i < batchSize; i++) { - size_t sampleIndex = * sampleIterator++; + size_t sampleIndex = *sampleIterator++; event = fData[sampleIndex]; for (size_t j = 0; j < n; j++) { // Copy output matrices. @@ -315,11 +303,298 @@ void TDataLoader>::CopyWeights(TCpuBuffer &buf } } +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorInput(TCpuBuffer &buffer, + IndexIterator_t sampleIterator) +{ + const std::vector> &inputTensor = std::get<0>(fData); + + if (fBatchDepth == 1) { + for (size_t i = 0; i < fBatchHeight; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchWidth; j++) { + size_t bufferIndex = j * fBatchHeight + i; + buffer[bufferIndex] = static_cast(inputTensor[0](sampleIndex, j)); + } + sampleIterator++; + } + } else { + for (size_t i = 0; i < fBatchDepth; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchHeight; j++) { + for (size_t k = 0; k < fBatchWidth; k++) { + size_t bufferIndex = i * fBatchHeight * fBatchWidth + k * fBatchHeight + j; + buffer[bufferIndex] = static_cast(inputTensor[sampleIndex](j, k)); + } + } + sampleIterator++; + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorOutput(TCpuBuffer &buffer, + IndexIterator_t sampleIterator) +{ + const TMatrixT &outputMatrix = std::get<1>(fData); + size_t n = outputMatrix.GetNcols(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < n; j++) { + size_t bufferIndex = j * fBatchSize + i; + buffer[bufferIndex] = static_cast(outputMatrix(sampleIndex, j)); + } + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorWeights(TCpuBuffer &buffer, + IndexIterator_t sampleIterator) +{ + const TMatrixT &outputMatrix = std::get<2>(fData); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + buffer[i] = static_cast(outputMatrix(sampleIndex, 0)); + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorInput(TCpuBuffer &buffer, + IndexIterator_t sampleIterator) +{ + const std::vector> &inputTensor = std::get<0>(fData); + + if (fBatchDepth == 1) { + for (size_t i = 0; i < fBatchHeight; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchWidth; j++) { + size_t bufferIndex = j * fBatchHeight + i; + buffer[bufferIndex] = inputTensor[0](sampleIndex, j); + } + sampleIterator++; + } + } else { + for (size_t i = 0; i < fBatchDepth; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchHeight; j++) { + for (size_t k = 0; k < fBatchWidth; k++) { + size_t bufferIndex = i * fBatchHeight * fBatchWidth + k * fBatchHeight + j; + buffer[bufferIndex] = inputTensor[sampleIndex](j, k); + } + } + sampleIterator++; + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorOutput(TCpuBuffer &buffer, + IndexIterator_t sampleIterator) +{ + const TMatrixT &outputMatrix = std::get<1>(fData); + size_t n = outputMatrix.GetNcols(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < n; j++) { + size_t bufferIndex = j * fBatchSize + i; + buffer[bufferIndex] = outputMatrix(sampleIndex, j); + } + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorWeights(TCpuBuffer &buffer, + IndexIterator_t sampleIterator) +{ + const TMatrixT &outputMatrix = std::get<2>(fData); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + buffer[i] = static_cast(outputMatrix(sampleIndex, 0)); + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorInput(TCpuBuffer &buffer, + IndexIterator_t sampleIterator) +{ + // one event, one example in the batch + Event *event = fData.front(); + + if (fBatchDepth == 1) { + for (size_t i = 0; i < fBatchHeight; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchWidth; j++) { + event = fData[sampleIndex]; + size_t bufferIndex = j * fBatchHeight + i; + buffer[bufferIndex] = event->GetValue(j); + } + sampleIterator++; + } + } else { + for (size_t i = 0; i < fBatchDepth; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchHeight; j++) { + for (size_t k = 0; k < fBatchWidth; k++) { + event = fData[sampleIndex]; + // because of the column-major ordering + size_t bufferIndex = i * fBatchHeight * fBatchWidth + k * fBatchHeight + j; + buffer[bufferIndex] = event->GetValue(j * fBatchHeight + k); + } + } + sampleIterator++; + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorOutput(TCpuBuffer &buffer, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + size_t n = buffer.GetSize() / fBatchSize; + + // Copy target(s). + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; + for (size_t j = 0; j < n; j++) { + // Copy output matrices. + size_t bufferIndex = j * fBatchSize + i; + // Classification + if (event->GetNTargets() == 0) { + if (n == 1) { + // Binary. + buffer[bufferIndex] = (event->GetClass() == 0) ? 1.0 : 0.0; + } else { + // Multiclass. + buffer[bufferIndex] = 0.0; + if (j == event->GetClass()) { + buffer[bufferIndex] = 1.0; + } + } + } else { + buffer[bufferIndex] = static_cast(event->GetTarget(j)); + } + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorWeights(TCpuBuffer &buffer, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; + buffer[i] = event->GetWeight(); + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorInput(TCpuBuffer &buffer, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + + if (fBatchDepth == 1) { + for (size_t i = 0; i < fBatchHeight; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchWidth; j++) { + event = fData[sampleIndex]; + size_t bufferIndex = j * fBatchHeight + i; + buffer[bufferIndex] = static_cast(event->GetValue(j)); + } + sampleIterator++; + } + } else { + for (size_t i = 0; i < fBatchDepth; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchHeight; j++) { + for (size_t k = 0; k < fBatchWidth; k++) { + event = fData[sampleIndex]; + // because of the column-major ordering + size_t bufferIndex = i * fBatchHeight * fBatchWidth + k * fBatchHeight + j; + buffer[bufferIndex] = static_cast(event->GetValue(j * fBatchHeight + k)); + } + } + sampleIterator++; + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorOutput(TCpuBuffer &buffer, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + size_t n = buffer.GetSize() / fBatchSize; + + // Copy target(s). + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; + for (size_t j = 0; j < n; j++) { + // Copy output matrices. + size_t bufferIndex = j * fBatchSize + i; + // Classification + if (event->GetNTargets() == 0) { + if (n == 1) { + // Binary. + buffer[bufferIndex] = (event->GetClass() == 0) ? 1.0 : 0.0; + } else { + // Multiclass. + buffer[bufferIndex] = 0.0; + if (j == event->GetClass()) { + buffer[bufferIndex] = 1.0; + } + } + } else { + buffer[bufferIndex] = static_cast(event->GetTarget(j)); + } + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorWeights(TCpuBuffer &buffer, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; + buffer[i] = static_cast(event->GetWeight()); + } +} + // Explicit instantiations. template class TCpuBuffer; template class TCpuBuffer; } // namespace DNN } // namespace TMVA - - diff --git a/tmva/tmva/src/DNN/Architectures/Cpu/CpuMatrix.cxx b/tmva/tmva/src/DNN/Architectures/Cpu/CpuMatrix.cxx index 3c708e377f077..1c42f7cac6dee 100644 --- a/tmva/tmva/src/DNN/Architectures/Cpu/CpuMatrix.cxx +++ b/tmva/tmva/src/DNN/Architectures/Cpu/CpuMatrix.cxx @@ -24,6 +24,11 @@ TCpuMatrix::TCpuMatrix(size_t nRows, size_t nCols) : fBuffer(nRows * nCols), fNCols(nCols), fNRows(nRows) { Initialize(); + for (size_t j = 0; j < fNCols; j++) { + for (size_t i = 0; i < fNRows; i++) { + (*this)(i, j) = 0; + } + } } //____________________________________________________________________________ diff --git a/tmva/tmva/src/DNN/Architectures/Cpu/Propagation.cxx b/tmva/tmva/src/DNN/Architectures/Cpu/Propagation.cxx index ef6e7a4168418..74184659a7166 100644 --- a/tmva/tmva/src/DNN/Architectures/Cpu/Propagation.cxx +++ b/tmva/tmva/src/DNN/Architectures/Cpu/Propagation.cxx @@ -18,76 +18,431 @@ #include "TMVA/DNN/Architectures/Cpu.h" #include "TMVA/DNN/Architectures/Cpu/Blas.h" -namespace TMVA -{ -namespace DNN -{ +namespace TMVA { +namespace DNN { -template -void TCpu::MultiplyTranspose(TCpuMatrix &output, - const TCpuMatrix &input, +template +void TCpu::MultiplyTranspose(TCpuMatrix &output, const TCpuMatrix &input, const TCpuMatrix &Weights) { - int m = (int) input.GetNrows(); - int k = (int) input.GetNcols(); - int n = (int) Weights.GetNrows(); - char transa = 'N'; - char transb = 'T'; + int m = (int)input.GetNrows(); + int k = (int)input.GetNcols(); + int n = (int)Weights.GetNrows(); + + R__ASSERT((int) output.GetNrows() == m); + R__ASSERT((int) output.GetNcols() == n); + R__ASSERT((int) Weights.GetNcols() == k); - AFloat alpha = 1.0; - AFloat beta = 0.0; + char transa = 'N'; + char transb = 'T'; - const AFloat *A = input.GetRawDataPointer(); - const AFloat *B = Weights.GetRawDataPointer(); - AFloat *C = output.GetRawDataPointer(); + AFloat alpha = 1.0; + AFloat beta = 0.0; - ::TMVA::DNN::Blas::Gemm(&transa, &transb, &m, &n, &k, &alpha, - A, &m, B, &n, &beta, C, &m); + const AFloat *A = input.GetRawDataPointer(); + const AFloat *B = Weights.GetRawDataPointer(); + AFloat *C = output.GetRawDataPointer(); + + ::TMVA::DNN::Blas::Gemm(&transa, &transb, &m, &n, &k, &alpha, A, &m, B, &n, &beta, C, &m); } -template -void TCpu::AddRowWise( - TCpuMatrix &output, - const TCpuMatrix &biases) +template +void TCpu::AddRowWise(TCpuMatrix &output, const TCpuMatrix &biases) { - int m = (int) output.GetNrows(); - int n = (int) output.GetNcols(); + int m = (int)output.GetNrows(); + int n = (int)output.GetNcols(); + + int inc = 1.0; + AFloat alpha = 1.0; - int inc = 1.0; - AFloat alpha = 1.0; + AFloat *A = output.GetRawDataPointer(); + const AFloat *x = TCpuMatrix::GetOnePointer(); + const AFloat *y = biases.GetRawDataPointer(); - AFloat * A = output.GetRawDataPointer(); - const AFloat * x = TCpuMatrix::GetOnePointer(); - const AFloat * y = biases.GetRawDataPointer(); + R__ASSERT(m <= (int)TCpuMatrix::GetOnePointerSize()); + R__ASSERT(n <= (int)biases.GetNcols()*biases.GetNrows()); - ::TMVA::DNN::Blas::Ger(&m, &n, &alpha, x, &inc, y, &inc, A, &m); + ::TMVA::DNN::Blas::Ger(&m, &n, &alpha, x, &inc, y, &inc, A, &m); } -template -void TCpu::Backward( - TCpuMatrix & activationGradientsBackward, - TCpuMatrix & weightGradients, - TCpuMatrix & biasGradients, - TCpuMatrix & df, - const TCpuMatrix & activationGradients, - const TCpuMatrix & weights, - const TCpuMatrix & activationsBackward) +template +void TCpu::Backward(TCpuMatrix &activationGradientsBackward, TCpuMatrix &weightGradients, + TCpuMatrix &biasGradients, TCpuMatrix &df, + const TCpuMatrix &activationGradients, const TCpuMatrix &weights, + const TCpuMatrix &activationsBackward) { // Compute element-wise product. Hadamard(df, activationGradients); // Activation gradients. - if (activationGradientsBackward.GetNElements() > 0) - Multiply(activationGradientsBackward, df, weights); + if (activationGradientsBackward.GetNElements() > 0) Multiply(activationGradientsBackward, df, weights); // Weight gradients. - if (weightGradients.GetNElements() > 0) - TransposeMultiply(weightGradients, df, activationsBackward); + if (weightGradients.GetNElements() > 0) TransposeMultiply(weightGradients, df, activationsBackward); // Bias gradients. - if (biasGradients.GetNElements() > 0) - SumColumns(biasGradients, df); + if (biasGradients.GetNElements() > 0) SumColumns(biasGradients, df); +} + +//____________________________________________________________________________ +template +void TCpu::Im2col(TCpuMatrix &A, TCpuMatrix &B, size_t imgHeight, size_t imgWidth, + size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, + size_t zeroPaddingHeight, size_t zeroPaddingWidth) +{ + + // image boudaries + int imgHeightBound = imgHeight + zeroPaddingHeight - (fltHeight - 1) / 2 - 1; + int imgWidthBound = imgWidth + zeroPaddingWidth - (fltWidth - 1) / 2 - 1; + size_t currLocalView = 0; + + // convolution centers + for (int i = -zeroPaddingHeight + fltHeight / 2; i <= imgHeightBound; i += strideRows) { + for (int j = -zeroPaddingWidth + fltWidth / 2; j <= imgWidthBound; j += strideCols) { + size_t currLocalViewPixel = 0; + + // within the local view + for (int m = 0; m < (Int_t)B.GetNrows(); m++) { + for (int k = i - fltHeight / 2; k <= Int_t(i + (fltHeight - 1) / 2); k++) { + for (int l = j - fltWidth / 2; l <= Int_t(j + (fltWidth - 1) / 2); l++) { + + // Check the boundaries + R__ASSERT(currLocalView < A.GetNrows() ); + R__ASSERT(currLocalViewPixel < A.GetNcols() ); + //R__ASSERT(k * imgWidth + l < B.GetNcols()); + if (k < 0 || k >= (Int_t)imgHeight || l < 0 || l >= (Int_t)imgWidth || k * imgWidth + l >= B.GetNcols()) + A(currLocalView, currLocalViewPixel++) = 0; + else + A(currLocalView, currLocalViewPixel++) = B(m, k * imgWidth + l); + } + } + } + //std::cout << " i " << i << " " << j << " increment currLocalView " << currLocalView << std::endl; + currLocalView++; + } + } +} + +//____________________________________________________________________________ +template +void TCpu::RotateWeights(TCpuMatrix &A, const TCpuMatrix &B, size_t filterDepth, + size_t filterHeight, size_t filterWidth, size_t numFilters) +{ + size_t jump = filterHeight * filterWidth; + for (size_t j = 0; j < filterDepth; j++) { + for (size_t k = 0; k < numFilters; k++) { + for (size_t i = 0; i < jump; i++) { + A(j, k * jump + i) = B(k, ((j + 1) * jump - 1) - i); + //A(j, k * jump + i) = B(k, j * jump + i); + } + } + } +} + +//____________________________________________________________________________ +template +void TCpu::AddConvBiases(TCpuMatrix &output, const TCpuMatrix &biases) +{ + int m = (int)output.GetNrows(); + int n = (int)output.GetNcols(); + + int inc = 1.0; + AFloat alpha = 1.0; + + AFloat *A = output.GetRawDataPointer(); + const AFloat *x = biases.GetRawDataPointer(); + const AFloat *y = TCpuMatrix::GetOnePointer(); + + ::TMVA::DNN::Blas::Ger(&m, &n, &alpha, x, &inc, y, &inc, A, &m); +} + +//____________________________________________________________________________ +template +void TCpu::ConvLayerBackward(std::vector> &activationGradientsBackward, + TCpuMatrix &weightGradients, TCpuMatrix &biasGradients, + std::vector> &df, + const std::vector> &activationGradients, + const TCpuMatrix &weights, + const std::vector> &activationsBackward, size_t batchSize, + size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, + size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t nLocalViews) +{ + // Update derivatives + // size_t m, n; + // m = activationGradients[0].GetNrows(); + // n = activationGradients[0].GetNcols(); + + for (size_t i = 0; i < batchSize; i++) { + // Compute element-wise product. + Hadamard(df[i], activationGradients[i]); + } + + // Calculate the activation gradients of the previous layer + CalculateConvActivationGradients(activationGradientsBackward, df, weights, batchSize, inputHeight, inputWidth, depth, + height, width, filterDepth, filterHeight, filterWidth); + + // Calculate the weight gradients + CalculateConvWeightGradients(weightGradients, df, activationsBackward, batchSize, inputHeight, inputWidth, depth, + height, width, filterDepth, filterHeight, filterWidth, nLocalViews); + + // Calculate the bias gradients + CalculateConvBiasGradients(biasGradients, df, batchSize, depth, nLocalViews); +} + +//____________________________________________________________________________ +template +void TCpu::CalculateConvActivationGradients(std::vector> &activationGradientsBackward, + std::vector> &df, + const TCpuMatrix &weights, size_t batchSize, + size_t inputHeight, size_t inputWidth, size_t depth, size_t height, + size_t width, size_t filterDepth, size_t filterHeight, + size_t filterWidth) +{ + if (activationGradientsBackward.size() == 0) return; + + + // Transform the weights + + PrintMatrix(weights,"weights"); + // filter depth must be same as input depth + TCpuMatrix rotWeights(filterDepth, depth * filterHeight * filterWidth); + RotateWeights(rotWeights, weights, filterDepth, filterHeight, filterWidth, weights.GetNrows()); + PrintMatrix(rotWeights,"rot-weights"); + + // Calculate the zero paddings + size_t tempZeroPaddingHeight = (size_t)(floor((inputHeight - height + filterHeight - 1) / 2)); + size_t tempZeroPaddingWidth = (size_t)(floor((inputWidth - width + filterWidth - 1) / 2)); + + // size_t tempZeroPaddingHeight = 1; + // size_t tempZeroPaddingWidth = 1; + + // Calculate the number of local views and the number of pixles in each view + size_t tempNLocalViews = inputHeight * inputWidth; + size_t tempNLocalViewPixels = depth * filterHeight * filterWidth; + + size_t tempStrideRows = 1; + size_t tempStrideCols = 1; + + // An entire convolution follows + for (size_t i = 0; i < batchSize; i++) { + TCpuMatrix dfTr(tempNLocalViews, tempNLocalViewPixels); + for (int j = 0; j < dfTr.GetNrows(); ++j) { + for (int k = 0; k < dfTr.GetNcols(); ++k) { + dfTr(j,k) = 0; + } + } + Im2col(dfTr, df[i], height, width, filterHeight, filterWidth, tempStrideRows, tempStrideCols, + tempZeroPaddingHeight, tempZeroPaddingWidth); + + PrintMatrix(df[i],"df[i]"); + PrintMatrix(dfTr,"dfTr"); + + MultiplyTranspose(activationGradientsBackward[i], rotWeights, dfTr); + + PrintMatrix(activationGradientsBackward[i],"activGrad-result"); + + } +} + +//____________________________________________________________________________ +template +void TCpu::CalculateConvWeightGradients(TCpuMatrix &weightGradients, + std::vector> &df, + const std::vector> &activationsBackward, + size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, + size_t height, size_t width, size_t filterDepth, size_t filterHeight, + size_t filterWidth, size_t nLocalViews) +{ + // reinitialize the weight gradients to 0 + for (size_t i = 0; i < weightGradients.GetNrows(); i++) { + for (size_t j = 0; j < weightGradients.GetNcols(); j++) { + weightGradients(i, j) = 0; + } + } + + size_t nLocalViewPixels = filterDepth * filterHeight * filterWidth; + R__ASSERT( weightGradients.GetNcols() == filterDepth * filterHeight * filterWidth); + + // convolution + TCpuMatrix res(depth, nLocalViewPixels); + //std::cout << "do back-propagation in conv layer - compute weight gradient" << std::endl; + for (size_t i = 0; i < batchSize; i++) { + + size_t tempStrideRows = 1; + size_t tempStrideCols = 1; + + // Calculate the zero paddings from the input height and width (assume stride =1 ) + size_t tempZeroPaddingHeight = (height - inputHeight + filterHeight - 1) / 2; + size_t tempZeroPaddingWidth = (width - inputWidth + filterWidth - 1) / 2; + + //PrintMatrix(df[i],"df-i"); + + //computing t he gradient is equivalent of doing a convolution of the input using as conv kernel the delta's (the df[] values) + //N.B. only stride values=1 are now supported + + TCpuMatrix xTr(nLocalViews, nLocalViewPixels); + Im2col(xTr, const_cast &>(activationsBackward[i]), inputHeight, inputWidth, filterHeight , filterWidth, + tempStrideRows, tempStrideCols, tempZeroPaddingHeight, tempZeroPaddingWidth); + + + //PrintMatrix(xTr,"xTr-i"); + //PrintMatrix(activationsBackward[i],"actbackward-i"); + Multiply(res, df[i], xTr); + //PrintMatrix(res,"res_ofMT"); + + for (size_t j = 0; j < depth; j++) { + for (size_t k = 0; k < filterDepth; k++) { + for (size_t l = 0; l < filterHeight * filterWidth; l++) { + //weightGradients(j, k * (filterHeight * filterWidth) + l) += res(k, (tempNLocalViews - 1) - l); + weightGradients(j, k * (filterHeight * filterWidth) + l) += res(j, k * (filterHeight * filterWidth) + l); + } + } + } + + } + //PrintMatrix(weightGradients,"W-Grad"); +} + +//____________________________________________________________________________ +template +void TCpu::CalculateConvBiasGradients(TCpuMatrix &biasGradients, std::vector> &df, + size_t batchSize, size_t depth, size_t nLocalViews) +{ + for (size_t i = 0; i < depth; i++) { + AFloat sum = 0; + for (size_t j = 0; j < nLocalViews; j++) { + for (size_t k = 0; k < batchSize; k++) { + sum += df[k](i, j); + } + } + biasGradients(i, 0) = sum; + } +} + +//____________________________________________________________________________ +template +void TCpu::Downsample(TCpuMatrix &A, TCpuMatrix &B, const TCpuMatrix &C, + size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, + size_t strideCols) +{ + // image boudaries + int imgHeightBound = imgHeight - (fltHeight - 1) / 2 - 1; + int imgWidthBound = imgWidth - (fltWidth - 1) / 2 - 1; + size_t currLocalView = 0; + + // centers + for (int i = fltHeight / 2; i <= imgHeightBound; i += strideRows) { + for (int j = fltWidth / 2; j <= imgWidthBound; j += strideCols) { + // within local views + for (int m = 0; m < (Int_t)C.GetNrows(); m++) { + AFloat value = -std::numeric_limits::max(); + + for (int k = i - fltHeight / 2; k <= Int_t(i + (fltHeight - 1) / 2); k++) { + for (int l = j - fltWidth / 2; l <= Int_t(j + (fltWidth - 1) / 2); l++) { + if (C(m, k * imgWidth + l) > value) { + value = C(m, k * imgWidth + l); + B(m, currLocalView) = k * imgWidth + l; + } + } + } + A(m, currLocalView) = value; + } + currLocalView++; + } + } +} + +//____________________________________________________________________________ +template +void TCpu::MaxPoolLayerBackward(std::vector> &activationGradientsBackward, + const std::vector> &activationGradients, + const std::vector> &indexMatrix, size_t batchSize, + size_t depth, size_t nLocalViews) +{ + for (size_t i = 0; i < batchSize; i++) { + for (size_t j = 0; j < depth; j++) { + + // initialize to zeros + for (size_t t = 0; t < (size_t)activationGradientsBackward[i].GetNcols(); t++) { + activationGradientsBackward[i](j, t) = 0; + } + + // set values + for (size_t k = 0; k < nLocalViews; k++) { + AFloat grad = activationGradients[i](j, k); + size_t winningIdx = indexMatrix[i](j, k); + activationGradientsBackward[i](j, winningIdx) += grad; + } + } + } +} + +//____________________________________________________________________________ +template +void TCpu::Reshape(TCpuMatrix &A, const TCpuMatrix &B) +{ + size_t nColsA = A.GetNcols(); + size_t nColsB = B.GetNcols(); + + for (size_t i = 0; i < A.GetNrows(); i++) { + for (size_t j = 0; j < A.GetNcols(); j++) { + size_t nElem = i * nColsA + j; + A(i, j) = B(nElem / nColsB, (nElem - 1) % nColsB); + } + } +} + +//____________________________________________________________________________ +template +void TCpu::Flatten(TCpuMatrix &A, const std::vector> &B, size_t size, size_t nRows, + size_t nCols) +{ + for (size_t i = 0; i < (size_t)size; i++) { + for (size_t j = 0; j < (size_t)nRows; j++) { + for (size_t k = 0; k < (size_t)nCols; k++) { + A(i, j * nCols + k) = B[i](j, k); + } + } + } +} + +//____________________________________________________________________________ +template +void TCpu::Deflatten(std::vector> &A, const TCpuMatrix &B, size_t size, size_t nRows, + size_t nCols) +{ + for (size_t i = 0; i < (size_t)size; i++) { + for (size_t j = 0; j < (size_t)nRows; j++) { + for (size_t k = 0; k < (size_t)nCols; k++) { + A[i](j, k) = B(i, j * nCols + k); + } + } + } +} + +//______________________________________________________________________________ +template +void TCpu::Rearrange(std::vector> &out, const std::vector> &in) +{ + // B x T x D out --- T x B x D in*/ + size_t B = out.size(); + size_t T = out[0].GetNrows(); + size_t D = out[0].GetNcols(); + if ((T != in.size()) || (B != in[0].GetNrows()) || (D != in[0].GetNcols())) { + std::cout << "Incompatible Dimensions\n" + << in.size() << "x" << in[0].GetNrows() << "x" << in[0].GetNcols() << " --> " << B << "x" << T << "x" + << D << "\n"; + return; + } + for (size_t i = 0; i < B; ++i) { + for (size_t j = 0; j < T; ++j) { + for (size_t k = 0; k < D; ++k) { + out[i](j, k) = in[j](i, k); + } + } + } + return; } } // namespace DNN diff --git a/tmva/tmva/src/DNN/Architectures/Cpu/RecurrentPropagation.cxx b/tmva/tmva/src/DNN/Architectures/Cpu/RecurrentPropagation.cxx new file mode 100644 index 0000000000000..c9f047516446b --- /dev/null +++ b/tmva/tmva/src/DNN/Architectures/Cpu/RecurrentPropagation.cxx @@ -0,0 +1,67 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Saurav Shekhar 23/06/17 + +/************************************************************************* + * Copyright (C) 2017, Saurav Shekhar * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +///////////////////////////////////////////////////////////////////// +// Implementation of the functions required for the forward and // +// backward propagation of activations through a recurrent neural // +// network in the TCpu architecture // +///////////////////////////////////////////////////////////////////// + + +#include "TMVA/DNN/Architectures/Cpu.h" +#include "TMVA/DNN/Architectures/Cpu/Blas.h" + +namespace TMVA +{ +namespace DNN +{ + +template +auto TCpu::RecurrentLayerBackward(TCpuMatrix & state_gradients_backward, // BxH + TCpuMatrix & input_weight_gradients, + TCpuMatrix & state_weight_gradients, + TCpuMatrix & bias_gradients, + TCpuMatrix & df, //DxH + const TCpuMatrix & state, // BxH + const TCpuMatrix & weights_input, // HxD + const TCpuMatrix & weights_state, // HxH + const TCpuMatrix & input, // BxD + TCpuMatrix & input_gradient) +-> TCpuMatrix & +{ + // Compute element-wise product. + Hadamard(df, state_gradients_backward); // B x H + + // Input gradients. + if (input_gradient.GetNElements() > 0) Multiply(input_gradient, df, weights_input); + + // State gradients. + if (state_gradients_backward.GetNElements() > 0) Multiply(state_gradients_backward, df, weights_state); + + // Weights gradients + if (input_weight_gradients.GetNElements() > 0) { + TCpuMatrix tmp(input_weight_gradients); + TransposeMultiply(input_weight_gradients, df, input); // H x B . B x D + ScaleAdd(input_weight_gradients, tmp, 1); + } + if (state_weight_gradients.GetNElements() > 0) { + TCpuMatrix tmp(state_weight_gradients); + TransposeMultiply(state_weight_gradients, df, state); // H x B . B x H + ScaleAdd(state_weight_gradients, tmp, 1); + } + + // Bias gradients. + if (bias_gradients.GetNElements() > 0) SumColumns(bias_gradients, df); + return input_gradient; +} + +} // namespace DNN +} // namespace TMVA diff --git a/tmva/tmva/src/DNN/Architectures/Cuda.cu b/tmva/tmva/src/DNN/Architectures/Cuda.cu index 7f791c613b487..ddc7a88da4f36 100644 --- a/tmva/tmva/src/DNN/Architectures/Cuda.cu +++ b/tmva/tmva/src/DNN/Architectures/Cuda.cu @@ -23,6 +23,7 @@ #include "Cuda/Regularization.cu" #include "Cuda/Initialization.cu" #include "Cuda/Dropout.cu" +#include "Cuda/RecurrentPropagation.cu" namespace TMVA { namespace DNN { diff --git a/tmva/tmva/src/DNN/Architectures/Cuda/Arithmetic.cu b/tmva/tmva/src/DNN/Architectures/Cuda/Arithmetic.cu index 43ffa15d327c9..5d5a72d6cbd62 100644 --- a/tmva/tmva/src/DNN/Architectures/Cuda/Arithmetic.cu +++ b/tmva/tmva/src/DNN/Architectures/Cuda/Arithmetic.cu @@ -234,5 +234,27 @@ void TCuda::ScaleAdd(TCudaMatrix & B, B.GetDataPointer(), 1); } +//____________________________________________________________________________ +template<> +void TCuda::ScaleAdd(std::vector> & B, + const std::vector> & A, + float alpha) +{ + for (size_t i = 0; i < A.size(); ++i) { + ScaleAdd(B[i], A[i], alpha); + } +} + +//____________________________________________________________________________ +template<> +void TCuda::ScaleAdd(std::vector> & B, + const std::vector> & A, + double alpha) +{ + for (size_t i = 0; i < A.size(); ++i) { + ScaleAdd(B[i], A[i], alpha); + } +} + } // DNN } // TMVA diff --git a/tmva/tmva/src/DNN/Architectures/Cuda/CudaBuffers.cxx b/tmva/tmva/src/DNN/Architectures/Cuda/CudaBuffers.cxx index 531f3dffed325..51f0e6245fd86 100644 --- a/tmva/tmva/src/DNN/Architectures/Cuda/CudaBuffers.cxx +++ b/tmva/tmva/src/DNN/Architectures/Cuda/CudaBuffers.cxx @@ -14,140 +14,132 @@ //////////////////////////////////////////////////////////////////////// #include "TMVA/DNN/DataLoader.h" + +#include "TMVA/DNN/TensorDataLoader.h" #include "TMVA/DNN/Architectures/Cuda.h" #include "TMVA/DNN/Architectures/Cuda/CudaBuffers.h" + #include "cuda_runtime.h" #include namespace TMVA { -namespace DNN { +namespace DNN { // // TCudaHostBuffer //______________________________________________________________________________ -template -void TCudaHostBuffer::TDestructor::operator()(AFloat ** devicePointer) +template +void TCudaHostBuffer::TDestructor::operator()(AFloat **devicePointer) { cudaFreeHost(*devicePointer); delete[] devicePointer; } //______________________________________________________________________________ -template -TCudaHostBuffer::TCudaHostBuffer(size_t size) - : fOffset(0), fSize(size), fComputeStream(0), fDestructor() +template +TCudaHostBuffer::TCudaHostBuffer(size_t size) : fOffset(0), fSize(size), fComputeStream(0), fDestructor() { - AFloat ** pointer = new AFloat * [1]; + AFloat **pointer = new AFloat *[1]; cudaMallocHost(pointer, size * sizeof(AFloat)); fHostPointer = std::shared_ptr(pointer, fDestructor); } //______________________________________________________________________________ -template -TCudaHostBuffer::operator AFloat * () const +template +TCudaHostBuffer::operator AFloat *() const { return *fHostPointer + fOffset; } //______________________________________________________________________________ -template -TCudaHostBuffer TCudaHostBuffer::GetSubBuffer(size_t offset, - size_t size) +template +TCudaHostBuffer TCudaHostBuffer::GetSubBuffer(size_t offset, size_t size) { TCudaHostBuffer buffer = *this; - buffer.fOffset = offset; - buffer.fSize = size; + buffer.fOffset = offset; + buffer.fSize = size; return buffer; } // // TCudaDevicePointer //______________________________________________________________________________ -template -void TCudaDeviceBuffer::TDestructor::operator()(AFloat ** devicePointer) +template +void TCudaDeviceBuffer::TDestructor::operator()(AFloat **devicePointer) { cudaFree(*devicePointer); delete[] devicePointer; } //______________________________________________________________________________ -template -TCudaDeviceBuffer::TCudaDeviceBuffer(size_t size) - : fOffset(0), fSize(size), fDestructor() +template +TCudaDeviceBuffer::TCudaDeviceBuffer(size_t size) : fOffset(0), fSize(size), fDestructor() { - AFloat ** pointer = new AFloat * [1]; + AFloat **pointer = new AFloat *[1]; cudaMalloc(pointer, size * sizeof(AFloat)); fDevicePointer = std::shared_ptr(pointer, fDestructor); cudaStreamCreate(&fComputeStream); } //______________________________________________________________________________ -template -TCudaDeviceBuffer::TCudaDeviceBuffer(size_t size, - cudaStream_t stream) - : fOffset(0), fSize(size), fComputeStream(stream), fDestructor() +template +TCudaDeviceBuffer::TCudaDeviceBuffer(size_t size, cudaStream_t stream) + : fOffset(0), fSize(size), fComputeStream(stream), fDestructor() { - AFloat ** pointer = new AFloat * [1]; + AFloat **pointer = new AFloat *[1]; cudaMalloc(pointer, size * sizeof(AFloat)); fDevicePointer = std::shared_ptr(pointer, fDestructor); } //______________________________________________________________________________ -template -TCudaDeviceBuffer::TCudaDeviceBuffer(AFloat * devicePointer, - size_t size, - cudaStream_t stream) - : fOffset(0), fSize(size), fComputeStream(stream), fDestructor() +template +TCudaDeviceBuffer::TCudaDeviceBuffer(AFloat *devicePointer, size_t size, cudaStream_t stream) + : fOffset(0), fSize(size), fComputeStream(stream), fDestructor() { - AFloat ** pointer = new AFloat * [1]; - *pointer = devicePointer; + AFloat **pointer = new AFloat *[1]; + *pointer = devicePointer; fDevicePointer = std::shared_ptr(pointer, fDestructor); } //______________________________________________________________________________ -template -TCudaDeviceBuffer TCudaDeviceBuffer::GetSubBuffer(size_t offset, - size_t size) +template +TCudaDeviceBuffer TCudaDeviceBuffer::GetSubBuffer(size_t offset, size_t size) { TCudaDeviceBuffer buffer = *this; - buffer.fOffset = offset; - buffer.fSize = size; + buffer.fOffset = offset; + buffer.fSize = size; return buffer; } //______________________________________________________________________________ -template -TCudaDeviceBuffer::operator AFloat * () const +template +TCudaDeviceBuffer::operator AFloat *() const { - return *fDevicePointer + fOffset; + return *fDevicePointer + fOffset; } //______________________________________________________________________________ -template +template void TCudaDeviceBuffer::CopyFrom(const TCudaHostBuffer &buffer) const { cudaStreamSynchronize(fComputeStream); - cudaMemcpyAsync(*this, buffer, fSize * sizeof(AFloat), - cudaMemcpyHostToDevice, fComputeStream); + cudaMemcpyAsync(*this, buffer, fSize * sizeof(AFloat), cudaMemcpyHostToDevice, fComputeStream); } //______________________________________________________________________________ -template +template void TCudaDeviceBuffer::CopyTo(const TCudaHostBuffer &buffer) const { - cudaMemcpyAsync(*this, buffer, fSize * sizeof(AFloat), - cudaMemcpyDeviceToHost, fComputeStream); + cudaMemcpyAsync(*this, buffer, fSize * sizeof(AFloat), cudaMemcpyDeviceToHost, fComputeStream); buffer.fComputeStream = fComputeStream; } //______________________________________________________________________________ -template<> -void TDataLoader>::CopyInput( - TCudaHostBuffer & buffer, - IndexIterator_t sampleIterator, - size_t batchSize) +template <> +void TDataLoader>::CopyInput(TCudaHostBuffer &buffer, IndexIterator_t sampleIterator, + size_t batchSize) { - const TMatrixT &inputMatrix = std::get<0>(fData); + const TMatrixT &inputMatrix = std::get<0>(fData); size_t n = inputMatrix.GetNcols(); for (size_t i = 0; i < batchSize; i++) { @@ -161,13 +153,11 @@ void TDataLoader>::CopyInput( } //______________________________________________________________________________ -template<> -void TDataLoader>::CopyOutput( - TCudaHostBuffer & buffer, - IndexIterator_t sampleIterator, - size_t batchSize) +template <> +void TDataLoader>::CopyOutput(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator, size_t batchSize) { - const TMatrixT &outputMatrix = std::get<1>(fData); + const TMatrixT &outputMatrix = std::get<1>(fData); size_t n = outputMatrix.GetNcols(); for (size_t i = 0; i < batchSize; i++) { @@ -193,19 +183,17 @@ void TDataLoader>::CopyWeights(TCudaHostBuffer -void TDataLoader>::CopyInput( - TCudaHostBuffer & buffer, - IndexIterator_t sampleIterator, - size_t batchSize) +template <> +void TDataLoader>::CopyInput(TCudaHostBuffer &buffer, IndexIterator_t sampleIterator, + size_t batchSize) { - Event * event = fData.front(); - size_t n = event->GetNVariables(); + Event *event = fData.front(); + size_t n = event->GetNVariables(); // Copy input variables. for (size_t i = 0; i < batchSize; i++) { - size_t sampleIndex = * sampleIterator++; + size_t sampleIndex = *sampleIterator++; event = fData[sampleIndex]; for (size_t j = 0; j < n; j++) { size_t bufferIndex = j * batchSize + i; @@ -215,20 +203,18 @@ void TDataLoader>::CopyInput( } //______________________________________________________________________________ -template<> -void TDataLoader>::CopyOutput( - TCudaHostBuffer & buffer, - IndexIterator_t sampleIterator, - size_t batchSize) +template <> +void TDataLoader>::CopyOutput(TCudaHostBuffer &buffer, IndexIterator_t sampleIterator, + size_t batchSize) { - Event * event = fData.front(); - size_t n = buffer.GetSize() / batchSize; + Event *event = fData.front(); + size_t n = buffer.GetSize() / batchSize; // Copy target(s). for (size_t i = 0; i < batchSize; i++) { - size_t sampleIndex = * sampleIterator++; - event = fData[sampleIndex]; + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; for (size_t j = 0; j < n; j++) { // Copy output matrices. size_t bufferIndex = j * batchSize + i; @@ -270,7 +256,7 @@ template <> void TDataLoader>::CopyInput(TCudaHostBuffer &buffer, IndexIterator_t sampleIterator, size_t batchSize) { - const TMatrixT &inputMatrix = std::get<0>(fData); + const TMatrixT &inputMatrix = std::get<0>(fData); size_t n = inputMatrix.GetNcols(); for (size_t i = 0; i < batchSize; i++) { @@ -284,13 +270,11 @@ void TDataLoader>::CopyInput(TCudaHostBuffer -void TDataLoader>::CopyOutput( - TCudaHostBuffer & buffer, - IndexIterator_t sampleIterator, - size_t batchSize) +template <> +void TDataLoader>::CopyOutput(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator, size_t batchSize) { - const TMatrixT &outputMatrix = std::get<1>(fData); + const TMatrixT &outputMatrix = std::get<1>(fData); size_t n = outputMatrix.GetNcols(); for (size_t i = 0; i < batchSize; i++) { @@ -320,13 +304,13 @@ template <> void TDataLoader>::CopyInput(TCudaHostBuffer &buffer, IndexIterator_t sampleIterator, size_t batchSize) { - Event * event = fData.front(); - size_t n = event->GetNVariables(); + Event *event = fData.front(); + size_t n = event->GetNVariables(); // Copy input variables. for (size_t i = 0; i < batchSize; i++) { - size_t sampleIndex = * sampleIterator++; + size_t sampleIndex = *sampleIterator++; event = fData[sampleIndex]; for (size_t j = 0; j < n; j++) { size_t bufferIndex = j * batchSize + i; @@ -336,26 +320,24 @@ void TDataLoader>::CopyInput(TCudaHostBuffer } //______________________________________________________________________________ -template<> -void TDataLoader>::CopyOutput( - TCudaHostBuffer & buffer, - IndexIterator_t sampleIterator, - size_t batchSize) +template <> +void TDataLoader>::CopyOutput(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator, size_t batchSize) { - Event * event = fData.front(); - size_t n = buffer.GetSize() / batchSize; + Event *event = fData.front(); + size_t n = buffer.GetSize() / batchSize; // Copy target(s). for (size_t i = 0; i < batchSize; i++) { - size_t sampleIndex = * sampleIterator++; - event = fData[sampleIndex]; + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; for (size_t j = 0; j < n; j++) { // Copy output matrices. size_t bufferIndex = j * batchSize + i; // Classification if (event->GetNTargets() == 0) { - // Binary. + // Binary. if (n == 1) { buffer[bufferIndex] = (event->GetClass() == 0) ? 1.0 : 0.0; } else { @@ -386,6 +368,244 @@ void TDataLoader>::CopyWeights(TCudaHostBuffer +void TTensorDataLoader>::CopyTensorInput(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator) +{ + const std::vector> &inputTensor = std::get<0>(fData); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchHeight; j++) { + for (size_t k = 0; k < fBatchWidth; k++) { + size_t bufferIndex = i * fBatchHeight * fBatchWidth + k * fBatchHeight + j; + buffer[bufferIndex] = static_cast(inputTensor[sampleIndex](j, k)); + } + } + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorOutput(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator) +{ + const TMatrixT &outputMatrix = std::get<1>(fData); + size_t n = outputMatrix.GetNcols(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < n; j++) { + size_t bufferIndex = j * fBatchSize + i; + buffer[bufferIndex] = static_cast(outputMatrix(sampleIndex, j)); + } + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorWeights(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator) +{ + const TMatrixT &weightMatrix = std::get<2>(fData); + for (size_t i = 0; i < fBatchSize; i++) { + buffer[i] = static_cast(weightMatrix(*sampleIterator, 0)); + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorInput(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchHeight; j++) { + for (size_t k = 0; k < fBatchWidth; k++) { + event = fData[sampleIndex]; + // because of the column-major ordering + size_t bufferIndex = i * fBatchHeight * fBatchWidth + k * fBatchHeight + j; + buffer[bufferIndex] = static_cast(event->GetValue(j * fBatchHeight + k)); + } + } + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorOutput(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + size_t n = buffer.GetSize() / fBatchSize; + + // Copy target(s). + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; + for (size_t j = 0; j < n; j++) { + // Copy output matrices. + size_t bufferIndex = j * fBatchSize + i; + // Classification + if (event->GetNTargets() == 0) { + if (n == 1) { + // Binary. + buffer[bufferIndex] = (event->GetClass() == 0) ? 1.0 : 0.0; + } else { + // Multiclass. + buffer[bufferIndex] = 0.0; + if (j == event->GetClass()) { + buffer[bufferIndex] = 1.0; + } + } + } else { + buffer[bufferIndex] = static_cast(event->GetTarget(j)); + } + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorWeights(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; + buffer[i] = static_cast(event->GetWeight()); + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorInput(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator) +{ + const std::vector> &inputTensor = std::get<0>(fData); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchHeight; j++) { + for (size_t k = 0; k < fBatchWidth; k++) { + size_t bufferIndex = i * fBatchHeight * fBatchWidth + k * fBatchHeight + j; + buffer[bufferIndex] = inputTensor[sampleIndex](j, k); + } + } + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorOutput(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator) +{ + const TMatrixT &outputMatrix = std::get<1>(fData); + size_t n = outputMatrix.GetNcols(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < n; j++) { + size_t bufferIndex = j * fBatchSize + i; + buffer[bufferIndex] = outputMatrix(sampleIndex, j); + } + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorWeights(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator) +{ + const TMatrixT &weightMatrix = std::get<2>(fData); + for (size_t i = 0; i < fBatchSize; i++) { + buffer[i] = static_cast(weightMatrix(*sampleIterator, 0)); + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorInput(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchHeight; j++) { + for (size_t k = 0; k < fBatchWidth; k++) { + event = fData[sampleIndex]; + // because of the column-major ordering + size_t bufferIndex = i * fBatchHeight * fBatchWidth + k * fBatchHeight + j; + buffer[bufferIndex] = event->GetValue(j * fBatchHeight + k); + } + } + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorOutput(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + size_t n = buffer.GetSize() / fBatchSize; + + // Copy target(s). + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; + for (size_t j = 0; j < n; j++) { + // Copy output matrices. + size_t bufferIndex = j * fBatchSize + i; + // Classification + if (event->GetNTargets() == 0) { + // Binary. + if (n == 1) { + buffer[bufferIndex] = (event->GetClass() == 0) ? 1.0 : 0.0; + } else { + // Multiclass. + buffer[bufferIndex] = 0.0; + if (j == event->GetClass()) { + buffer[bufferIndex] = 1.0; + } + } + } else { + buffer[bufferIndex] = event->GetTarget(j); + } + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorWeights(TCudaHostBuffer &buffer, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; + buffer[i] = static_cast(event->GetWeight()); + } +} + // Explicit Instantiations. template class TCudaDeviceBuffer; @@ -395,9 +615,9 @@ template class TCudaHostBuffer; template class TCudaHostBuffer; template class TDataLoader>; -template class TDataLoader>; +template class TDataLoader>; template class TDataLoader>; -template class TDataLoader>; +template class TDataLoader>; } // TMVA } // DNN diff --git a/tmva/tmva/src/DNN/Architectures/Cuda/Propagation.cu b/tmva/tmva/src/DNN/Architectures/Cuda/Propagation.cu index 35331b7855fd5..93cb9d4b9dab0 100644 --- a/tmva/tmva/src/DNN/Architectures/Cuda/Propagation.cu +++ b/tmva/tmva/src/DNN/Architectures/Cuda/Propagation.cu @@ -128,5 +128,207 @@ void TCuda::Copy(TCudaMatrix & B, m * n * sizeof(AFloat), cudaMemcpyDeviceToDevice, 0); } +//____________________________________________________________________________ +template +void TCuda::Copy(std::vector> & B, + const std::vector> & A) +{ + for (size_t i = 0; i < B.size(); ++i) { + Copy(B[i], A[i]); + } +} + +//____________________________________________________________________________ +template +void TCuda::Im2col(TCudaMatrix &A, + TCudaMatrix &B, + size_t imgHeight, + size_t imgWidth, + size_t fltHeight, + size_t fltWidth, + size_t strideRows, + size_t strideCols, + size_t zeroPaddingHeight, + size_t zeroPaddingWidth) +{ + +} + +//____________________________________________________________________________ +template +void TCuda::RotateWeights(TCudaMatrix &A, + const TCudaMatrix &B, + size_t filterDepth, + size_t filterHeight, + size_t filterWidth, + size_t numFilters) +{ + +} + + +//____________________________________________________________________________ +template +void TCuda::ConvLayerBackward(std::vector> & activation_gradients_backward, + TCudaMatrix & weight_gradients, + TCudaMatrix & bias_gradients, + std::vector> & df, + const std::vector> & activation_gradients, + const TCudaMatrix & weights, + const std::vector> & activation_backward, + size_t batchSize, + size_t inputHeight, + size_t inputWidth, + size_t depth, + size_t height, + size_t width, + size_t filterDepth, + size_t filterHeight, + size_t filterWidth, + size_t nLocalViews) +{ + + +} + +//____________________________________________________________________________ +template +void TCuda::CalculateConvActivationGradients( + std::vector> & activation_gradients_backward, + std::vector> & df, + const TCudaMatrix & weights, + size_t batchSize, + size_t inputHeight, + size_t inputWidth, + size_t depth, + size_t height, + size_t width, + size_t filterDepth, + size_t filterHeight, + size_t filterWidth) +{ + +} + +//____________________________________________________________________________ +template +void TCuda::CalculateConvWeightGradients(TCudaMatrix & weight_gradients, + std::vector> & df, + const std::vector> & activations_backward, + size_t batchSize, + size_t inputHeight, + size_t inputWidth, + size_t depth, + size_t height, + size_t width, + size_t filterDepth, + size_t filterHeight, + size_t filterWidth, + size_t nLocalViews) +{ + +} + +//____________________________________________________________________________ +template +void TCuda::CalculateConvBiasGradients(TCudaMatrix & bias_gradients, + std::vector> & df, + size_t batchSize, + size_t depth, + size_t nLocalViews) +{ + +} + +//____________________________________________________________________________ +template +void TCuda::AddConvBiases(TCudaMatrix &output, + const TCudaMatrix &biases) +{ + +} + + +//____________________________________________________________________________ +template +void TCuda::Downsample(TCudaMatrix &A, + TCudaMatrix &B, + const TCudaMatrix &C, + size_t imgHeight, + size_t imgWidth, + size_t fltHeight, + size_t fltWidth, + size_t strideRows, + size_t strideCols) +{ + +} + +/____________________________________________________________________________ +template +void TCuda::MaxPoolLayerBackward(std::vector> & activationGradientsBackward, + const std::vector> & activationGradients, + const std::vector> & indexMatrix, + size_t batchSize, + size_t depth, + size_t nLocalViews) +{ + +} + +//____________________________________________________________________________ +template +void TCuda::Reshape(TCudaMatrix &A, const TCudaMatrix &B) +{ + //TODO +} + +//______________________________________________________________________________ +template +void TCuda::Rearrange(std::vector> &out, const std::vector> &in) +{ + // B x T x D out --- T x B x D in*/ + size_t B = out.size(); + size_t T = out[0].GetNrows(); + size_t D = out[0].GetNcols(); + if ((T != in.size()) || (B != in[0].GetNrows()) + || (D != in[0].GetNcols())) { + std::cout << "Incompatible Dimensions\n" + << in.size() << "x" << in[0].GetNrows() << "x" << in[0].GetNcols() + << " --> " << B << "x" << T << "x" << D << "\n"; + return; + } + for (size_t i = 0; i < B; ++i) { + for (size_t j = 0; j < T; ++j) { + for (size_t k = 0; k < D; ++k) { + out[i](j, k) = in[j](i, k); + } + } + } + return; +} + +//____________________________________________________________________________ +template +void TCuda::Flatten(TCudaMatrix &A, + const std::vector> &B, + size_tt size, + size_t nRows, + size_t nCols) +{ + +} + +//____________________________________________________________________________ +template +void TCuda::Deflatten(std::vector> &A, + const TCudaMatrix &B, + size_t index, + size_t nRows, + size_t nCols) +{ + +} + } // namespace DNN } // namespace TMVA diff --git a/tmva/tmva/src/DNN/Architectures/Cuda/RecurrentPropagation.cu b/tmva/tmva/src/DNN/Architectures/Cuda/RecurrentPropagation.cu new file mode 100644 index 0000000000000..be944d3dc1c3a --- /dev/null +++ b/tmva/tmva/src/DNN/Architectures/Cuda/RecurrentPropagation.cu @@ -0,0 +1,75 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Saurav Shekhar 23/06/17 + +/************************************************************************* + * Copyright (C) 2017, Saurav Shekhar * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + + ////////////////////////////////////////////////////////////////// + // Implementation of the functions required for the forward and // + // backward propagation of activations through a neural network // + // for CUDA architectures. // + ////////////////////////////////////////////////////////////////// + +#include "TMVA/DNN/Architectures/Cuda.h" +#include "TMVA/DNN/Architectures/Cuda/Device.h" +#include "Kernels.cuh" + +namespace TMVA +{ +namespace DNN +{ + +//____________________________________________________________________________ +template +auto TCuda::RecurrentLayerBackward(TCudaMatrix & state_gradients_backward, // BxH + TCudaMatrix & input_weight_gradients, + TCudaMatrix & state_weight_gradients, + TCudaMatrix & bias_gradients, + TCudaMatrix & df, //DxH + const TCudaMatrix & state, // BxH + const TCudaMatrix & weights_input, // HxD + const TCudaMatrix & weights_state, // HxH + const TCudaMatrix & input, // BxD + TCudaMatrix & input_gradient); +-> TCudaMatrix & +{ + // Compute element-wise product. + TCuda::Hadamard(df, state_gradients_backward); // B x H + + // Input gradients. + if (input_gradient.GetNoElements() > 0) { + TCuda::Multiply(input_gradient, df, weights_input); + } + + // State gradients. + if (state_gradients_backward.GetNoElements() > 0) { + TCuda::Multiply(state_gradients_backward, df, weights_state); + } + + // Weights gradients + if (input_weight_gradients.GetNoElements() > 0) { + TCudaMatrix tmp(input_weight_gradients); + TCuda::TransposeMultiply(input_weight_gradients, df, input); // H x B . B x D + TCuda::ScaleAdd(input_weight_gradients, tmp, 1); + } + if (state_weight_gradients.GetNoElements() > 0) { + TCpuMatrix tmp(state_weight_gradients); + TCuda::TransposeMultiply(state_weight_gradients, df, state); // H x B . B x H + TCuda::ScaleAdd(state_weight_gradients, tmp, 1); + } + + // Bias gradients. + if (bias_gradients.GetNoElements() > 0) { + TCuda::SumColumns(bias_gradients, df); + } + return input_gradient; +} + +} // namespace DNN +} // namespace TMVA + diff --git a/tmva/tmva/src/DNN/Architectures/Reference.cxx b/tmva/tmva/src/DNN/Architectures/Reference.cxx index b7620d088f74b..48e590d1e98bf 100644 --- a/tmva/tmva/src/DNN/Architectures/Reference.cxx +++ b/tmva/tmva/src/DNN/Architectures/Reference.cxx @@ -24,6 +24,8 @@ #include "Reference/Regularization.cxx" #include "Reference/Initialization.cxx" #include "Reference/Dropout.cxx" +#include "Reference/DenoisePropagation.cxx" +#include "Reference/RecurrentPropagation.cxx" namespace TMVA { namespace DNN { diff --git a/tmva/tmva/src/DNN/Architectures/Reference/DenoisePropagation.cxx b/tmva/tmva/src/DNN/Architectures/Reference/DenoisePropagation.cxx new file mode 100644 index 0000000000000..06af93c3cc00f --- /dev/null +++ b/tmva/tmva/src/DNN/Architectures/Reference/DenoisePropagation.cxx @@ -0,0 +1,219 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Akshay Vashistha(ajatgd) + +/************************************************************************* + * Copyright (C) 2017 ajatgd * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +////////////////////////////////////////////////////////////////// +// Implementation of the Denoise Autoencoder functions for the // +// reference implementation. // +////////////////////////////////////////////////////////////////// + +#include "TMVA/DNN/Architectures/Reference.h" + +#include +#include +#include +namespace TMVA +{ +namespace DNN +{ + +//______________________________________________________________________________ + +template +void TReference::AddBiases(TMatrixT &A, + const TMatrixT &biases) +{ + size_t m,n; + m = A.GetNrows(); + n= A.GetNcols(); + for(size_t i = 0; i +void TReference::UpdateParams( + TMatrixT &x, TMatrixT &tildeX, TMatrixT &y, + TMatrixT &z, TMatrixT &fVBiases, TMatrixT &fHBiases, + TMatrixT &fWeights, TMatrixT &VBiasError, + TMatrixT &HBiasError, Real_t learningRate, size_t fBatchSize) { + + //updating fVBiases + for (size_t i = 0; i < (size_t)fVBiases.GetNrows(); i++) + { + for (size_t j = 0; j < (size_t)fVBiases.GetNcols(); j++) { + VBiasError(i, j) = x(i, j) - z(i, j); + fVBiases(i, j) += learningRate * VBiasError(i, j) / fBatchSize; + } + } + + //updating fHBiases + for (Int_t i = 0; i < fHBiases.GetNrows(); i++) { + HBiasError(i,0) = 0; + for (Int_t j = 0; j < fVBiases.GetNrows(); j++) { + HBiasError(i, 0) += fWeights(i, j) * VBiasError(j, 0); + } + HBiasError(i, 0) *= y(i, 0) * (1 - y(i, 0)); + fHBiases(i, 0) += learningRate * HBiasError(i, 0) / fBatchSize; + } + + //updating weights + for (Int_t i = 0; i < fHBiases.GetNrows(); i++) { + for (Int_t j = 0; j < fVBiases.GetNrows(); j++) { + fWeights(i, j) += learningRate * (HBiasError(i, 0) * tildeX(j, 0) + + VBiasError(j, 0) * y(i, 0)) / fBatchSize; + } + } +} + +//______________________________________________________________________________ + +template +void TReference::SoftmaxAE(TMatrixT & A) +{ + size_t m,n; + m = A.GetNrows(); + n = A.GetNcols(); + + Real_t sum = 0.0; + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + sum += exp(A(i, j)); + } + } + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + A(i, j) = exp(A(i, j)) / sum; + } + } +} + +//______________________________________________________________________________ + +template +void TReference::CorruptInput(TMatrixT &input, + TMatrixT &corruptedInput, + Real_t corruptionLevel) { + for(size_t i=0; i< (size_t)input.GetNrows(); i++) + { + for(size_t j=0; j<(size_t)input.GetNcols(); j++ ) + { + + if ((size_t)((rand() / (RAND_MAX + 1.0)) * 100) % + ((size_t)(corruptionLevel * 10)) == + 0) + { + corruptedInput(i, j) = 0; + } + else + { + corruptedInput(i, j) = input(i, j); + } + } + } +} + + +//______________________________________________________________________________ + +template +void TReference::EncodeInput(TMatrixT &input, + TMatrixT &compressedInput, + TMatrixT &Weights) { + + size_t m, a; + m = compressedInput.GetNrows(); + a = input.GetNrows(); + + for (size_t i = 0; i < m; i++) { + compressedInput(i, 0) = 0; + for (size_t j = 0; j < a; j++) { + compressedInput(i, 0) = + compressedInput(i, 0) + (Weights(i, j) * input(j, 0)); + } + } +} +//______________________________________________________________________________ +template +void TReference::ReconstructInput(TMatrixT &compressedInput, + TMatrixT &reconstructedInput, + TMatrixT &fWeights) { + for (size_t i=0; i<(size_t)reconstructedInput.GetNrows(); i++) + { + reconstructedInput(i, 0) = 0; + for(size_t j=0; j<(size_t)compressedInput.GetNrows();j++) + { + reconstructedInput(i, 0) += fWeights(j, i) * compressedInput(j, 0); + } + } +} + +//______________________________________________________________________________ +// Logistic Regression Layer Methods +// +//______________________________________________________________________________ + +template +void TReference::ForwardLogReg(TMatrixT &input, + TMatrixT &p, + TMatrixT &fWeights) +{ + size_t m,n; + m = p.GetNrows(); + n = input.GetNrows(); + for(size_t i= 0; i < m; i++) + { + p(i, 0) = 0; + for(size_t j=0; j < n; j++) + { + p(i, 0) += fWeights(i, j) * input(j, 0); + } + } +} + +//______________________________________________________________________________ + +template +void TReference::UpdateParamsLogReg(TMatrixT &input, + TMatrixT &output, + TMatrixT &difference, + TMatrixT &p, + TMatrixT &fWeights, + TMatrixT &fBiases, + Real_t learningRate, + size_t fBatchSize) +{ + size_t m,n; + m = p.GetNrows(); + n = input.GetNrows(); + + for(size_t i= 0; i void TReference::MultiplyTranspose(TMatrixT &output, const TMatrixT &input, const TMatrixT &weights) { - output.MultT(input, weights); + output.MultT(input, weights); } template void TReference::AddRowWise(TMatrixT &output, const TMatrixT &biases) { - for (size_t i = 0; i < (size_t) output.GetNrows(); i++) { - for (size_t j = 0; j < (size_t) output.GetNcols(); j++) { - output(i,j) += biases(j,0); + for (size_t i = 0; i < (size_t)output.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)output.GetNcols(); j++) { + output(i, j) += biases(j, 0); } } } @@ -46,15 +44,15 @@ void TReference::Backward(TMatrixT &activation_gradients_backward, { // Compute element-wise product. - for (size_t i = 0; i < (size_t) df.GetNrows(); i++) { - for (size_t j = 0; j < (size_t) df.GetNcols(); j++) { - df(i,j) *= activation_gradients(i,j); + for (size_t i = 0; i < (size_t)df.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)df.GetNcols(); j++) { + df(i, j) *= activation_gradients(i, j); } } // Activation gradients. if (activation_gradients_backward.GetNoElements() > 0) { - activation_gradients_backward.Mult(df, weights); + activation_gradients_backward.Mult(df, weights); } // Weights gradients. @@ -64,12 +62,12 @@ void TReference::Backward(TMatrixT &activation_gradients_backward, // Bias gradients. if (bias_gradients.GetNoElements() > 0) { - for (size_t j = 0; j < (size_t) df.GetNcols(); j++) { + for (size_t j = 0; j < (size_t)df.GetNcols(); j++) { AReal sum = 0.0; - for (size_t i = 0; i < (size_t) df.GetNrows(); i++) { - sum += df(i,j); + for (size_t i = 0; i < (size_t)df.GetNrows(); i++) { + sum += df(i, j); } - bias_gradients(j,0) = sum; + bias_gradients(j, 0) = sum; } } } @@ -77,9 +75,9 @@ void TReference::Backward(TMatrixT &activation_gradients_backward, template void TReference::ScaleAdd(TMatrixT &A, const TMatrixT &B, AReal beta) { - for (size_t i = 0; i < (size_t) A.GetNrows(); i++) { - for (size_t j = 0; j < (size_t) A.GetNcols(); j++) { - A(i,j) += beta * B(i,j); + for (size_t i = 0; i < (size_t)A.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)A.GetNcols(); j++) { + A(i, j) += beta * B(i, j); } } } @@ -90,6 +88,22 @@ void TReference::Copy(TMatrixT &A, const TMatrixT &B) A = B; } +template +void TReference::ScaleAdd(std::vector> &A, const std::vector> &B, AReal beta) +{ + for (size_t i = 0; i < A.size(); ++i) { + ScaleAdd(A[i], B[i], beta); + } +} + +template +void TReference::Copy(std::vector> &A, const std::vector> &B) +{ + for (size_t i = 0; i < A.size(); ++i) { + Copy(A[i], B[i]); + } +} + template void TReference::SumColumns(TMatrixT &B, const TMatrixT &A) { @@ -101,5 +115,334 @@ void TReference::SumColumns(TMatrixT &B, const TMatrixT &A) } } +//______________________________________________________________________________ +template +void TReference::Im2col(TMatrixT &A, TMatrixT &B, size_t imgHeight, size_t imgWidth, + size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, + size_t zeroPaddingHeight, size_t zeroPaddingWidth) +{ + // image boudaries + int imgHeightBound = imgHeight + zeroPaddingHeight - (fltHeight - 1) / 2 - 1; + int imgWidthBound = imgWidth + zeroPaddingWidth - (fltWidth - 1) / 2 - 1; + size_t currLocalView = 0; + + // convolution centers + for (int i = -zeroPaddingHeight + fltHeight / 2; i <= imgHeightBound; i += strideRows) { + for (int j = -zeroPaddingWidth + fltWidth / 2; j <= imgWidthBound; j += strideCols) { + size_t currLocalViewPixel = 0; + + // within the local view + for (int m = 0; m < B.GetNrows(); m++) { + for (Int_t k = i - Int_t(fltHeight) / 2; k <= i + (Int_t(fltHeight) - 1) / 2; k++) { + for (Int_t l = j - Int_t(fltWidth) / 2; l <= j + (Int_t(fltWidth) - 1) / 2; l++) { + + // Check the boundaries + if (k < 0 || k >= Int_t(imgHeight) || l < 0 || l >= Int_t(imgWidth)) + A(currLocalView, currLocalViewPixel++) = 0; + else + A(currLocalView, currLocalViewPixel++) = B(m, k * imgWidth + l); + } + } + } + + currLocalView++; + } + } +} + +//______________________________________________________________________________ +template +void TReference::RotateWeights(TMatrixT &A, const TMatrixT &B, size_t filterDepth, + size_t filterHeight, size_t filterWidth, size_t numFilters) +{ + size_t jump = filterHeight * filterWidth; + for (size_t j = 0; j < filterDepth; j++) { + for (size_t k = 0; k < numFilters; k++) { + for (size_t i = 0; i < jump; i++) { + A(j, k * jump + i) = B(k, ((j + 1) * jump - 1) - i); + } + } + } +} + +//______________________________________________________________________________ +template +void TReference::AddConvBiases(TMatrixT &output, const TMatrixT &biases) +{ + for (size_t i = 0; i < (size_t)output.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)output.GetNcols(); j++) { + output(i, j) += biases(i, 0); + } + } +} + +//______________________________________________________________________________ +template +void TReference::ConvLayerBackward(std::vector> &activation_gradients_backward, + TMatrixT &weight_gradients, TMatrixT &bias_gradients, + std::vector> &df, + const std::vector> &activation_gradients, + const TMatrixT &weights, + const std::vector> &activations_backward, size_t batchSize, + size_t inputHeight, size_t inputWidth, size_t depth, size_t height, + size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth, + size_t nLocalViews) +{ + + // Update derivatives + size_t m, n; + m = activation_gradients[0].GetNrows(); + n = activation_gradients[0].GetNcols(); + + for (size_t i = 0; i < batchSize; i++) { + for (size_t j = 0; j < (size_t)m; j++) { + for (size_t k = 0; k < (size_t)n; k++) { + df[i](j, k) *= activation_gradients[i](j, k); + } + } + } + + // Calculate the activation gradients of the previous layer + CalculateConvActivationGradients(activation_gradients_backward, df, weights, batchSize, inputHeight, inputWidth, + depth, height, width, filterDepth, filterHeight, filterWidth); + + // Calculate the weight gradients + CalculateConvWeightGradients(weight_gradients, df, activations_backward, batchSize, inputHeight, inputWidth, depth, + height, width, filterDepth, filterHeight, filterWidth, nLocalViews); + + // Calculate the bias gradients + CalculateConvBiasGradients(bias_gradients, df, batchSize, depth, nLocalViews); +} + +//______________________________________________________________________________ +template +void TReference::CalculateConvActivationGradients(std::vector> &activation_gradients_backward, + std::vector> &df, + const TMatrixT &weights, size_t batchSize, + size_t inputHeight, size_t inputWidth, size_t depth, + size_t height, size_t width, size_t filterDepth, + size_t filterHeight, size_t filterWidth) +{ + if (activation_gradients_backward.size() == 0) return; + + // Transform the weights + TMatrixT rotWeights(filterDepth, depth * filterHeight * filterWidth); + RotateWeights(rotWeights, weights, filterDepth, filterHeight, filterWidth, weights.GetNrows()); + + // Calculate the zero paddings + size_t tempZeroPaddingHeight = (size_t)(floor((inputHeight - height + filterHeight - 1) / 2)); + size_t tempZeroPaddingWidth = (size_t)(floor((inputWidth - width + filterWidth - 1) / 2)); + + // Calculate the number of local views and the number of pixles in each view + size_t tempNLocalViews = inputHeight * inputWidth; + size_t tempNLocalViewPixels = depth * filterHeight * filterWidth; + + size_t tempStrideRows = 1; + size_t tempStrideCols = 1; + + // An entire convolution follows + for (size_t i = 0; i < batchSize; i++) { + TMatrixT dfTr(tempNLocalViews, tempNLocalViewPixels); + Im2col(dfTr, df[i], inputHeight, inputWidth, filterHeight, filterWidth, tempStrideRows, tempStrideCols, + tempZeroPaddingHeight, tempZeroPaddingWidth); + + activation_gradients_backward[i].MultT(rotWeights, dfTr); + } +} + +//______________________________________________________________________________ +template +void TReference::CalculateConvWeightGradients(TMatrixT &weight_gradients, + std::vector> &df, + const std::vector> &activations_backward, + size_t batchSize, size_t inputHeight, size_t inputWidth, + size_t depth, size_t height, size_t width, size_t filterDepth, + size_t filterHeight, size_t filterWidth, size_t nLocalViews) +{ + // reinitialize the weight gradients to 0 + for (Int_t i = 0; i < weight_gradients.GetNrows(); i++) { + for (Int_t j = 0; j < weight_gradients.GetNcols(); j++) { + weight_gradients(i, j) = 0; + } + } + + for (size_t i = 0; i < batchSize; i++) { + // Calculate the zero paddings + size_t tempZeroPaddingHeight = (filterHeight - height + inputHeight - 1) / 2; + size_t tempZeroPaddingWidth = (filterWidth - width + inputWidth - 1) / 2; + + size_t tempNLocalViews = filterHeight * filterWidth; + size_t tempNLocalViewPixels = inputHeight * inputWidth; + + size_t tempStrideRows = 1; + size_t tempStrideCols = 1; + + for (size_t j = 0; j < depth; j++) { + + // row matrix + TMatrixT rowDelta(1, nLocalViews); + for (size_t k = 0; k < nLocalViews; k++) { + rowDelta(0, k) = df[i](j, k); + } + + // convolution + TMatrixT res(filterDepth, filterHeight * filterWidth); + + TMatrixT rowDeltaTr(tempNLocalViews, tempNLocalViewPixels); + Im2col(rowDeltaTr, rowDelta, height, width, inputHeight, inputWidth, tempStrideRows, tempStrideCols, + tempZeroPaddingHeight, tempZeroPaddingWidth); + + res.MultT(activations_backward[i], rowDeltaTr); + + for (size_t k = 0; k < filterDepth; k++) { + for (size_t l = 0; l < filterHeight * filterWidth; l++) { + weight_gradients(j, k * (filterHeight * filterWidth) + l) += res(k, (tempNLocalViews - 1) - l); + } + } + } + } +} + +//______________________________________________________________________________ +template +void TReference::CalculateConvBiasGradients(TMatrixT &bias_gradients, std::vector> &df, + size_t batchSize, size_t depth, size_t nLocalViews) +{ + for (size_t i = 0; i < depth; i++) { + AReal sum = 0; + for (size_t j = 0; j < nLocalViews; j++) { + for (size_t k = 0; k < batchSize; k++) { + sum += df[k](i, j); + } + } + bias_gradients(i, 0) = sum; + } +} + +//______________________________________________________________________________ +template +void TReference::Downsample(TMatrixT &A, TMatrixT &B, const TMatrixT &C, size_t imgHeight, + size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, + size_t strideCols) +{ + // image boudaries + int imgHeightBound = imgHeight - (fltHeight - 1) / 2 - 1; + int imgWidthBound = imgWidth - (fltWidth - 1) / 2 - 1; + size_t currLocalView = 0; + + // centers + for (int i = fltHeight / 2; i <= imgHeightBound; i += strideRows) { + for (int j = fltWidth / 2; j <= imgWidthBound; j += strideCols) { + // within local views + for (int m = 0; m < C.GetNrows(); m++) { + AReal value = -std::numeric_limits::max(); + + for (int k = i - Int_t(fltHeight) / 2; k <= i + (Int_t(fltHeight) - 1) / 2; k++) { + for (int l = j - Int_t(fltWidth) / 2; l <= j + (Int_t(fltWidth) - 1) / 2; l++) { + if (C(m, k * imgWidth + l) > value) { + value = C(m, k * imgWidth + l); + B(m, currLocalView) = k * imgWidth + l; + } + } + } + A(m, currLocalView) = value; + } + currLocalView++; + } + } +} + +//______________________________________________________________________________ +template +void TReference::MaxPoolLayerBackward(std::vector> &activationGradientsBackward, + const std::vector> &activationGradients, + const std::vector> &indexMatrix, size_t batchSize, + size_t depth, size_t nLocalViews) +{ + for (size_t i = 0; i < batchSize; i++) { + for (size_t j = 0; j < depth; j++) { + + // initialize to zeros + for (size_t t = 0; t < (size_t)activationGradientsBackward[i].GetNcols(); t++) { + activationGradientsBackward[i][j][t] = 0; + } + + // set values + for (size_t k = 0; k < nLocalViews; k++) { + AReal grad = activationGradients[i][j][k]; + size_t winningIdx = indexMatrix[i][j][k]; + activationGradientsBackward[i][j][winningIdx] = grad; + } + } + } +} + +//______________________________________________________________________________ +template +void TReference::Reshape(TMatrixT &A, const TMatrixT &B) +{ + auto nColsA = A.GetNcols(); + auto nColsB = B.GetNcols(); + + for (Int_t i = 0; i < A.GetNrows(); i++) { + for (Int_t j = 0; j < A.GetNcols(); j++) { + auto nElem = i * nColsA + j; + A(i, j) = B(nElem / nColsB, (nElem - 1) % nColsB); + } + } +} + +//______________________________________________________________________________ +template +void TReference::Flatten(TMatrixT &A, const std::vector> &B, size_t size, size_t nRows, + size_t nCols) +{ + for (size_t i = 0; i < (size_t)size; i++) { + for (size_t j = 0; j < (size_t)nRows; j++) { + for (size_t k = 0; k < (size_t)nCols; k++) { + A(i, j * nCols + k) = B[i](j, k); + } + } + } +} + +//______________________________________________________________________________ +template +void TReference::Deflatten(std::vector> &A, const TMatrixT &B, size_t size, size_t nRows, + size_t nCols) +{ + for (size_t i = 0; i < (size_t)size; i++) { + for (size_t j = 0; j < (size_t)nRows; j++) { + for (size_t k = 0; k < (size_t)nCols; k++) { + A[i](j, k) = B(i, j * nCols + k); + } + } + } +} + +//______________________________________________________________________________ +template +void TReference::Rearrange(std::vector> &out, const std::vector> &in) +{ + // B x T x D out --- T x B x D in*/ + auto B = out.size(); + auto T = out[0].GetNrows(); + auto D = out[0].GetNcols(); + if ((T != (Int_t)in.size()) || (Int_t(B) != in[0].GetNrows()) || (D != in[0].GetNcols())) { + std::cout << "Incompatible Dimensions\n" + << in.size() << "x" << in[0].GetNrows() << "x" << in[0].GetNcols() << " --> " << B << "x" << T << "x" + << D << "\n"; + return; + } + for (size_t i = 0; i < B; ++i) { + for (Int_t j = 0; j < T; ++j) { + for (Int_t k = 0; k < D; ++k) { + out[i](j, k) = in[j](i, k); + } + } + } + return; +} + } // namespace DNN } // namespace TMVA diff --git a/tmva/tmva/src/DNN/Architectures/Reference/RecurrentPropagation.cxx b/tmva/tmva/src/DNN/Architectures/Reference/RecurrentPropagation.cxx new file mode 100644 index 0000000000000..66521f28d8be9 --- /dev/null +++ b/tmva/tmva/src/DNN/Architectures/Reference/RecurrentPropagation.cxx @@ -0,0 +1,80 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Saurav Shekhar 23/06/17 + +/************************************************************************* + * Copyright (C) 2017, Saurav Shekhar * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +///////////////////////////////////////////////////////////////////// +// Implementation of the functions required for the forward and // +// backward propagation of activations through a recurrent neural // +// network in the reference implementation. // +///////////////////////////////////////////////////////////////////// + +#include "TMVA/DNN/Architectures/Reference.h" + +namespace TMVA { +namespace DNN { + +//______________________________________________________________________________ +template +auto TReference::RecurrentLayerBackward(TMatrixT & state_gradients_backward, // BxH + TMatrixT & input_weight_gradients, + TMatrixT & state_weight_gradients, + TMatrixT & bias_gradients, + TMatrixT & df, //BxH + const TMatrixT & state, // BxH + const TMatrixT & weights_input, // HxD + const TMatrixT & weights_state, // HxH + const TMatrixT & input, // BxD + TMatrixT & input_gradient) +-> Matrix_t & +{ + // Compute element-wise product. + for (size_t i = 0; i < (size_t) df.GetNrows(); i++) { + for (size_t j = 0; j < (size_t) df.GetNcols(); j++) { + df(i,j) *= state_gradients_backward(i,j); // B x H + } + } + + // Input gradients. + if (input_gradient.GetNoElements() > 0) { + input_gradient.Mult(df, weights_input); // B x H . H x D = B x D + } + // State gradients + if (state_gradients_backward.GetNoElements() > 0) { + state_gradients_backward.MultT(df, weights_state); // B x H . H x H = B x H + } + + // Weights gradients. + if (input_weight_gradients.GetNoElements() > 0) { + TMatrixT tmp(input_weight_gradients); + input_weight_gradients.TMult(df, input); // H x B . B x D + input_weight_gradients += tmp; + } + if (state_weight_gradients.GetNoElements() > 0) { + TMatrixT tmp(state_weight_gradients); + state_weight_gradients.TMult(df, state); // H x B . B x H + state_weight_gradients += tmp; + } + + // Bias gradients. B x H -> H x 1 + if (bias_gradients.GetNoElements() > 0) { + for (size_t j = 0; j < (size_t) df.GetNcols(); j++) { + Scalar_t sum = 0.0; + for (size_t i = 0; i < (size_t) df.GetNrows(); i++) { + sum += df(i,j); + } + bias_gradients(j,0) = sum; + } + } + return input_gradient; +} + + +} // namespace DNN +} // namespace TMVA diff --git a/tmva/tmva/src/DNN/Architectures/Reference/TensorDataLoader.cxx b/tmva/tmva/src/DNN/Architectures/Reference/TensorDataLoader.cxx new file mode 100644 index 0000000000000..1e77bad3b4372 --- /dev/null +++ b/tmva/tmva/src/DNN/Architectures/Reference/TensorDataLoader.cxx @@ -0,0 +1,306 @@ +// @(#)root/tmva/tmva/dnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : TTensorDataLoader * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Specialization of the Tensor Data Loader Class * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +/////////////////////////////////////////////////////////////////// +// Specializations of Copy functions for the TensorDataLoader // +// specialized for the reference architecture. // +/////////////////////////////////////////////////////////////////// + +#include "TMVA/DNN/Architectures/Reference.h" + +namespace TMVA { +namespace DNN { + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorInput(std::vector> &tensor, + IndexIterator_t sampleIterator) +{ + const std::vector> &linputTensor = std::get<0>(fData); + + if (fBatchDepth == 1) { + for (size_t i = 0; i < fBatchHeight; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchWidth; j++) { + tensor[0](i, j) = static_cast(linputTensor[0](sampleIndex, j)); + } + sampleIterator++; + } + } else { + for (size_t i = 0; i < fBatchDepth; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchHeight; j++) { + for (size_t k = 0; k < fBatchWidth; k++) { + tensor[i](j, k) = static_cast(linputTensor[sampleIndex](j, k)); + } + } + sampleIterator++; + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorOutput(TMatrixT &matrix, + IndexIterator_t sampleIterator) +{ + const TMatrixT &loutputMatrix = std::get<1>(fData); + size_t n = loutputMatrix.GetNcols(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < n; j++) { + matrix(i, j) = static_cast(loutputMatrix(sampleIndex, j)); + } + + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorWeights(TMatrixT &matrix, + IndexIterator_t sampleIterator) +{ + const TMatrixT &lweightMatrix = std::get<2>(fData); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + matrix(i, 0) = static_cast(lweightMatrix(sampleIndex, 0)); + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorInput(std::vector> &tensor, + IndexIterator_t sampleIterator) +{ + const std::vector> &linputTensor = std::get<0>(fData); + + if (fBatchDepth == 1) { + for (size_t i = 0; i < fBatchHeight; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchWidth; j++) { + tensor[0](i, j) = linputTensor[0](sampleIndex, j); + } + sampleIterator++; + } + } else { + for (size_t i = 0; i < fBatchDepth; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchHeight; j++) { + for (size_t k = 0; k < fBatchWidth; k++) { + tensor[i](j, k) = linputTensor[sampleIndex](j, k); + } + } + sampleIterator++; + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorOutput(TMatrixT &matrix, + IndexIterator_t sampleIterator) +{ + const TMatrixT &loutputMatrix = std::get<1>(fData); + size_t n = loutputMatrix.GetNcols(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < n; j++) { + matrix(i, j) = loutputMatrix(sampleIndex, j); + } + + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorWeights(TMatrixT &matrix, + IndexIterator_t sampleIterator) +{ + const TMatrixT &lweightMatrix = std::get<2>(fData); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator; + matrix(i, 0) = lweightMatrix(sampleIndex, 0); + sampleIterator++; + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorInput(std::vector> &tensor, + IndexIterator_t sampleIterator) +{ + // one event, one example in the batch + Event *event = fData.front(); + + if (fBatchDepth == 1) { + for (size_t i = 0; i < fBatchHeight; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchWidth; j++) { + event = fData[sampleIndex]; + tensor[0](i, j) = static_cast(event->GetValue(j)); + } + sampleIterator++; + } + } else { + for (size_t i = 0; i < fBatchDepth; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchHeight; j++) { + for (size_t k = 0; k < fBatchWidth; k++) { + event = fData[sampleIndex]; + tensor[i](j, k) = static_cast(event->GetValue(j * fBatchHeight + k)); + } + } + sampleIterator++; + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorOutput(TMatrixT &matrix, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + Int_t n = matrix.GetNcols(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; + + for (Int_t j = 0; j < n; j++) { + if (event->GetNTargets() == 0) { + if (n == 1) { + matrix(i, j) = (event->GetClass() == 0) ? 1.0 : 0.0; + } else { + matrix(i, j) = 0.0; + if (j == (Int_t)event->GetClass()) { + matrix(i, j) = 1.0; + } + } + } else { + matrix(i, j) = static_cast(event->GetTarget(j)); + } + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorWeights(TMatrixT &matrix, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; + matrix(i, 0) = static_cast(event->GetWeight()); + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorInput(std::vector> &tensor, + IndexIterator_t sampleIterator) +{ + // one event, one example in the batch + Event *event = fData.front(); + + if (fBatchDepth == 1) { + for (size_t i = 0; i < fBatchHeight; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchWidth; j++) { + event = fData[sampleIndex]; + tensor[0](i, j) = event->GetValue(j); + } + sampleIterator++; + } + } else { + for (size_t i = 0; i < fBatchDepth; i++) { + size_t sampleIndex = *sampleIterator; + for (size_t j = 0; j < fBatchHeight; j++) { + for (size_t k = 0; k < fBatchWidth; k++) { + event = fData[sampleIndex]; + tensor[i](j, k) = event->GetValue(j * fBatchHeight + k); + } + } + sampleIterator++; + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorOutput(TMatrixT &matrix, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + Int_t n = matrix.GetNcols(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; + + for (Int_t j = 0; j < n; j++) { + if (event->GetNTargets() == 0) { + if (n == 1) { + matrix(i, j) = (event->GetClass() == 0) ? 1.0 : 0.0; + } else { + matrix(i, j) = 0.0; + if (j == (Int_t)event->GetClass()) { + matrix(i, j) = 1.0; + } + } + } else { + matrix(i, j) = event->GetTarget(j); + } + } + } +} + +//______________________________________________________________________________ +template <> +void TTensorDataLoader>::CopyTensorWeights(TMatrixT &matrix, + IndexIterator_t sampleIterator) +{ + Event *event = fData.front(); + + for (size_t i = 0; i < fBatchSize; i++) { + size_t sampleIndex = *sampleIterator++; + event = fData[sampleIndex]; + matrix(i, 0) = event->GetWeight(); + } +} + +} // namespace DNN +} // namespace TMVA diff --git a/tmva/tmva/src/MethodDL.cxx b/tmva/tmva/src/MethodDL.cxx new file mode 100644 index 0000000000000..6bf29f03e6587 --- /dev/null +++ b/tmva/tmva/src/MethodDL.cxx @@ -0,0 +1,1363 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski, Saurav Shekhar + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : MethodDL * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Deep Neural Network Method * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * Saurav Shekhar - ETH Zurich, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#include "TFormula.h" +#include "TString.h" +#include "TMath.h" + +#include "TMVA/Configurable.h" +#include "TMVA/IMethod.h" +#include "TMVA/ClassifierFactory.h" +#include "TMVA/MethodDL.h" +#include "TMVA/Types.h" +#include "TMVA/DNN/TensorDataLoader.h" +#include "TMVA/DNN/DLMinimizers.h" + +REGISTER_METHOD(DL) +ClassImp(TMVA::MethodDL); + +using namespace TMVA::DNN::CNN; +using namespace TMVA::DNN; + +using TMVA::DNN::EActivationFunction; +using TMVA::DNN::ELossFunction; +using TMVA::DNN::EInitialization; +using TMVA::DNN::EOutputFunction; + +namespace TMVA { + +//////////////////////////////////////////////////////////////////////////////// +TString fetchValueTmp(const std::map &keyValueMap, TString key) +{ + key.ToUpper(); + std::map::const_iterator it = keyValueMap.find(key); + if (it == keyValueMap.end()) { + return TString(""); + } + return it->second; +} + +//////////////////////////////////////////////////////////////////////////////// +template +T fetchValueTmp(const std::map &keyValueMap, TString key, T defaultValue); + +//////////////////////////////////////////////////////////////////////////////// +template <> +int fetchValueTmp(const std::map &keyValueMap, TString key, int defaultValue) +{ + TString value(fetchValueTmp(keyValueMap, key)); + if (value == "") { + return defaultValue; + } + return value.Atoi(); +} + +//////////////////////////////////////////////////////////////////////////////// +template <> +double fetchValueTmp(const std::map &keyValueMap, TString key, double defaultValue) +{ + TString value(fetchValueTmp(keyValueMap, key)); + if (value == "") { + return defaultValue; + } + return value.Atof(); +} + +//////////////////////////////////////////////////////////////////////////////// +template <> +TString fetchValueTmp(const std::map &keyValueMap, TString key, TString defaultValue) +{ + TString value(fetchValueTmp(keyValueMap, key)); + if (value == "") { + return defaultValue; + } + return value; +} + +//////////////////////////////////////////////////////////////////////////////// +template <> +bool fetchValueTmp(const std::map &keyValueMap, TString key, bool defaultValue) +{ + TString value(fetchValueTmp(keyValueMap, key)); + if (value == "") { + return defaultValue; + } + + value.ToUpper(); + if (value == "TRUE" || value == "T" || value == "1") { + return true; + } + + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +template <> +std::vector fetchValueTmp(const std::map &keyValueMap, TString key, + std::vector defaultValue) +{ + TString parseString(fetchValueTmp(keyValueMap, key)); + if (parseString == "") { + return defaultValue; + } + + parseString.ToUpper(); + std::vector values; + + const TString tokenDelim("+"); + TObjArray *tokenStrings = parseString.Tokenize(tokenDelim); + TIter nextToken(tokenStrings); + TObjString *tokenString = (TObjString *)nextToken(); + for (; tokenString != NULL; tokenString = (TObjString *)nextToken()) { + std::stringstream sstr; + double currentValue; + sstr << tokenString->GetString().Data(); + sstr >> currentValue; + values.push_back(currentValue); + } + return values; +} + +//////////////////////////////////////////////////////////////////////////////// +void MethodDL::DeclareOptions() +{ + // Set default values for all option strings + + DeclareOptionRef(fInputLayoutString = "0|0|0", "InputLayout", "The Layout of the input"); + + DeclareOptionRef(fBatchLayoutString = "0|0|0", "BatchLayout", "The Layout of the batch"); + + DeclareOptionRef(fLayoutString = "DENSE|(N+100)*2|SOFTSIGN,DENSE|0|LINEAR", "Layout", "Layout of the network."); + + DeclareOptionRef(fErrorStrategy = "CROSSENTROPY", "ErrorStrategy", "Loss function: Mean squared error (regression)" + " or cross entropy (binary classification)."); + AddPreDefVal(TString("CROSSENTROPY")); + AddPreDefVal(TString("SUMOFSQUARES")); + AddPreDefVal(TString("MUTUALEXCLUSIVE")); + + DeclareOptionRef(fWeightInitializationString = "XAVIER", "WeightInitialization", "Weight initialization strategy"); + AddPreDefVal(TString("XAVIER")); + AddPreDefVal(TString("XAVIERUNIFORM")); + + DeclareOptionRef(fArchitectureString = "CPU", "Architecture", "Which architecture to perform the training on."); + AddPreDefVal(TString("STANDARD")); + AddPreDefVal(TString("CPU")); + AddPreDefVal(TString("GPU")); + AddPreDefVal(TString("OPENCL")); + + DeclareOptionRef(fTrainingStrategyString = "LearningRate=1e-1," + "Momentum=0.3," + "Repetitions=3," + "ConvergenceSteps=50," + "BatchSize=30," + "TestRepetitions=7," + "WeightDecay=0.0," + "Renormalize=L2," + "DropConfig=0.0," + "DropRepetitions=5|LearningRate=1e-4," + "Momentum=0.3," + "Repetitions=3," + "ConvergenceSteps=50," + "BatchSize=20," + "TestRepetitions=7," + "WeightDecay=0.001," + "Renormalize=L2," + "DropConfig=0.0+0.5+0.5," + "DropRepetitions=5," + "Multithreading=True", + "TrainingStrategy", "Defines the training strategies."); +} + +//////////////////////////////////////////////////////////////////////////////// +void MethodDL::ProcessOptions() +{ + if (IgnoreEventsWithNegWeightsInTraining()) { + Log() << kINFO << "Will ignore negative events in training!" << Endl; + } + + if (fArchitectureString == "STANDARD") { + Log() << kERROR << "The STANDARD architecture has been deprecated. " + "Please use Architecture=CPU or Architecture=CPU." + "See the TMVA Users' Guide for instructions if you " + "encounter problems." + << Endl; + Log() << kFATAL << "The STANDARD architecture has been deprecated. " + "Please use Architecture=CPU or Architecture=CPU." + "See the TMVA Users' Guide for instructions if you " + "encounter problems." + << Endl; + } + + if (fArchitectureString == "OPENCL") { + Log() << kERROR << "The OPENCL architecture has not been implemented yet. " + "Please use Architecture=CPU or Architecture=CPU for the " + "time being. See the TMVA Users' Guide for instructions " + "if you encounter problems." + << Endl; + Log() << kFATAL << "The OPENCL architecture has not been implemented yet. " + "Please use Architecture=CPU or Architecture=CPU for the " + "time being. See the TMVA Users' Guide for instructions " + "if you encounter problems." + << Endl; + } + + if (fArchitectureString == "GPU") { +#ifndef DNNCUDA // Included only if DNNCUDA flag is _not_ set. + Log() << kERROR << "CUDA backend not enabled. Please make sure " + "you have CUDA installed and it was successfully " + "detected by CMAKE." + << Endl; + Log() << kFATAL << "CUDA backend not enabled. Please make sure " + "you have CUDA installed and it was successfully " + "detected by CMAKE." + << Endl; +#endif // DNNCUDA + } + + if (fArchitectureString == "CPU") { +#ifndef DNNCPU // Included only if DNNCPU flag is _not_ set. + Log() << kERROR << "Multi-core CPU backend not enabled. Please make sure " + "you have a BLAS implementation and it was successfully " + "detected by CMake as well that the imt CMake flag is set." + << Endl; + Log() << kFATAL << "Multi-core CPU backend not enabled. Please make sure " + "you have a BLAS implementation and it was successfully " + "detected by CMake as well that the imt CMake flag is set." + << Endl; +#endif // DNNCPU + } + + // Input Layout + ParseInputLayout(); + ParseBatchLayout(); + + // Loss function and output. + fOutputFunction = EOutputFunction::kSigmoid; + if (fAnalysisType == Types::kClassification) { + if (fErrorStrategy == "SUMOFSQUARES") { + fLossFunction = ELossFunction::kMeanSquaredError; + } + if (fErrorStrategy == "CROSSENTROPY") { + fLossFunction = ELossFunction::kCrossEntropy; + } + fOutputFunction = EOutputFunction::kSigmoid; + } else if (fAnalysisType == Types::kRegression) { + if (fErrorStrategy != "SUMOFSQUARES") { + Log() << kWARNING << "For regression only SUMOFSQUARES is a valid " + << " neural net error function. Setting error function to " + << " SUMOFSQUARES now." << Endl; + } + + fLossFunction = ELossFunction::kMeanSquaredError; + fOutputFunction = EOutputFunction::kIdentity; + } else if (fAnalysisType == Types::kMulticlass) { + if (fErrorStrategy == "SUMOFSQUARES") { + fLossFunction = ELossFunction::kMeanSquaredError; + } + if (fErrorStrategy == "CROSSENTROPY") { + fLossFunction = ELossFunction::kCrossEntropy; + } + if (fErrorStrategy == "MUTUALEXCLUSIVE") { + fLossFunction = ELossFunction::kSoftmaxCrossEntropy; + } + fOutputFunction = EOutputFunction::kSoftmax; + } + + // Initialization + if (fWeightInitializationString == "XAVIER") { + fWeightInitialization = DNN::EInitialization::kGauss; + } else if (fWeightInitializationString == "XAVIERUNIFORM") { + fWeightInitialization = DNN::EInitialization::kUniform; + } else { + fWeightInitialization = DNN::EInitialization::kGauss; + } + + // Training settings. + + KeyValueVector_t strategyKeyValues = ParseKeyValueString(fTrainingStrategyString, TString("|"), TString(",")); + for (auto &block : strategyKeyValues) { + TTrainingSettings settings; + + settings.convergenceSteps = fetchValueTmp(block, "ConvergenceSteps", 100); + settings.batchSize = fetchValueTmp(block, "BatchSize", 30); + settings.testInterval = fetchValueTmp(block, "TestRepetitions", 7); + settings.weightDecay = fetchValueTmp(block, "WeightDecay", 0.0); + settings.learningRate = fetchValueTmp(block, "LearningRate", 1e-5); + settings.momentum = fetchValueTmp(block, "Momentum", 0.3); + settings.dropoutProbabilities = fetchValueTmp(block, "DropConfig", std::vector()); + + TString regularization = fetchValueTmp(block, "Regularization", TString("NONE")); + if (regularization == "L1") { + settings.regularization = DNN::ERegularization::kL1; + } else if (regularization == "L2") { + settings.regularization = DNN::ERegularization::kL2; + } + + TString strMultithreading = fetchValueTmp(block, "Multithreading", TString("True")); + + if (strMultithreading.BeginsWith("T")) { + settings.multithreading = true; + } else { + settings.multithreading = false; + } + + fTrainingSettings.push_back(settings); + } +} + +//////////////////////////////////////////////////////////////////////////////// +/// default initializations +void MethodDL::Init() +{ + // Nothing to do here +} + +//////////////////////////////////////////////////////////////////////////////// +/// Parse the input layout +void MethodDL::ParseInputLayout() +{ + // Define the delimiter + const TString delim("|"); + + // Get the input layout string + TString inputLayoutString = this->GetInputLayoutString(); + + size_t depth = 0; + size_t height = 0; + size_t width = 0; + + // Split the input layout string + TObjArray *inputDimStrings = inputLayoutString.Tokenize(delim); + TIter nextInputDim(inputDimStrings); + TObjString *inputDimString = (TObjString *)nextInputDim(); + int idxToken = 0; + + for (; inputDimString != nullptr; inputDimString = (TObjString *)nextInputDim()) { + switch (idxToken) { + case 0: // input depth + { + TString strDepth(inputDimString->GetString()); + depth = (size_t)strDepth.Atoi(); + } break; + case 1: // input height + { + TString strHeight(inputDimString->GetString()); + height = (size_t)strHeight.Atoi(); + } break; + case 2: // input width + { + TString strWidth(inputDimString->GetString()); + width = (size_t)strWidth.Atoi(); + } break; + } + ++idxToken; + } + + this->SetInputDepth(depth); + this->SetInputHeight(height); + this->SetInputWidth(width); +} + +//////////////////////////////////////////////////////////////////////////////// +/// Parse the input layout +void MethodDL::ParseBatchLayout() +{ + // Define the delimiter + const TString delim("|"); + + // Get the input layout string + TString batchLayoutString = this->GetBatchLayoutString(); + + size_t batchDepth = 0; + size_t batchHeight = 0; + size_t batchWidth = 0; + + // Split the input layout string + TObjArray *batchDimStrings = batchLayoutString.Tokenize(delim); + TIter nextBatchDim(batchDimStrings); + TObjString *batchDimString = (TObjString *)nextBatchDim(); + int idxToken = 0; + + for (; batchDimString != nullptr; batchDimString = (TObjString *)nextBatchDim()) { + switch (idxToken) { + case 0: // input depth + { + TString strDepth(batchDimString->GetString()); + batchDepth = (size_t)strDepth.Atoi(); + } break; + case 1: // input height + { + TString strHeight(batchDimString->GetString()); + batchHeight = (size_t)strHeight.Atoi(); + } break; + case 2: // input width + { + TString strWidth(batchDimString->GetString()); + batchWidth = (size_t)strWidth.Atoi(); + } break; + } + ++idxToken; + } + + this->SetBatchDepth(batchDepth); + this->SetBatchHeight(batchHeight); + this->SetBatchWidth(batchWidth); +} + +//////////////////////////////////////////////////////////////////////////////// +/// Create a deep net based on the layout string +template +void MethodDL::CreateDeepNet(DNN::TDeepNet &deepNet, + std::vector> &nets) +{ + // Layer specification, layer details + const TString layerDelimiter(","); + const TString subDelimiter("|"); + + TString layoutString = this->GetLayoutString(); + + // Split layers + TObjArray *layerStrings = layoutString.Tokenize(layerDelimiter); + TIter nextLayer(layerStrings); + TObjString *layerString = (TObjString *)nextLayer(); + + for (; layerString != nullptr; layerString = (TObjString *)nextLayer()) { + // Split layer details + TObjArray *subStrings = layerString->GetString().Tokenize(subDelimiter); + TIter nextToken(subStrings); + TObjString *token = (TObjString *)nextToken(); + + // Determine the type of the layer + TString strLayerType = token->GetString(); + + if (strLayerType == "DENSE") { + ParseDenseLayer(deepNet, nets, layerString->GetString(), subDelimiter); + } else if (strLayerType == "CONV") { + ParseConvLayer(deepNet, nets, layerString->GetString(), subDelimiter); + } else if (strLayerType == "MAXPOOL") { + ParseMaxPoolLayer(deepNet, nets, layerString->GetString(), subDelimiter); + } else if (strLayerType == "RESHAPE") { + ParseReshapeLayer(deepNet, nets, layerString->GetString(), subDelimiter); + } else if (strLayerType == "RNN") { + ParseRnnLayer(deepNet, nets, layerString->GetString(), subDelimiter); + } else if (strLayerType == "LSTM") { + ParseLstmLayer(deepNet, nets, layerString->GetString(), subDelimiter); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +/// Pases the layer string and creates the appropriate dense layer +template +void MethodDL::ParseDenseLayer(DNN::TDeepNet &deepNet, + std::vector> & /*nets*/, TString layerString, + TString delim) +{ + int width = 0; + EActivationFunction activationFunction = EActivationFunction::kTanh; + + // not sure about this + const size_t inputSize = GetNvar(); + + // Split layer details + TObjArray *subStrings = layerString.Tokenize(delim); + TIter nextToken(subStrings); + TObjString *token = (TObjString *)nextToken(); + int idxToken = 0; + + // jump the first token + for (; token != nullptr; token = (TObjString *)nextToken()) { + switch (idxToken) { + case 1: // number of nodes + { + // not sure + TString strNumNodes(token->GetString()); + TString strN("x"); + strNumNodes.ReplaceAll("N", strN); + strNumNodes.ReplaceAll("n", strN); + TFormula fml("tmp", strNumNodes); + width = fml.Eval(inputSize); + } break; + case 2: // actiovation function + { + TString strActFnc(token->GetString()); + if (strActFnc == "RELU") { + activationFunction = DNN::EActivationFunction::kRelu; + } else if (strActFnc == "TANH") { + activationFunction = DNN::EActivationFunction::kTanh; + } else if (strActFnc == "SYMMRELU") { + activationFunction = DNN::EActivationFunction::kSymmRelu; + } else if (strActFnc == "SOFTSIGN") { + activationFunction = DNN::EActivationFunction::kSoftSign; + } else if (strActFnc == "SIGMOID") { + activationFunction = DNN::EActivationFunction::kSigmoid; + } else if (strActFnc == "LINEAR") { + activationFunction = DNN::EActivationFunction::kIdentity; + } else if (strActFnc == "GAUSS") { + activationFunction = DNN::EActivationFunction::kGauss; + } + } break; + } + ++idxToken; + } + + // Add the dense layer, initialize the weights and biases and copy + TDenseLayer *denseLayer = deepNet.AddDenseLayer(width, activationFunction); + denseLayer->Initialize(); + + // add same layer to fNet + fNet->AddDenseLayer(width, activationFunction); + + //TDenseLayer *copyDenseLayer = new TDenseLayer(*denseLayer); + + // add the copy to all slave nets + //for (size_t i = 0; i < nets.size(); i++) { + // nets[i].AddDenseLayer(copyDenseLayer); + //} +} + +//////////////////////////////////////////////////////////////////////////////// +/// Pases the layer string and creates the appropriate convolutional layer +template +void MethodDL::ParseConvLayer(DNN::TDeepNet &deepNet, + std::vector> & /*nets*/, TString layerString, + TString delim) +{ + int depth = 0; + int fltHeight = 0; + int fltWidth = 0; + int strideRows = 0; + int strideCols = 0; + int zeroPadHeight = 0; + int zeroPadWidth = 0; + EActivationFunction activationFunction = EActivationFunction::kTanh; + + // Split layer details + TObjArray *subStrings = layerString.Tokenize(delim); + TIter nextToken(subStrings); + TObjString *token = (TObjString *)nextToken(); + int idxToken = 0; + + for (; token != nullptr; token = (TObjString *)nextToken()) { + switch (idxToken) { + case 1: // depth + { + TString strDepth(token->GetString()); + depth = strDepth.Atoi(); + } break; + case 2: // filter height + { + TString strFltHeight(token->GetString()); + fltHeight = strFltHeight.Atoi(); + } break; + case 3: // filter width + { + TString strFltWidth(token->GetString()); + fltWidth = strFltWidth.Atoi(); + } break; + case 4: // stride in rows + { + TString strStrideRows(token->GetString()); + strideRows = strStrideRows.Atoi(); + } break; + case 5: // stride in cols + { + TString strStrideCols(token->GetString()); + strideCols = strStrideCols.Atoi(); + } break; + case 6: // zero padding height + { + TString strZeroPadHeight(token->GetString()); + zeroPadHeight = strZeroPadHeight.Atoi(); + } break; + case 7: // zero padding width + { + TString strZeroPadWidth(token->GetString()); + zeroPadWidth = strZeroPadWidth.Atoi(); + } break; + case 8: // activation function + { + TString strActFnc(token->GetString()); + if (strActFnc == "RELU") { + activationFunction = DNN::EActivationFunction::kRelu; + } else if (strActFnc == "TANH") { + activationFunction = DNN::EActivationFunction::kTanh; + } else if (strActFnc == "SYMMRELU") { + activationFunction = DNN::EActivationFunction::kSymmRelu; + } else if (strActFnc == "SOFTSIGN") { + activationFunction = DNN::EActivationFunction::kSoftSign; + } else if (strActFnc == "SIGMOID") { + activationFunction = DNN::EActivationFunction::kSigmoid; + } else if (strActFnc == "LINEAR") { + activationFunction = DNN::EActivationFunction::kIdentity; + } else if (strActFnc == "GAUSS") { + activationFunction = DNN::EActivationFunction::kGauss; + } + } break; + } + ++idxToken; + } + + // Add the convolutional layer, initialize the weights and biases and copy + TConvLayer *convLayer = deepNet.AddConvLayer(depth, fltHeight, fltWidth, strideRows, strideCols, + zeroPadHeight, zeroPadWidth, activationFunction); + convLayer->Initialize(); + + // Add same layer to fNet + fNet->AddConvLayer(depth, fltHeight, fltWidth, strideRows, strideCols, + zeroPadHeight, zeroPadWidth, activationFunction); + + //TConvLayer *copyConvLayer = new TConvLayer(*convLayer); + + //// add the copy to all slave nets + //for (size_t i = 0; i < nets.size(); i++) { + // nets[i].AddConvLayer(copyConvLayer); + //} +} + +//////////////////////////////////////////////////////////////////////////////// +/// Pases the layer string and creates the appropriate max pool layer +template +void MethodDL::ParseMaxPoolLayer(DNN::TDeepNet &deepNet, + std::vector> & /*nets*/, TString layerString, + TString delim) +{ + + int frameHeight = 0; + int frameWidth = 0; + int strideRows = 0; + int strideCols = 0; + + // Split layer details + TObjArray *subStrings = layerString.Tokenize(delim); + TIter nextToken(subStrings); + TObjString *token = (TObjString *)nextToken(); + int idxToken = 0; + + for (; token != nullptr; token = (TObjString *)nextToken()) { + switch (idxToken) { + case 1: // frame height + { + TString strFrmHeight(token->GetString()); + frameHeight = strFrmHeight.Atoi(); + } break; + case 2: // frame width + { + TString strFrmWidth(token->GetString()); + frameWidth = strFrmWidth.Atoi(); + } break; + case 3: // stride in rows + { + TString strStrideRows(token->GetString()); + strideRows = strStrideRows.Atoi(); + } break; + case 4: // stride in cols + { + TString strStrideCols(token->GetString()); + strideCols = strStrideCols.Atoi(); + } break; + } + ++idxToken; + } + + // Add the Max pooling layer + // TMaxPoolLayer *maxPoolLayer = + deepNet.AddMaxPoolLayer(frameHeight, frameWidth, strideRows, strideCols); + + // Add the same layer to fNet + fNet->AddMaxPoolLayer(frameHeight, frameWidth, strideRows, strideCols); + + //TMaxPoolLayer *copyMaxPoolLayer = new TMaxPoolLayer(*maxPoolLayer); + + //// add the copy to all slave nets + //for (size_t i = 0; i < nets.size(); i++) { + // nets[i].AddMaxPoolLayer(copyMaxPoolLayer); + //} +} + +//////////////////////////////////////////////////////////////////////////////// +/// Pases the layer string and creates the appropriate reshape layer +template +void MethodDL::ParseReshapeLayer(DNN::TDeepNet &deepNet, + std::vector> & /*nets*/, TString layerString, + TString delim) +{ + int depth = 0; + int height = 0; + int width = 0; + bool flattening = false; + + // Split layer details + TObjArray *subStrings = layerString.Tokenize(delim); + TIter nextToken(subStrings); + TObjString *token = (TObjString *)nextToken(); + int idxToken = 0; + + for (; token != nullptr; token = (TObjString *)nextToken()) { + switch (idxToken) { + case 1: { + TString strDepth(token->GetString()); + depth = strDepth.Atoi(); + } break; + case 2: // height + { + TString strHeight(token->GetString()); + height = strHeight.Atoi(); + } break; + case 3: // width + { + TString strWidth(token->GetString()); + width = strWidth.Atoi(); + } break; + case 4: // flattening + { + TString flat(token->GetString()); + if (flat == "FLAT") { + flattening = true; + } + } break; + } + ++idxToken; + } + + // Add the reshape layer + // TReshapeLayer *reshapeLayer = + deepNet.AddReshapeLayer(depth, height, width, flattening); + + // Add the same layer to fNet + fNet->AddReshapeLayer(depth, height, width, flattening); + + //TReshapeLayer *copyReshapeLayer = new TReshapeLayer(*reshapeLayer); + + //// add the copy to all slave nets + //for (size_t i = 0; i < nets.size(); i++) { + // nets[i].AddReshapeLayer(copyReshapeLayer); + //} +} + +//////////////////////////////////////////////////////////////////////////////// +/// Pases the layer string and creates the appropriate rnn layer +template +void MethodDL::ParseRnnLayer(DNN::TDeepNet &deepNet, + std::vector> & /*nets*/, TString layerString, + TString delim) +{ + // int depth = 0; + int stateSize = 0; + int inputSize = 0; + int timeSteps = 0; + bool rememberState = false; + + // Split layer details + TObjArray *subStrings = layerString.Tokenize(delim); + TIter nextToken(subStrings); + TObjString *token = (TObjString *)nextToken(); + int idxToken = 0; + + for (; token != nullptr; token = (TObjString *)nextToken()) { + switch (idxToken) { + case 1: // state size + { + TString strstateSize(token->GetString()); + stateSize = strstateSize.Atoi(); + } break; + case 2: // input size + { + TString strinputSize(token->GetString()); + inputSize = strinputSize.Atoi(); + } break; + case 3: // time steps + { + TString strtimeSteps(token->GetString()); + timeSteps = strtimeSteps.Atoi(); + } + case 4: // remember state (1 or 0) + { + TString strrememberState(token->GetString()); + rememberState = (bool) strrememberState.Atoi(); + } break; + } + ++idxToken; + } + + // Add the recurrent layer, initialize the weights and biases and copy + TBasicRNNLayer *basicRNNLayer = deepNet.AddBasicRNNLayer(stateSize, inputSize, + timeSteps, rememberState); + basicRNNLayer->Initialize(); + + // Add same layer to fNet + fNet->AddBasicRNNLayer(stateSize, inputSize, timeSteps, rememberState); + + //TBasicRNNLayer *copyRNNLayer = new TBasicRNNLayer(*basicRNNLayer); + + //// add the copy to all slave nets + //for (size_t i = 0; i < nets.size(); i++) { + // nets[i].AddBasicRNNLayer(copyRNNLayer); + //} +} + +//////////////////////////////////////////////////////////////////////////////// +/// Pases the layer string and creates the appropriate lstm layer +template +void MethodDL::ParseLstmLayer(DNN::TDeepNet & /*deepNet*/, + std::vector> & /*nets*/, TString layerString, + TString delim) +{ + // Split layer details + TObjArray *subStrings = layerString.Tokenize(delim); + TIter nextToken(subStrings); + TObjString *token = (TObjString *)nextToken(); + int idxToken = 0; + + for (; token != nullptr; token = (TObjString *)nextToken()) { + switch (idxToken) { + } + ++idxToken; + } +} + +//////////////////////////////////////////////////////////////////////////////// +/// Standard constructor. +MethodDL::MethodDL(const TString &jobName, const TString &methodTitle, DataSetInfo &theData, const TString &theOption) + : MethodBase(jobName, Types::kDL, methodTitle, theData, theOption), fInputDepth(), fInputHeight(), fInputWidth(), + fBatchDepth(), fBatchHeight(), fBatchWidth(), fWeightInitialization(), fOutputFunction(), fLossFunction(), + fInputLayoutString(), fBatchLayoutString(), fLayoutString(), fErrorStrategy(), fTrainingStrategyString(), + fWeightInitializationString(), fArchitectureString(), fResume(false), fTrainingSettings() +{ + // Nothing to do here +} + +//////////////////////////////////////////////////////////////////////////////// +/// Constructor from a weight file. +MethodDL::MethodDL(DataSetInfo &theData, const TString &theWeightFile) + : MethodBase(Types::kDL, theData, theWeightFile), fInputDepth(), fInputHeight(), fInputWidth(), fBatchDepth(), + fBatchHeight(), fBatchWidth(), fWeightInitialization(), fOutputFunction(), fLossFunction(), fInputLayoutString(), + fBatchLayoutString(), fLayoutString(), fErrorStrategy(), fTrainingStrategyString(), fWeightInitializationString(), + fArchitectureString(), fResume(false), fTrainingSettings() +{ + // Nothing to do here +} + +//////////////////////////////////////////////////////////////////////////////// +/// Destructor. +MethodDL::~MethodDL() +{ + // Nothing to do here +} + +//////////////////////////////////////////////////////////////////////////////// +/// Parse key value pairs in blocks -> return vector of blocks with map of key value pairs. +auto MethodDL::ParseKeyValueString(TString parseString, TString blockDelim, TString tokenDelim) -> KeyValueVector_t +{ + KeyValueVector_t blockKeyValues; + const TString keyValueDelim("="); + + TObjArray *blockStrings = parseString.Tokenize(blockDelim); + TIter nextBlock(blockStrings); + TObjString *blockString = (TObjString *)nextBlock(); + + for (; blockString != nullptr; blockString = (TObjString *)nextBlock()) { + blockKeyValues.push_back(std::map()); + std::map ¤tBlock = blockKeyValues.back(); + + TObjArray *subStrings = blockString->GetString().Tokenize(tokenDelim); + TIter nextToken(subStrings); + TObjString *token = (TObjString *)nextToken(); + + for (; token != nullptr; token = (TObjString *)nextToken()) { + TString strKeyValue(token->GetString()); + int delimPos = strKeyValue.First(keyValueDelim.Data()); + if (delimPos <= 0) continue; + + TString strKey = TString(strKeyValue(0, delimPos)); + strKey.ToUpper(); + TString strValue = TString(strKeyValue(delimPos + 1, strKeyValue.Length())); + + strKey.Strip(TString::kBoth, ' '); + strValue.Strip(TString::kBoth, ' '); + + currentBlock.insert(std::make_pair(strKey, strValue)); + } + } + return blockKeyValues; +} + +//////////////////////////////////////////////////////////////////////////////// +/// What kind of analysis type can handle the CNN +Bool_t MethodDL::HasAnalysisType(Types::EAnalysisType type, UInt_t numberClasses, UInt_t /*numberTargets*/) +{ + if (type == Types::kClassification && numberClasses == 2) return kTRUE; + if (type == Types::kMulticlass) return kTRUE; + if (type == Types::kRegression) return kTRUE; + + return kFALSE; +} + +//////////////////////////////////////////////////////////////////////////////// +void MethodDL::Train() +{ + if (fInteractive) { + Log() << kFATAL << "Not implemented yet" << Endl; + return; + } + + if (this->GetArchitectureString() == "GPU") { + TrainGpu(); + return; + } else if (this->GetArchitectureString() == "OpenCL") { + Log() << kFATAL << "OpenCL backend not yet supported." << Endl; + return; + } else if (this->GetArchitectureString() == "CPU") { + TrainCpu(); + return; + } +} + +//////////////////////////////////////////////////////////////////////////////// +void MethodDL::TrainGpu() +{ +#ifdef DNNCUDA // Included only if DNNCUDA flag is set. + + using Architecture_t = DNN::TCuda; + using Scalar_t = Architecture_t::Scalar_t; + using Matrix_t = typename Architecture_t::Matrix_t; + using DeepNet_t = TMVA::DNN::TDeepNet; + using TensorDataLoader_t = TTensorDataLoader; + + Log() << kINFO << "Start of deep neural network training on GPU." << Endl << Endl; + + // Determine the number of training and testing examples + size_t nTrainingSamples = GetEventCollection(Types::kTraining).size(); + size_t nTestSamples = GetEventCollection(Types::kTesting).size(); + + // Determine the number of outputs + size_t outputSize = 1; + if (fAnalysisType == Types::kRegression && GetNTargets() != 0) { + outputSize = GetNTargets(); + } else if (fAnalysisType == Types::kMulticlass && DataInfo().GetNClasses() >= 2) { + outputSize = DataInfo().GetNClasses(); + } + + size_t trainingPhase = 1; + for (TTrainingSettings &settings : this->GetTrainingSettings()) { + + size_t nThreads = 1; + + Log() << "Training phase " << trainingPhase << " of " << this->GetTrainingSettings().size() << ":" << Endl; + trainingPhase++; + + // After the processing of the options, initialize the master deep net + size_t batchSize = settings.batchSize; + // Should be replaced by actual implementation. No support for this now. + size_t inputDepth = this->GetInputDepth(); + size_t inputHeight = this->GetInputHeight(); + size_t inputWidth = this->GetInputWidth(); + size_t batchDepth = this->GetBatchDepth(); + size_t batchHeight = this->GetBatchHeight(); + size_t batchWidth = this->GetBatchWidth(); + ELossFunction J = this->GetLossFunction(); + EInitialization I = this->GetWeightInitialization(); + ERegularization R = settings.regularization; + Scalar_t weightDecay = settings.weightDecay; + DeepNet_t deepNet(batchSize, inputDepth, inputHeight, inputWidth, batchDepth, batchHeight, batchWidth, J, I, R, + weightDecay); + + // Initialize the vector of slave nets + std::vector nets{}; + nets.reserve(nThreads); + for (size_t i = 0; i < nThreads; i++) { + // create a copies of the master deep net + nets.push_back(deepNet); + } + + // Add all appropriate layers + CreateDeepNet(deepNet, nets); + + // Loading the training and testing datasets + TensorDataLoader_t trainingData(GetEventCollection(Types::kTraining), nTrainingSamples, deepNet.GetBatchSize(), + deepNet.GetBatchDepth(), deepNet.GetBatchHeight(), deepNet.GetBatchWidth(), + deepNet.GetOutputWidth(), nThreads); + + TensorDataLoader_t testingData(GetEventCollection(Types::kTesting), nTestSamples, deepNet.GetBatchSize(), + deepNet.GetBatchDepth(), deepNet.GetBatchHeight(), deepNet.GetBatchWidth(), + deepNet.GetOutputWidth(), nThreads); + + // Initialize the minimizer + DNN::TDLGradientDescent minimizer(settings.learningRate, settings.convergenceSteps, + settings.testInterval); + + // Initialize the vector of batches, one batch for one slave network + std::vector> batches{}; + + bool converged = false; + // count the steps until the convergence + size_t stepCount = 0; + size_t batchesInEpoch = nTrainingSamples / deepNet.GetBatchSize(); + + // start measuring + std::chrono::time_point start, end; + start = std::chrono::system_clock::now(); + + if (!fInteractive) { + Log() << std::setw(10) << "Epoch" + << " | " << std::setw(12) << "Train Err." << std::setw(12) << "Test Err." << std::setw(12) << "GFLOP/s" + << std::setw(12) << "Conv. Steps" << Endl; + std::string separator(62, '-'); + Log() << separator << Endl; + } + + while (!converged) { + stepCount++; + trainingData.Shuffle(); + + // execute all epochs + for (size_t i = 0; i < batchesInEpoch; i += nThreads) { + // Clean and load new batches, one batch for one slave net + batches.clear(); + batches.reserve(nThreads); + for (size_t j = 0; j < nThreads; j++) { + batches.push_back(trainingData.GetTensorBatch()); + } + + // execute one minimization step + if (settings.momentum > 0.0) { + minimizer.StepMomentum(deepNet, nets, batches, settings.momentum); + } else { + minimizer.Step(deepNet, nets, batches); + } + } + + if ((stepCount % minimizer.GetTestInterval()) == 0) { + // Compute test error. + Double_t testError = 0.0; + for (auto batch : testingData) { + auto inputTensor = batch.GetInput(); + auto outputMatrix = batch.GetOutput(); + testError += deepNet.Loss(inputTensor, outputMatrix); + } + + testError /= (Double_t)(nTestSamples / settings.batchSize); + + // stop measuring + end = std::chrono::system_clock::now(); + + // Compute training error. + Double_t trainingError = 0.0; + for (auto batch : trainingData) { + auto inputTensor = batch.GetInput(); + auto outputMatrix = batch.GetOutput(); + trainingError += deepNet.Loss(inputTensor, outputMatrix); + } + trainingError /= (Double_t)(nTrainingSamples / settings.batchSize); + + // Compute numerical throughput. + std::chrono::duration elapsed_seconds = end - start; + double seconds = elapsed_seconds.count(); + double nFlops = (double)(settings.testInterval * batchesInEpoch); + // nFlops *= deepNet.GetNFlops() * 1e-9; + + converged = minimizer.HasConverged(testError); + + Log() << std::setw(10) << stepCount << " | " << std::setw(12) << trainingError << std::setw(12) << testError + << std::setw(12) << nFlops / seconds << std::setw(12) << minimizer.GetConvergenceCount() << Endl; + + if (converged) { + Log() << Endl; + } + } + } + } + +#else // DNNCUDA flag not set. + + Log() << kFATAL << "CUDA backend not enabled. Please make sure " + "you have CUDA installed and it was successfully " + "detected by CMAKE." + << Endl; +#endif // DNNCUDA +} + +//////////////////////////////////////////////////////////////////////////////// +void MethodDL::TrainCpu() +{ +#ifdef DNNCPU // Included only if DNNCPU flag is set. + + std::cout << "Train CPU Method" << std::endl; + using Architecture_t = DNN::TCpu; + using Scalar_t = Architecture_t::Scalar_t; + // using Matrix_t = typename Architecture_t::Matrix_t; + using DeepNet_t = TMVA::DNN::TDeepNet; + using TensorDataLoader_t = TTensorDataLoader; + + Log() << kINFO << "Start of deep neural network training on CPU." << Endl << Endl; + + // Determine the number of training and testing examples + size_t nTrainingSamples = GetEventCollection(Types::kTraining).size(); + size_t nTestSamples = GetEventCollection(Types::kTesting).size(); + + // Determine the number of outputs + // // size_t outputSize = 1; + // // if (fAnalysisType == Types::kRegression && GetNTargets() != 0) { + // // outputSize = GetNTargets(); + // // } else if (fAnalysisType == Types::kMulticlass && DataInfo().GetNClasses() >= 2) { + // // outputSize = DataInfo().GetNClasses(); + // // } + + size_t trainingPhase = 1; + for (TTrainingSettings &settings : this->GetTrainingSettings()) { + + size_t nThreads = 1; // FIXME threads are hard coded to 1, no use of slave threads or multi-threading + + Log() << "Training phase " << trainingPhase << " of " << this->GetTrainingSettings().size() << ":" << Endl; + trainingPhase++; + + // After the processing of the options, initialize the master deep net + size_t batchSize = settings.batchSize; + // Should be replaced by actual implementation. No support for this now. + size_t inputDepth = this->GetInputDepth(); + size_t inputHeight = this->GetInputHeight(); + size_t inputWidth = this->GetInputWidth(); + size_t batchDepth = this->GetBatchDepth(); + size_t batchHeight = this->GetBatchHeight(); + size_t batchWidth = this->GetBatchWidth(); + ELossFunction J = this->GetLossFunction(); + EInitialization I = this->GetWeightInitialization(); + ERegularization R = settings.regularization; + Scalar_t weightDecay = settings.weightDecay; + + //fNet = std::unique_ptr(new DeepNet_t(batchSize, inputDepth, inputHeight, inputWidth, batchDepth, + // batchHeight, batchWidth, J, I, R, weightDecay)); + //DeepNet_t &deepNet = *(fNet.get()); + + DeepNet_t deepNet(batchSize, inputDepth, inputHeight, inputWidth, batchDepth, batchHeight, batchWidth, J, I, R, weightDecay); + + fNet = std::unique_ptr(new DeepNet_t(1, inputDepth, inputHeight, inputWidth, batchDepth, + batchHeight, batchWidth, J, I, R, weightDecay)); + + // Initialize the vector of slave nets + std::vector nets{}; + nets.reserve(nThreads); + for (size_t i = 0; i < nThreads; i++) { + // create a copies of the master deep net + nets.push_back(deepNet); + } + + // Add all appropriate layers to deepNet and copies to fNet + CreateDeepNet(deepNet, nets); + + // Loading the training and testing datasets + TensorDataLoader_t trainingData(GetEventCollection(Types::kTraining), nTrainingSamples, deepNet.GetBatchSize(), + deepNet.GetBatchDepth(), deepNet.GetBatchHeight(), deepNet.GetBatchWidth(), + deepNet.GetOutputWidth(), nThreads); + + TensorDataLoader_t testingData(GetEventCollection(Types::kTesting), nTestSamples, deepNet.GetBatchSize(), + deepNet.GetBatchDepth(), deepNet.GetBatchHeight(), deepNet.GetBatchWidth(), + deepNet.GetOutputWidth(), nThreads); + + // Initialize the minimizer + DNN::TDLGradientDescent minimizer(settings.learningRate, settings.convergenceSteps, + settings.testInterval); + + // Initialize the vector of batches, one batch for one slave network + std::vector> batches{}; + + bool converged = false; + // count the steps until the convergence + size_t stepCount = 0; + size_t batchesInEpoch = nTrainingSamples / deepNet.GetBatchSize(); + + // start measuring + std::chrono::time_point start, end; + start = std::chrono::system_clock::now(); + + if (!fInteractive) { + Log() << std::setw(10) << "Epoch" + << " | " << std::setw(12) << "Train Err." << std::setw(12) << "Test Err." << std::setw(12) << "GFLOP/s" + << std::setw(12) << "Conv. Steps" << Endl; + std::string separator(62, '-'); + Log() << separator << Endl; + } + + while (!converged) { + stepCount++; + trainingData.Shuffle(); + + // execute all epochs + //for (size_t i = 0; i < batchesInEpoch; i += nThreads) { + // Clean and load new batches, one batch for one slave net + //batches.clear(); + //batches.reserve(nThreads); + //for (size_t j = 0; j < nThreads; j++) { + // batches.push_back(trainingData.GetTensorBatch()); + //} + auto my_batch = trainingData.GetTensorBatch(); + + // execute one minimization step + // StepMomentum is currently not written for single thread, TODO write it + if (settings.momentum > 0.0) { + //minimizer.StepMomentum(deepNet, nets, batches, settings.momentum); + minimizer.Step(deepNet, my_batch.GetInput(), my_batch.GetOutput(), my_batch.GetWeights()); + } else { + //minimizer.Step(deepNet, nets, batches); + minimizer.Step(deepNet, my_batch.GetInput(), my_batch.GetOutput(), my_batch.GetWeights()); + } + //} + + if ((stepCount % minimizer.GetTestInterval()) == 0) { + // Compute test error. + Double_t testError = 0.0; + for (auto batch : testingData) { + auto inputTensor = batch.GetInput(); + auto outputMatrix = batch.GetOutput(); + auto weights = batch.GetWeights(); + testError += deepNet.Loss(inputTensor, outputMatrix, weights); + } + + testError /= (Double_t)(nTestSamples / settings.batchSize); + + // stop measuring + end = std::chrono::system_clock::now(); + + // Compute training error. + Double_t trainingError = 0.0; + for (auto batch : trainingData) { + auto inputTensor = batch.GetInput(); + auto outputMatrix = batch.GetOutput(); + auto weights = batch.GetWeights(); + trainingError += deepNet.Loss(inputTensor, outputMatrix, weights); + } + trainingError /= (Double_t)(nTrainingSamples / settings.batchSize); + + // Compute numerical throughput. + std::chrono::duration elapsed_seconds = end - start; + double seconds = elapsed_seconds.count(); + double nFlops = (double)(settings.testInterval * batchesInEpoch); + // nFlops *= net.GetNFlops() * 1e-9; + + converged = minimizer.HasConverged(testError); + + Log() << std::setw(10) << stepCount << " | " << std::setw(12) << trainingError << std::setw(12) << testError + << std::setw(12) << nFlops / seconds << std::setw(12) << minimizer.GetConvergenceCount() << Endl; + + if (converged) { + Log() << Endl; + } + } + } + + // Copy weights from deepNet to fNet + for (size_t i = 0; i < deepNet.GetDepth(); ++i) { + const auto & fLayer = fNet->GetLayerAt(i); + const auto & dLayer = deepNet.GetLayerAt(i); + fLayer->CopyWeights(dLayer->GetWeights()); + fLayer->CopyBiases(dLayer->GetBiases()); + } + + } + +#else // DNNCPU flag not set. + Log() << kFATAL << "Multi-core CPU backend not enabled. Please make sure " + "you have a BLAS implementation and it was successfully " + "detected by CMake as well that the imt CMake flag is set." + << Endl; +#endif // DNNCPU +} + +//////////////////////////////////////////////////////////////////////////////// +Double_t MethodDL::GetMvaValue(Double_t * /*errLower*/, Double_t * /*errUpper*/) +{ +#ifdef DNNCPU + using Architecture_t = DNN::TCpu; + using Matrix_t = typename Architecture_t::Matrix_t; + + size_t nVariables = GetEvent()->GetNVariables(); + int ntime = fNet->GetBatchHeight(); + int nvar = fNet->GetBatchWidth(); + int nb = fNet->GetBatchSize(); + int noutput = fNet->GetOutputWidth(); + + std::vector X{}; + Matrix_t YHat(nb, noutput); + + // get current event + const std::vector &inputValues = GetEvent()->GetValues(); + + for (int i = 0; i < nb; ++i) X.push_back(Matrix_t(ntime, nvar)); + + // need to fill lal lbatch with same event + for (int i = 0; i < nb; ++i) { // batch size loop + for (int j = 0; j < ntime; ++j) { + for (int k = 0; k < (Int_t)nvar; k++) { + int ivar = j * ntime + k; + R__ASSERT(ivar < (Int_t)nVariables); + X[i](j, k) = inputValues[ivar]; + } + } + } + + // perform the prediction + fNet->Prediction(YHat, X, fOutputFunction); + + double mvaValue = YHat(0, 0); + return (TMath::IsNaN(mvaValue)) ? -999. : mvaValue; + +#else + return 0.0; +#endif +} + +//////////////////////////////////////////////////////////////////////////////// +void MethodDL::AddWeightsXMLTo(void * /*parent*/) const +{ + // TODO +} + +//////////////////////////////////////////////////////////////////////////////// +void MethodDL::ReadWeightsFromXML(void * /*rootXML*/) +{ + // TODO +} + +//////////////////////////////////////////////////////////////////////////////// +void MethodDL::ReadWeightsFromStream(std::istream & /*istr*/) +{ +} + +//////////////////////////////////////////////////////////////////////////////// +const Ranking *TMVA::MethodDL::CreateRanking() +{ + // TODO + return NULL; +} + +//////////////////////////////////////////////////////////////////////////////// +void MethodDL::GetHelpMessage() const +{ + // TODO +} + +} // namespace TMVA diff --git a/tmva/tmva/test/CMakeLists.txt b/tmva/tmva/test/CMakeLists.txt index fc2a3e633d359..249c689879936 100644 --- a/tmva/tmva/test/CMakeLists.txt +++ b/tmva/tmva/test/CMakeLists.txt @@ -7,3 +7,6 @@ project(tmva-tests) find_package(ROOT REQUIRED) ROOT_ADD_TEST_SUBDIRECTORY(DNN) +ROOT_ADD_TEST_SUBDIRECTORY(DNN/CNN) +ROOT_ADD_TEST_SUBDIRECTORY(DNN/DAE) +ROOT_ADD_TEST_SUBDIRECTORY(DNN/RNN) diff --git a/tmva/tmva/test/DNN/CMakeLists.txt b/tmva/tmva/test/DNN/CMakeLists.txt index 4411d8e70658e..eadb887c0d1c6 100644 --- a/tmva/tmva/test/DNN/CMakeLists.txt +++ b/tmva/tmva/test/DNN/CMakeLists.txt @@ -9,6 +9,9 @@ find_package(ROOT REQUIRED) set(Libraries Core MathCore Matrix TMVA) include_directories(${ROOT_INCLUDE_DIRS}) +ROOT_ADD_TEST_SUBDIRECTORY(CNN) +ROOT_ADD_TEST_SUBDIRECTORY(DAE) +ROOT_ADD_TEST_SUBDIRECTORY(RNN) #--- CUDA tests. --------------------------- if (CUDA_FOUND) @@ -79,6 +82,11 @@ if (BLAS_FOUND AND imt) LIBRARIES ${Libraries}) ROOT_ADD_TEST(TMVA-DNN-Backpropagation-Cpu COMMAND testBackpropagationCpu) + # DNN - BackpropagationDL CPU + ROOT_EXECUTABLE(testBackpropagationDLCpu TestBackpropagationDLCpu.cxx + LIBRARIES ${Libraries}) + ROOT_ADD_TEST(TMVA-DNN-Backpropagation-DL-Cpu COMMAND testBackpropagationDLCpu) + # DNN - DataLoader CPU ROOT_EXECUTABLE(testDataLoaderCpu TestDataLoaderCpu.cxx LIBRARIES ${Libraries}) @@ -107,6 +115,10 @@ endif (BLAS_FOUND AND imt) ROOT_EXECUTABLE(testBackpropagation TestBackpropagation.cxx LIBRARIES ${Libraries}) ROOT_ADD_TEST(TMVA-DNN-Backpropagation COMMAND testBackpropagation) + # DNN - Backpropagation DL + ROOT_EXECUTABLE(testBackpropagationDL TestBackpropagationDL.cxx LIBRARIES ${Libraries}) + ROOT_ADD_TEST(TMVA-DNN-Backpropagation-DL COMMAND testBackpropagationDL) + # DNN - DataLoader ROOT_EXECUTABLE(testDataLoader TestDataLoader.cxx LIBRARIES ${Libraries}) ROOT_ADD_TEST(TMVA-DNN-Data-Loader COMMAND testDataLoader) diff --git a/tmva/tmva/test/DNN/CNN/CMakeLists.txt b/tmva/tmva/test/DNN/CNN/CMakeLists.txt new file mode 100644 index 0000000000000..3d13cf2c286c0 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/CMakeLists.txt @@ -0,0 +1,74 @@ +############################################################################ +# CMakeLists.txt file for building TMVA/DNN/CNN tests. +# @author Vladimir Ilievski +############################################################################ + +project(tmva-tests) +find_package(ROOT REQUIRED) + +set(Libraries Core MathCore Matrix TMVA) +include_directories(${ROOT_INCLUDE_DIRS}) + +ROOT_EXECUTABLE(testIm2Col TestIm2Col.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Im2Col COMMAND testIm2Col) + +ROOT_EXECUTABLE(testDownsample TestDownsample.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Downsample COMMAND testDownsample) + +ROOT_EXECUTABLE(testFlatten TestFlatten.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-testFlatten COMMAND testFlatten) + +ROOT_EXECUTABLE(testRotWeights TestRotateWeights.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-rotWeights COMMAND testrotWeights) + +ROOT_EXECUTABLE(testForwardPass TestForwardPass.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Forward COMMAND testForwardPass) + +ROOT_EXECUTABLE(testConvNetLoss TestConvNetLoss.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Loss COMMAND testConvNetLoss) + +ROOT_EXECUTABLE(testConvNetPred TestConvNetPrediction.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Pred COMMAND testConvNetPred) + +ROOT_EXECUTABLE(testConvBackpropagation TestConvBackpropagation.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Backpropagation COMMAND testConvBackpropagation) + +ROOT_EXECUTABLE(testTensorDataLoader TestTensorDataLoader.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-Tensor-Data-Loader COMMAND testTensorDataLoader) + +ROOT_EXECUTABLE(testDLMinimization TestMinimization.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Minimization COMMAND testDLMinimization) + +ROOT_EXECUTABLE(testMethodDL TestMethodDL.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-MethodDL COMMAND testMethodDL) + +#--- CPU tests. ---------------------------- +if (BLAS_FOUND AND imt) + +include_directories(SYSTEM ${TBB_INCLUDE_DIRS}) + +ROOT_EXECUTABLE(testIm2ColCpu TestIm2ColCpu.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Im2Col-CPU COMMAND testIm2ColCpu) + +ROOT_EXECUTABLE(testDownsampleCpu TestDownsampleCpu.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Downsample-CPU COMMAND testDownsampleCpu) + +ROOT_EXECUTABLE(testRotWeightsCpu TestRotateWeightsCpu.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-rotWeights-CPU COMMAND testrotWeightsCpu) + +ROOT_EXECUTABLE(testForwardPassCpu TestForwardPassCpu.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Forward-CPU COMMAND testForwardPassCpu) + +ROOT_EXECUTABLE(testConvNetLossCpu TestConvNetLossCpu.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Loss-CPU COMMAND testConvNetLossCpu) + +ROOT_EXECUTABLE(testConvNetPredCpu TestConvNetPredictionCpu.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Pred-CPU COMMAND testConvNetPredCpu) + +ROOT_EXECUTABLE(testTensorDataLoaderCpu TestTensorDataLoaderCpu.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-Tensor-Data-Loader-CPU COMMAND testTensorDataLoaderCpu) + +ROOT_EXECUTABLE(testDLMinimizationCpu TestMinimizationCpu.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-CNN-Minimization-CPU COMMAND testDLMinimizationCpu) + +endif (BLAS_FOUND AND imt) diff --git a/tmva/tmva/test/DNN/CNN/TestConvBackpropagation.cxx b/tmva/tmva/test/DNN/CNN/TestConvBackpropagation.cxx new file mode 100644 index 0000000000000..2d8fa781e819e --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestConvBackpropagation.cxx @@ -0,0 +1,98 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Conv Net Backpropagation * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the Conv Net Backward Pass // +//////////////////////////////////////////////////////////////////// + +#include +#include "TMVA/DNN/Architectures/Reference.h" +#ifdef DEBUG +// to debug the test and print matrices +#define DEBUG_TMVA_TCPUMATRIX +#endif +#include "TMVA/DNN/Architectures/Cpu.h" +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; + + +// test first in a simple network with linear activations +bool test1() +{ + + size_t batchSizeTest = 2; + size_t imgDepthTest = 2; + size_t imgHeightTest = 4; + size_t imgWidthTest = 4; + size_t batchDepth = batchSizeTest; + size_t batchHeight = imgDepthTest; + size_t batchWidth = imgWidthTest* imgHeightTest; + double stepSize = 1.E-5; // for computing derivatives with finate differences + + ETestType type = kLinearNet; + + return testConvBackwardPass>(batchSizeTest, imgDepthTest, imgHeightTest, imgWidthTest, batchDepth, + batchHeight, batchWidth,stepSize,type); +} +// test in a more complex network +bool test2() +{ + + size_t batchSizeTest = 4; + size_t imgDepthTest = 1; + size_t imgHeightTest = 8; + size_t imgWidthTest = 8; + size_t batchDepth = batchSizeTest; + size_t batchHeight = imgDepthTest; + size_t batchWidth = imgHeightTest * imgWidthTest; + + // testConvBackwardPass>(batchSizeTest1, imgDepthTest1, imgHeightTest1, imgWidthTest1, batchDepth, + // batchHeight, batchWidth); + + double stepSize = 1.E-5; // for computing derivatives with finate differences + ETestType type = kRndmActNet; + + return testConvBackwardPass>(batchSizeTest, imgDepthTest, imgHeightTest, imgWidthTest, batchDepth, + batchHeight, batchWidth,stepSize,type); +} + +int main() +{ + bool ret = true; + std::cout << "Testing CNN Backward Pass:" << std::endl; + std::cout << "Test1, backward pass with linear activation network - compare with finite difference" << std::endl; + + ret &= test1(); + if (!ret) { + std::cerr << "ERROR - test1 failed " << std::endl; + return -1; + } + std::cout << "Test2, more complex network architecture no dropout" << std::endl; + ret &= test2(); +} + diff --git a/tmva/tmva/test/DNN/CNN/TestConvNet.h b/tmva/tmva/test/DNN/CNN/TestConvNet.h new file mode 100644 index 0000000000000..07f064489615a --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestConvNet.h @@ -0,0 +1,483 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Conv Net Features * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_TEST_DNN_TEST_CNN_TEST_CONV_NET_H +#define TMVA_TEST_DNN_TEST_CNN_TEST_CONV_NET_H + +#include "../Utility.h" + +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DeepNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; + +enum ETestType { kLinearNet, kRndmActNet }; + +/** Testing the image to column function. Check wheether the matrix A after + * the transformation will be equal to the matrix B. */ +//______________________________________________________________________________ +template +auto testIm2col(typename Architecture::Matrix_t &A, typename Architecture::Matrix_t &B, size_t imgHeight, + size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, + size_t zeroPaddingHeight, size_t zeroPaddingWidth) -> bool +{ + + size_t m, n; + m = B.GetNrows(); + n = B.GetNcols(); + + typename Architecture::Matrix_t ATr(m, n); + Architecture::Im2col(ATr, A, imgHeight, imgWidth, fltHeight, fltWidth, strideRows, strideCols, zeroPaddingHeight, + zeroPaddingWidth); + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + if (ATr(i, j) != B(i, j)) { + return false; + } + } + } + + return true; +} + +/** Testing the rotation of the weights function. Check whether the rotated + * weight matrix A, will be equal to the matrix B. */ +//______________________________________________________________________________ +template +auto testRotateWeights(typename Architecture::Matrix_t &A, typename Architecture::Matrix_t &B, size_t filterDepth, + size_t filterHeight, size_t filterWidth, size_t numFilters) -> bool +{ + + size_t m, n; + m = B.GetNrows(); + n = B.GetNcols(); + + typename Architecture::Matrix_t ARot(m, n); + Architecture::RotateWeights(ARot, A, filterDepth, filterHeight, filterWidth, numFilters); + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + if (ARot(i, j) != B(i, j)) { + return false; + } + } + } + + return true; +} + +/** Downsample the matrix A and check whether the downsampled version + * is equal to B, and if the winning indices are equal to the matrix ind. */ +//______________________________________________________________________________ +template +auto testDownsample(const typename Architecture::Matrix_t &A, const typename Architecture::Matrix_t &ind, + const typename Architecture::Matrix_t &B, size_t imgHeight, size_t imgWidth, size_t fltHeight, + size_t fltWidth, size_t strideRows, size_t strideCols) -> bool +{ + + size_t m1, n1; + m1 = B.GetNrows(); + n1 = B.GetNcols(); + + typename Architecture::Matrix_t ADown(m1, n1); + + size_t m2, n2; + m2 = ind.GetNrows(); + n2 = ind.GetNcols(); + + typename Architecture::Matrix_t AInd(m2, n2); + + Architecture::Downsample(ADown, AInd, A, imgHeight, imgWidth, fltHeight, fltWidth, strideRows, strideCols); + + for (size_t i = 0; i < m1; i++) { + for (size_t j = 0; j < n1; j++) { + if (ADown(i, j) != B(i, j)) { + return false; + } + } + } + + for (size_t i = 0; i < m2; i++) { + for (size_t j = 0; j < n2; j++) { + if (AInd(i, j) != ind(i, j)) { + return false; + } + } + } + + return true; +} + +/** Flatten the 3D tensor A using the Flatten function and compare it to + * the result in the flat matrix B. */ +//______________________________________________________________________________ +template +auto testFlatten(std::vector &A, const typename Architecture::Matrix_t &B, size_t size, + size_t nRows, size_t nCols) -> bool +{ + + size_t m, n; + m = B.GetNrows(); + n = B.GetNcols(); + + typename Architecture::Matrix_t AFlat(m, n); + Architecture::Flatten(AFlat, A, size, nRows, nCols); + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + if (AFlat(i, j) != B(i, j)) { + return false; + } + } + } + + return true; +} + +/*! Generate a conv net, perform forward pass */ +//______________________________________________________________________________ +template +auto testConvForwardPass(size_t batchSize, size_t imgDepth, size_t imgHeight, size_t imgWidth, size_t batchDepth, + size_t batchHeight, size_t batchWidth) -> void +{ + using Matrix_t = typename Architecture::Matrix_t; + using Net_t = TDeepNet; + + Net_t convNet(batchSize, imgDepth, imgHeight, imgWidth, batchDepth, batchHeight, batchWidth, + ELossFunction::kMeanSquaredError, EInitialization::kGauss); + constructConvNet(convNet); + convNet.Initialize(); + + std::vector X; + for (size_t i = 0; i < batchSize; i++) { + X.emplace_back(imgDepth, imgHeight * imgWidth); + randomMatrix(X[i]); + } + + convNet.Forward(X); +} + +/*! Generate a conv net, get the loss. */ +//______________________________________________________________________________ +template +auto testConvLossFunction(size_t batchSize, size_t imgDepth, size_t imgHeight, size_t imgWidth, size_t batchDepth, + size_t batchHeight, size_t batchWidth) -> void +{ + using Scalar_t = typename Architecture::Scalar_t; + using Matrix_t = typename Architecture::Matrix_t; + using Net_t = TDeepNet; + + Net_t convNet(batchSize, imgDepth, imgHeight, imgWidth, batchDepth, batchHeight, batchWidth, + ELossFunction::kMeanSquaredError, EInitialization::kGauss); + constructConvNet(convNet); + convNet.Initialize(); + + std::vector X; + for (size_t i = 0; i < batchSize; i++) { + X.emplace_back(imgDepth, imgHeight * imgWidth); + randomMatrix(X[i]); + } + + Matrix_t Y(batchSize, convNet.GetOutputWidth()); + Matrix_t W(batchSize, 1); + randomMatrix(Y); + randomMatrix(W); + + Scalar_t loss = convNet.Loss(X, Y, W); + std::cout << "The loss is: " << loss << std::endl; +} + +/*! Generate a conv net, get the predictios */ +//______________________________________________________________________________ +template +auto testConvPrediction(size_t batchSize, size_t imgDepth, size_t imgHeight, size_t imgWidth, size_t batchDepth, + size_t batchHeight, size_t batchWidth, EOutputFunction f) -> void +{ + using Matrix_t = typename Architecture::Matrix_t; + using Net_t = TDeepNet; + + Net_t convNet(batchSize, imgDepth, imgHeight, imgWidth, batchDepth, batchHeight, batchWidth, + ELossFunction::kMeanSquaredError, EInitialization::kGauss); + constructConvNet(convNet); + convNet.Initialize(); + + std::vector X; + for (size_t i = 0; i < batchSize; i++) { + X.emplace_back(imgDepth, imgHeight * imgWidth); + randomMatrix(X[i]); + } + + Matrix_t Predictions(batchSize, convNet.GetOutputWidth()); + convNet.Prediction(Predictions, X, f); + + for (size_t i = 0; i < batchSize; i++) { + for (size_t j = 0; j < convNet.GetOutputWidth(); j++) { + std::cout << Predictions(i, j) << " "; + } + std::cout << "" << std::endl; + } +} + +/*! Generate a conv net, test the backward pass, always with stride 1. */ +//______________________________________________________________________________ +template +auto testConvBackwardPassOnly(size_t batchSize, size_t imgDepth, size_t imgHeight, size_t imgWidth, size_t batchDepth, + size_t batchHeight, size_t batchWidth) -> void +{ + using Matrix_t = typename Architecture::Matrix_t; + using Net_t = TDeepNet; + + Net_t convNet(batchSize, imgDepth, imgHeight, imgWidth, batchDepth, batchHeight, batchWidth, + ELossFunction::kMeanSquaredError, EInitialization::kGauss); + constructConvNet(convNet); + convNet.Initialize(); + + std::vector X; + for (size_t i = 0; i < batchSize; i++) { + X.emplace_back(imgDepth, imgHeight * imgWidth); + randomMatrix(X[i]); + } + + Matrix_t Y(batchSize, convNet.GetOutputWidth()); + Matrix_t W(batchSize, 1); + randomMatrix(Y); + randomMatrix(W); + + convNet.Forward(X); + convNet.Backward(X, Y, W); +} + + +/*! Compute the loss of the net as a function of the weight at index (i,j) in + * layer l. dx is added as an offset to the current value of the weight. */ +//______________________________________________________________________________ +template +auto evaluate_net_weight(TDeepNet &net, std::vector &X, + const typename Architecture::Matrix_t &Y, const typename Architecture::Matrix_t &W, size_t l, + size_t i, size_t j, size_t k, typename Architecture::Scalar_t dx) -> typename Architecture::Scalar_t +{ + using Scalar_t = typename Architecture::Scalar_t; + //using Matrix_t = typename Architecture::Matrix_t; + + // shift the weight value and compute the Loss + auto & netW = net.GetLayerAt(l)->GetWeights(); + netW[k](i,j) += dx; + Scalar_t res = net.Loss(X, Y, W); + // rest weight to original value + netW[k](i,j) -= dx; + //std::cout << "loss(w+dx = " << res << " loss(w) " << net.Loss(X,Y,W) << std::endl; + return res; +} + +/*! Compute the loss of the net as a function of the weight at index i in + * layer l. dx is added as an offset to the current value of the weight. */ +//______________________________________________________________________________ +template +auto evaluate_net_bias(TDeepNet &net, std::vector &X, + const typename Architecture::Matrix_t &Y, const typename Architecture::Matrix_t &W, size_t l, + size_t i, size_t k, typename Architecture::Scalar_t dx) -> typename Architecture::Scalar_t +{ + using Scalar_t = typename Architecture::Scalar_t; + //using Matrix_t = typename Architecture::Matrix_t; + + auto & netB = net.GetLayerAt(l)->GetBiases(); + netB[k](i,0) += dx; + Scalar_t res = net.Loss(X, Y, W); + netB[k](i,0) -= dx; + return res; +} + + + +template +auto testConvBackwardPass(size_t batchSize, size_t imgDepth, size_t imgHeight, size_t imgWidth, size_t batchDepth, + size_t batchHeight, size_t batchWidth, typename Architecture::Scalar_t dx, ETestType testType) -> bool +{ + using Matrix_t = typename Architecture::Matrix_t; + using Net_t = TDeepNet; + using Scalar_t = typename Architecture::Scalar_t; + + Net_t convNet(batchSize, imgDepth, imgHeight, imgWidth, batchDepth, batchHeight, batchWidth, + ELossFunction::kMeanSquaredError, EInitialization::kGauss); + // tyoe of network + if (testType == kLinearNet) + constructLinearConvNet(convNet); + else + constructConvNet(convNet); + + convNet.Initialize(); + +// auto & w0 = convNet.GetLayerAt(0)->GetWeights(); +// #ifdef DEBUG +// std::cout << "Netwrok weights for Layer 0 " << std::endl; +// std::cout << "weight depth = " << w0.size() << std::endl; +// for (size_t i = 0; i < w0.size(); ++i) +// PrintMatrix(w0[i],"weight-layer0"); +// #endif + + std::vector X; + for (size_t i = 0; i < batchSize; i++) { + X.emplace_back(batchHeight , batchWidth); + randomMatrix(X[i]); + + // print input +#ifdef DEBUGH + std::cout << "INPUT - batch " << i << std::endl; + PrintMatrix(X[i],"input"); +#endif + } + + Matrix_t Y(batchSize, convNet.GetOutputWidth()); + Matrix_t W(batchSize, 1); // this are the data weights + randomMatrix(Y); + //randomMatrix(W); + // for the moment use weights equal to 1 + fillMatrix(W, 1.0); + + std::cout << "Do Forward Pass " << std::endl; + convNet.Forward(X); + + // print layer derivatives +#ifdef DEBUG + using ConvLayer_t = TConvLayer; + auto convLayer = dynamic_cast(convNet.GetLayerAt(0) ); + if (convLayer) { + auto & df = convLayer->GetDerivatives(); + std::cout << "Derivatives - size " << df.size() << std::endl; + for (size_t ii=0; ii< df.size(); ++ii) + PrintMatrix(df[ii],"Derivatives"); + } +#endif + + //if (testType == kRndmActNet) return true; + + std::cout << "Do Backward Pass " << std::endl; + convNet.Backward(X, Y, W); + + + // now compare derivatives using finite differences and compare the result + Scalar_t maximum_error = 0.0; + + for (size_t l = 0; l < convNet.GetDepth(); l++) { + std::cout << "\rTesting weight gradients: layer: " << l << " / " << convNet.GetDepth(); + std::cout << std::flush; + auto & layer = *(convNet.GetLayerAt(l)); + std::vector &gw = layer.GetWeightGradients(); + + std::cout << std::endl; + + if (gw.size() > 0) { + std::cout << "Weight gradient from back-propagation - vector size is " << gw.size() << std::endl; + if (gw[0].GetNElements() < 100 ) { + PrintMatrix(gw[0],"WeightGrad"); + } + else + std::cout << "BP Weight Gradient ( " << gw[0].GetNrows() << " x " << gw[0].GetNcols() << " ) , ...... skip printing (too many elements ) " << std::endl; + } + else { + std::cout << "Layer " << l << " has no weights " << std::endl; + } + auto & actGrad = layer.GetActivationGradients(); + if (actGrad.size() > 0) { + std::cout << "Activation gradient from back-propagation - vector size is " << actGrad.size() << std::endl; + if (actGrad[0].GetNElements() < 100 ) { + for (size_t ii = 0; ii < actGrad.size(); ++ii) + PrintMatrix(actGrad[ii],"ActivationGrad"); + } else + std::cout << "Activation Gradient ( " << actGrad[0].GetNrows() << " x " << actGrad[0].GetNcols() << " ) , ...... skip printing (too many elements ) " << std::endl; + } + + std::cout << "Layer " << l << " : output D x H x W " << layer.GetDepth() << " " << layer.GetHeight() << " " << layer.GetWidth() + << "\t input D x H x W " << layer.GetInputDepth() << " " << layer.GetInputHeight() << " " << layer.GetInputWidth() << std::endl; + + + // print output + auto & outL = layer.GetOutput(); + std::cout << "layer output size " << outL.size() << std::endl; + if (outL.size() > 0) { + if (outL[0].GetNElements() < 100 ) { + PrintMatrix(outL[0],"LayerOutput-Matrix0"); + } else + std::cout << "Layer Output ( " << outL[0].GetNrows() << " x " << outL[0].GetNcols() << " ) , ...... skip printing (too many elements ) " << std::endl; + } + + std::cout << "Evaluate the Derivatives with Finite difference and compare with BP for Layer " << l << std::endl; + int nerrors = 0; + int ngood = 0; +#ifdef DEBUG + int ngoodPrint = 10000; +#else + int ngoodPrint = 3; +#endif + for (size_t k = 0; k < gw.size() ; ++k) { + //for (size_t i = 0; i < layer.GetWidth(); i++) { + for (size_t i = 0; i < gw[k].GetNrows(); i++) { + //for (size_t j = 0; j < layer.GetInputWidth(); j++) { + for (size_t j = 0; j < gw[k].GetNcols(); j++) { + auto f = [&convNet, &X, &Y, &W, l, i, j, k](Scalar_t x) { + return evaluate_net_weight(convNet, X, Y, W, l, i, j, k, x); + }; + Scalar_t dy = finiteDifference(f, dx) / (2.0 * dx); + Scalar_t dy_ref = gw[k](i, j); + // Compute the relative error if dy != 0. + Scalar_t error; + if (std::fabs(dy_ref) > 1e-15) { + error = std::fabs((dy - dy_ref) / dy_ref); + } else { + error = std::fabs(dy - dy_ref); + } + maximum_error = std::max(error, maximum_error); + if (error > 1.E-3) { + std::cout << k << " - " << i << " , " << j << " : " << dy << " from BP " << dy_ref << " " << error << " ERROR " << std::endl; + nerrors ++; + } + else { + if (ngood < ngoodPrint) std::cout << k << " - " << i << " , " << j << " : " << dy << " from BP " << dy_ref << " " << error << std::endl; + ngood++; + } + if (nerrors > 10) { + std::cout << "Reached error limit skip..." << std::endl; + break; + } + } + if (nerrors > 10) break; + } + if (nerrors > 10) break; + } + } + std::cout << "\rTesting weight gradients: "; + std::cout << "maximum relative error: " << print_error(maximum_error) << std::endl; + if (maximum_error > 1.E-3) { + std::cout << "ERROR - BackPropagation test failed in computing weight Derivatives " << std::endl; + return false; + } + //return maximum_error; + return true; +} + +#endif diff --git a/tmva/tmva/test/DNN/CNN/TestConvNetLoss.cxx b/tmva/tmva/test/DNN/CNN/TestConvNetLoss.cxx new file mode 100644 index 0000000000000..eeecc1351bbf6 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestConvNetLoss.cxx @@ -0,0 +1,96 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Conv Net Loss * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the Conv Net Loss Function // +//////////////////////////////////////////////////////////////////// + +#include +#include "TMVA/DNN/Architectures/Reference.h" + +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; + +void test1() +{ + + size_t batchSizeTest1 = 50; + size_t imgDepthTest1 = 3; + size_t imgHeightTest1 = 32; + size_t imgWidthTest1 = 32; + size_t batchDepth = batchSizeTest1; + size_t batchHeight = imgDepthTest1; + size_t batchWidth = imgHeightTest1 * imgWidthTest1; + + testConvLossFunction>(batchSizeTest1, imgDepthTest1, imgHeightTest1, imgWidthTest1, batchDepth, + batchHeight, batchWidth); +} + +void test2() +{ + + size_t batchSizeTest2 = 50; + size_t imgDepthTest2 = 3; + size_t imgHeightTest2 = 32; + size_t imgWidthTest2 = 32; + size_t batchDepth = batchSizeTest2; + size_t batchHeight = imgDepthTest2; + size_t batchWidth = imgHeightTest2 * imgWidthTest2; + + testConvLossFunction>(batchSizeTest2, imgDepthTest2, imgHeightTest2, imgWidthTest2, batchDepth, + batchHeight, batchWidth); +} + +void test3() +{ + + size_t batchSizeTest3 = 50; + size_t imgDepthTest3 = 3; + size_t imgHeightTest3 = 32; + size_t imgWidthTest3 = 32; + size_t batchDepth = batchSizeTest3; + size_t batchHeight = imgDepthTest3; + size_t batchWidth = imgHeightTest3 * imgWidthTest3; + + testConvLossFunction>(batchSizeTest3, imgDepthTest3, imgHeightTest3, imgWidthTest3, batchDepth, + batchHeight, batchWidth); +} + +int main() +{ + std::cout << "Testing CNN Loss:" << std::endl; + + std::cout << "Test1" << std::endl; + test1(); + + std::cout << "Test2" << std::endl; + test2(); + + std::cout << "Test3" << std::endl; + test3(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestConvNetLossCpu.cxx b/tmva/tmva/test/DNN/CNN/TestConvNetLossCpu.cxx new file mode 100644 index 0000000000000..3a2196a960580 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestConvNetLossCpu.cxx @@ -0,0 +1,96 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Conv Net Loss for CPU * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the Conv Net Loss Function // +//////////////////////////////////////////////////////////////////// + +#include +#include "TMVA/DNN/Architectures/Cpu.h" + +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; + +void test1() +{ + + size_t batchSizeTest1 = 50; + size_t imgDepthTest1 = 3; + size_t imgHeightTest1 = 32; + size_t imgWidthTest1 = 32; + size_t batchDepth = batchSizeTest1; + size_t batchHeight = imgDepthTest1; + size_t batchWidth = imgHeightTest1 * imgWidthTest1; + + testConvLossFunction>(batchSizeTest1, imgDepthTest1, imgHeightTest1, imgWidthTest1, batchDepth, + batchHeight, batchWidth); +} + +void test2() +{ + + size_t batchSizeTest2 = 50; + size_t imgDepthTest2 = 3; + size_t imgHeightTest2 = 32; + size_t imgWidthTest2 = 32; + size_t batchDepth = batchSizeTest2; + size_t batchHeight = imgDepthTest2; + size_t batchWidth = imgHeightTest2 * imgWidthTest2; + + testConvLossFunction>(batchSizeTest2, imgDepthTest2, imgHeightTest2, imgWidthTest2, batchDepth, + batchHeight, batchWidth); +} + +void test3() +{ + + size_t batchSizeTest3 = 50; + size_t imgDepthTest3 = 3; + size_t imgHeightTest3 = 32; + size_t imgWidthTest3 = 32; + size_t batchDepth = batchSizeTest3; + size_t batchHeight = imgDepthTest3; + size_t batchWidth = imgHeightTest3 * imgWidthTest3; + + testConvLossFunction>(batchSizeTest3, imgDepthTest3, imgHeightTest3, imgWidthTest3, batchDepth, + batchHeight, batchWidth); +} + +int main() +{ + std::cout << "Testing CNN Loss:" << std::endl; + + std::cout << "Test1" << std::endl; + test1(); + + std::cout << "Test2" << std::endl; + test2(); + + std::cout << "Test3" << std::endl; + test3(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestConvNetPrediction.cxx b/tmva/tmva/test/DNN/CNN/TestConvNetPrediction.cxx new file mode 100644 index 0000000000000..d07c48df2313e --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestConvNetPrediction.cxx @@ -0,0 +1,99 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Conv Net Prediction * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the Conv Net Prediction // +//////////////////////////////////////////////////////////////////// + +#include +#include "TMVA/DNN/Architectures/Reference.h" + +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; + +void test1() +{ + + size_t batchSizeTest1 = 50; + size_t imgDepthTest1 = 3; + size_t imgHeightTest1 = 32; + size_t imgWidthTest1 = 32; + size_t batchDepth = batchSizeTest1; + size_t batchHeight = imgDepthTest1; + size_t batchWidth = imgHeightTest1 * imgWidthTest1; + EOutputFunction f = EOutputFunction::kIdentity; + + testConvPrediction>(batchSizeTest1, imgDepthTest1, imgHeightTest1, imgWidthTest1, batchDepth, + batchHeight, batchWidth, f); +} + +void test2() +{ + + size_t batchSizeTest2 = 50; + size_t imgDepthTest2 = 3; + size_t imgHeightTest2 = 32; + size_t imgWidthTest2 = 32; + size_t batchDepth = batchSizeTest2; + size_t batchHeight = imgDepthTest2; + size_t batchWidth = imgHeightTest2 * imgWidthTest2; + EOutputFunction f = EOutputFunction::kSigmoid; + + testConvPrediction>(batchSizeTest2, imgDepthTest2, imgHeightTest2, imgWidthTest2, batchDepth, + batchHeight, batchWidth, f); +} + +void test3() +{ + + size_t batchSizeTest3 = 50; + size_t imgDepthTest3 = 3; + size_t imgHeightTest3 = 32; + size_t imgWidthTest3 = 32; + size_t batchDepth = batchSizeTest3; + size_t batchHeight = imgDepthTest3; + size_t batchWidth = imgHeightTest3 * imgWidthTest3; + EOutputFunction f = EOutputFunction::kSoftmax; + + testConvPrediction>(batchSizeTest3, imgDepthTest3, imgHeightTest3, imgWidthTest3, batchDepth, + batchHeight, batchWidth, f); +} + +int main() +{ + std::cout << "Testing CNN Prediction:" << std::endl; + + std::cout << "Test1, identity output function" << std::endl; + test1(); + + std::cout << "Test2, sigmoid output function" << std::endl; + test2(); + + std::cout << "Test3, softmax output function" << std::endl; + test3(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestConvNetPredictionCpu.cxx b/tmva/tmva/test/DNN/CNN/TestConvNetPredictionCpu.cxx new file mode 100644 index 0000000000000..b4db9e38f329f --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestConvNetPredictionCpu.cxx @@ -0,0 +1,99 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Conv Net Prediction for CPU * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the Conv Net Prediction // +//////////////////////////////////////////////////////////////////// + +#include +#include "TMVA/DNN/Architectures/Cpu.h" + +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; + +void test1() +{ + + size_t batchSizeTest1 = 50; + size_t imgDepthTest1 = 3; + size_t imgHeightTest1 = 32; + size_t imgWidthTest1 = 32; + size_t batchDepth = batchSizeTest1; + size_t batchHeight = imgDepthTest1; + size_t batchWidth = imgHeightTest1 * imgWidthTest1; + EOutputFunction f = EOutputFunction::kIdentity; + + testConvPrediction>(batchSizeTest1, imgDepthTest1, imgHeightTest1, imgWidthTest1, batchDepth, + batchHeight, batchWidth, f); +} + +void test2() +{ + + size_t batchSizeTest2 = 50; + size_t imgDepthTest2 = 3; + size_t imgHeightTest2 = 32; + size_t imgWidthTest2 = 32; + size_t batchDepth = batchSizeTest2; + size_t batchHeight = imgDepthTest2; + size_t batchWidth = imgHeightTest2 * imgWidthTest2; + EOutputFunction f = EOutputFunction::kSigmoid; + + testConvPrediction>(batchSizeTest2, imgDepthTest2, imgHeightTest2, imgWidthTest2, batchDepth, + batchHeight, batchWidth, f); +} + +void test3() +{ + + size_t batchSizeTest3 = 50; + size_t imgDepthTest3 = 3; + size_t imgHeightTest3 = 32; + size_t imgWidthTest3 = 32; + size_t batchDepth = batchSizeTest3; + size_t batchHeight = imgDepthTest3; + size_t batchWidth = imgHeightTest3 * imgWidthTest3; + EOutputFunction f = EOutputFunction::kSoftmax; + + testConvPrediction>(batchSizeTest3, imgDepthTest3, imgHeightTest3, imgWidthTest3, batchDepth, + batchHeight, batchWidth, f); +} + +int main() +{ + std::cout << "Testing CNN Prediction:" << std::endl; + + std::cout << "Test1, identity output function" << std::endl; + test1(); + + std::cout << "Test2, sigmoid output function" << std::endl; + test2(); + + std::cout << "Test3, softmax output function" << std::endl; + test3(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestDownsample.cxx b/tmva/tmva/test/DNN/CNN/TestDownsample.cxx new file mode 100644 index 0000000000000..582b74cc21649 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestDownsample.cxx @@ -0,0 +1,250 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Downsample method * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + + + +//////////////////////////////////////////////////////////////////// +// Testing the Downsample function // +//////////////////////////////////////////////////////////////////// + +#include +#include + +#include "TMVA/DNN/Architectures/Reference.h" +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; +using Matrix_t = typename TReference::Matrix_t; + + +inline bool isInteger(double x) {return x == floor(x);} + +size_t calculateDimension(size_t imgDim, + size_t fltDim, + size_t padding, + size_t stride) +{ + double dimension = ((imgDim - fltDim + 2 * padding) / stride) + 1; + if(!isInteger(dimension)) { + std::cout << "Not compatible hyper parameters" << std::endl; + std::exit(EXIT_FAILURE); + } + + return (size_t) dimension; +} + + + +/************************************************************************* + * Test 1: + * depth = 2, image height = 4, image width = 5, + * frame depth = 2, filter height = 2, filter width = 2, + * stride rows = 2, stride cols = 1, + * zero-padding height = 0, zero-padding width = 0, + *************************************************************************/ + +void test1() +{ + + double imgTest1[][20] = + { + {166, 212, 213, 150, 114, + 119, 109, 115, 88, 144, + 227, 208, 208, 235, 57, + 57, 165, 250, 139, 76}, + + { 57, 255, 184, 162, 204, + 220, 11, 192, 183, 174, + 2, 153, 183, 175, 10, + 55, 123, 246, 138, 80} + }; + + + double answerTest1[][8] = + { + {212, 213, 213, 150, + 227, 250, 250, 235}, + + {255, 255, 192, 204, + 153, 246, 246, 175} + }; + + double answerIdxTest1[][8] = + { + { 1, 2, 2, 3, + 10, 17, 17, 13}, + + { 1, 1, 7, 4, + 11, 17, 17, 13} + }; + + size_t imgDepthTest1 = 2; + size_t imgHeightTest1 = 4; + size_t imgWidthTest1 = 5; + size_t fltHeightTest1 = 2; + size_t fltWidthTest1 = 2; + size_t strideRowsTest1 = 2; + size_t strideColsTest1 = 1; + + + Matrix_t A(imgDepthTest1, imgHeightTest1 * imgWidthTest1); + + for(size_t i = 0; i < (size_t) A.GetNrows(); i++){ + for(size_t j = 0; j < (size_t) A.GetNcols(); j++){ + A(i, j) = imgTest1[i][j]; + } + } + + + size_t height = calculateDimension(imgHeightTest1, fltHeightTest1, + 0, strideRowsTest1); + + size_t width = calculateDimension(imgWidthTest1, fltWidthTest1, + 0, strideColsTest1); + + + Matrix_t idx(imgDepthTest1, height * width); + Matrix_t B(imgDepthTest1, height * width); + + for(size_t i = 0; i < (size_t) B.GetNrows(); i++){ + for(size_t j = 0; j < (size_t) B.GetNcols(); j++){ + idx(i, j) = answerIdxTest1[i][j]; + B(i, j) = answerTest1[i][j]; + } + } + + + + bool status = testDownsample>(A, idx, B, + imgHeightTest1, imgWidthTest1, + fltHeightTest1, fltWidthTest1, + strideRowsTest1, strideColsTest1); + + if(status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + +/************************************************************************* + * Test 1: + * depth = 1, image height = 6, image width = 6, + * frame depth = 1, filter height = 2, filter width = 3, + * stride rows = 1, stride cols = 3, + * zero-padding height = 0, zero-padding width = 0, + *************************************************************************/ + +void test2() +{ + + double imgTest2[][36] = + { + {200, 79, 69, 58, 98, 168, + 49, 230, 21, 141, 218, 38, + 72, 224, 14, 65, 147, 105, + 38, 27, 111, 160, 200, 48, + 109, 104, 153, 149, 233, 11, + 16, 91, 236, 183, 166, 155} + }; + + + double answerTest2[][10] = + { + {230, 218, + 230, 218, + 224, 200, + 153, 233, + 236, 233} + }; + + double answerIdxTest2[][10] = + { + { 7, 10, + 7, 10, + 13, 22, + 26, 28, + 32, 28} + }; + + size_t imgDepthTest2 = 1; + size_t imgHeightTest2 = 6; + size_t imgWidthTest2 = 6; + size_t fltHeightTest2 = 2; + size_t fltWidthTest2 = 3; + size_t strideRowsTest2 = 1; + size_t strideColsTest2 = 3; + + + Matrix_t A(imgDepthTest2, imgHeightTest2 * imgWidthTest2); + + for(size_t i = 0; i < (size_t) A.GetNrows(); i++){ + for(size_t j = 0; j < (size_t) A.GetNcols(); j++){ + A(i, j) = imgTest2[i][j]; + } + } + + + size_t height = calculateDimension(imgHeightTest2, fltHeightTest2, + 0, strideRowsTest2); + + size_t width = calculateDimension(imgWidthTest2, fltWidthTest2, + 0, strideColsTest2); + + + Matrix_t idx(imgDepthTest2, height * width); + Matrix_t B(imgDepthTest2, height * width); + + for(size_t i = 0; i < (size_t) B.GetNrows(); i++){ + for(size_t j = 0; j < (size_t) B.GetNcols(); j++){ + idx(i, j) = answerIdxTest2[i][j]; + B(i, j) = answerTest2[i][j]; + } + } + + + + bool status = testDownsample>(A, idx, B, + imgHeightTest2, imgWidthTest2, + fltHeightTest2, fltWidthTest2, + strideRowsTest2, strideColsTest2); + + if(status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + + +int main(){ + std::cout << "Testing Downsample function:" << std::endl; + + std::cout << "Test 1: " << std::endl; + test1(); + + std::cout << "Test 2: " << std::endl; + test2(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestDownsampleCpu.cxx b/tmva/tmva/test/DNN/CNN/TestDownsampleCpu.cxx new file mode 100644 index 0000000000000..4a685a9ec9e99 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestDownsampleCpu.cxx @@ -0,0 +1,186 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Downsample method on a CPU architecture * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the Downsample function // +//////////////////////////////////////////////////////////////////// + +#include +#include + +#include "TMVA/DNN/Architectures/Cpu.h" +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; +using Matrix_t = typename TCpu::Matrix_t; + +inline bool isInteger(double x) +{ + return x == floor(x); +} + +size_t calculateDimension(size_t imgDim, size_t fltDim, size_t padding, size_t stride) +{ + double dimension = ((imgDim - fltDim + 2 * padding) / stride) + 1; + if (!isInteger(dimension)) { + std::cout << "Not compatible hyper parameters" << std::endl; + std::exit(EXIT_FAILURE); + } + + return (size_t)dimension; +} + +/************************************************************************* + * Test 1: + * depth = 2, image height = 4, image width = 5, + * frame depth = 2, filter height = 2, filter width = 2, + * stride rows = 2, stride cols = 1, + * zero-padding height = 0, zero-padding width = 0, + *************************************************************************/ + +void test1() +{ + + double imgTest1[][20] = { + {166, 212, 213, 150, 114, 119, 109, 115, 88, 144, 227, 208, 208, 235, 57, 57, 165, 250, 139, 76}, + + {57, 255, 184, 162, 204, 220, 11, 192, 183, 174, 2, 153, 183, 175, 10, 55, 123, 246, 138, 80}}; + + double answerTest1[][8] = {{212, 213, 213, 150, 227, 250, 250, 235}, + + {255, 255, 192, 204, 153, 246, 246, 175}}; + + double answerIdxTest1[][8] = {{1, 2, 2, 3, 10, 17, 17, 13}, + + {1, 1, 7, 4, 11, 17, 17, 13}}; + + size_t imgDepthTest1 = 2; + size_t imgHeightTest1 = 4; + size_t imgWidthTest1 = 5; + size_t fltHeightTest1 = 2; + size_t fltWidthTest1 = 2; + size_t strideRowsTest1 = 2; + size_t strideColsTest1 = 1; + + Matrix_t A(imgDepthTest1, imgHeightTest1 * imgWidthTest1); + + for (size_t i = 0; i < (size_t)A.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)A.GetNcols(); j++) { + A(i, j) = imgTest1[i][j]; + } + } + + size_t height = calculateDimension(imgHeightTest1, fltHeightTest1, 0, strideRowsTest1); + + size_t width = calculateDimension(imgWidthTest1, fltWidthTest1, 0, strideColsTest1); + + Matrix_t idx(imgDepthTest1, height * width); + Matrix_t B(imgDepthTest1, height * width); + + for (size_t i = 0; i < (size_t)B.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)B.GetNcols(); j++) { + idx(i, j) = answerIdxTest1[i][j]; + B(i, j) = answerTest1[i][j]; + } + } + + bool status = testDownsample>(A, idx, B, imgHeightTest1, imgWidthTest1, fltHeightTest1, fltWidthTest1, + strideRowsTest1, strideColsTest1); + + if (status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + +/************************************************************************* + * Test 1: + * depth = 1, image height = 6, image width = 6, + * frame depth = 1, filter height = 2, filter width = 3, + * stride rows = 1, stride cols = 3, + * zero-padding height = 0, zero-padding width = 0, + *************************************************************************/ + +void test2() +{ + + double imgTest2[][36] = {{200, 79, 69, 58, 98, 168, 49, 230, 21, 141, 218, 38, 72, 224, 14, 65, 147, 105, + 38, 27, 111, 160, 200, 48, 109, 104, 153, 149, 233, 11, 16, 91, 236, 183, 166, 155}}; + + double answerTest2[][10] = {{230, 218, 230, 218, 224, 200, 153, 233, 236, 233}}; + + double answerIdxTest2[][10] = {{7, 10, 7, 10, 13, 22, 26, 28, 32, 28}}; + + size_t imgDepthTest2 = 1; + size_t imgHeightTest2 = 6; + size_t imgWidthTest2 = 6; + size_t fltHeightTest2 = 2; + size_t fltWidthTest2 = 3; + size_t strideRowsTest2 = 1; + size_t strideColsTest2 = 3; + + Matrix_t A(imgDepthTest2, imgHeightTest2 * imgWidthTest2); + + for (size_t i = 0; i < (size_t)A.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)A.GetNcols(); j++) { + A(i, j) = imgTest2[i][j]; + } + } + + size_t height = calculateDimension(imgHeightTest2, fltHeightTest2, 0, strideRowsTest2); + + size_t width = calculateDimension(imgWidthTest2, fltWidthTest2, 0, strideColsTest2); + + Matrix_t idx(imgDepthTest2, height * width); + Matrix_t B(imgDepthTest2, height * width); + + for (size_t i = 0; i < (size_t)B.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)B.GetNcols(); j++) { + idx(i, j) = answerIdxTest2[i][j]; + B(i, j) = answerTest2[i][j]; + } + } + + bool status = testDownsample>(A, idx, B, imgHeightTest2, imgWidthTest2, fltHeightTest2, fltWidthTest2, + strideRowsTest2, strideColsTest2); + + if (status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + +int main() +{ + std::cout << "Testing Downsample function:" << std::endl; + + std::cout << "Test 1: " << std::endl; + test1(); + + std::cout << "Test 2: " << std::endl; + test2(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestFlatten.cxx b/tmva/tmva/test/DNN/CNN/TestFlatten.cxx new file mode 100644 index 0000000000000..e11205c19e7d3 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestFlatten.cxx @@ -0,0 +1,113 @@ +// @(#)root/tmva $Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Flatten function for Reference backend * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the Flatten Function // +//////////////////////////////////////////////////////////////////// + +#include + +#include "TMVA/DNN/Architectures/Reference.h" +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; +using Matrix_t = typename TReference::Matrix_t; + +/************************************************************************* + * Test 1: + * depth = 3, width = 5, height = 5 + *************************************************************************/ + +void test1() +{ + + double imgTest1[][5][5] = {{ + {158, 157, 22, 166, 179}, + {68, 179, 233, 110, 163}, + {168, 216, 76, 8, 102}, + {159, 163, 25, 78, 119}, + {116, 50, 206, 102, 247}, + }, + + {{187, 166, 121, 112, 136}, + {237, 30, 180, 7, 248}, + {52, 172, 146, 130, 92}, + {124, 244, 214, 175, 9}, + {80, 232, 139, 224, 237}}, + + {{53, 147, 103, 53, 110}, + {112, 222, 19, 156, 232}, + {81, 19, 188, 224, 220}, + {255, 190, 76, 219, 95}, + {245, 4, 217, 22, 22}}}; + + double answerTest1[][25] = {{158, 157, 22, 166, 179, 68, 179, 233, 110, 163, 168, 216, 76, + 8, 102, 159, 163, 25, 78, 119, 116, 50, 206, 102, 247}, + + {187, 166, 121, 112, 136, 237, 30, 180, 7, 248, 52, 172, 146, + 130, 92, 124, 244, 214, 175, 9, 80, 232, 139, 224, 237}, + + {53, 147, 103, 53, 110, 112, 222, 19, 156, 232, 81, 19, 188, + 224, 220, 255, 190, 76, 219, 95, 245, 4, 217, 22, 22}}; + + size_t sizeTest1 = 3; + size_t nRowsTest1 = 5; + size_t nColsTest1 = 5; + + std::vector A; + for (size_t i = 0; i < sizeTest1; i++) { + Matrix_t temp(nRowsTest1, nColsTest1); + for (size_t j = 0; j < nRowsTest1; j++) { + for (size_t k = 0; k < nColsTest1; k++) { + temp(j, k) = imgTest1[i][j][k]; + } + } + A.push_back(temp); + } + + Matrix_t B(sizeTest1, nRowsTest1 * nColsTest1); + for (size_t i = 0; i < sizeTest1; i++) { + for (size_t j = 0; j < nRowsTest1 * nColsTest1; j++) { + B(i, j) = answerTest1[i][j]; + } + } + + bool status = testFlatten>(A, B, sizeTest1, nRowsTest1, nColsTest1); + + if (status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + +int main() +{ + std::cout << "Testing Flatten function:" << std::endl; + + std::cout << "Test 1: " << std::endl; + test1(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestForwardPass.cxx b/tmva/tmva/test/DNN/CNN/TestForwardPass.cxx new file mode 100644 index 0000000000000..a557a583415e2 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestForwardPass.cxx @@ -0,0 +1,78 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Conv Net Forward Pass * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the Conv Net Forward Pass // +//////////////////////////////////////////////////////////////////// + +#include +#include "TMVA/DNN/Architectures/Reference.h" + +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; + +void test1() +{ + + size_t batchSizeTest1 = 50; + size_t imgDepthTest1 = 3; + size_t imgHeightTest1 = 32; + size_t imgWidthTest1 = 32; + size_t batchDepth = batchSizeTest1; + size_t batchHeight = imgDepthTest1; + size_t batchWidth = imgHeightTest1 * imgWidthTest1; + + testConvForwardPass>(batchSizeTest1, imgDepthTest1, imgHeightTest1, imgWidthTest1, batchDepth, + batchHeight, batchWidth); +} + +void test2() +{ + + size_t batchSizeTest2 = 50; + size_t imgDepthTest2 = 3; + size_t imgHeightTest2 = 32; + size_t imgWidthTest2 = 32; + size_t batchDepth = batchSizeTest2; + size_t batchHeight = imgDepthTest2; + size_t batchWidth = imgHeightTest2 * imgWidthTest2; + + testConvForwardPass>(batchSizeTest2, imgDepthTest2, imgHeightTest2, imgWidthTest2, batchDepth, + batchHeight, batchWidth); +} + +int main() +{ + std::cout << "Testing CNN Forward Pass:" << std::endl; + + std::cout << "Test1" << std::endl; + test1(); + + std::cout << "Test2" << std::endl; + test2(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestForwardPassCpu.cxx b/tmva/tmva/test/DNN/CNN/TestForwardPassCpu.cxx new file mode 100644 index 0000000000000..9fe97f9cb95fb --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestForwardPassCpu.cxx @@ -0,0 +1,78 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Conv Net Forward Pass for the CPU * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the Conv Net Forward Pass // +//////////////////////////////////////////////////////////////////// + +#include +#include "TMVA/DNN/Architectures/Cpu.h" + +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; + +void test1() +{ + + size_t batchSizeTest1 = 50; + size_t imgDepthTest1 = 3; + size_t imgHeightTest1 = 32; + size_t imgWidthTest1 = 32; + size_t batchDepth = batchSizeTest1; + size_t batchHeight = imgDepthTest1; + size_t batchWidth = imgHeightTest1 * imgWidthTest1; + + testConvForwardPass>(batchSizeTest1, imgDepthTest1, imgHeightTest1, imgWidthTest1, batchDepth, + batchHeight, batchWidth); +} + +void test2() +{ + + size_t batchSizeTest2 = 50; + size_t imgDepthTest2 = 3; + size_t imgHeightTest2 = 32; + size_t imgWidthTest2 = 32; + size_t batchDepth = batchSizeTest2; + size_t batchHeight = imgDepthTest2; + size_t batchWidth = imgHeightTest2 * imgWidthTest2; + + testConvForwardPass>(batchSizeTest2, imgDepthTest2, imgHeightTest2, imgWidthTest2, batchDepth, + batchHeight, batchWidth); +} + +int main() +{ + std::cout << "Testing CNN Forward Pass:" << std::endl; + + std::cout << "Test1" << std::endl; + test1(); + + std::cout << "Test2" << std::endl; + test2(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestIm2Col.cxx b/tmva/tmva/test/DNN/CNN/TestIm2Col.cxx new file mode 100644 index 0000000000000..6e8a9a272ec5c --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestIm2Col.cxx @@ -0,0 +1,294 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Im2Col method * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the method Im2col // +//////////////////////////////////////////////////////////////////// + +#include +#include + +#include "TMVA/DNN/Architectures/Reference.h" +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; +using Matrix_t = typename TReference::Matrix_t; + +inline bool isInteger(double x) +{ + return x == floor(x); +} + +int calculateDimension(size_t imgDim, size_t fltDim, size_t padding, size_t stride) +{ + double dimension = ((imgDim - fltDim + 2 * padding) / stride) + 1; + if (!isInteger(dimension)) { + std::cout << "Not compatible hyper parameters" << std::endl; + std::exit(EXIT_FAILURE); + } + + return (size_t)dimension; +} + +/************************************************************************* + * Test 1: + * depth = 1, image height = 5, image width = 5, + * filter depth = 1, filter height = 2, filter width = 2, + * stride rows = 1, stride cols = 1, + * zero-padding height = 1, zero-padding width = 1, + *************************************************************************/ + +void test1() +{ + double imgTest1[][25] = {{244, 198, 134, 194, 86, 104, 156, 52, 126, 39, 56, 250, 68, + 247, 251, 93, 160, 61, 8, 81, 204, 113, 107, 206, 146} + + }; + + double answerTest1[][4] = {{0, 0, 0, 244}, {0, 0, 244, 198}, {0, 0, 198, 134}, {0, 0, 134, 194}, + {0, 0, 194, 86}, {0, 0, 86, 0}, {0, 244, 0, 104}, {244, 198, 104, 156}, + {198, 134, 156, 52}, {134, 194, 52, 126}, {194, 86, 126, 39}, {86, 0, 39, 0}, + {0, 104, 0, 56}, {104, 156, 56, 250}, {156, 52, 250, 68}, {52, 126, 68, 247}, + {126, 39, 247, 251}, {39, 0, 251, 0}, {0, 56, 0, 93}, {56, 250, 93, 160}, + {250, 68, 160, 61}, {68, 247, 61, 8}, {247, 251, 8, 81}, {251, 0, 81, 0}, + {0, 93, 0, 204}, {93, 160, 204, 113}, {160, 61, 113, 107}, {61, 8, 107, 206}, + {8, 81, 206, 146}, {81, 0, 146, 0}, {0, 204, 0, 0}, {204, 113, 0, 0}, + {113, 107, 0, 0}, {107, 206, 0, 0}, {206, 146, 0, 0}, {146, 0, 0, 0}}; + + size_t imgDepthTest1 = 1; + size_t imgHeightTest1 = 5; + size_t imgWidthTest1 = 5; + size_t fltHeightTest1 = 2; + size_t fltWidthTest1 = 2; + size_t strideRowsTest1 = 1; + size_t strideColsTest1 = 1; + size_t zeroPaddingHeightTest1 = 1; + size_t zeroPaddingWidthTest1 = 1; + + Matrix_t A(imgDepthTest1, imgHeightTest1 * imgWidthTest1); + + for (size_t i = 0; i < (size_t)A.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)A.GetNcols(); j++) { + A(i, j) = imgTest1[i][j]; + } + } + + size_t height = calculateDimension(imgHeightTest1, fltHeightTest1, zeroPaddingHeightTest1, strideRowsTest1); + + size_t width = calculateDimension(imgWidthTest1, fltWidthTest1, zeroPaddingWidthTest1, strideColsTest1); + + Matrix_t B(height * width, imgDepthTest1 * fltHeightTest1 * fltWidthTest1); + + for (size_t i = 0; i < (size_t)B.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)B.GetNcols(); j++) { + B(i, j) = answerTest1[i][j]; + } + } + + bool status = + testIm2col>(A, B, imgHeightTest1, imgWidthTest1, fltHeightTest1, fltWidthTest1, + strideRowsTest1, strideColsTest1, zeroPaddingHeightTest1, zeroPaddingWidthTest1); + + if (status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + +/************************************************************************* + * Test 2: + * depth = 2, image height = 5, image width = 5, + * filter depth = 2, filter height = 2, filter width = 3, + * stride rows = 1, stride cols = 1, + * zero-padding height = 1, zero-padding width = 1, + *************************************************************************/ + +void test2() +{ + + // 2 x 5 x 5 image + double imgTest2[][25] = { + + {244, 198, 134, 194, 86, 104, 156, 52, 126, 39, 56, 250, 68, + 247, 251, 93, 160, 61, 8, 81, 204, 113, 107, 206, 146}, + + {205, 136, 184, 196, 42, 157, 10, 62, 201, 46, 250, 78, 43, + 185, 82, 95, 218, 128, 104, 71, 118, 215, 228, 199, 52} + + }; + + double answerTest2[][12] = {{0, 0, 0, 0, 244, 198, 0, 0, 0, 0, 205, 136}, + {0, 0, 0, 244, 198, 134, 0, 0, 0, 205, 136, 184}, + {0, 0, 0, 198, 134, 194, 0, 0, 0, 136, 184, 196}, + {0, 0, 0, 134, 194, 86, 0, 0, 0, 184, 196, 42}, + {0, 0, 0, 194, 86, 0, 0, 0, 0, 196, 42, 0}, + {0, 244, 198, 0, 104, 156, 0, 205, 136, 0, 157, 10}, + {244, 198, 134, 104, 156, 52, 205, 136, 184, 157, 10, 62}, + {198, 134, 194, 156, 52, 126, 136, 184, 196, 10, 62, 201}, + {134, 194, 86, 52, 126, 39, 184, 196, 42, 62, 201, 46}, + {194, 86, 0, 126, 39, 0, 196, 42, 0, 201, 46, 0}, + {0, 104, 156, 0, 56, 250, 0, 157, 10, 0, 250, 78}, + {104, 156, 52, 56, 250, 68, 157, 10, 62, 250, 78, 43}, + {156, 52, 126, 250, 68, 247, 10, 62, 201, 78, 43, 185}, + {52, 126, 39, 68, 247, 251, 62, 201, 46, 43, 185, 82}, + {126, 39, 0, 247, 251, 0, 201, 46, 0, 185, 82, 0}, + {0, 56, 250, 0, 93, 160, 0, 250, 78, 0, 95, 218}, + {56, 250, 68, 93, 160, 61, 250, 78, 43, 95, 218, 128}, + {250, 68, 247, 160, 61, 8, 78, 43, 185, 218, 128, 104}, + {68, 247, 251, 61, 8, 81, 43, 185, 82, 128, 104, 71}, + {247, 251, 0, 8, 81, 0, 185, 82, 0, 104, 71, 0}, + {0, 93, 160, 0, 204, 113, 0, 95, 218, 0, 118, 215}, + {93, 160, 61, 204, 113, 107, 95, 218, 128, 118, 215, 228}, + {160, 61, 8, 113, 107, 206, 218, 128, 104, 215, 228, 199}, + {61, 8, 81, 107, 206, 146, 128, 104, 71, 228, 199, 52}, + {8, 81, 0, 206, 146, 0, 104, 71, 0, 199, 52, 0}, + {0, 204, 113, 0, 0, 0, 0, 118, 215, 0, 0, 0}, + {204, 113, 107, 0, 0, 0, 118, 215, 228, 0, 0, 0}, + {113, 107, 206, 0, 0, 0, 215, 228, 199, 0, 0, 0}, + {107, 206, 146, 0, 0, 0, 228, 199, 52, 0, 0, 0}, + {206, 146, 0, 0, 0, 0, 199, 52, 0, 0, 0, 0}}; + + size_t imgDepthTest2 = 2; + size_t imgHeightTest2 = 5; + size_t imgWidthTest2 = 5; + size_t fltHeightTest2 = 2; + size_t fltWidthTest2 = 3; + size_t strideRowsTest2 = 1; + size_t strideColsTest2 = 1; + size_t zeroPaddingHeightTest2 = 1; + size_t zeroPaddingWidthTest2 = 1; + + Matrix_t A(imgDepthTest2, imgHeightTest2 * imgWidthTest2); + + for (size_t i = 0; i < (size_t)A.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)A.GetNcols(); j++) { + A(i, j) = imgTest2[i][j]; + } + } + + size_t height = calculateDimension(imgHeightTest2, fltHeightTest2, zeroPaddingHeightTest2, strideRowsTest2); + + size_t width = calculateDimension(imgWidthTest2, fltWidthTest2, zeroPaddingWidthTest2, strideColsTest2); + + Matrix_t B(height * width, imgDepthTest2 * fltHeightTest2 * fltWidthTest2); + + for (size_t i = 0; i < (size_t)B.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)B.GetNcols(); j++) { + B(i, j) = answerTest2[i][j]; + } + } + + bool status = + testIm2col>(A, B, imgHeightTest2, imgWidthTest2, fltHeightTest2, fltWidthTest2, + strideRowsTest2, strideColsTest2, zeroPaddingHeightTest2, zeroPaddingWidthTest2); + + if (status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + +/************************************************************************* + * Test 3: + * depth = 3, image height = 2, image width = 3, + * filter depth = 3, filter height = 3, filter width = 2, + * stride rows = 3, stride cols = 1, + * zero-padding height = 2, zero-padding width = 1, + *************************************************************************/ + +void test3() +{ + + // 3 x 2 x 3 image + double imgTest3[][6] = {{235, 213, 185, 144, 235, 212}, + + {158, 168, 116, 68, 159, 157}, + + {240, 135, 195, 252, 36, 77}}; + + double answerTest3[][18] = {{0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 240}, + {0, 0, 0, 0, 235, 213, 0, 0, 0, 0, 158, 168, 0, 0, 0, 0, 240, 135}, + {0, 0, 0, 0, 213, 185, 0, 0, 0, 0, 168, 116, 0, 0, 0, 0, 135, 195}, + {0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 195, 0}, + {0, 144, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0}, + {144, 235, 0, 0, 0, 0, 68, 159, 0, 0, 0, 0, 252, 36, 0, 0, 0, 0}, + {235, 212, 0, 0, 0, 0, 159, 157, 0, 0, 0, 0, 36, 77, 0, 0, 0, 0}, + {212, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0}}; + + size_t imgDepthTest3 = 3; + size_t imgHeightTest3 = 2; + size_t imgWidthTest3 = 3; + size_t fltHeightTest3 = 3; + size_t fltWidthTest3 = 2; + size_t strideRowsTest3 = 3; + size_t strideColsTest3 = 1; + size_t zeroPaddingHeightTest3 = 2; + size_t zeroPaddingWidthTest3 = 1; + + Matrix_t A(imgDepthTest3, imgHeightTest3 * imgWidthTest3); + + for (size_t i = 0; i < (size_t)A.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)A.GetNcols(); j++) { + A(i, j) = imgTest3[i][j]; + } + } + + size_t height = calculateDimension(imgHeightTest3, fltHeightTest3, zeroPaddingHeightTest3, strideRowsTest3); + + size_t width = calculateDimension(imgWidthTest3, fltWidthTest3, zeroPaddingWidthTest3, strideColsTest3); + + Matrix_t B(height * width, imgDepthTest3 * fltHeightTest3 * fltWidthTest3); + + for (size_t i = 0; i < (size_t)B.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)B.GetNcols(); j++) { + B(i, j) = answerTest3[i][j]; + } + } + + bool status = + testIm2col>(A, B, imgHeightTest3, imgWidthTest3, fltHeightTest3, fltWidthTest3, + strideRowsTest3, strideColsTest3, zeroPaddingHeightTest3, zeroPaddingWidthTest3); + + if (status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + +int main() +{ + std::cout << "Testing Im2Col function:" << std::endl; + + std::cout << "Test 1: " << std::endl; + test1(); + + std::cout << "Test 2: " << std::endl; + test2(); + + std::cout << "Test 3: " << std::endl; + test3(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestIm2ColCpu.cxx b/tmva/tmva/test/DNN/CNN/TestIm2ColCpu.cxx new file mode 100644 index 0000000000000..0757a24401ac4 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestIm2ColCpu.cxx @@ -0,0 +1,294 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Im2Col method on a CPU architecture * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the method Im2col // +//////////////////////////////////////////////////////////////////// + +#include +#include + +#include "TMVA/DNN/Architectures/Cpu.h" +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; +using Matrix_t = typename TCpu::Matrix_t; + +inline bool isInteger(double x) +{ + return x == floor(x); +} + +int calculateDimension(size_t imgDim, size_t fltDim, size_t padding, size_t stride) +{ + double dimension = ((imgDim - fltDim + 2 * padding) / stride) + 1; + if (!isInteger(dimension)) { + std::cout << "Not compatible hyper parameters" << std::endl; + std::exit(EXIT_FAILURE); + } + + return (size_t)dimension; +} + +/************************************************************************* + * Test 1: + * depth = 1, image height = 5, image width = 5, + * filter depth = 1, filter height = 2, filter width = 2, + * stride rows = 1, stride cols = 1, + * zero-padding height = 1, zero-padding width = 1, + *************************************************************************/ + +void test1() +{ + double imgTest1[][25] = {{244, 198, 134, 194, 86, 104, 156, 52, 126, 39, 56, 250, 68, + 247, 251, 93, 160, 61, 8, 81, 204, 113, 107, 206, 146} + + }; + + double answerTest1[][4] = {{0, 0, 0, 244}, {0, 0, 244, 198}, {0, 0, 198, 134}, {0, 0, 134, 194}, + {0, 0, 194, 86}, {0, 0, 86, 0}, {0, 244, 0, 104}, {244, 198, 104, 156}, + {198, 134, 156, 52}, {134, 194, 52, 126}, {194, 86, 126, 39}, {86, 0, 39, 0}, + {0, 104, 0, 56}, {104, 156, 56, 250}, {156, 52, 250, 68}, {52, 126, 68, 247}, + {126, 39, 247, 251}, {39, 0, 251, 0}, {0, 56, 0, 93}, {56, 250, 93, 160}, + {250, 68, 160, 61}, {68, 247, 61, 8}, {247, 251, 8, 81}, {251, 0, 81, 0}, + {0, 93, 0, 204}, {93, 160, 204, 113}, {160, 61, 113, 107}, {61, 8, 107, 206}, + {8, 81, 206, 146}, {81, 0, 146, 0}, {0, 204, 0, 0}, {204, 113, 0, 0}, + {113, 107, 0, 0}, {107, 206, 0, 0}, {206, 146, 0, 0}, {146, 0, 0, 0}}; + + size_t imgDepthTest1 = 1; + size_t imgHeightTest1 = 5; + size_t imgWidthTest1 = 5; + size_t fltHeightTest1 = 2; + size_t fltWidthTest1 = 2; + size_t strideRowsTest1 = 1; + size_t strideColsTest1 = 1; + size_t zeroPaddingHeightTest1 = 1; + size_t zeroPaddingWidthTest1 = 1; + + Matrix_t A(imgDepthTest1, imgHeightTest1 * imgWidthTest1); + + for (size_t i = 0; i < (size_t)A.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)A.GetNcols(); j++) { + A(i, j) = imgTest1[i][j]; + } + } + + size_t height = calculateDimension(imgHeightTest1, fltHeightTest1, zeroPaddingHeightTest1, strideRowsTest1); + + size_t width = calculateDimension(imgWidthTest1, fltWidthTest1, zeroPaddingWidthTest1, strideColsTest1); + + Matrix_t B(height * width, imgDepthTest1 * fltHeightTest1 * fltWidthTest1); + + for (size_t i = 0; i < (size_t)B.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)B.GetNcols(); j++) { + B(i, j) = answerTest1[i][j]; + } + } + + bool status = + testIm2col>(A, B, imgHeightTest1, imgWidthTest1, fltHeightTest1, fltWidthTest1, strideRowsTest1, + strideColsTest1, zeroPaddingHeightTest1, zeroPaddingWidthTest1); + + if (status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + +/************************************************************************* + * Test 2: + * depth = 2, image height = 5, image width = 5, + * filter depth = 2, filter height = 2, filter width = 3, + * stride rows = 1, stride cols = 1, + * zero-padding height = 1, zero-padding width = 1, + *************************************************************************/ + +void test2() +{ + + // 2 x 5 x 5 image + double imgTest2[][25] = { + + {244, 198, 134, 194, 86, 104, 156, 52, 126, 39, 56, 250, 68, + 247, 251, 93, 160, 61, 8, 81, 204, 113, 107, 206, 146}, + + {205, 136, 184, 196, 42, 157, 10, 62, 201, 46, 250, 78, 43, + 185, 82, 95, 218, 128, 104, 71, 118, 215, 228, 199, 52} + + }; + + double answerTest2[][12] = {{0, 0, 0, 0, 244, 198, 0, 0, 0, 0, 205, 136}, + {0, 0, 0, 244, 198, 134, 0, 0, 0, 205, 136, 184}, + {0, 0, 0, 198, 134, 194, 0, 0, 0, 136, 184, 196}, + {0, 0, 0, 134, 194, 86, 0, 0, 0, 184, 196, 42}, + {0, 0, 0, 194, 86, 0, 0, 0, 0, 196, 42, 0}, + {0, 244, 198, 0, 104, 156, 0, 205, 136, 0, 157, 10}, + {244, 198, 134, 104, 156, 52, 205, 136, 184, 157, 10, 62}, + {198, 134, 194, 156, 52, 126, 136, 184, 196, 10, 62, 201}, + {134, 194, 86, 52, 126, 39, 184, 196, 42, 62, 201, 46}, + {194, 86, 0, 126, 39, 0, 196, 42, 0, 201, 46, 0}, + {0, 104, 156, 0, 56, 250, 0, 157, 10, 0, 250, 78}, + {104, 156, 52, 56, 250, 68, 157, 10, 62, 250, 78, 43}, + {156, 52, 126, 250, 68, 247, 10, 62, 201, 78, 43, 185}, + {52, 126, 39, 68, 247, 251, 62, 201, 46, 43, 185, 82}, + {126, 39, 0, 247, 251, 0, 201, 46, 0, 185, 82, 0}, + {0, 56, 250, 0, 93, 160, 0, 250, 78, 0, 95, 218}, + {56, 250, 68, 93, 160, 61, 250, 78, 43, 95, 218, 128}, + {250, 68, 247, 160, 61, 8, 78, 43, 185, 218, 128, 104}, + {68, 247, 251, 61, 8, 81, 43, 185, 82, 128, 104, 71}, + {247, 251, 0, 8, 81, 0, 185, 82, 0, 104, 71, 0}, + {0, 93, 160, 0, 204, 113, 0, 95, 218, 0, 118, 215}, + {93, 160, 61, 204, 113, 107, 95, 218, 128, 118, 215, 228}, + {160, 61, 8, 113, 107, 206, 218, 128, 104, 215, 228, 199}, + {61, 8, 81, 107, 206, 146, 128, 104, 71, 228, 199, 52}, + {8, 81, 0, 206, 146, 0, 104, 71, 0, 199, 52, 0}, + {0, 204, 113, 0, 0, 0, 0, 118, 215, 0, 0, 0}, + {204, 113, 107, 0, 0, 0, 118, 215, 228, 0, 0, 0}, + {113, 107, 206, 0, 0, 0, 215, 228, 199, 0, 0, 0}, + {107, 206, 146, 0, 0, 0, 228, 199, 52, 0, 0, 0}, + {206, 146, 0, 0, 0, 0, 199, 52, 0, 0, 0, 0}}; + + size_t imgDepthTest2 = 2; + size_t imgHeightTest2 = 5; + size_t imgWidthTest2 = 5; + size_t fltHeightTest2 = 2; + size_t fltWidthTest2 = 3; + size_t strideRowsTest2 = 1; + size_t strideColsTest2 = 1; + size_t zeroPaddingHeightTest2 = 1; + size_t zeroPaddingWidthTest2 = 1; + + Matrix_t A(imgDepthTest2, imgHeightTest2 * imgWidthTest2); + + for (size_t i = 0; i < (size_t)A.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)A.GetNcols(); j++) { + A(i, j) = imgTest2[i][j]; + } + } + + size_t height = calculateDimension(imgHeightTest2, fltHeightTest2, zeroPaddingHeightTest2, strideRowsTest2); + + size_t width = calculateDimension(imgWidthTest2, fltWidthTest2, zeroPaddingWidthTest2, strideColsTest2); + + Matrix_t B(height * width, imgDepthTest2 * fltHeightTest2 * fltWidthTest2); + + for (size_t i = 0; i < (size_t)B.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)B.GetNcols(); j++) { + B(i, j) = answerTest2[i][j]; + } + } + + bool status = + testIm2col>(A, B, imgHeightTest2, imgWidthTest2, fltHeightTest2, fltWidthTest2, strideRowsTest2, + strideColsTest2, zeroPaddingHeightTest2, zeroPaddingWidthTest2); + + if (status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + +/************************************************************************* + * Test 3: + * depth = 3, image height = 2, image width = 3, + * filter depth = 3, filter height = 3, filter width = 2, + * stride rows = 3, stride cols = 1, + * zero-padding height = 2, zero-padding width = 1, + *************************************************************************/ + +void test3() +{ + + // 3 x 2 x 3 image + double imgTest3[][6] = {{235, 213, 185, 144, 235, 212}, + + {158, 168, 116, 68, 159, 157}, + + {240, 135, 195, 252, 36, 77}}; + + double answerTest3[][18] = {{0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 240}, + {0, 0, 0, 0, 235, 213, 0, 0, 0, 0, 158, 168, 0, 0, 0, 0, 240, 135}, + {0, 0, 0, 0, 213, 185, 0, 0, 0, 0, 168, 116, 0, 0, 0, 0, 135, 195}, + {0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 195, 0}, + {0, 144, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0}, + {144, 235, 0, 0, 0, 0, 68, 159, 0, 0, 0, 0, 252, 36, 0, 0, 0, 0}, + {235, 212, 0, 0, 0, 0, 159, 157, 0, 0, 0, 0, 36, 77, 0, 0, 0, 0}, + {212, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0}}; + + size_t imgDepthTest3 = 3; + size_t imgHeightTest3 = 2; + size_t imgWidthTest3 = 3; + size_t fltHeightTest3 = 3; + size_t fltWidthTest3 = 2; + size_t strideRowsTest3 = 3; + size_t strideColsTest3 = 1; + size_t zeroPaddingHeightTest3 = 2; + size_t zeroPaddingWidthTest3 = 1; + + Matrix_t A(imgDepthTest3, imgHeightTest3 * imgWidthTest3); + + for (size_t i = 0; i < (size_t)A.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)A.GetNcols(); j++) { + A(i, j) = imgTest3[i][j]; + } + } + + size_t height = calculateDimension(imgHeightTest3, fltHeightTest3, zeroPaddingHeightTest3, strideRowsTest3); + + size_t width = calculateDimension(imgWidthTest3, fltWidthTest3, zeroPaddingWidthTest3, strideColsTest3); + + Matrix_t B(height * width, imgDepthTest3 * fltHeightTest3 * fltWidthTest3); + + for (size_t i = 0; i < (size_t)B.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)B.GetNcols(); j++) { + B(i, j) = answerTest3[i][j]; + } + } + + bool status = + testIm2col>(A, B, imgHeightTest3, imgWidthTest3, fltHeightTest3, fltWidthTest3, strideRowsTest3, + strideColsTest3, zeroPaddingHeightTest3, zeroPaddingWidthTest3); + + if (status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + +int main() +{ + std::cout << "Testing Im2Col function:" << std::endl; + + std::cout << "Test 1: " << std::endl; + test1(); + + std::cout << "Test 2: " << std::endl; + test2(); + + std::cout << "Test 3: " << std::endl; + test3(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestMethodDL.cxx b/tmva/tmva/test/DNN/CNN/TestMethodDL.cxx new file mode 100644 index 0000000000000..d33a02d4ad7f5 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestMethodDL.cxx @@ -0,0 +1,37 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Method DL for Conv Net for the Reference backend * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#include "TestMethodDL.h" +#include "TString.h" + +int main() +{ + std::cout << "Testing Method DL for CPU backend: " << std::endl; + + TString archCPU = "CPU"; + //testMethodDL(archCPU); + testMethodDL_DNN(archCPU); +} \ No newline at end of file diff --git a/tmva/tmva/test/DNN/CNN/TestMethodDL.h b/tmva/tmva/test/DNN/CNN/TestMethodDL.h new file mode 100644 index 0000000000000..26900db2f8584 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestMethodDL.h @@ -0,0 +1,267 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Method DL for Conv Net * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_TEST_DNN_TEST_CNN_TEST_METHOD_DL_H +#define TMVA_TEST_DNN_TEST_CNN_TEST_METHOD_DL_H + +#include "TFile.h" +#include "TTree.h" +#include "TString.h" +#include "TROOT.h" + +#include "TMVA/MethodDL.h" +#include "TMVA/DataLoader.h" +#include "TMVA/Factory.h" +#include "TMVA/Config.h" + +#include + +/** Testing the entire pipeline of the Method DL, when only a Convolutional Net + * is constructed. */ +//______________________________________________________________________________ +void testMethodDL(TString architectureStr) +{ + // Load the photon input + TFile *photonInput(0); + TString photonFileName = + "/Users/vladimirilievski/Desktop/Vladimir/GSoC/ROOT-CI/common-version/root/tmva/tmva/test/DNN/CNN/" + "dataset/SinglePhotonPt50_FEVTDEBUG_n250k_IMG_CROPS32.root"; + photonInput = TFile::Open(photonFileName); + + // Load the electron input + TFile *electronInput(0); + TString electronFileName = "/Users/vladimirilievski/Desktop/Vladimir/GSoC/ROOT-CI/common-version/root/tmva/tmva/" + "test/DNN/CNN/dataset/SingleElectronPt50_FEVTDEBUG_n250k_IMG_CROPS32.root"; + electronInput = TFile::Open(electronFileName); + + // Get the trees + TTree *signalTree = (TTree *)photonInput->Get("RHTree"); + TTree *background = (TTree *)electronInput->Get("RHTree"); + + // Create a ROOT output file where TMVA will store ntuples, histograms, etc. + TString outfileName("TMVA_MethodDL.root"); + TFile *outputFile = TFile::Open(outfileName, "RECREATE"); + + TMVA::DataLoader *dataloader = new TMVA::DataLoader("dataset"); + + // global event weights per tree + Double_t signalWeight = 1.0; + Double_t backgroundWeight = 1.0; + + // Add signal and background trees + dataloader->AddSignalTree(signalTree, signalWeight); + dataloader->AddBackgroundTree(background, backgroundWeight); + + // dataloader->SetBackgroundWeightExpression("weight"); + + TCut mycuts = ""; + TCut mycutb = ""; + + dataloader->PrepareTrainingAndTestTree( + mycuts, mycutb, "nTrain_Signal=10000:nTrain_Background=10000:SplitMode=Random:NormMode=NumEvents:!V"); + + // Input Layout + TString inputLayoutString("InputLayout=1|32|32"); + + // Input Layout + TString batchLayoutString("BatchLayout=256|1|1024"); + + // General layout. + TString layoutString("Layout=CONV|12|2|2|1|1|1|1|TANH,MAXPOOL|6|6|1|1,RESHAPE|1|1|9408|FLAT,DENSE|512|TANH,DENSE|32|" + "TANH,DENSE|2|LINEAR"); + + // Training strategies. + TString training0("LearningRate=1e-1,Momentum=0.9,Repetitions=1," + "ConvergenceSteps=20,BatchSize=256,TestRepetitions=10," + "WeightDecay=1e-4,Regularization=L2," + "DropConfig=0.0+0.5+0.5+0.5, Multithreading=True"); + TString training1("LearningRate=1e-2,Momentum=0.9,Repetitions=1," + "ConvergenceSteps=20,BatchSize=256,TestRepetitions=10," + "WeightDecay=1e-4,Regularization=L2," + "DropConfig=0.0+0.0+0.0+0.0, Multithreading=True"); + TString training2("LearningRate=1e-3,Momentum=0.0,Repetitions=1," + "ConvergenceSteps=20,BatchSize=256,TestRepetitions=10," + "WeightDecay=1e-4,Regularization=L2," + "DropConfig=0.0+0.0+0.0+0.0, Multithreading=True"); + TString trainingStrategyString("TrainingStrategy="); + trainingStrategyString += training0 + "|" + training1 + "|" + training2; + + // General Options. + TString dnnOptions("!H:V:ErrorStrategy=CROSSENTROPY:" + "WeightInitialization=XAVIERUNIFORM"); + + // Concatenate all option strings + dnnOptions.Append(":"); + dnnOptions.Append(inputLayoutString); + + dnnOptions.Append(":"); + dnnOptions.Append(batchLayoutString); + + dnnOptions.Append(":"); + dnnOptions.Append(layoutString); + + dnnOptions.Append(":"); + dnnOptions.Append(trainingStrategyString); + + dnnOptions.Append(":Architecture="); + dnnOptions.Append(architectureStr); + + // Create a factory for booking the method + TMVA::Factory *factory = + new TMVA::Factory("TMVAClassification", outputFile, + "!V:!Silent:Color:DrawProgressBar:Transformations=I:AnalysisType=Classification"); + + TString methodTitle = "DL_" + architectureStr; + factory->BookMethod(dataloader, TMVA::Types::kDL, methodTitle, dnnOptions); + + // Train MVAs using the set of training events + factory->TrainAllMethods(); + + // Save the output + outputFile->Close(); + + std::cout << "==> Wrote root file: " << outputFile->GetName() << std::endl; + std::cout << "==> TMVAClassification is done!" << std::endl; + + delete factory; + delete dataloader; +} + +/** Testing the entire pipeline of the Method DL, when only a Multilayer Percepton + * is constructed. */ +//______________________________________________________________________________ +void testMethodDL_DNN(TString architectureStr) +{ + + ROOT::EnableImplicitMT(1); + TMVA::Config::Instance(); + + TFile *input(0); + // TString fname = "/Users/vladimirilievski/Desktop/Vladimir/GSoC/ROOT-CI/common-version/root/tmva/tmva/test/DNN/CNN/" + // "dataset/tmva_class_example.root"; + TString fname = "http://root.cern.ch/files/tmva_class_example.root"; + TString fopt = "CACHEREAD"; + input = TFile::Open(fname,fopt); + + // Register the training and test trees + TTree *signalTree = (TTree *)input->Get("TreeS"); + TTree *background = (TTree *)input->Get("TreeB"); + + TString outfileName("TMVA_DNN.root"); + TFile *outputFile = TFile::Open(outfileName, "RECREATE"); + + TMVA::DataLoader *dataloader = new TMVA::DataLoader("dataset"); + + dataloader->AddVariable("myvar1 := var1+var2", 'F'); + dataloader->AddVariable("myvar2 := var1-var2", "Expression 2", "", 'F'); + dataloader->AddVariable("var3", "Variable 3", "units", 'F'); + dataloader->AddVariable("var4", "Variable 4", "units", 'F'); + dataloader->AddSpectator("spec1 := var1*2", "Spectator 1", "units", 'F'); + dataloader->AddSpectator("spec2 := var1*3", "Spectator 2", "units", 'F'); + + // global event weights per tree + Double_t signalWeight = 1.0; + Double_t backgroundWeight = 1.0; + + // Add signal and background trees + dataloader->AddSignalTree(signalTree, signalWeight); + dataloader->AddBackgroundTree(background, backgroundWeight); + + dataloader->SetBackgroundWeightExpression("weight"); + + TCut mycuts = ""; + TCut mycutb = ""; + + dataloader->PrepareTrainingAndTestTree( + mycuts, mycutb, "nTrain_Signal=1000:nTrain_Background=1000:SplitMode=Random:NormMode=NumEvents:!V"); + + // Input Layout + TString inputLayoutString("InputLayout=1|1|4"); + + // Batch Layout + TString batchLayoutString("BatchLayout=256|1|4"); + + // General layout. + TString layoutString("Layout=RESHAPE|1|1|4|FLAT,DENSE|128|TANH,DENSE|128|TANH,DENSE|128|TANH,DENSE|1|LINEAR"); + + // Training strategies. + TString training0("LearningRate=1e-1,Momentum=0.9,Repetitions=1," + "ConvergenceSteps=20,BatchSize=256,TestRepetitions=10," + "WeightDecay=1e-4,Regularization=L2," + "DropConfig=0.0+0.5+0.5+0.5, Multithreading=True"); + TString training1("LearningRate=1e-2,Momentum=0.9,Repetitions=1," + "ConvergenceSteps=20,BatchSize=256,TestRepetitions=10," + "WeightDecay=1e-4,Regularization=L2," + "DropConfig=0.0+0.0+0.0+0.0, Multithreading=True"); + TString training2("LearningRate=1e-3,Momentum=0.9,Repetitions=1," + "ConvergenceSteps=20,BatchSize=256,TestRepetitions=10," + "WeightDecay=1e-4,Regularization=L2," + "DropConfig=0.0+0.0+0.0+0.0, Multithreading=True"); + TString trainingStrategyString("TrainingStrategy="); + trainingStrategyString += training0 + "|" + training1 + "|" + training2; + + // General Options. + TString dnnOptions("!H:V:ErrorStrategy=SUMOFSQUARES:" + "WeightInitialization=XAVIERUNIFORM"); + + // Concatenate all option strings + dnnOptions.Append(":"); + dnnOptions.Append(inputLayoutString); + + dnnOptions.Append(":"); + dnnOptions.Append(batchLayoutString); + + dnnOptions.Append(":"); + dnnOptions.Append(layoutString); + + dnnOptions.Append(":"); + dnnOptions.Append(trainingStrategyString); + + dnnOptions.Append(":Architecture="); + dnnOptions.Append(architectureStr); + + // create factory + TMVA::Factory *factory = + new TMVA::Factory("TMVAClassification", outputFile, + "!V:!Silent:Color:DrawProgressBar:Transformations=I;D;P;G,D:AnalysisType=Classification"); + + TString methodTitle = "DL_" + architectureStr; + factory->BookMethod(dataloader, TMVA::Types::kDL, methodTitle, dnnOptions); + + // Train MVAs using the set of training events + factory->TrainAllMethods(); + + // Save the output + outputFile->Close(); + + std::cout << "==> Wrote root file: " << outputFile->GetName() << std::endl; + std::cout << "==> TMVAClassification is done!" << std::endl; + + delete factory; + delete dataloader; +} + +#endif diff --git a/tmva/tmva/test/DNN/CNN/TestMinimization.cxx b/tmva/tmva/test/DNN/CNN/TestMinimization.cxx new file mode 100644 index 0000000000000..6c0f7493610cf --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestMinimization.cxx @@ -0,0 +1,48 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Deep Learning Minimizer for the Reference backend * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#include "TestMinimization.h" +#include "TMVA/DNN/Architectures/Reference.h" + +#include + +using namespace TMVA::DNN; + +int main() +{ + using Scalar_t = Real_t; + + Scalar_t momentum = 0.0; + std::cout << "Minimizer, no momentum" << std::endl; + testMinimization>(momentum, false); + + momentum = 0.1; + std::cout << "Minimizer, with momentum" << std::endl; + testMinimization>(momentum, false); + + std::cout << "Minimizer, with Nestorov momentum" << std::endl; + testMinimization>(momentum, true); +} \ No newline at end of file diff --git a/tmva/tmva/test/DNN/CNN/TestMinimization.h b/tmva/tmva/test/DNN/CNN/TestMinimization.h new file mode 100644 index 0000000000000..ee1094d255c24 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestMinimization.h @@ -0,0 +1,139 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Deep Learning Minimizer * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_TEST_DNN_TEST_CNN_TEST_DL_MINIMIZER_H +#define TMVA_TEST_DNN_TEST_CNN_TEST_DL_MINIMIZER_H + +#include "TMatrix.h" +#include "TMVA/DNN/DLMinimizers.h" +#include "TMVA/DNN/DeepNet.h" +#include "../Utility.h" +#include "TMVA/DNN/TensorDataLoader.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; + +template +auto testMinimization(typename Architecture_t::Scalar_t momentum, bool nestorov) -> void +{ + using DeepNet_t = TDeepNet; + using DataLoader_t = TTensorDataLoader; + + size_t nSamples = 10000; + size_t nChannels = 20; + size_t imgWidth = 32; + size_t imgHeight = 32; + size_t batchSize = 256; + + // Initialize train and test input + std::vector> XTrain, XTest; + XTrain.reserve(nSamples); + XTest.reserve(nSamples); + for (size_t i = 0; i < nSamples; i++) { + XTrain.emplace_back(nChannels, imgWidth * imgHeight); + XTest.emplace_back(nChannels, imgWidth * imgHeight); + } + + // Initialize train and test output + size_t nOutput = 5; + TMatrixT YTrain(nSamples, nOutput), YTest(nSamples, nOutput); + + // Initialize train and test weights + TMatrixT WTrain(nSamples, 1), WTest(nSamples, 1); + + // Fill-in the input + for (size_t i = 0; i < nSamples; i++) { + randomMatrix(XTrain[i]); + randomMatrix(XTest[i]); + } + + // Fill-in the output + randomMatrix(YTrain); + randomMatrix(YTest); + + // Fill-in the batch weights + fillMatrix(WTrain, 1.0); + fillMatrix(WTest, 1.0); + + // Construct the Linear Net + size_t batchDepth = batchSize; + size_t batchHeight = nChannels; + size_t batchWidth = imgHeight * imgWidth; + + DeepNet_t convNet(batchSize, nChannels, imgHeight, imgWidth, batchDepth, batchHeight, batchWidth, + ELossFunction::kMeanSquaredError); + + size_t nThreads = 5; + std::vector nets{}; + nets.reserve(nThreads); + + for (size_t i = 0; i < nThreads; i++) { + // create a copies of the master deep net + nets.push_back(convNet); + } + + // construct the master and slave conv nets + constructMasterSlaveConvNets(convNet, nets); + + // Initialize the minimizers + TDLGradientDescent minimizer(0.0001, 5, 1); + + // Initialize the tensor inputs + TensorInput trainingInput(XTrain, YTrain, WTrain); + TensorInput testInput(XTest, YTest, WTest); + + DataLoader_t trainingData(trainingInput, nSamples, batchSize, batchDepth, batchHeight, batchWidth, nOutput, + nThreads); + DataLoader_t testingData(testInput, nSamples, batchSize, batchDepth, batchHeight, batchWidth, nOutput, nThreads); + + // Initialize the vector of batches, one batch for one slave network + std::vector> batches{}; + + size_t batchesInEpoch = nSamples / convNet.GetBatchSize(); + + // execute all epochs + for (size_t i = 0; i < batchesInEpoch; i += nThreads) { + batches.clear(); + batches.reserve(nThreads); + + for (size_t j = 0; j < nThreads; j++) { + batches.push_back(trainingData.GetTensorBatch()); + } + + if (momentum != 0.0) { + if (nestorov) { + minimizer.StepNesterov(convNet, nets, batches, momentum); + } else { + minimizer.StepMomentum(convNet, nets, batches, momentum); + } + } else { + minimizer.Step(convNet, nets, batches); + } + } + // no returning then putting this void +} + +#endif diff --git a/tmva/tmva/test/DNN/CNN/TestMinimizationCpu.cxx b/tmva/tmva/test/DNN/CNN/TestMinimizationCpu.cxx new file mode 100644 index 0000000000000..a59c1de9ef40f --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestMinimizationCpu.cxx @@ -0,0 +1,48 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Deep Learning Minimizer for the CPU backend * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#include "TestMinimization.h" +#include "TMVA/DNN/Architectures/Cpu.h" + +#include + +using namespace TMVA::DNN; + +int main() +{ + using Scalar_t = Real_t; + + Scalar_t momentum = 0.0; + std::cout << "Minimizer, no momentum" << std::endl; + testMinimization>(momentum, false); + + momentum = 0.1; + std::cout << "Minimizer, with momentum momentum" << std::endl; + testMinimization>(momentum, false); + + std::cout << "Minimizer, with Nestorov momentum" << std::endl; + testMinimization>(momentum, true); +} \ No newline at end of file diff --git a/tmva/tmva/test/DNN/CNN/TestRotateWeights.cxx b/tmva/tmva/test/DNN/CNN/TestRotateWeights.cxx new file mode 100644 index 0000000000000..5e9af00b40d26 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestRotateWeights.cxx @@ -0,0 +1,92 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing RotateWeights method * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the Rotate Weights function // +//////////////////////////////////////////////////////////////////// + +#include +#include + +#include "TMVA/DNN/Architectures/Reference.h" +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; +using Matrix_t = typename TReference::Matrix_t; + +/************************************************************************* + * Test 1: + * filter depth = 3, filter height = 2, filter width = 2, num. filters = 4 + *************************************************************************/ +void test1() +{ + double weightsTest1[][12] = {{252, 116, 155, 246, 170, 149, 227, 113, 166, 227, 119, 57}, + {92, 103, 151, 37, 110, 46, 70, 8, 88, 182, 43, 236}, + {153, 246, 216, 102, 179, 248, 187, 227, 66, 102, 180, 169}, + {5, 215, 115, 103, 35, 138, 193, 28, 213, 93, 117, 208}}; + + double answerTest1[][16] = {{246, 155, 116, 252, 37, 151, 103, 92, 102, 216, 246, 153, 103, 115, 215, 5}, + {113, 227, 149, 170, 8, 70, 46, 110, 227, 187, 248, 179, 28, 193, 138, 35}, + {57, 119, 227, 166, 236, 43, 182, 88, 169, 180, 102, 66, 208, 117, 93, 213}}; + + size_t filterDepthTest1 = 3; + size_t filterHeightTest1 = 2; + size_t filterWidthTest1 = 2; + size_t numFiltersTest1 = 4; + + Matrix_t A(numFiltersTest1, filterDepthTest1 * filterHeightTest1 * filterWidthTest1); + + for (size_t i = 0; i < (size_t)A.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)A.GetNcols(); j++) { + A(i, j) = weightsTest1[i][j]; + } + } + + Matrix_t B(filterDepthTest1, numFiltersTest1 * filterHeightTest1 * filterWidthTest1); + + for (size_t i = 0; i < (size_t)B.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)B.GetNcols(); j++) { + B(i, j) = answerTest1[i][j]; + } + } + + bool status = testRotateWeights>(A, B, filterDepthTest1, filterHeightTest1, filterWidthTest1, + numFiltersTest1); + + if (status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + +int main() +{ + std::cout << "Testing Rotate Weights function:" << std::endl; + + std::cout << "Test 1: " << std::endl; + test1(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestRotateWeightsCpu.cxx b/tmva/tmva/test/DNN/CNN/TestRotateWeightsCpu.cxx new file mode 100644 index 0000000000000..94bde7bcddd6f --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestRotateWeightsCpu.cxx @@ -0,0 +1,92 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing RotateWeights method on a CPU architecture * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Testing the Rotate Weights function // +//////////////////////////////////////////////////////////////////// + +#include +#include + +#include "TMVA/DNN/Architectures/Cpu.h" +#include "TestConvNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; +using Matrix_t = typename TCpu::Matrix_t; + +/************************************************************************* + * Test 1: + * filter depth = 3, filter height = 2, filter width = 2, num. filters = 4 + *************************************************************************/ +void test1() +{ + double weightsTest1[][12] = {{252, 116, 155, 246, 170, 149, 227, 113, 166, 227, 119, 57}, + {92, 103, 151, 37, 110, 46, 70, 8, 88, 182, 43, 236}, + {153, 246, 216, 102, 179, 248, 187, 227, 66, 102, 180, 169}, + {5, 215, 115, 103, 35, 138, 193, 28, 213, 93, 117, 208}}; + + double answerTest1[][16] = {{246, 155, 116, 252, 37, 151, 103, 92, 102, 216, 246, 153, 103, 115, 215, 5}, + {113, 227, 149, 170, 8, 70, 46, 110, 227, 187, 248, 179, 28, 193, 138, 35}, + {57, 119, 227, 166, 236, 43, 182, 88, 169, 180, 102, 66, 208, 117, 93, 213}}; + + size_t filterDepthTest1 = 3; + size_t filterHeightTest1 = 2; + size_t filterWidthTest1 = 2; + size_t numFiltersTest1 = 4; + + Matrix_t A(numFiltersTest1, filterDepthTest1 * filterHeightTest1 * filterWidthTest1); + + for (size_t i = 0; i < (size_t)A.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)A.GetNcols(); j++) { + A(i, j) = weightsTest1[i][j]; + } + } + + Matrix_t B(filterDepthTest1, numFiltersTest1 * filterHeightTest1 * filterWidthTest1); + + for (size_t i = 0; i < (size_t)B.GetNrows(); i++) { + for (size_t j = 0; j < (size_t)B.GetNcols(); j++) { + B(i, j) = answerTest1[i][j]; + } + } + + bool status = + testRotateWeights>(A, B, filterDepthTest1, filterHeightTest1, filterWidthTest1, numFiltersTest1); + + if (status) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test not passed!" << std::endl; +} + +int main() +{ + std::cout << "Testing Rotate Weights function on a CPU architecture:" << std::endl; + + std::cout << "Test 1: " << std::endl; + test1(); +} diff --git a/tmva/tmva/test/DNN/CNN/TestTensorDataLoader.cxx b/tmva/tmva/test/DNN/CNN/TestTensorDataLoader.cxx new file mode 100644 index 0000000000000..278796c3fa215 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestTensorDataLoader.cxx @@ -0,0 +1,53 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Tensor Data Loader Features * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +////////////////////////////////////////////////////////// +// Test the reference tensor data loader implementation. // +////////////////////////////////////////////////////////// + +#include "TestTensorDataLoader.h" +#include "TMVA/DNN/Architectures/Reference.h" + +using namespace TMVA::DNN; + +int main() +{ + + using Scalar_t = Real_t; + + std::cout << "Testing tensor data loader:" << std::endl; + + // Scalar_t maximumError = 0.0; + // Scalar_t error = testSum>(); + // std::cout << "Sum: Maximum relative error = " << error << std::endl; + // maximumError = std::max(error, maximumError); + + // error = testIdentity>(); + // std::cout << "Identity: Maximum relative error = " << error << std::endl; + // maximumError = std::max(error, maximumError); + + testDataLoaderDataSet>(); +} \ No newline at end of file diff --git a/tmva/tmva/test/DNN/CNN/TestTensorDataLoader.h b/tmva/tmva/test/DNN/CNN/TestTensorDataLoader.h new file mode 100644 index 0000000000000..d2fbf5a4bc399 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestTensorDataLoader.h @@ -0,0 +1,295 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Tensor Data Loader Features * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_TEST_DNN_TEST_CNN_TEST_TENSOR_DATA_LOADER_H +#define TMVA_TEST_DNN_TEST_CNN_TEST_TENSOR_DATA_LOADER_H + +#include "../Utility.h" + +#include "TFile.h" +#include "TTree.h" +#include "TString.h" +#include "TBranch.h" +#include "TTreeReader.h" +#include "TTreeReaderValue.h" +#include "TTreeReaderArray.h" + +#include "TMVA/Event.h" + +#include "TMVA/DNN/TensorDataLoader.h" +#include "TMVA/DNN/DeepNet.h" + +#include +#include + +using namespace TMVA::DNN; +using namespace TMVA::DNN::CNN; + +/** Test that the tensor data loader loads all data in the data set by summing + * up all elements batch wise and comparing to the result obtained by summing + * over the complete dataset. + */ +//______________________________________________________________________________ +template +auto testSum() -> typename Architecture_t::Scalar_t +{ + using Scalar_t = typename Architecture_t::Scalar_t; + using Matrix_t = typename Architecture_t::Matrix_t; + using DataLoader_t = TTensorDataLoader; + + size_t nSamples = 10000; + size_t nRows = 1000; + size_t nCols = 1; + + // Create the tensor input + std::vector> inputTensor; + std::vector inputTensorArch; + inputTensor.reserve(nSamples); + inputTensorArch.reserve(nSamples); + + for (size_t i = 0; i < nSamples; i++) { + inputTensor.emplace_back(nRows, nCols); + inputTensorArch.emplace_back(nRows, nCols); + } + + for (size_t i = 0; i < nSamples; i++) { + for (size_t j = 0; j < nRows; j++) { + for (size_t k = 0; k < nCols; k++) { + inputTensor[i](j, k) = i; + inputTensorArch[i](j, k) = i; + } + } + } + + // Create the output + TMatrixT output(nSamples, 1); + Matrix_t outputArch(nSamples, 1); + for (size_t i = 0; i < nSamples; i++) { + output(i, 0) = i; + outputArch(i, 0) = i; + } + + // Create the weights + TMatrixT weights(nSamples, 1); + Matrix_t weightsArch(nSamples, 1); + for (size_t i = 0; i < nSamples; i++) { + weights(i, 0) = i; + weightsArch(i, 0) = i; + } + + TensorInput input(inputTensor, output, weights); + DataLoader_t loader(input, nSamples, 5, 5, 1000, 1, 1); + + Matrix_t Sum(1, 1), SumTotal(1, 1); + Scalar_t sum = 0.0, sumTotal = 0.0; + + for (auto b : loader) { + for (size_t i = 0; i < b.GetInput().size(); i++) { + Architecture_t::SumColumns(Sum, b.GetInput()[i]); + sum += Sum(0, 0); + } + + Architecture_t::SumColumns(Sum, b.GetOutput()); + sum += Sum(0, 0); + + Architecture_t::SumColumns(Sum, b.GetWeights()); + sum += Sum(0, 0); + } + + for (size_t i = 0; i < inputTensorArch.size(); i++) { + Architecture_t::SumColumns(SumTotal, inputTensorArch[i]); + sumTotal += SumTotal(0, 0); + } + + Architecture_t::SumColumns(SumTotal, outputArch); + sumTotal += SumTotal(0, 0); + + Architecture_t::SumColumns(SumTotal, weightsArch); + sumTotal += SumTotal(0, 0); + + return fabs(sumTotal - sum) / sumTotal; +} + +//______________________________________________________________________________ +template +auto testIdentity() -> typename Architecture_t::Scalar_t +{ + using Scalar_t = typename Architecture_t::Scalar_t; + using Net_t = TDeepNet; + using DataLoader_t = TTensorDataLoader; + + size_t nSamples = 2000; + size_t nRows = 3; + size_t nCols = 1024; + + std::vector> inputTensor; + inputTensor.reserve(nSamples); + + for (size_t i = 0; i < nSamples; i++) { + inputTensor.emplace_back(nRows, nCols); + } + + for (size_t i = 0; i < nSamples; i++) { + randomMatrix(inputTensor[i]); + } + + // Create the output + TMatrixT output(nSamples, 1); + for (size_t i = 0; i < nSamples; i++) { + output(i, 0) = 0; + } + + // Create the weights + TMatrixT weights(nSamples, 1); + for (size_t i = 0; i < nSamples; i++) { + weights(i, 0) = 1; + } + + TensorInput input(inputTensor, output, weights); + DataLoader_t loader(input, nSamples, 5, 5, 3, 1024, 1); + + Net_t convNet(5, 3, 32, 32, 5, 3, 1024, ELossFunction::kMeanSquaredError, EInitialization::kIdentity); + constructLinearConvNet(convNet); + convNet.Initialize(); + + Scalar_t maximumError = 0.0; + for (auto b : loader) { + auto linputTensor = b.GetInput(); + auto outputMatrix = b.GetOutput(); + auto weightMatrix = b.GetWeights(); + Scalar_t error = convNet.Loss(linputTensor, outputMatrix, weightMatrix); + maximumError = std::max(error, maximumError); + } + + return maximumError; +} + +//______________________________________________________________________________ +template +void testDataLoaderDataSet() +{ + using DeepNet_t = TDeepNet; + using DataLoader_t = TTensorDataLoader; + + // Load the photon input + TFile *photonInput(0); + TString photonFileName = + "/Users/vladimirilievski/Desktop/Vladimir/GSoC/ROOT-CI/common-version/root/tmva/tmva/test/DNN/CNN/" + "dataset/SinglePhotonPt50_FEVTDEBUG_n250k_IMG_CROPS32.root"; + photonInput = TFile::Open(photonFileName); + TTreeReader photonReader("RHTree", photonInput); + + // Load the electron input + TFile *electronInput(0); + TString electronFileName = "/Users/vladimirilievski/Desktop/Vladimir/GSoC/ROOT-CI/common-version/root/tmva/tmva/" + "test/DNN/CNN/dataset/SingleElectronPt50_FEVTDEBUG_n250k_IMG_CROPS32.root"; + electronInput = TFile::Open(electronFileName); + TTreeReader electronReader("RHTree", electronInput); + + // Calculate the number of samples + size_t nSamples = (size_t)(photonReader.GetEntries(true) + electronReader.GetEntries(true)); + size_t nChannels = 1; + size_t imgWidth = 32; + size_t imgHeight = 32; + size_t batchSize = 256; + + // Initialize the input + std::vector> inputTensor; + inputTensor.reserve(nSamples); + for (size_t i = 0; i < nSamples; i++) { + inputTensor.emplace_back(nChannels, imgWidth * imgHeight); + } + + size_t nOutput = 2; + // Initialize the output and the weights + TMatrixT output(nSamples, nOutput); + TMatrixT weights(nSamples, 1); + + // Get the Energy Red Branches + TTreeReaderArray photonEnergyRedBr(photonReader, "EBenergyRed"); + TTreeReaderArray electronEnergyRedBr(electronReader, "EBenergyRed"); + + size_t counter = 0; + + // Read the photons first + while (photonReader.Next()) { + // Set the input + for (size_t i = 0; i < photonEnergyRedBr.GetSize(); i++) { + inputTensor[counter](0, i) = static_cast(photonEnergyRedBr[i]); + } + + // Set the output + output(counter, 0) = 1.0; + output(counter, 1) = 0.0; + + // Set the weights + weights(counter, 0) = 1.0; + + counter++; + } + + // Read the electrons + while (electronReader.Next()) { + // Set the input + for (size_t i = 0; i < electronEnergyRedBr.GetSize(); i++) { + inputTensor[counter](0, i) = static_cast(electronEnergyRedBr[i]); + } + + // Set the output + output(counter, 0) = 0.0; + output(counter, 1) = 1.0; + + // Set the weights + weights(counter, 0) = 1.0; + + counter++; + } + + // Construct the Linear Net + size_t batchDepth = batchSize; + size_t batchHeight = nChannels; + size_t batchWidth = imgHeight * imgWidth; + size_t nOutputs = 2; + + TensorInput input(inputTensor, output, weights); + DataLoader_t loader(input, nSamples, batchSize, batchDepth, batchHeight, batchWidth, nOutputs); + + DeepNet_t convNet(batchSize, nChannels, imgHeight, imgWidth, batchDepth, batchHeight, batchWidth, + ELossFunction::kMeanSquaredError, EInitialization::kIdentity); + constructConvNet(convNet); + convNet.Initialize(); + + for (auto b : loader) { + auto linputTensor = b.GetInput(); + auto outputMatrix = b.GetOutput(); + auto weightMatrix = b.GetWeights(); + + convNet.Forward(linputTensor); + convNet.Backward(linputTensor, outputMatrix, weightMatrix); + } +} + +#endif diff --git a/tmva/tmva/test/DNN/CNN/TestTensorDataLoaderCpu.cxx b/tmva/tmva/test/DNN/CNN/TestTensorDataLoaderCpu.cxx new file mode 100644 index 0000000000000..070668e4902f4 --- /dev/null +++ b/tmva/tmva/test/DNN/CNN/TestTensorDataLoaderCpu.cxx @@ -0,0 +1,53 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Vladimir Ilievski + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing Tensor Data Loader Features for CPU backend * + * * + * Authors (alphabetical): * + * Vladimir Ilievski - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +////////////////////////////////////////////////////////// +// Test the CPU tensor data loader implementation. // +////////////////////////////////////////////////////////// + +#include "TestTensorDataLoader.h" +#include "TMVA/DNN/Architectures/Cpu.h" + +using namespace TMVA::DNN; + +int main() +{ + + using Scalar_t = Real_t; + + std::cout << "Testing tensor data loader:" << std::endl; + + // Scalar_t maximumError = 0.0; + // Scalar_t error = testSum>(); + // std::cout << "Sum: Maximum relative error = " << error << std::endl; + // maximumError = std::max(error, maximumError); + + // error = testIdentity>(); + // std::cout << "Identity: Maximum relative error = " << error << std::endl; + // maximumError = std::max(error, maximumError); + + testDataLoaderDataSet>(); +} \ No newline at end of file diff --git a/tmva/tmva/test/DNN/DAE/CMakeLists.txt b/tmva/tmva/test/DNN/DAE/CMakeLists.txt new file mode 100644 index 0000000000000..cee037ba69a80 --- /dev/null +++ b/tmva/tmva/test/DNN/DAE/CMakeLists.txt @@ -0,0 +1,28 @@ +############################################################################ +# CMakeLists.txt file for building TMVA/DNN/CNN tests. +# @author Akshay Vashistha +############################################################################ + +project(tmva-tests) +find_package(ROOT REQUIRED) + +set(Libraries Core MathCore Matrix TMVA) +include_directories(${ROOT_INCLUDE_DIRS}) + +ROOT_EXECUTABLE(testCorruptionLayer TestCorruptionLayer.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-DAE-CorruptionLayer COMMAND testCorruptionLayer) + +ROOT_EXECUTABLE(testCompressionLayer TestCompressionLayer.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-DAE-CompressionLayer COMMAND testCompressionLayer) + +ROOT_EXECUTABLE(testReconstructionLayer TestReconstructionLayer.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-DAE-ReconstructionLayer COMMAND testReconstructionLayer) + +ROOT_EXECUTABLE(testLogisticRegressionLayer TestLogisticRegressionLayer.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-DAE-LogisticRegressionLayer COMMAND testLogisticRegressionLayer) + +ROOT_EXECUTABLE(testNetPreTrain TestNetPreTrain.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-DAE-NetPreTrain COMMAND testNetPreTrain) + +ROOT_EXECUTABLE(testNetFineTune TestNetFineTune.cxx LIBRARIES ${Libraries}) +ROOT_ADD_TEST(TMVA-DNN-DAE-NetFineTune COMMAND testNetFineTune) diff --git a/tmva/tmva/test/DNN/DAE/TestCompressionLayer.cxx b/tmva/tmva/test/DNN/DAE/TestCompressionLayer.cxx new file mode 100644 index 0000000000000..15fcc73465f12 --- /dev/null +++ b/tmva/tmva/test/DNN/DAE/TestCompressionLayer.cxx @@ -0,0 +1,41 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Akshay Vashistha + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing the Compression Layer * + * * + * Authors (alphabetical): * + * Akshay Vashistha - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * *` + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#include "TestCompressionLayer.h" +#include "TMVA/DNN/Architectures/Reference.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::DAE; + +int main() + +{ + std::cout << "Testing for Corruption Layer started" << std::endl; + + testLayer>(5, 4,2); + + return 0; +} diff --git a/tmva/tmva/test/DNN/DAE/TestCompressionLayer.h b/tmva/tmva/test/DNN/DAE/TestCompressionLayer.h new file mode 100644 index 0000000000000..e7037516664a8 --- /dev/null +++ b/tmva/tmva/test/DNN/DAE/TestCompressionLayer.h @@ -0,0 +1,106 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Akshay Vashistha + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing the Compression Layer. * + * * + * Authors (alphabetical): * + * Akshay Vashistha - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_TEST_DNN_TEST_COMPRESSION_LAYER_H +#define TMVA_TEST_DNN_TEST_COMPRESSION_LAYER_H + +#include "../Utility.h" + +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DAE/CompressionLayer.h" + +#include "TMVA/DNN/Functions.h" +#include + +using namespace TMVA::DNN; +using namespace TMVA::DNN::DAE; + +template auto testLayer(size_t batchSize, size_t visibleUnits, size_t hiddenUnits) +-> void +{ + //using Scalar_t = typename Architecture::Scalar_t; + using Matrix_t = typename Architecture::Matrix_t; + + using TCompressionLayer = TCompressionLayer; + + Matrix_t weights(hiddenUnits,visibleUnits); + Matrix_t biases(hiddenUnits,1); + randomMatrix(weights); + randomMatrix(biases); + + std::vector Weights, Biases; + Weights.emplace_back(weights); + Biases.emplace_back(biases); + + for(size_t j=0; j< (size_t)weights.GetNrows(); j++) + { + for(size_t k=0; k< (size_t)weights.GetNcols(); k++) + { + std::cout< input, compressedInput; + for(size_t i=0; i - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * *` + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#include "TestCorruptionLayer.h" +#include "TMVA/DNN/Architectures/Reference.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::DAE; + +int main() + +{ + std::cout << "Testing for Corruption Layer started" << std::endl; + + testLayer>(5, 4); + + return 0; +} diff --git a/tmva/tmva/test/DNN/DAE/TestCorruptionLayer.h b/tmva/tmva/test/DNN/DAE/TestCorruptionLayer.h new file mode 100644 index 0000000000000..36cb22f489d38 --- /dev/null +++ b/tmva/tmva/test/DNN/DAE/TestCorruptionLayer.h @@ -0,0 +1,86 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Akshay Vashistha + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing the Corruption Layer. * + * * + * Authors (alphabetical): * + * Akshay Vashistha - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_TEST_DNN_TEST_CORRUPTION_LAYER_H +#define TMVA_TEST_DNN_TEST_CORRUPTION_LAYER_H + +#include "../Utility.h" + +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DAE/CorruptionLayer.h" + +#include "TMVA/DNN/Functions.h" +#include + +using namespace TMVA::DNN; +using namespace TMVA::DNN::DAE; + +template auto testLayer(size_t batchSize, size_t visibleUnits) +-> void +{ + //using Scalar_t = typename Architecture::Scalar_t; + using Matrix_t = typename Architecture::Matrix_t; + + using TCorruptionLayer = TCorruptionLayer; + + TCorruptionLayer dae(batchSize, visibleUnits,2, 1, 0.3); + + std::vector input, corruptedInput; + for(size_t i=0; i - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * *` + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ +#include "TestLogisticRegressionLayer.h" +#include "TMVA/DNN/Architectures/Reference.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::DAE; + +int main() + +{ + std::cout << "Testing for Logistic Regression Layer started" << std::endl; + + testLayer1>(); + + return 0; +} diff --git a/tmva/tmva/test/DNN/DAE/TestLogisticRegressionLayer.h b/tmva/tmva/test/DNN/DAE/TestLogisticRegressionLayer.h new file mode 100644 index 0000000000000..8700b9efd18c5 --- /dev/null +++ b/tmva/tmva/test/DNN/DAE/TestLogisticRegressionLayer.h @@ -0,0 +1,143 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Akshay Vashistha + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing the Logistic Regression Layer functionalities that are * + * responsible for propagation. * + * * + * Authors (alphabetical): * + * Akshay Vashistha - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_TEST_DNN_TEST_LOGISTIC_REGRESSION_LAYER_H +#define TMVA_TEST_DNN_TEST_LOGISTIC_REGRESSION_LAYER_H + +#include "../Utility.h" + +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DAE/LogisticRegressionLayer.h" + +#include "TMVA/DNN/Functions.h" +#include + +using namespace TMVA::DNN; +using namespace TMVA::DNN::DAE; + +template auto testLayer1() +-> void +{ + //using Scalar_t = typename Architecture::Scalar_t; + using Matrix_t = typename Architecture::Matrix_t; + + using TLogisticRegressionLayer = TLogisticRegressionLayer; + + TLogisticRegressionLayer logistic(6, 6, 2, 2,0.1); + logistic.Initialize(); + + double train_X[][6] = { + {1, 1, 1, 0, 0, 0}, + {1, 0, 1, 0, 0, 0}, + {1, 1, 1, 0, 0, 0}, + {0, 0, 1, 1, 1, 0}, + {0, 0, 1, 1, 0, 0}, + {0, 0, 1, 1, 1, 0} + }; + double train_Y[][2] = { + {1, 0}, + {1, 0}, + {1, 0}, + {0, 1}, + {0, 1}, + {0, 1} + }; + + double test_X[][6] = { + {1, 0, 1, 0, 0, 0}, + {0, 0, 1, 1, 1, 0} + }; + + std::vectorinput, inputLabel, testInput, output; + for(size_t i=0; i<6;i++) + { + input.emplace_back(6,1); + inputLabel.emplace_back(2,1); + } + for(size_t i=0; i<2;i++) + { + testInput.emplace_back(6,1); + output.emplace_back(2,1); + } + + for(size_t i=0; i<6; i++) + { + for(size_t j=0; j<(size_t)input[i].GetNrows(); j++) + { + for(size_t k=0; k<(size_t)input[i].GetNcols(); k++) + { + input[i](j,k)=train_X[i][j]; + } + } + } + + for(size_t i=0; i<6; i++) + { + for(size_t j=0; j<(size_t)inputLabel[i].GetNrows(); j++) + { + for(size_t k=0; k<(size_t)inputLabel[i].GetNcols(); k++) + { + inputLabel[i](j,k)=train_Y[i][j]; + } + } + } + + for(size_t i=0; i<2; i++) + { + for(size_t j=0; j<(size_t)testInput[i].GetNrows(); j++) + { + for(size_t k=0; k<(size_t)testInput[i].GetNcols(); k++) + { + testInput[i](j,k)=test_X[i][j]; + } + } + } + std::vectora,b; + for(size_t i=0;i<1000;i++) + { + logistic.Backward(inputLabel,a,input,b); + } + logistic.Forward(testInput, false); + std::cout< - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * *` + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#include "TestNetFineTune.h" +#include "TMVA/DNN/Architectures/Reference.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::DAE; + +int main() + +{ + std::cout << "Testing for DeepNet started" << std::endl; + + testNet>(); + + return 0; +} diff --git a/tmva/tmva/test/DNN/DAE/TestNetFineTune.h b/tmva/tmva/test/DNN/DAE/TestNetFineTune.h new file mode 100644 index 0000000000000..15d7b6dc296aa --- /dev/null +++ b/tmva/tmva/test/DNN/DAE/TestNetFineTune.h @@ -0,0 +1,130 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Akshay Vashistha + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing the AutoEncoder DeepNet . * + * * + * Authors (alphabetical): * + * Akshay Vashistha - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_TEST_DNN_TEST_DEEPNET_AE_FINETUNE_H +#define TMVA_TEST_DNN_TEST_DEEPNET_AE_FINETUNE_H + +#include "../Utility.h" + +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DeepNet.h" + +#include "TMVA/DNN/Functions.h" +#include + +using namespace TMVA::DNN; +using namespace TMVA::DNN::DAE; + +template auto constructDeepAutoEncoderNet(TDeepNet &net) +-> void +{ + using Matrix_t = typename Architecture::Matrix_t; + + double train_X[][6] = { + {1, 1, 1, 0, 0, 0}, + {1, 0, 1, 0, 0, 0}, + {1, 1, 1, 0, 0, 0}, + {0, 0, 1, 1, 1, 0}, + {0, 0, 1, 1, 0, 0}, + {0, 0, 1, 1, 1, 0} + }; + double train_Y[][2] = { + {1, 0}, + {1, 0}, + {1, 0}, + {0, 1}, + {0, 1}, + {0, 1} + }; + + double test_X[][6] = { + {1, 0, 1, 0, 0, 0}, + {0, 0, 1, 1, 1, 0} + }; + + std::vectorinput, inputLabel, testInput; + for(size_t i=0; i<6;i++) + { + input.emplace_back(6,1); + inputLabel.emplace_back(2,1); + } + for(size_t i=0; i<2;i++) + { + testInput.emplace_back(6,1); + } + + for(size_t i=0; i<6; i++) + { + for(size_t j=0; j<(size_t)input[i].GetNrows(); j++) + { + for(size_t k=0; k<(size_t)input[i].GetNcols(); k++) + { + input[i](j,k)=train_X[i][j]; + } + } + } + + for(size_t i=0; i<6; i++) + { + for(size_t j=0; j<(size_t)inputLabel[i].GetNrows(); j++) + { + for(size_t k=0; k<(size_t)inputLabel[i].GetNcols(); k++) + { + inputLabel[i](j,k)=train_Y[i][j]; + } + } + } + + for(size_t i=0; i<2; i++) + { + for(size_t j=0; j<(size_t)testInput[i].GetNrows(); j++) + { + for(size_t k=0; k<(size_t)testInput[i].GetNcols(); k++) + { + testInput[i](j,k)=test_X[i][j]; + } + } + } + + + size_t testDataBatchSize =2; + + net.FineTune(input, + testInput, + inputLabel,2,testDataBatchSize, 0.1, 1000); +} + +template auto testNet() +-> void +{ + using Net_t = TDeepNet; + size_t batchSize = 6; + Net_t convNet(batchSize, 1, 1, 1, 1, 1, 1, + ELossFunction::kMeanSquaredError, EInitialization::kGauss); + + constructDeepAutoEncoderNet(convNet); + } +#endif diff --git a/tmva/tmva/test/DNN/DAE/TestNetPreTrain.cxx b/tmva/tmva/test/DNN/DAE/TestNetPreTrain.cxx new file mode 100644 index 0000000000000..d35032cb6d536 --- /dev/null +++ b/tmva/tmva/test/DNN/DAE/TestNetPreTrain.cxx @@ -0,0 +1,41 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Akshay Vashistha + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing the DeepNet * + * * + * Authors (alphabetical): * + * Akshay Vashistha - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * *` + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#include "TestNetPreTrain.h" +#include "TMVA/DNN/Architectures/Reference.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::DAE; + +int main() + +{ + std::cout << "Testing for DeepNet started" << std::endl; + + testNet>(); + + return 0; +} diff --git a/tmva/tmva/test/DNN/DAE/TestNetPreTrain.h b/tmva/tmva/test/DNN/DAE/TestNetPreTrain.h new file mode 100644 index 0000000000000..7206991d6dacf --- /dev/null +++ b/tmva/tmva/test/DNN/DAE/TestNetPreTrain.h @@ -0,0 +1,83 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Akshay Vashistha + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing the AutoEncoder DeepNet . * + * * + * Authors (alphabetical): * + * Akshay Vashistha - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_TEST_DNN_TEST_DEEPNET_AE_H +#define TMVA_TEST_DNN_TEST_DEEPNET_AE_H + +#include "../Utility.h" + +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DeepNet.h" + +#include "TMVA/DNN/Functions.h" +#include + +using namespace TMVA::DNN; +using namespace TMVA::DNN::DAE; + +template auto constructDeepAutoEncoderNet(TDeepNet &net) +-> void +{ + using Scalar_t = typename Architecture::Scalar_t; + using Matrix_t = typename Architecture::Matrix_t; + + std::vector input; + size_t batchSize=1; + size_t visibleUnits=6; + std::vector numHiddenUnitsPerLayer = {4,3,2}; + Scalar_t learningRate =0.1; + Scalar_t corruptionLevel = 0.3; + Scalar_t dropoutProbability = 1; + size_t epochs=5; + + Matrix_t inputMatrix(visibleUnits,1); + for(size_t i=0; i auto testNet() +-> void +{ + using Net_t = TDeepNet; + size_t batchSize = 1; + Net_t convNet(batchSize, 1, 1, 1, 1, 1, 1, + ELossFunction::kMeanSquaredError, EInitialization::kGauss); + + constructDeepAutoEncoderNet(convNet); + } +#endif diff --git a/tmva/tmva/test/DNN/DAE/TestReconstructionLayer.cxx b/tmva/tmva/test/DNN/DAE/TestReconstructionLayer.cxx new file mode 100644 index 0000000000000..64c6942df62ac --- /dev/null +++ b/tmva/tmva/test/DNN/DAE/TestReconstructionLayer.cxx @@ -0,0 +1,41 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Akshay Vashistha + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing the Compression Layer * + * * + * Authors (alphabetical): * + * Akshay Vashistha - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * *` + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#include "TestReconstructionLayer.h" +#include "TMVA/DNN/Architectures/Reference.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::DAE; + +int main() + +{ + std::cout << "Testing for Reconstruction Layer started" << std::endl; + + testLayer>(5, 4,2); + + return 0; +} diff --git a/tmva/tmva/test/DNN/DAE/TestReconstructionLayer.h b/tmva/tmva/test/DNN/DAE/TestReconstructionLayer.h new file mode 100644 index 0000000000000..3164b8e7e1a0a --- /dev/null +++ b/tmva/tmva/test/DNN/DAE/TestReconstructionLayer.h @@ -0,0 +1,129 @@ +// @(#)root/tmva/tmva/cnn:$Id$ +// Author: Akshay Vashistha + +/********************************************************************************** + * Project: TMVA - a Root-integrated toolkit for multivariate data analysis * + * Package: TMVA * + * Class : * + * Web : http://tmva.sourceforge.net * + * * + * Description: * + * Testing the ReconstructionLayer. * + * * + * Authors (alphabetical): * + * Akshay Vashistha - CERN, Switzerland * + * * + * Copyright (c) 2005-2015: * + * CERN, Switzerland * + * U. of Victoria, Canada * + * MPI-K Heidelberg, Germany * + * U. of Bonn, Germany * + * * + * Redistribution and use in source and binary forms, with or without * + * modification, are permitted according to the terms listed in LICENSE * + * (http://tmva.sourceforge.net/LICENSE) * + **********************************************************************************/ + +#ifndef TMVA_TEST_DNN_TEST_RECONSTRUCTION_LAYER_H +#define TMVA_TEST_DNN_TEST_RECONSTRUCTION_LAYER_H + +#include "../Utility.h" + +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DAE/ReconstructionLayer.h" +#include "TMVA/DNN/DAE/CorruptionLayer.h" + +#include "TMVA/DNN/Functions.h" +#include + +using namespace TMVA::DNN; +using namespace TMVA::DNN::DAE; + +template auto testLayer(size_t batchSize, size_t visibleUnits, size_t hiddenUnits) +-> void +{ + //using Scalar_t = typename Architecture::Scalar_t; + using Matrix_t = typename Architecture::Matrix_t; + + using TReconstructionLayer = TReconstructionLayer; + + + Matrix_t weights(hiddenUnits,visibleUnits); + Matrix_t biases1(visibleUnits,1); + Matrix_t biases2(visibleUnits,1); + randomMatrix(weights); + randomMatrix(biases1); + randomMatrix(biases2); + + std::vector Weights, Biases; + Weights.emplace_back(weights); + Biases.emplace_back(biases1); + Biases.emplace_back(biases2); + + + TReconstructionLayer dae(batchSize, visibleUnits, hiddenUnits,0.1, EActivationFunction::kSigmoid,Weights,Biases,0.3,1); + + + std::vector input, compressedInput, corruptedInput; + for(size_t i=0; i +#include "TMVA/DNN/Architectures/Reference.h" +#include "TestForwardPass.h" +//#include "gtest/gtest.h" +//#include "gmock/gmock.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::RNN; + +//TEST(RNNTest, ForwardPass) +//{ +// EXPECT_EQ(testForwardPass>(3, 8, 100, 50), 0.0); +//} + +int main() { + std::cout << "Testing RNN Forward pass\n"; + + // timesteps, batchsize, statesize, inputsize + std::cout << testForwardPass>(1, 2, 3, 2) << "\n"; + std::cout << testForwardPass>(1, 8, 100, 50) << "\n"; + std::cout << testForwardPass>(5, 9, 128, 64) << "\n"; + + return 0; +} diff --git a/tmva/tmva/test/DNN/RNN/TestForwardPass.h b/tmva/tmva/test/DNN/RNN/TestForwardPass.h new file mode 100644 index 0000000000000..866df8af61fd9 --- /dev/null +++ b/tmva/tmva/test/DNN/RNN/TestForwardPass.h @@ -0,0 +1,126 @@ +// @(#)root/tmva $Id$ +// Author: Saurav Shekhar + +/************************************************************************* + * Copyright (C) 2017, Saurav Shekhar * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Generic tests of the RNNLayer Forward pass // +//////////////////////////////////////////////////////////////////// + +#ifndef TMVA_TEST_DNN_TEST_RNN_TEST_FWDPASS_H +#define TMVA_TEST_DNN_TEST_RNN_TEST_FWDPASS_H + +#include +#include + +#include "../Utility.h" + +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DeepNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::RNN; + +template +auto printTensor(const std::vector &A, const std::string name = "matrix") +-> void +{ + std::cout << name << "\n"; + for (size_t l = 0; l < A.size(); ++l) { + for (size_t i = 0; i < A[l].GetNrows(); ++i) { + for (size_t j = 0; j < A[l].GetNcols(); ++j) { + std::cout << A[l](i, j) << " "; + } + std::cout << "\n"; + } + std::cout << "********\n"; + } +} + +template +auto printMatrix(const typename Architecture::Matrix_t &A, const std::string name = "matrix") +-> void +{ + std::cout << name << "\n"; + for (size_t i = 0; i < A.GetNrows(); ++i) { + for (size_t j = 0; j < A.GetNcols(); ++j) { + std::cout << A(i, j) << " "; + } + std::cout << "\n"; + } + std::cout << "********\n"; +} + + +/*! Generate a DeepNet, test forward pass */ +//______________________________________________________________________________ +template +auto testForwardPass(size_t timeSteps, size_t batchSize, size_t stateSize, + size_t inputSize) +-> Double_t +{ + using Matrix_t = typename Architecture::Matrix_t; + using Tensor_t = std::vector; + using RNNLayer_t = TBasicRNNLayer; + using Net_t = TDeepNet; + + std::vector> XRef(timeSteps, TMatrixT(batchSize, inputSize)); // T x B x D + Tensor_t XArch, arr_XArch; + // arr_XArch(batchSize, Matrix_t(timeSteps, inputSize)) does not work! initializes both + // elements of array to same matrix!! + for (size_t i = 0; i < batchSize; ++i) arr_XArch.emplace_back(timeSteps, inputSize); // B x T x D + + for (size_t i = 0; i < timeSteps; ++i) { + randomMatrix(XRef[i]); + XArch.emplace_back(XRef[i]); + } + + + Architecture::Rearrange(arr_XArch, XArch); // B x T x D + + Net_t rnn(batchSize, batchSize, timeSteps, inputSize, 0, 0, 0, ELossFunction::kMeanSquaredError, EInitialization::kGauss); + RNNLayer_t* layer = rnn.AddBasicRNNLayer(stateSize, inputSize, timeSteps); + + layer->Initialize(); + + TMatrixT weightsInput = layer->GetWeightsInput(); // H x D + TMatrixT weightsState = layer->GetWeightsState(); // H x H + TMatrixT biases = layer->GetBiasesAt(0); // H x 1 + TMatrixT state = layer->GetState(); // B x H + TMatrixT tmp(batchSize, stateSize); + + rnn.Forward(arr_XArch); + Tensor_t outputArch = layer->GetOutput(); // B x T x H + Tensor_t arr_outputArch; + for (size_t t = 0; t < timeSteps; ++t) arr_outputArch.emplace_back(batchSize, stateSize); // T x B x H + Architecture::Rearrange(arr_outputArch, outputArch); + + Double_t maximumError = 0.0; + for (size_t t = 0; t < timeSteps; ++t) { + tmp.MultT(state, weightsState); + state.MultT(XRef[t], weightsInput); + state += tmp; + // adding bias + for (size_t i = 0; i < (size_t) state.GetNrows(); i++) { + for (size_t j = 0; j < (size_t) state.GetNcols(); j++) { + state(i,j) += biases(j,0); + } + } + // activation fn + applyMatrix(state, [](double x){return tanh(x);}); + TMatrixT output = arr_outputArch[t]; + + Double_t error = maximumRelativeError(output, state); + std::cout << "Time " << t << " Error: " << error << "\n"; + maximumError = std::max(error, maximumError); + } + return maximumError; +} + +#endif diff --git a/tmva/tmva/test/DNN/RNN/TestForwardPassCpu.cxx b/tmva/tmva/test/DNN/RNN/TestForwardPassCpu.cxx new file mode 100644 index 0000000000000..9324bcbf75ea0 --- /dev/null +++ b/tmva/tmva/test/DNN/RNN/TestForwardPassCpu.cxx @@ -0,0 +1,42 @@ +// @(#)root/tmva $Id$ +// Author: Saurav Shekhar 01/08/17 + +/************************************************************************* + * Copyright (C) 2017, Saurav Shekhar * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +//////////////////////////////////////////////////////////////////// +//Testing RNNLayer forward pass for Reference implementation // +//////////////////////////////////////////////////////////////////// + +#include +#include "TMVA/DNN/Architectures/Cpu.h" +#include "TestForwardPass.h" +//#include "gtest/gtest.h" +//#include "gmock/gmock.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::RNN; + +//TEST(RNNTest, ForwardPass) +//{ +// EXPECT_EQ(testForwardPass>(3, 8, 100, 50), 0.0); +//} + +int main() { + + using Scalar_t = Double_t; + + std::cout << "Testing RNN Forward pass\n"; + + // timesteps, batchsize, statesize, inputsize + std::cout << testForwardPass>(2, 2, 3, 2) << "\n"; + std::cout << testForwardPass>(1, 8, 100, 50) << "\n"; + std::cout << testForwardPass>(5, 9, 128, 64) << "\n"; + + return 0; +} diff --git a/tmva/tmva/test/DNN/RNN/TestForwardPassCuda.cxx b/tmva/tmva/test/DNN/RNN/TestForwardPassCuda.cxx new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tmva/tmva/test/DNN/RNN/TestFullRNN.cxx b/tmva/tmva/test/DNN/RNN/TestFullRNN.cxx new file mode 100644 index 0000000000000..78d4223f023d9 --- /dev/null +++ b/tmva/tmva/test/DNN/RNN/TestFullRNN.cxx @@ -0,0 +1,35 @@ +// @(#)root/tmva $Id$ +// Author: Saurav Shekhar 02/08/17 + +/************************************************************************* + * Copyright (C) 2017, Saurav Shekhar * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +//////////////////////////////////////////////////////////////////// +//Testing RNNLayer for incrementing a number // +//////////////////////////////////////////////////////////////////// + +#include +#include "TMVA/DNN/Architectures/Reference.h" +#include "TestFullRNN.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::RNN; + + +int main() { + std::cout << "Training RNN to identity first"; + + //testFullRNN(size_t batchSize, size_t stateSize, size_t inputSize, size_t outputSize) + // reconstruct 8 bit vector + // batchsize, statesize, inputsize, outputsize + testFullRNN>(2, 3, 2, 2) ; + //testFullRNN>(64, 10, 8, 8) ; + //testFullRNN>(3, 8, 100, 50) ; + + return 0; +} diff --git a/tmva/tmva/test/DNN/RNN/TestFullRNN.h b/tmva/tmva/test/DNN/RNN/TestFullRNN.h new file mode 100644 index 0000000000000..50a0ad5f6ae15 --- /dev/null +++ b/tmva/tmva/test/DNN/RNN/TestFullRNN.h @@ -0,0 +1,124 @@ +// @(#)root/tmva $Id$ +// Author: Saurav Shekhar + +/************************************************************************* + * Copyright (C) 2017, Saurav Shekhar * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Generic tests of the RNNLayer Forward pass // +//////////////////////////////////////////////////////////////////// + +#ifndef TMVA_TEST_DNN_TEST_RNN_TEST_FULL +#define TMVA_TEST_DNN_TEST_RNN_TEST_FULL + +#include +#include + +#include "../Utility.h" + +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DeepNet.h" +#include "TMVA/DNN/Net.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::RNN; + +template +auto printTensor(const std::vector &A, const std::string name = "matrix") +-> void +{ + std::cout << name << "\n"; + for (size_t l = 0; l < A.size(); ++l) { + for (size_t i = 0; i < A[l].GetNrows(); ++i) { + for (size_t j = 0; j < A[l].GetNcols(); ++j) { + std::cout << A[l](i, j) << " "; + } + std::cout << "\n"; + } + std::cout << "********\n"; + } +} + +template +auto printMatrix(const typename Architecture::Matrix_t &A, const std::string name = "matrix") +-> void +{ + std::cout << name << "\n"; + for (size_t i = 0; i < A.GetNrows(); ++i) { + for (size_t j = 0; j < A.GetNcols(); ++j) { + std::cout << A(i, j) << " "; + } + std::cout << "\n"; + } + std::cout << "********\n"; +} + +/* Generate a full recurrent neural net + * like a word generative model */ +//______________________________________________________________________________ +template +auto testFullRNN(size_t batchSize, size_t stateSize, + size_t inputSize, size_t outputSize) +-> void +{ + using Matrix_t = typename Architecture::Matrix_t; + using Tensor_t = std::vector; + // using RNNLayer_t = TBasicRNNLayer; + // using FCLayer_t = TDenseLayer; + // using Reshape_t = TReshapeLayer; + using Net_t = TDeepNet; + using Scalar_t = typename Architecture::Scalar_t; + + // check, denselayer takes only first one as input, + // so make sure time = 1, in the current case + size_t timeSteps = 1; + std::vector> XRef(batchSize, TMatrixT(timeSteps, inputSize)); // B x T x D + //TMatrixT YRef(batchSize, outputSize); // B x O (D = O) + Tensor_t XArch; + Matrix_t YArch(batchSize, outputSize); // B x O (D = O) + for (size_t i = 0; i < batchSize; ++i) { + randomMatrix(XRef[i]); + std::cerr << "Copying output into input\n"; + XArch.emplace_back(XRef[i]); + for (size_t j = 0; j < outputSize; ++j) { + YArch(i, j) = XArch[i](0, j); + } + } + + Net_t rnn(batchSize, batchSize, timeSteps, inputSize, 0, 0, 0, ELossFunction::kMeanSquaredError, EInitialization::kGauss); + // RNNLayer_t* layer = rnn.AddBasicRNNLayer(stateSize, inputSize, timeSteps, false); + // Reshape_t* reshape = rnn.AddReshapeLayer(1, 1, stateSize, true); + // FCLayer_t* classifier = rnn.AddDenseLayer(outputSize, EActivationFunction::kIdentity); + rnn.AddBasicRNNLayer(stateSize, inputSize, timeSteps, false); + rnn.AddReshapeLayer(1, 1, stateSize, true); + rnn.AddDenseLayer(outputSize, EActivationFunction::kIdentity); + + Matrix_t W(batchSize, 1); + for (size_t i = 0; i < batchSize; ++i) W(i, 0) = 1.0; + rnn.Initialize(); + + size_t iter = 0; + while (iter++ < 50) { + rnn.Forward(XArch); + Scalar_t loss = rnn.Loss(YArch, W, false); + + //if (iter % 20 == 0) { + //for (size_t i = 0; i < inputSize; ++i) std::cout << XRef[0](0, i) << " "; std::cout << "\n"; + //for (size_t i = 0; i < inputSize; ++i) std::cout << rnn.GetLayers().back()->GetOutputAt(0)(0, i) << " "; std::cout << "\n"; + //} + std::cout << "loss: " << loss << std::endl; + + rnn.Backward(XArch, YArch, W); + + rnn.Update(0.1); + + } +} + + +#endif diff --git a/tmva/tmva/test/DNN/RNN/TestFullRNNCpu.cxx b/tmva/tmva/test/DNN/RNN/TestFullRNNCpu.cxx new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tmva/tmva/test/DNN/RNN/TestFullRNNCuda.cxx b/tmva/tmva/test/DNN/RNN/TestFullRNNCuda.cxx new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tmva/tmva/test/DNN/RNN/TestRecurrentBackpropagation.cxx b/tmva/tmva/test/DNN/RNN/TestRecurrentBackpropagation.cxx new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tmva/tmva/test/DNN/RNN/TestRecurrentBackpropagation.h b/tmva/tmva/test/DNN/RNN/TestRecurrentBackpropagation.h new file mode 100644 index 0000000000000..a0950df712d0c --- /dev/null +++ b/tmva/tmva/test/DNN/RNN/TestRecurrentBackpropagation.h @@ -0,0 +1,48 @@ +// @(#)root/tmva $Id$ +// Author: Saurav Shekhar + +/************************************************************************* + * Copyright (C) 2017, Saurav Shekhar * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Generic tests of the RNNLayer Backward pass // +//////////////////////////////////////////////////////////////////// + +#ifndef TMVA_TEST_DNN_TEST_RNN_TEST_BWDPASS_H +#define TMVA_TEST_DNN_TEST_RNN_TEST_BWDPASS_H + +#include +#include + +#include "../Utility.h" + +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DeepNet.h" + +using namespace TMVA::DNN; +using namespace TMVA::DNN::RNN; + +/*! Generate a DeepNet, test forward pass */ +//______________________________________________________________________________ +template +auto testForwardPass(size_t timeSteps, size_t batchSize, size_t stateSize, + size_t inputSize) +-> Double_t +{ + using Matrix_t = typename Architecture::Matrix_t; + using Tensor_t = std::vector; + using RNNLayer_t = TBasicRNNLayer; + using Net_t = TDeepNet; + + + + +} + + +#endif diff --git a/tmva/tmva/test/DNN/RNN/TestRecurrentBackpropagationCpu.cxx b/tmva/tmva/test/DNN/RNN/TestRecurrentBackpropagationCpu.cxx new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tmva/tmva/test/DNN/RNN/TestRecurrentBackpropagationCuda.cxx b/tmva/tmva/test/DNN/RNN/TestRecurrentBackpropagationCuda.cxx new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tmva/tmva/test/DNN/TestBackpropagationDL.cxx b/tmva/tmva/test/DNN/TestBackpropagationDL.cxx new file mode 100644 index 0000000000000..54be1ce78030c --- /dev/null +++ b/tmva/tmva/test/DNN/TestBackpropagationDL.cxx @@ -0,0 +1,50 @@ +// @(#)root/tmva $Id$ +// Author: Simon Pfreundschuh + +/************************************************************************* + * Copyright (C) 2016, Simon Pfreundschuh + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Concrete instantiation of the generic backpropagation test for // +// the reference architecture. // +//////////////////////////////////////////////////////////////////// + +#include +#include "TMVA/DNN/Architectures/Reference.h" +#include "TestBackpropagationDL.h" + +using namespace TMVA::DNN; + +int main() +{ + std::cout << "Testing Backpropagation:" << std::endl; + + double error; + + // + // Test backpropagation for linear net. + // + + error = testBackpropagationWeightsLinear>(1.0); + if (error > 1e-3) + return 1; + + error = testBackpropagationL1Regularization>(1e-2); + if (error > 1e-3) + return 1; + + error = testBackpropagationL2Regularization>(1.0); + if (error > 1e-3) + return 1; + + error = testBackpropagationBiasesLinear>(1.0); + if (error > 1e-3) + return 1; + + return 0; +} diff --git a/tmva/tmva/test/DNN/TestBackpropagationDL.h b/tmva/tmva/test/DNN/TestBackpropagationDL.h new file mode 100644 index 0000000000000..d5379ef3a4064 --- /dev/null +++ b/tmva/tmva/test/DNN/TestBackpropagationDL.h @@ -0,0 +1,349 @@ +// @(#)root/tmva $Id$ +// Author: Simon Pfreundschuh + +/************************************************************************* + * Copyright (C) 2016, Simon Pfreundschuh, Saurav Shekhar * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Generic tests of the backpropagation algorithm. // +// // +// All tests randomly generate a net with identity activation // +// functions, i.e. which is completely linear and then tests the // +// computed gradients for each layer using numerical // +// derivation. The restriction to linear nets is to avoid the // +// required division by the finite difference interval used to // +// approximate the numerical derivatives, which would otherwise // +// cause precision loss. // +//////////////////////////////////////////////////////////////////// + +#include +#include "TMVA/DNN/Functions.h" +#include "TMVA/DNN/DeepNet.h" +#include "Utility.h" + +using namespace TMVA::DNN; + +/*! Compute the loss of the net as a function of the weight at index (i,j) in + * layer l. dx is added as an offset to the current value of the weight. */ +//______________________________________________________________________________ +template +auto evaluate_net_weight(TDeepNet &net, typename Architecture::Matrix_t & /*X*/, + const typename Architecture::Matrix_t &Y, const typename Architecture::Matrix_t &W, size_t l, + size_t k, size_t i, size_t j, typename Architecture::Scalar_t dx) -> + typename Architecture::Scalar_t +{ + using Scalar_t = typename Architecture::Scalar_t; + + net.GetLayerAt(l)->GetWeightsAt(k).operator()(i,j) += dx; + Scalar_t res = net.Loss(Y, W, false); + net.GetLayerAt(l)->GetWeightsAt(k).operator()(i,j) -= dx; + return res; +} + +/*! Compute the loss of the net as a function of the weight at index i in + * layer l. dx is added as an offset to the current value of the weight. */ +//______________________________________________________________________________ +template +auto evaluate_net_bias(TDeepNet &net, typename Architecture::Matrix_t & /*X*/, + const typename Architecture::Matrix_t &Y, const typename Architecture::Matrix_t &W, size_t l, + size_t k, size_t i, typename Architecture::Scalar_t dx) -> typename Architecture::Scalar_t +{ + using Scalar_t = typename Architecture::Scalar_t; + + net.GetLayerAt(l)->GetBiasesAt(k).operator()(i,0) += dx; + Scalar_t res = net.Loss(Y, W, false); + net.GetLayerAt(l)->GetBiasesAt(k).operator()(i,0) -= dx; + return res; +} + +// TODO pass as function params +size_t tbatchSize = 2, timeSteps = 1, inputSize = 2, outputSize = 2; + +/*! Generate a random net, perform forward and backward propagation and check + * the weight gradients using numerical differentiation. Returns the maximum + * relative gradient error and also prints it to stdout. */ +//______________________________________________________________________________ +template +auto testBackpropagationWeightsLinear(typename Architecture::Scalar_t dx) +-> typename Architecture::Scalar_t +{ + using Scalar_t = typename Architecture::Scalar_t; + using Matrix_t = typename Architecture::Matrix_t; + using Net_t = TDeepNet; + // using FCLayer_t = TDenseLayer; + + // Random net. + Net_t net(tbatchSize, timeSteps, tbatchSize, inputSize, 0, 0, 0, ELossFunction::kMeanSquaredError, + EInitialization::kGauss); + // FCLayer_t* l1 = net.AddDenseLayer(outputSize, EActivationFunction::kIdentity); + net.AddDenseLayer(outputSize, EActivationFunction::kIdentity); + + // Random training data. + std::vector X(timeSteps, Matrix_t(tbatchSize, inputSize)); // T x B x D + Matrix_t Y(tbatchSize, outputSize), weights(tbatchSize, 1); + net.Initialize(); + randomBatch(X[0]); + randomMatrix(Y); + fillMatrix(weights, 1.0); + + net.Forward(X); + net.Backward(X, Y, weights); + + Scalar_t maximum_error = 0.0; + + // Compute derivatives for all weights using finite differences and + // compare to result obtained from backpropagation. + for (size_t l = 0; l < net.GetDepth(); l++) { + std::cout << "\rTesting weight gradients: layer: " << l << " / " << net.GetDepth(); + std::cout << std::flush; + auto layer = net.GetLayerAt(l); + auto &W = layer->GetWeightGradientsAt(0); + + for (size_t i = 0; i < layer->GetWidth(); i++) { + for (size_t j = 0; j < layer->GetInputWidth(); j++) { + auto f = [&net, &X, &Y, &weights, l, i, j](Scalar_t x) { + return evaluate_net_weight(net, X[0], Y, weights, l, 0, i, j, x); + }; + Scalar_t dy = finiteDifference(f, dx) / (2.0 * dx); + Scalar_t dy_ref = W(i, j); + + // Compute the relative error if dy != 0. + Scalar_t error; + if (std::fabs(dy_ref) > 1e-15) { + error = std::fabs((dy - dy_ref) / dy_ref); + } else { + error = std::fabs(dy - dy_ref); + } + + maximum_error = std::max(error, maximum_error); + } + } + } + + std::cout << "\rTesting weight gradients: "; + std::cout << "maximum relative error: " << print_error(maximum_error) << std::endl; + return maximum_error; +} + +/*! Generate a random, linear net, perform forward and backward propagation with + * L1 regularization and check the weight gradients using numerical + * differentiation. Returns the maximum relative gradient error and + * also prints it to stdout. */ +//______________________________________________________________________________ +template +auto testBackpropagationL1Regularization(typename Architecture::Scalar_t dx) +-> typename Architecture::Scalar_t +{ + using Scalar_t = typename Architecture::Scalar_t; + using Matrix_t = typename Architecture::Matrix_t; + using Net_t = TDeepNet; + // using FCLayer_t = TDenseLayer; + + // Random net. + Net_t net(tbatchSize, timeSteps, tbatchSize, inputSize, 0, 0, 0, ELossFunction::kMeanSquaredError, + EInitialization::kGauss); + // FCLayer_t* l1 = net.AddDenseLayer(outputSize, EActivationFunction::kIdentity); + net.AddDenseLayer(outputSize, EActivationFunction::kIdentity); + // Random training data. + std::vector X(timeSteps, Matrix_t(tbatchSize, inputSize)); // T x B x D + Matrix_t Y(tbatchSize, outputSize), weights(tbatchSize, 1); + net.Initialize(); + // Random training data. + randomBatch(X[0]); + randomMatrix(Y); + fillMatrix(weights, 1.0); + + net.Forward(X); + net.Backward(X, Y, weights); + + Scalar_t maximum_error = 0.0; + + // Compute derivatives for all weights using finite differences and + // compare to result obtained from backpropagation. + for (size_t l = 0; l < net.GetDepth(); l++) + { + std::cout << "\rTesting weight gradients (L1): layer: " + << l << " / " << net.GetDepth(); + std::cout << std::flush; + auto layer = net.GetLayerAt(l); + auto & W = layer->GetWeightsAt(0); + auto & dW = layer->GetWeightGradientsAt(0); + + for (size_t i = 0; i < layer->GetWidth(); i++) { + for (size_t j = 0; j < layer->GetInputWidth(); j++) { + // Avoid running into the non-derivable point at 0.0. + if (std::abs(W(i,j)) > dx) { + auto f = [&net, &X, &Y, &weights, l, i, j](Scalar_t x) { + return evaluate_net_weight(net, X[0], Y, weights, l, 0, i, j, x); + }; + Scalar_t dy = finiteDifference(f, dx) / (2.0 * dx); + Scalar_t dy_ref = dW(i,j); + + // Compute the relative error if dy != 0. + Scalar_t error; + if (std::fabs(dy_ref) > 1e-15) + { + error = std::fabs((dy - dy_ref) / dy_ref); + } + else + { + error = std::fabs(dy - dy_ref); + } + + maximum_error = std::max(error, maximum_error); + } + } + } + } + + std::cout << "\rTesting weight gradients (L1): "; + std::cout << "maximum relative error: " << print_error(maximum_error) << std::endl; + return maximum_error; +} + +/*! Generate a random, linear net, perform forward and backward propagation with + * L2 regularization and check the weight gradients using numerical + * differentiation. Returns the maximum relative gradient error and + * also prints it to stdout. */ +//______________________________________________________________________________ +template +auto testBackpropagationL2Regularization(typename Architecture::Scalar_t dx) +-> typename Architecture::Scalar_t +{ + using Scalar_t = typename Architecture::Scalar_t; + using Matrix_t = typename Architecture::Matrix_t; + using Net_t = TDeepNet; + // using FCLayer_t = TDenseLayer; + + Net_t net(tbatchSize, timeSteps, tbatchSize, inputSize, 0, 0, 0, ELossFunction::kMeanSquaredError, + EInitialization::kGauss); + // FCLayer_t* l1 = net.AddDenseLayer(outputSize, EActivationFunction::kIdentity); + net.AddDenseLayer(outputSize, EActivationFunction::kIdentity); + + // Random training data. + std::vector X(timeSteps, Matrix_t(tbatchSize, inputSize)); // T x B x D + Matrix_t Y(tbatchSize, outputSize), weights(tbatchSize, 1); + net.Initialize(); + // Random training data. + randomBatch(X[0]); + randomMatrix(Y); + fillMatrix(weights, 1.0); + + net.Forward(X); + net.Backward(X, Y, weights); + + Scalar_t maximum_error = 0.0; + + // Compute derivatives for all weights using finite differences and + // compare to result obtained from backpropagation. + for (size_t l = 0; l < net.GetDepth(); l++) + { + std::cout << "\rTesting weight gradients (L2): layer: " + << l << " / " << net.GetDepth(); + std::cout << std::flush; + auto layer = net.GetLayerAt(l); + auto & W = layer->GetWeightGradientsAt(0); + + for (size_t i = 0; i < layer->GetWidth(); i++) + { + for (size_t j = 0; j < layer->GetInputWidth(); j++) + { + auto f = [&net, &X, &Y, &weights, l, i, j](Scalar_t x) { + return evaluate_net_weight(net, X[0], Y, weights, l, 0, i, j, x); + }; + Scalar_t dy = finiteDifference(f, dx) / (2.0 * dx); + Scalar_t dy_ref = W(i,j); + + // Compute the relative error if dy != 0. + Scalar_t error; + if (std::fabs(dy_ref) > 1e-15) + { + error = std::fabs((dy - dy_ref) / dy_ref); + } + else + { + error = std::fabs(dy - dy_ref); + } + + maximum_error = std::max(error, maximum_error); + } + } + } + + std::cout << "\rTesting weight gradients (L2): "; + std::cout << "maximum relative error: " << print_error(maximum_error) << std::endl; + return maximum_error; +} + +/*! Generate a random net, perform forward and backward propagation and check + * the bias gradients using numerical differentiation. Returns the maximum + * relative gradient error and also prints it to stdout. */ +//______________________________________________________________________________ +template +auto testBackpropagationBiasesLinear(typename Architecture::Scalar_t dx) +-> typename Architecture::Scalar_t +{ + using Net_t = TDeepNet; + using Scalar_t = typename Architecture::Scalar_t; + using Matrix_t = typename Architecture::Matrix_t; + // using FCLayer_t = TDenseLayer; + + Net_t net(tbatchSize, timeSteps, tbatchSize, inputSize, 0, 0, 0, ELossFunction::kMeanSquaredError, + EInitialization::kGauss); + // FCLayer_t* l1 = net.AddDenseLayer(outputSize, EActivationFunction::kIdentity); + net.AddDenseLayer(outputSize, EActivationFunction::kIdentity); + + // Random training data. + std::vector X(timeSteps, Matrix_t(tbatchSize, inputSize)); // T x B x D + Matrix_t Y(tbatchSize, outputSize), weights(tbatchSize, 1); + net.Initialize(); + // Random training data. + randomBatch(X[0]); + randomMatrix(Y); + fillMatrix(weights, 1.0); + + net.Forward(X); + net.Backward(X, Y, weights); + + Scalar_t maximum_error = 0.0; + + // Compute derivatives for all bias terms using finite differences and + // compare to result obtained from backpropagation. + for (size_t l = 0; l < net.GetDepth(); l++) + { + std::cout << "\rTesting bias gradients: layer: " + << l << " / " << net.GetDepth(); + std::cout << std::flush; + auto layer = net.GetLayerAt(l); + auto & dtheta = layer->GetBiasGradientsAt(0); + + for (size_t i = 0; i < layer->GetWidth(); i++) + { + auto f = [&net, &X, &Y, &weights, l, i](Scalar_t x) { return evaluate_net_bias(net, X[0], Y, weights, l, 0, i, x); }; + Scalar_t dy = finiteDifference(f, dx); + Scalar_t dy_ref = dtheta(i,0) * 2.0 * dx; + + // Compute the relative error if dy != 0. + Scalar_t error; + if (std::fabs(dy_ref) > 1e-10) + { + error = std::fabs((dy - dy_ref) / dy_ref); + } + else + { + error = std::fabs(dy - dy_ref); + } + + maximum_error = std::max(error, maximum_error); + } + } + + std::cout << "\rTesting bias gradients: "; + std::cout << "maximum relative error: " << print_error(maximum_error) << std::endl; + return maximum_error; +} diff --git a/tmva/tmva/test/DNN/TestBackpropagationDLCpu.cxx b/tmva/tmva/test/DNN/TestBackpropagationDLCpu.cxx new file mode 100644 index 0000000000000..4f849ee4902ef --- /dev/null +++ b/tmva/tmva/test/DNN/TestBackpropagationDLCpu.cxx @@ -0,0 +1,48 @@ +// @(#)root/tmva $Id$ +// Author: Simon Pfreundschuh + +/************************************************************************* + * Copyright (C) 2016, Simon Pfreundschuh * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Concrete instantiation of the generic backpropagation test for // +// multi-threaded CPU architectures. // +//////////////////////////////////////////////////////////////////// + +#include "TMatrix.h" +#include "TMVA/DNN/Architectures/Cpu.h" +#include "TestBackpropagationDL.h" + +using namespace TMVA::DNN; + +int main() +{ + using Scalar_t = Double_t; + std::cout << "Testing Backpropagation:" << std::endl; + + double error; + int iret = 0; + + error = testBackpropagationWeightsLinear>(1.0); + if (error > 1e-3) + iret++; + + error = testBackpropagationL1Regularization>(1e-2); + if (error > 1e-3) + iret++; + + error = testBackpropagationL2Regularization>(1.0); + if (error > 1e-3) + iret++; + + error = testBackpropagationBiasesLinear>(1.0); + if (error > 1e-3) + iret++; + + return iret; +} diff --git a/tmva/tmva/test/DNN/TestBackpropagationDLCuda.cxx b/tmva/tmva/test/DNN/TestBackpropagationDLCuda.cxx new file mode 100644 index 0000000000000..eec0f6c356653 --- /dev/null +++ b/tmva/tmva/test/DNN/TestBackpropagationDLCuda.cxx @@ -0,0 +1,43 @@ +// @(#)root/tmva $Id$ +// Author: Simon Pfreundschuh + +/************************************************************************* + * Copyright (C) 2016, Simon Pfreundschuh * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +//////////////////////////////////////////////////////////////////// +// Concrete instantiation of the generic backpropagation test for // +// CUDA architectures. // +//////////////////////////////////////////////////////////////////// + +#include +#include "TMVA/DNN/Architectures/Cuda.h" +#include "TMatrix.h" +#include "TestBackpropagationDL.h" + +using namespace TMVA::DNN; + +int main() +{ + using Scalar_t = Double_t; + + std::cout << "Testing Backpropagation:" << std::endl; + double error; + error = testBackpropagationWeightsLinear>(1.0); + if (error > 1e-3) + return 1; + error = testBackpropagationL1Regularization>(1e-2); + if (error > 1e-3) + return 1; + error = testBackpropagationL2Regularization>(1.0); + if (error > 1e-3) + return 1; + error = testBackpropagationBiasesLinear>(1.0); + if (error > 1e-3) + return 1; + return 0; +} diff --git a/tmva/tmva/test/DNN/Utility.h b/tmva/tmva/test/DNN/Utility.h index d554b31dbe6bb..e0bb75a4bda64 100644 --- a/tmva/tmva/test/DNN/Utility.h +++ b/tmva/tmva/test/DNN/Utility.h @@ -10,28 +10,277 @@ #include "TMVA/DNN/Architectures/Reference.h" #include "TMVA/DNN/Functions.h" #include "TMVA/DNN/Net.h" +#include "TMVA/DNN/DeepNet.h" +#include "TMVA/DNN/CNN/ConvLayer.h" +#include "TMVA/DNN/CNN/MaxPoolLayer.h" +#include "TMVA/DNN/DenseLayer.h" -namespace TMVA +namespace TMVA { +namespace DNN { + +/** Construct a convolutional neural network with one convolutional layer, + * one pooling layer and two fully connected layers. The dimensions are + * predetermined. The activation functions are chosen randomly. */ +//______________________________________________________________________________ +template +void constructConvNet(TDeepNet &net) +{ + /* For random selection */ + std::vector ActivationFunctions = {EActivationFunction::kIdentity, EActivationFunction::kRelu, + EActivationFunction::kSigmoid, EActivationFunction::kTanh}; + + size_t depth1 = 12; + size_t filterHeightConv1 = 4; + size_t filterWidthConv1 = 4; + size_t strideRowsConv1 = 1; + size_t strideColsConv1 = 1; + size_t zeroPaddingHeight1 = 1; + size_t zeroPaddingWidth1 = 1; + + EActivationFunction fConv1 = EActivationFunction::kIdentity; + + //EActivationFunction fConv1 = ActivationFunctions[rand() % ActivationFunctions.size()]; + + net.AddConvLayer(depth1, filterHeightConv1, filterWidthConv1, strideRowsConv1, strideColsConv1, zeroPaddingHeight1, + zeroPaddingWidth1, fConv1); + + std::cout << "added Conv layer " << net.GetLayerAt(net.GetDepth() - 1)->GetDepth() << " x " << net.GetLayerAt(net.GetDepth() - 1)->GetHeight() + << " x " << net.GetLayerAt(net.GetDepth() - 1)->GetWidth() << std::endl; + + + size_t depth2 = 6; + size_t filterHeightConv2 = 3; + size_t filterWidthConv2 = 3; + size_t strideRowsConv2 = 1; + size_t strideColsConv2 = 1; + size_t zeroPaddingHeight2 = 0; + size_t zeroPaddingWidth2 = 0; + + EActivationFunction fConv2 = EActivationFunction::kIdentity; +// EActivationFunction fConv2 = ActivationFunctions[rand() % ActivationFunctions.size()]; + + net.AddConvLayer(depth2, filterHeightConv2, filterWidthConv2, strideRowsConv2, strideColsConv2, zeroPaddingHeight2, + zeroPaddingWidth2, fConv2); + + std::cout << "added Conv layer " << net.GetLayerAt(net.GetDepth() - 1)->GetDepth() << " x " << net.GetLayerAt(net.GetDepth() - 1)->GetHeight() + << " x " << net.GetLayerAt(net.GetDepth() - 1)->GetWidth() << std::endl; + + + size_t filterHeightPool = 3; + size_t filterWidthPool = 3; + size_t strideRowsPool = 1; + size_t strideColsPool = 1; + + net.AddMaxPoolLayer(filterHeightPool, filterWidthPool, strideRowsPool, strideColsPool); + + std::cout << "added MaxPool layer " << net.GetLayerAt(net.GetDepth() - 1)->GetDepth() << " x " << net.GetLayerAt(net.GetDepth() - 1)->GetHeight() + << " x " << net.GetLayerAt(net.GetDepth() - 1)->GetWidth() << std::endl; + + + size_t depthReshape = 1; + size_t heightReshape = 1; + size_t widthReshape = net.GetLayerAt(net.GetDepth() - 1)->GetDepth() * + net.GetLayerAt(net.GetDepth() - 1)->GetHeight() * + net.GetLayerAt(net.GetDepth() - 1)->GetWidth(); + + net.AddReshapeLayer(depthReshape, heightReshape, widthReshape, true); + + size_t widthFC1 = 20; + + EActivationFunction fFC1 = EActivationFunction::kIdentity; + + //EActivationFunction fFC1 = ActivationFunctions[rand() % ActivationFunctions.size()]; + net.AddDenseLayer(widthFC1, fFC1); + + size_t widthFC2 = 2; + EActivationFunction fFC2 = EActivationFunction::kIdentity; + net.AddDenseLayer(widthFC2, fFC2); +} + +/** Construct a linear convolutional neural network with one convolutional layer, + * one pooling layer and two fully connected layers. The dimensions are + * predetermined. The activation functions are all linear. */ +//______________________________________________________________________________ +template +void constructLinearConvNet(TDeepNet &net) { -namespace DNN + + size_t depth1 = 2; + size_t filterHeightConv1 = 2; + size_t filterWidthConv1 = 2; + size_t strideRowsConv1 = 1; + size_t strideColsConv1 = 1; + size_t zeroPaddingHeight1 = 1; + size_t zeroPaddingWidth1 = 1; + + EActivationFunction fConv1 = EActivationFunction::kIdentity; + + net.AddConvLayer(depth1, filterHeightConv1, filterWidthConv1, strideRowsConv1, strideColsConv1, zeroPaddingHeight1, + zeroPaddingWidth1, fConv1); + + std::cout << "added Conv layer " << net.GetLayerAt(net.GetDepth() - 1)->GetDepth() << " x " << net.GetLayerAt(net.GetDepth() - 1)->GetHeight() + << " x " << net.GetLayerAt(net.GetDepth() - 1)->GetWidth() << std::endl; + + + + size_t depth2 = 2; + size_t filterHeightConv2 = 3; + size_t filterWidthConv2 = 3; + size_t strideRowsConv2 = 1; + size_t strideColsConv2 = 1; + size_t zeroPaddingHeight2 = 0; + size_t zeroPaddingWidth2 = 0; + + EActivationFunction fConv2 = EActivationFunction::kIdentity; + + net.AddConvLayer(depth2, filterHeightConv2, filterWidthConv2, strideRowsConv2, strideColsConv2, zeroPaddingHeight2, + zeroPaddingWidth2, fConv2); + + std::cout << "added Conv layer " << net.GetLayerAt(net.GetDepth() - 1)->GetDepth() << " x " << net.GetLayerAt(net.GetDepth() - 1)->GetHeight() + << " x " << net.GetLayerAt(net.GetDepth() - 1)->GetWidth() << std::endl; + + // size_t depth3 = 12; + // size_t filterHeightConv3 = 3; + // size_t filterWidthConv3 = 3; + // size_t strideRowsConv3 = 1; + // size_t strideColsConv3 = 1; + // size_t zeroPaddingHeight3 = 1; + // size_t zeroPaddingWidth3 = 1; + + // EActivationFunction fConv3 = EActivationFunction::kIdentity; + + // net.AddConvLayer(depth3, filterHeightConv3, filterWidthConv3, strideRowsConv3, strideColsConv3, zeroPaddingHeight3, + // zeroPaddingWidth3, fConv3); + + + size_t filterHeightPool = 2; + size_t filterWidthPool = 2; + size_t strideRowsPool = 1; + size_t strideColsPool = 1; + + net.AddMaxPoolLayer(filterHeightPool, filterWidthPool, strideRowsPool, strideColsPool); + + std::cout << "added MaxPool layer " << net.GetLayerAt(net.GetDepth() - 1)->GetDepth() << " x " << net.GetLayerAt(net.GetDepth() - 1)->GetHeight() + << " x " << net.GetLayerAt(net.GetDepth() - 1)->GetWidth() << std::endl; + + size_t depthReshape = 1; + size_t heightReshape = 1; + size_t widthReshape = net.GetLayerAt(net.GetDepth() - 1)->GetDepth() * + net.GetLayerAt(net.GetDepth() - 1)->GetHeight() * + net.GetLayerAt(net.GetDepth() - 1)->GetWidth(); + + net.AddReshapeLayer(depthReshape, heightReshape, widthReshape, true); + + size_t widthFC1 = 3; + EActivationFunction fFC1 = EActivationFunction::kIdentity; + net.AddDenseLayer(widthFC1, fFC1); + + size_t widthFC2 = 1; + EActivationFunction fFC2 = EActivationFunction::kIdentity; + net.AddDenseLayer(widthFC2, fFC2); +} + +//______________________________________________________________________________ +template +void constructMasterSlaveConvNets(TDeepNet &master, std::vector> &nets) { + /* For random selection */ + std::vector ActivationFunctions = {EActivationFunction::kIdentity, EActivationFunction::kRelu, + EActivationFunction::kSigmoid, EActivationFunction::kTanh}; + + // Add Convolutional Layer + size_t depth = 12; + size_t filterHeightConv = 2; + size_t filterWidthConv = 2; + size_t strideRowsConv = 1; + size_t strideColsConv = 1; + size_t zeroPaddingHeight = 1; + size_t zeroPaddingWidth = 1; + + EActivationFunction fConv = ActivationFunctions[rand() % ActivationFunctions.size()]; + + TConvLayer *convLayer = + master.AddConvLayer(depth, filterHeightConv, filterWidthConv, strideRowsConv, strideColsConv, zeroPaddingHeight, + zeroPaddingWidth, fConv); + + convLayer->Initialize(); + TConvLayer *copyConvLayer = new TConvLayer(*convLayer); + + // add the copy to all slave nets + for (size_t i = 0; i < nets.size(); i++) { + nets[i].AddConvLayer(copyConvLayer); + } + + // Add Max Pooling Layer + size_t filterHeightPool = 6; + size_t filterWidthPool = 6; + size_t strideRowsPool = 1; + size_t strideColsPool = 1; + + // Add the Max pooling layer + TMaxPoolLayer *maxPoolLayer = + master.AddMaxPoolLayer(filterHeightPool, filterWidthPool, strideRowsPool, strideColsPool); + TMaxPoolLayer *copyMaxPoolLayer = new TMaxPoolLayer(*maxPoolLayer); + + // Add the copy to all slave nets + for (size_t i = 0; i < nets.size(); i++) { + nets[i].AddMaxPoolLayer(copyMaxPoolLayer); + } + + // Add the reshape layer + size_t depthReshape = 1; + size_t heightReshape = 1; + size_t widthReshape = master.GetLayerAt(master.GetDepth() - 1)->GetDepth() * + master.GetLayerAt(master.GetDepth() - 1)->GetHeight() * + master.GetLayerAt(master.GetDepth() - 1)->GetWidth(); + + TReshapeLayer *reshapeLayer = master.AddReshapeLayer(depthReshape, heightReshape, widthReshape, true); + TReshapeLayer *copyReshapeLayer = new TReshapeLayer(*reshapeLayer); + + // Add the copy to all slave nets + for (size_t i = 0; i < nets.size(); i++) { + nets[i].AddReshapeLayer(copyReshapeLayer); + } + + // Add Dense Layer + size_t widthFC1 = 20; + EActivationFunction fFC1 = ActivationFunctions[rand() % ActivationFunctions.size()]; + TDenseLayer *denseLayer = master.AddDenseLayer(widthFC1, fFC1); + denseLayer->Initialize(); + TDenseLayer *copyDenseLayer = new TDenseLayer(*denseLayer); + + // add the copy to all slave nets + for (size_t i = 0; i < nets.size(); i++) { + nets[i].AddDenseLayer(copyDenseLayer); + } + + // Add the final Dense Layer + size_t widthFC2 = 5; + EActivationFunction fFC2 = EActivationFunction::kIdentity; + TDenseLayer *finalDenseLayer = master.AddDenseLayer(widthFC2, fFC2); + finalDenseLayer->Initialize(); + TDenseLayer *copyFinalDenseLayer = new TDenseLayer(*finalDenseLayer); + + // add the copy to all slave nets + for (size_t i = 0; i < nets.size(); i++) { + nets[i].AddDenseLayer(copyFinalDenseLayer); + } +} /** Construct a random linear neural network with up to five layers.*/ //______________________________________________________________________________ template -void constructRandomLinearNet(TNet & net) +void constructRandomLinearNet(TNet &net) { - int nlayers = rand() % 5 + 1; + int nlayers = rand() % 5 + 1; - std::vector ActivationFunctions - = {EActivationFunction::kIdentity}; + std::vector ActivationFunctions = {EActivationFunction::kIdentity}; - for (int i = 0; i < nlayers; i++) { - int width = rand() % 20 + 1; - EActivationFunction f = - ActivationFunctions[rand() % ActivationFunctions.size()]; - net.AddLayer(width, f); - } + for (int i = 0; i < nlayers; i++) { + int width = rand() % 20 + 1; + EActivationFunction f = ActivationFunctions[rand() % ActivationFunctions.size()]; + net.AddLayer(width, f); + } } /*! Set matrix to the identity matrix */ @@ -39,19 +288,18 @@ void constructRandomLinearNet(TNet & net) template void identityMatrix(AMatrix &X) { - size_t m, n; - m = X.GetNrows(); - n = X.GetNcols(); - - - for (size_t i = 0; i < m; i++) { - for (size_t j = 0; j < n; j++) { - X(i,j) = 0.0; - } - if (i < n) { - X(i,i) = 1.0; - } - } + size_t m, n; + m = X.GetNrows(); + n = X.GetNcols(); + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + X(i, j) = 0.0; + } + if (i < n) { + X(i, i) = 1.0; + } + } } /*! Fill matrix with given value.*/ @@ -75,19 +323,37 @@ void fillMatrix(AMatrix &X, AReal x) template void randomMatrix(AMatrix &X) { - size_t m,n; - m = X.GetNrows(); - n = X.GetNcols(); + size_t m, n; + m = X.GetNrows(); + n = X.GetNcols(); + + TRandom rand(clock()); + + Double_t sigma = sqrt(10.0); + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + X(i, j) = rand.Gaus(0.0, sigma); + } + } +} - TRandom rand(clock()); +/*! Fill matrix with random, uniform-distributed values in [-1, 1] */ +//______________________________________________________________________________ +template +void uniformMatrix(AMatrix &X) +{ + size_t m, n; + m = X.GetNrows(); + n = X.GetNcols(); - Double_t sigma = sqrt(10.0); + TRandom rand(clock()); - for (size_t i = 0; i < m; i++) { - for (size_t j = 0; j < n; j++) { - X(i,j) = rand.Gaus(0.0, sigma); - } - } + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + X(i, j) = rand.Uniform(-1, 1); + } + } } /*! Generate a random batch as input for a neural net. */ @@ -95,7 +361,7 @@ void randomMatrix(AMatrix &X) template void randomBatch(AMatrix &X) { - randomMatrix(X); + randomMatrix(X); } /*! Generate a random batch as input for a neural net. */ @@ -103,15 +369,15 @@ void randomBatch(AMatrix &X) template void copyMatrix(AMatrix &X, const AMatrix &Y) { - size_t m,n; - m = X.GetNrows(); - n = X.GetNcols(); - - for (size_t i = 0; i < m; i++) { - for (size_t j = 0; j < n; j++) { - X(i,j) = Y(i,j); - } - } + size_t m, n; + m = X.GetNrows(); + n = X.GetNcols(); + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + X(i, j) = Y(i, j); + } + } } /*! Apply functional to each element in the matrix. */ @@ -119,35 +385,32 @@ void copyMatrix(AMatrix &X, const AMatrix &Y) template void applyMatrix(AMatrix &X, F f) { - size_t m,n; - m = X.GetNrows(); - n = X.GetNcols(); - - for (size_t i = 0; i < m; i++) { - for (size_t j = 0; j < n; j++) { - X(i,j) = f(X(i,j)); - } - } + size_t m, n; + m = X.GetNrows(); + n = X.GetNcols(); + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + X(i, j) = f(X(i, j)); + } + } } /*! Combine elements of two given matrices into a single matrix using * the given function f. */ //______________________________________________________________________________ template -void zipWithMatrix(AMatrix &Z, - F f, - const AMatrix &X, - const AMatrix &Y) +void zipWithMatrix(AMatrix &Z, F f, const AMatrix &X, const AMatrix &Y) { - size_t m,n; - m = X.GetNrows(); - n = X.GetNcols(); - - for (size_t i = 0; i < m; i++) { - for (size_t j = 0; j < n; j++) { - Z(i,j) = f(X(i,j), Y(i,j)); - } - } + size_t m, n; + m = X.GetNrows(); + n = X.GetNcols(); + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + Z(i, j) = f(X(i, j), Y(i, j)); + } + } } /** Generate a random batch as input for a neural net. */ @@ -155,18 +418,18 @@ void zipWithMatrix(AMatrix &Z, template AFloat reduce(F f, AFloat start, const AMatrix &X) { - size_t m,n; - m = X.GetNrows(); - n = X.GetNcols(); - - AFloat result = start; - - for (size_t i = 0; i < m; i++) { - for (size_t j = 0; j < n; j++) { - result = f(result, X(i,j)); - } - } - return result; + size_t m, n; + m = X.GetNrows(); + n = X.GetNcols(); + + AFloat result = start; + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + result = f(result, X(i, j)); + } + } + return result; } /** Apply function to matrix element-wise and compute the mean of the resulting @@ -175,18 +438,18 @@ AFloat reduce(F f, AFloat start, const AMatrix &X) template AFloat reduceMean(F f, AFloat start, const AMatrix &X) { - size_t m,n; - m = X.GetNrows(); - n = X.GetNcols(); - - AFloat result = start; - - for (size_t i = 0; i < m; i++) { - for (size_t j = 0; j < n; j++) { - result = f(result, X(i,j)); - } - } - return result / (AFloat) (m * n); + size_t m, n; + m = X.GetNrows(); + n = X.GetNcols(); + + AFloat result = start; + + for (size_t i = 0; i < m; i++) { + for (size_t j = 0; j < n; j++) { + result = f(result, X(i, j)); + } + } + return result / (AFloat)(m * n); } /** Compute the relative error of x and y */ @@ -194,18 +457,15 @@ AFloat reduceMean(F f, AFloat start, const AMatrix &X) template inline T relativeError(const T &x, const T &y) { - using std::abs; + using std::abs; - if (x == y) - return T(0.0); + if (x == y) return T(0.0); - T diff = abs(x - y); + T diff = abs(x - y); - if (x * y == T(0.0) || - diff < std::numeric_limits::epsilon()) - return diff; + if (x * y == T(0.0) || diff < std::numeric_limits::epsilon()) return diff; - return diff / (abs(x) + abs(y)); + return diff / (abs(x) + abs(y)); } /*! Compute the maximum, element-wise relative error of the matrices @@ -228,9 +488,9 @@ auto maximumRelativeError(const Matrix1 &X, const Matrix2 &Y) -> Double_t curError = relativeError(X(i, j), Y(i, j)); maxError = std::max(curError, maxError); } - } + } - return maxError; + return maxError; } /*! Numerically compute the derivative of the functional f using finite @@ -239,7 +499,7 @@ auto maximumRelativeError(const Matrix1 &X, const Matrix2 &Y) -> Double_t template inline AFloat finiteDifference(F f, AFloat dx) { - return f(dx) - f(0.0 - dx); + return f(dx) - f(0.0 - dx); } /*! Color code error. */ @@ -247,23 +507,22 @@ inline AFloat finiteDifference(F f, AFloat dx) template std::string print_error(AFloat &e) { - std::ostringstream out{}; + std::ostringstream out{}; - out << ("\e["); + out << ("\e["); - if (e > 1e-5) - out << "31m"; - else if (e > 1e-9) - out << "33m"; - else - out << "32m"; + if (e > 1e-5) + out << "31m"; + else if (e > 1e-9) + out << "33m"; + else + out << "32m"; - out << e; - out << "\e[39m"; + out << e; + out << "\e[39m"; - return out.str(); + return out.str(); } - } }