Add GPU configuration for TensorFlow models (Sequential and Functional) (#572)

* Fix grammar in documentation and add GPU configuration class

* Updated Tensorflow dependencies in build.gradle and cleaned up LeNetClassicWithGPUMemoryConfig.kt

Author: zaleslaw

examples/build.gradle

@@ -18,10 +18,10 @@ dependencies {
     testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.8.2'
     testImplementation 'org.junit.jupiter:junit-jupiter-params:5.8.2'
     // to run on GPU (if CUDA is updated and machine with NVIDIA onboard)
-    /*implementation 'org.tensorflow:libtensorflow:1.15.0'
-    implementation 'org.tensorflow:libtensorflow_jni_gpu:1.15.0'
-    api 'com.microsoft.onnxruntime:onnxruntime_gpu:1.12.1'
-    */
+    // implementation 'org.tensorflow:libtensorflow:1.15.0'
+    // implementation 'org.tensorflow:libtensorflow_jni_gpu:1.15.0'
+    // api 'com.microsoft.onnxruntime:onnxruntime_gpu:1.12.1'
+
 }
 
 def publishedArtifactsVersion = System.getenv("KOTLIN_DL_RELEASE_VERSION")

examples/src/main/kotlin/examples/cnn/mnist/advanced/LeNetClassicWithGPUMemoryConfig.kt

@@ -0,0 +1,130 @@
+/*
+ * Copyright 2020-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
+ */
+
+package examples.cnn.mnist.advanced
+
+import org.jetbrains.kotlinx.dl.api.core.GpuConfiguration
+import org.jetbrains.kotlinx.dl.api.core.Sequential
+import org.jetbrains.kotlinx.dl.api.core.activation.Activations
+import org.jetbrains.kotlinx.dl.api.core.initializer.Constant
+import org.jetbrains.kotlinx.dl.api.core.initializer.GlorotNormal
+import org.jetbrains.kotlinx.dl.api.core.initializer.Zeros
+import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv2D
+import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.ConvPadding
+import org.jetbrains.kotlinx.dl.api.core.layer.core.Dense
+import org.jetbrains.kotlinx.dl.api.core.layer.core.Input
+import org.jetbrains.kotlinx.dl.api.core.layer.pooling.AvgPool2D
+import org.jetbrains.kotlinx.dl.api.core.layer.reshaping.Flatten
+import org.jetbrains.kotlinx.dl.api.core.loss.Losses
+import org.jetbrains.kotlinx.dl.api.core.metric.Metrics
+import org.jetbrains.kotlinx.dl.api.core.optimizer.Adam
+import org.jetbrains.kotlinx.dl.api.core.optimizer.ClipGradientByValue
+import org.jetbrains.kotlinx.dl.dataset.embedded.NUMBER_OF_CLASSES
+import org.jetbrains.kotlinx.dl.dataset.embedded.mnist
+import org.jetbrains.kotlinx.dl.impl.summary.logSummary
+
+private const val EPOCHS = 3
+private const val TRAINING_BATCH_SIZE = 1000
+private const val NUM_CHANNELS = 1L
+private const val IMAGE_SIZE = 28L
+private const val SEED = 12L
+private const val TEST_BATCH_SIZE = 1000
+
+/**
+ * This example shows how to do image classification from scratch using [lenet5Classic], without leveraging pre-trained weights or a pre-made model.
+ * We demonstrate the workflow on the Mnist classification dataset.
+ *
+ * It could be run only with enabled tensorflow GPU dependencies
+ *
+ * It includes:
+ * - dataset loading from S3
+ * - model compilation
+ * - model summary
+ * - model training
+ * - model evaluation
+ */
+fun lenetClassicWithGPUMemoryConfig() {
+
+    val layersActivation = Activations.Tanh
+    val classifierActivation = Activations.Linear
+
+    val model = Sequential.of(
+        Input(
+            IMAGE_SIZE,
+            IMAGE_SIZE,
+            NUM_CHANNELS,
+        ),
+        Conv2D(
+            filters = 6,
+            kernelSize = 5,
+            strides = 1,
+            activation = layersActivation,
+            kernelInitializer = GlorotNormal(SEED),
+            biasInitializer = Zeros(),
+            padding = ConvPadding.SAME,
+        ),
+        AvgPool2D(
+            poolSize = 2,
+            strides = 2,
+            padding = ConvPadding.VALID,
+        ),
+        Conv2D(
+            filters = 16,
+            kernelSize = 5,
+            strides = 1,
+            activation = layersActivation,
+            kernelInitializer = GlorotNormal(SEED),
+            biasInitializer = Zeros(),
+            padding = ConvPadding.SAME,
+        ),
+        AvgPool2D(
+            poolSize = 2,
+            strides = 2,
+            padding = ConvPadding.VALID,
+        ),
+        Flatten(),
+        Dense(
+            outputSize = 120,
+            activation = layersActivation,
+            kernelInitializer = GlorotNormal(SEED),
+            biasInitializer = Constant(0.1f),
+        ),
+        Dense(
+            outputSize = 84,
+            activation = Activations.Tanh,
+            kernelInitializer = GlorotNormal(SEED),
+            biasInitializer = Constant(0.1f),
+        ),
+        Dense(
+            outputSize = NUMBER_OF_CLASSES,
+            activation = classifierActivation,
+            kernelInitializer = GlorotNormal(SEED),
+            biasInitializer = Constant(0.1f),
+        ),
+        gpuConfiguration = GpuConfiguration(allowGrowth = true)
+    )
+
+    val (train, test) = mnist()
+
+    model.use {
+        it.compile(
+            optimizer = Adam(clipGradient = ClipGradientByValue(0.1f)),
+            loss = Losses.SOFT_MAX_CROSS_ENTROPY_WITH_LOGITS,
+            metric = Metrics.ACCURACY
+        )
+
+        it.logSummary()
+
+        it.fit(dataset = train, epochs = EPOCHS, batchSize = TRAINING_BATCH_SIZE)
+
+        val accuracy = it.evaluate(dataset = test, batchSize = TEST_BATCH_SIZE).metrics[Metrics.ACCURACY]
+
+        println("Accuracy: $accuracy")
+    }
+}
+
+/** */
+fun main(): Unit = lenetClassicWithGPUMemoryConfig()
+

examples/src/main/kotlin/examples/transferlearning/lenet/Example_1_Load_model_with_weights_and_evaluate_it.kt

@@ -17,7 +17,7 @@ import org.jetbrains.kotlinx.dl.impl.summary.logSummary
 import java.io.File
 
 /**
- * This examples demonstrates the inference concept:
+ * This example demonstrates the inference concept:
  * - Weights are loaded from .h5 file, configuration is loaded from .json file.
  * - Model is evaluated after loading to obtain accuracy value.
  * - No additional training.

examples/src/main/kotlin/examples/transferlearning/lenet/Example_2_Load_model_without_weights_and_evaluate_it.kt

@@ -13,7 +13,7 @@ import org.jetbrains.kotlinx.dl.dataset.embedded.fashionMnist
 import org.jetbrains.kotlinx.dl.impl.summary.logSummary
 
 /**
- * This examples demonstrates the weird inference case:
+ * This example demonstrates the weird inference case:
  * - Weights are not loaded, but initialized via initialized defined in configuration, configuration is loaded from .json file.
  * - Model is evaluated after loading to obtain accuracy value.
  * - No additional training.

examples/src/main/kotlin/examples/transferlearning/lenet/Example_3_Additional_training.kt

@@ -14,7 +14,7 @@ import org.jetbrains.kotlinx.dl.dataset.embedded.fashionMnist
 import org.jetbrains.kotlinx.dl.impl.summary.logSummary
 
 /**
- * This examples demonstrates the transfer learning concept:
+ * This example demonstrates the transfer learning concept:
  * - Weights are loaded from .h5 file, configuration is loaded from .json file.
  * - All model weights are not frozen, and can be changed during the training.
  * - No new layers are added.

examples/src/main/kotlin/examples/transferlearning/lenet/Example_4_Additional_training_and_freezing.kt

@@ -17,9 +17,9 @@ import org.jetbrains.kotlinx.dl.dataset.embedded.fashionMnist
 import org.jetbrains.kotlinx.dl.impl.summary.logSummary
 
 /**
- * This examples demonstrates the transfer learning concept:
+ * This example demonstrates the transfer learning concept:
  * - Weights are loaded from .h5 file, configuration is loaded from .json file.
- * - Conv2D layer are added to the new Neural Network, its weights are frozen, Dense layers are added too and its weights are not frozen, and can be changed during the training.
+ * - Conv2D layer is added to the new Neural Network, its weights are frozen, Dense layers are added too and its weights are not frozen, and can be changed during the training.
  * - No new layers are added.
  *
  * NOTE: Model and weights are resources in `examples` module.

examples/src/main/kotlin/examples/transferlearning/lenet/Example_5_Additional_training_and_freezing_and_init.kt

@@ -17,9 +17,9 @@ import org.jetbrains.kotlinx.dl.dataset.embedded.fashionMnist
 import org.jetbrains.kotlinx.dl.impl.summary.logSummary
 
 /**
- * This examples demonstrates the transfer learning concept:
+ * This example demonstrates the transfer learning concept:
  * - Weights are loaded from .h5 file for a pre-filtered list of layers (Conv2D only), configuration is loaded from .json file.
- * - Conv2D layer are added to the new Neural Network, its weights are frozen, Dense layers are added too and its weights are initialized via defined initializers.
+ * - Conv2D layer is added to the new Neural Network, its weights are frozen, Dense layers are added too, and its weights are initialized via defined initializers.
  * - No new layers are added.
  *
  * NOTE: Model and weights are resources in `examples` module.

examples/src/main/kotlin/examples/transferlearning/lenet/Example_6_Additional_training_and_new_dense_layers.kt

@@ -23,9 +23,9 @@ import org.jetbrains.kotlinx.dl.dataset.embedded.fashionMnist
 import org.jetbrains.kotlinx.dl.impl.summary.logSummary
 
 /**
- * This examples demonstrates the transfer learning concept:
+ * This example demonstrates the transfer learning concept:
  * - Weights are loaded from .h5 file, configuration is loaded from .json file.
- * - Conv2D layer are added to the new Neural Network, its weights are frozen.
+ * - Conv2D layer is added to the new Neural Network, its weights are frozen.
  * - Flatten and new Dense layers are added and initialized via defined initializers.
  *
  * NOTE: Model and weights are resources in `examples` module.

examples/src/main/kotlin/examples/transferlearning/modelhub/densenet/DenseNet121.kt

@@ -22,7 +22,7 @@ import org.jetbrains.kotlinx.dl.impl.summary.logSummary
 import java.io.File
 
 /**
- * This examples demonstrates the inference concept on DenseNet121 model:
+ * This example demonstrates the inference concept on DenseNet121 model:
  * - Model configuration, model weights and labels are obtained from [TFModelHub].
  * - Weights are loaded from .h5 file, configuration is loaded from .json file.
  * - Model predicts on a few images located in resources.

examples/src/main/kotlin/examples/transferlearning/modelhub/densenet/DenseNet169.kt

@@ -24,7 +24,7 @@ import org.jetbrains.kotlinx.dl.impl.summary.logSummary
 import java.io.File
 
 /**
- * This examples demonstrates the inference concept on DenseNet169 model:
+ * This example demonstrates the inference concept on DenseNet169 model:
  * - Model configuration, model weights and labels are obtained from [TFModelHub].
  * - Weights are loaded from .h5 file, configuration is loaded from .json file.
  * - Model predicts on a few images located in resources.