update attention

course_UvA-DL/01-introduction-to-pytorch/Introduction_to_PyTorch.py

@@ -1,3 +1,15 @@
+# ---
+# jupyter:
+#   jupytext:
+#     cell_metadata_filter: -all
+#     formats: ipynb,py:percent
+#     text_representation:
+#       extension: .py
+#       format_name: percent
+#       format_version: '1.3'
+#       jupytext_version: 1.14.5
+# ---
+
 # %% [markdown]
 # <div class="center-wrapper"><div class="video-wrapper"><iframe src="https://www.youtube.com/embed/wnKZZgFQY-E" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></div></div>
 # Welcome to our PyTorch tutorial for the Deep Learning course 2020 at the University of Amsterdam!
@@ -31,12 +43,12 @@
 import torch.utils.data as data
 
 # %matplotlib inline
-from IPython.display import set_matplotlib_formats
+import matplotlib_inline.backend_inline
 from matplotlib.colors import to_rgba
 from torch import Tensor
 from tqdm.notebook import tqdm  # Progress bar
 
-set_matplotlib_formats("svg", "pdf")
+matplotlib_inline.backend_inline.set_matplotlib_formats("svg", "pdf")  # For export
 
 # %% [markdown]
 # ## The Basics of PyTorch

course_UvA-DL/02-activation-functions/Activation_Functions.py

@@ -21,12 +21,12 @@
 import torchvision
 
 # %matplotlib inline
-from IPython.display import set_matplotlib_formats
+import matplotlib_inline.backend_inline
 from torchvision import transforms
 from torchvision.datasets import FashionMNIST
 from tqdm.notebook import tqdm
 
-set_matplotlib_formats("svg", "pdf")  # For export
+matplotlib_inline.backend_inline.set_matplotlib_formats("svg", "pdf")  # For export
 sns.set()
 
 # %% [markdown]

course_UvA-DL/03-initialization-and-optimization/Initialization_and_Optimization.py

@@ -35,13 +35,13 @@
 import torch.utils.data as data
 
 # %matplotlib inline
-from IPython.display import set_matplotlib_formats
+import matplotlib_inline.backend_inline
 from matplotlib import cm
 from torchvision import transforms
 from torchvision.datasets import FashionMNIST
 from tqdm.notebook import tqdm
 
-set_matplotlib_formats("svg", "pdf")  # For export
+matplotlib_inline.backend_inline.set_matplotlib_formats("svg", "pdf")  # For export
 sns.set()
 
 # %% [markdown]

course_UvA-DL/04-inception-resnet-densenet/Inception_ResNet_DenseNet.py

@@ -1,3 +1,15 @@
+# ---
+# jupyter:
+#   jupytext:
+#     cell_metadata_filter: -all
+#     formats: ipynb,py:percent
+#     text_representation:
+#       extension: .py
+#       format_name: percent
+#       format_version: '1.3'
+#       jupytext_version: 1.14.5
+# ---
+
 # %% [markdown]
 # <div class="center-wrapper"><div class="video-wrapper"><iframe src="https://www.youtube.com/embed/vjSSyGxlczs" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></div></div>
 # Let's start with importing our standard libraries here.

course_UvA-DL/05-transformers-and-MH-attention/Transformers_MHAttention.py

@@ -28,7 +28,7 @@
 import numpy as np
 
 # PyTorch Lightning
-import pytorch_lightning as pl
+import lightning as L
 import seaborn as sns
 
 # PyTorch
@@ -40,15 +40,15 @@
 
 # Torchvision
 import torchvision
-from IPython.display import set_matplotlib_formats
-from pytorch_lightning.callbacks import ModelCheckpoint
+import matplotlib_inline.backend_inline
+from lightning.pytorch.callbacks import ModelCheckpoint
 from torchvision import transforms
 from torchvision.datasets import CIFAR100
 from tqdm.notebook import tqdm
 
 plt.set_cmap("cividis")
 # %matplotlib inline
-set_matplotlib_formats("svg", "pdf")  # For export
+matplotlib_inline.backend_inline.set_matplotlib_formats("svg", "pdf")  # For export
 matplotlib.rcParams["lines.linewidth"] = 2.0
 sns.reset_orig()
 
@@ -58,7 +58,7 @@
 CHECKPOINT_PATH = os.environ.get("PATH_CHECKPOINT", "saved_models/Transformers/")
 
 # Setting the seed
-pl.seed_everything(42)
+L.seed_everything(42)
 
 # Ensure that all operations are deterministic on GPU (if used) for reproducibility
 torch.backends.cudnn.determinstic = True
@@ -246,7 +246,7 @@ def scaled_dot_product(q, k, v, mask=None):
 
 # %%
 seq_len, d_k = 3, 2
-pl.seed_everything(42)
+L.seed_everything(42)
 q = torch.randn(seq_len, d_k)
 k = torch.randn(seq_len, d_k)
 v = torch.randn(seq_len, d_k)
@@ -744,7 +744,7 @@ def get_lr_factor(self, epoch):
 
 
 # %%
-class TransformerPredictor(pl.LightningModule):
+class TransformerPredictor(L.LightningModule):
     def __init__(
         self,
         input_dim,
@@ -976,13 +976,13 @@ def train_reverse(**kwargs):
     # Create a PyTorch Lightning trainer with the generation callback
     root_dir = os.path.join(CHECKPOINT_PATH, "ReverseTask")
     os.makedirs(root_dir, exist_ok=True)
-    trainer = pl.Trainer(
+    trainer = L.Trainer(
         default_root_dir=root_dir,
         callbacks=[ModelCheckpoint(save_weights_only=True, mode="max", monitor="val_acc")],
-        gpus=1 if str(device).startswith("cuda") else 0,
+        accelerator=("cuda" if str(device).startswith("cuda") else "cpu"),
+        devices=1,
         max_epochs=10,
         gradient_clip_val=5,
-        progress_bar_refresh_rate=1,
     )
     trainer.logger._default_hp_metric = None  # Optional logging argument that we don't need
 
@@ -1436,13 +1436,13 @@ def train_anomaly(**kwargs):
     # Create a PyTorch Lightning trainer with the generation callback
     root_dir = os.path.join(CHECKPOINT_PATH, "SetAnomalyTask")
     os.makedirs(root_dir, exist_ok=True)
-    trainer = pl.Trainer(
+    trainer = L.Trainer(
         default_root_dir=root_dir,
         callbacks=[ModelCheckpoint(save_weights_only=True, mode="max", monitor="val_acc")],
-        gpus=1 if str(device).startswith("cuda") else 0,
+        accelerator=("cuda" if str(device).startswith("cuda") else "cpu"),
+        devices=1,
         max_epochs=100,
         gradient_clip_val=2,
-        progress_bar_refresh_rate=1,
     )
     trainer.logger._default_hp_metric = None  # Optional logging argument that we don't need