{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## makemore: becoming a backprop ninja\n", "\n", "swole doge style" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "# there no change change in the first several cells from last lecture" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import torch\n", "import torch.nn.functional as F\n", "import matplotlib.pyplot as plt # for making figures\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "32033\n", "15\n", "['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia']\n" ] } ], "source": [ "# read in all the words\n", "words = open('names.txt', 'r').read().splitlines()\n", "print(len(words))\n", "print(max(len(w) for w in words))\n", "print(words[:8])" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n", "27\n" ] } ], "source": [ "# build the vocabulary of characters and mappings to/from integers\n", "chars = sorted(list(set(''.join(words))))\n", "stoi = {s:i+1 for i,s in enumerate(chars)}\n", "stoi['.'] = 0\n", "itos = {i:s for s,i in stoi.items()}\n", "vocab_size = len(itos)\n", "print(itos)\n", "print(vocab_size)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "torch.Size([182625, 3]) torch.Size([182625])\n", "torch.Size([22655, 3]) torch.Size([22655])\n", "torch.Size([22866, 3]) torch.Size([22866])\n" ] } ], "source": [ "# build the dataset\n", "block_size = 3 # context length: how many characters do we take to predict the next one?\n", "\n", "def build_dataset(words): \n", " X, Y = [], []\n", " \n", " for w in words:\n", " context = [0] * block_size\n", " for ch in w + '.':\n", " ix = stoi[ch]\n", " X.append(context)\n", " Y.append(ix)\n", " context = context[1:] + [ix] # crop and append\n", "\n", " X = torch.tensor(X)\n", " Y = torch.tensor(Y)\n", " print(X.shape, Y.shape)\n", " return X, Y\n", "\n", "import random\n", "random.seed(42)\n", "random.shuffle(words)\n", "n1 = int(0.8*len(words))\n", "n2 = int(0.9*len(words))\n", "\n", "Xtr, Ytr = build_dataset(words[:n1]) # 80%\n", "Xdev, Ydev = build_dataset(words[n1:n2]) # 10%\n", "Xte, Yte = build_dataset(words[n2:]) # 10%" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "# ok biolerplate done, now we get to the action:" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "# utility function we will use later when comparing manual gradients to PyTorch gradients\n", "def cmp(s, dt, t):\n", " ex = torch.all(dt == t.grad).item()\n", " app = torch.allclose(dt, t.grad)\n", " maxdiff = (dt - t.grad).abs().max().item()\n", " print(f'{s:15s} | exact: {str(ex):5s} | approximate: {str(app):5s} | maxdiff: {maxdiff}')" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "4137\n" ] } ], "source": [ "n_embd = 10 # the dimensionality of the character embedding vectors\n", "n_hidden = 64 # the number of neurons in the hidden layer of the MLP\n", "\n", "g = torch.Generator().manual_seed(2147483647) # for reproducibility\n", "C = torch.randn((vocab_size, n_embd), generator=g)\n", "# Layer 1\n", "W1 = torch.randn((n_embd * block_size, n_hidden), generator=g) * (5/3)/((n_embd * block_size)**0.5)\n", "b1 = torch.randn(n_hidden, generator=g) * 0.1 # using b1 just for fun, it's useless because of BN\n", "# Layer 2\n", "W2 = torch.randn((n_hidden, vocab_size), generator=g) * 0.1\n", "b2 = torch.randn(vocab_size, generator=g) * 0.1\n", "# BatchNorm parameters\n", "bngain = torch.randn((1, n_hidden))*0.1 + 1.0\n", "bnbias = torch.randn((1, n_hidden))*0.1\n", "\n", "# Note: I am initializating many of these parameters in non-standard ways\n", "# because sometimes initializating with e.g. all zeros could mask an incorrect\n", "# implementation of the backward pass.\n", "\n", "parameters = [C, W1, b1, W2, b2, bngain, bnbias]\n", "print(sum(p.nelement() for p in parameters)) # number of parameters in total\n", "for p in parameters:\n", " p.requires_grad = True" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "batch_size = 32\n", "n = batch_size # a shorter variable also, for convenience\n", "# construct a minibatch\n", "ix = torch.randint(0, Xtr.shape[0], (batch_size,), generator=g)\n", "Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor(3.3377, grad_fn=)" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# forward pass, \"chunkated\" into smaller steps that are possible to backward one at a time\n", "\n", "emb = C[Xb] # embed the characters into vectors\n", "embcat = emb.view(emb.shape[0], -1) # concatenate the vectors\n", "# Linear layer 1\n", "hprebn = embcat @ W1 + b1 # hidden layer pre-activation\n", "# BatchNorm layer\n", "bnmeani = 1/n*hprebn.sum(0, keepdim=True)\n", "bndiff = hprebn - bnmeani\n", "bndiff2 = bndiff**2\n", "bnvar = 1/(n-1)*(bndiff2).sum(0, keepdim=True) # note: Bessel's correction (dividing by n-1, not n)\n", "bnvar_inv = (bnvar + 1e-5)**-0.5\n", "bnraw = bndiff * bnvar_inv\n", "hpreact = bngain * bnraw + bnbias\n", "# Non-linearity\n", "h = torch.tanh(hpreact) # hidden layer\n", "# Linear layer 2\n", "logits = h @ W2 + b2 # output layer\n", "# cross entropy loss (same as F.cross_entropy(logits, Yb))\n", "logit_maxes = logits.max(1, keepdim=True).values\n", "norm_logits = logits - logit_maxes # subtract max for numerical stability\n", "counts = norm_logits.exp()\n", "counts_sum = counts.sum(1, keepdims=True)\n", "counts_sum_inv = counts_sum**-1 # if I use (1.0 / counts_sum) instead then I can't get backprop to be bit exact...\n", "probs = counts * counts_sum_inv\n", "logprobs = probs.log()\n", "loss = -logprobs[range(n), Yb].mean()\n", "\n", "# PyTorch backward pass\n", "for p in parameters:\n", " p.grad = None\n", "for t in [logprobs, probs, counts, counts_sum, counts_sum_inv, # afaik there is no cleaner way\n", " norm_logits, logit_maxes, logits, h, hpreact, bnraw,\n", " bnvar_inv, bnvar, bndiff2, bndiff, hprebn, bnmeani,\n", " embcat, emb]:\n", " t.retain_grad()\n", "loss.backward()\n", "loss" ] }, { "cell_type": "code", "execution_count": 210, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "logprobs | exact: True | approximate: True | maxdiff: 0.0\n", "probs | exact: True | approximate: True | maxdiff: 0.0\n", "counts_sum_inv | exact: True | approximate: True | maxdiff: 0.0\n", "counts_sum | exact: True | approximate: True | maxdiff: 0.0\n", "counts | exact: True | approximate: True | maxdiff: 0.0\n", "norm_logits | exact: True | approximate: True | maxdiff: 0.0\n", "logit_maxes | exact: True | approximate: True | maxdiff: 0.0\n", "logits | exact: True | approximate: True | maxdiff: 0.0\n", "h | exact: True | approximate: True | maxdiff: 0.0\n", "W2 | exact: True | approximate: True | maxdiff: 0.0\n", "b2 | exact: True | approximate: True | maxdiff: 0.0\n", "hpreact | exact: True | approximate: True | maxdiff: 0.0\n", "bngain | exact: True | approximate: True | maxdiff: 0.0\n", "bnbias | exact: True | approximate: True | maxdiff: 0.0\n", "bnraw | exact: True | approximate: True | maxdiff: 0.0\n", "bnvar_inv | exact: True | approximate: True | maxdiff: 0.0\n", "bnvar | exact: True | approximate: True | maxdiff: 0.0\n", "bndiff2 | exact: True | approximate: True | maxdiff: 0.0\n", "bndiff | exact: True | approximate: True | maxdiff: 0.0\n", "bnmeani | exact: True | approximate: True | maxdiff: 0.0\n", "hprebn | exact: True | approximate: True | maxdiff: 0.0\n", "embcat | exact: True | approximate: True | maxdiff: 0.0\n", "W1 | exact: True | approximate: True | maxdiff: 0.0\n", "b1 | exact: True | approximate: True | maxdiff: 0.0\n", "emb | exact: True | approximate: True | maxdiff: 0.0\n", "C | exact: True | approximate: True | maxdiff: 0.0\n" ] } ], "source": [ "# Exercise 1: backprop through the whole thing manually, \n", "# backpropagating through exactly all of the variables \n", "# as they are defined in the forward pass above, one by one\n", "\n", "dlogprobs = torch.zeros_like(logprobs)\n", "dlogprobs[range(n), Yb] = -1.0/n\n", "dprobs = (1.0 / probs) * dlogprobs\n", "dcounts_sum_inv = (counts * dprobs).sum(1, keepdim=True)\n", "dcounts = counts_sum_inv * dprobs\n", "dcounts_sum = (-counts_sum**-2) * dcounts_sum_inv\n", "dcounts += torch.ones_like(counts) * dcounts_sum\n", "dnorm_logits = counts * dcounts\n", "dlogits = dnorm_logits.clone()\n", "dlogit_maxes = (-dnorm_logits).sum(1, keepdim=True)\n", "dlogits += F.one_hot(logits.max(1).indices, num_classes=logits.shape[1]) * dlogit_maxes\n", "dh = dlogits @ W2.T\n", "dW2 = h.T @ dlogits\n", "db2 = dlogits.sum(0)\n", "dhpreact = (1.0 - h**2) * dh\n", "dbngain = (bnraw * dhpreact).sum(0, keepdim=True)\n", "dbnraw = bngain * dhpreact\n", "dbnbias = dhpreact.sum(0, keepdim=True)\n", "dbndiff = bnvar_inv * dbnraw\n", "dbnvar_inv = (bndiff * dbnraw).sum(0, keepdim=True)\n", "dbnvar = (-0.5*(bnvar + 1e-5)**-1.5) * dbnvar_inv\n", "dbndiff2 = (1.0/(n-1))*torch.ones_like(bndiff2) * dbnvar\n", "dbndiff += (2*bndiff) * dbndiff2\n", "dhprebn = dbndiff.clone()\n", "dbnmeani = (-dbndiff).sum(0)\n", "dhprebn += 1.0/n * (torch.ones_like(hprebn) * dbnmeani)\n", "dembcat = dhprebn @ W1.T\n", "dW1 = embcat.T @ dhprebn\n", "db1 = dhprebn.sum(0)\n", "demb = dembcat.view(emb.shape)\n", "dC = torch.zeros_like(C)\n", "for k in range(Xb.shape[0]):\n", " for j in range(Xb.shape[1]):\n", " ix = Xb[k,j]\n", " dC[ix] += demb[k,j]\n", " \n", "cmp('logprobs', dlogprobs, logprobs)\n", "cmp('probs', dprobs, probs)\n", "cmp('counts_sum_inv', dcounts_sum_inv, counts_sum_inv)\n", "cmp('counts_sum', dcounts_sum, counts_sum)\n", "cmp('counts', dcounts, counts)\n", "cmp('norm_logits', dnorm_logits, norm_logits)\n", "cmp('logit_maxes', dlogit_maxes, logit_maxes)\n", "cmp('logits', dlogits, logits)\n", "cmp('h', dh, h)\n", "cmp('W2', dW2, W2)\n", "cmp('b2', db2, b2)\n", "cmp('hpreact', dhpreact, hpreact)\n", "cmp('bngain', dbngain, bngain)\n", "cmp('bnbias', dbnbias, bnbias)\n", "cmp('bnraw', dbnraw, bnraw)\n", "cmp('bnvar_inv', dbnvar_inv, bnvar_inv)\n", "cmp('bnvar', dbnvar, bnvar)\n", "cmp('bndiff2', dbndiff2, bndiff2)\n", "cmp('bndiff', dbndiff, bndiff)\n", "cmp('bnmeani', dbnmeani, bnmeani)\n", "cmp('hprebn', dhprebn, hprebn)\n", "cmp('embcat', dembcat, embcat)\n", "cmp('W1', dW1, W1)\n", "cmp('b1', db1, b1)\n", "cmp('emb', demb, emb)\n", "cmp('C', dC, C)" ] }, { "cell_type": "code", "execution_count": 211, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3.3377387523651123 diff: 2.384185791015625e-07\n" ] } ], "source": [ "# Exercise 2: backprop through cross_entropy but all in one go\n", "# to complete this challenge look at the mathematical expression of the loss,\n", "# take the derivative, simplify the expression, and just write it out\n", "\n", "# forward pass\n", "\n", "# before:\n", "# logit_maxes = logits.max(1, keepdim=True).values\n", "# norm_logits = logits - logit_maxes # subtract max for numerical stability\n", "# counts = norm_logits.exp()\n", "# counts_sum = counts.sum(1, keepdims=True)\n", "# counts_sum_inv = counts_sum**-1 # if I use (1.0 / counts_sum) instead then I can't get backprop to be bit exact...\n", "# probs = counts * counts_sum_inv\n", "# logprobs = probs.log()\n", "# loss = -logprobs[range(n), Yb].mean()\n", "\n", "# now:\n", "loss_fast = F.cross_entropy(logits, Yb)\n", "print(loss_fast.item(), 'diff:', (loss_fast - loss).item())" ] }, { "cell_type": "code", "execution_count": 229, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "logits | exact: False | approximate: True | maxdiff: 5.122274160385132e-09\n" ] } ], "source": [ "# backward pass\n", "\n", "dlogits = F.softmax(logits, 1)\n", "dlogits[range(n), Yb] -= 1\n", "dlogits /= n\n", "\n", "cmp('logits', dlogits, logits) # I can only get approximate to be true, my maxdiff is 6e-9" ] }, { "cell_type": "code", "execution_count": 225, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(torch.Size([32, 27]), torch.Size([32]))" ] }, "execution_count": 225, "metadata": {}, "output_type": "execute_result" } ], "source": [ "logits.shape, Yb.shape" ] }, { "cell_type": "code", "execution_count": 235, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([0.0719, 0.0881, 0.0193, 0.0493, 0.0169, 0.0864, 0.0226, 0.0356, 0.0165,\n", " 0.0314, 0.0364, 0.0383, 0.0424, 0.0279, 0.0317, 0.0142, 0.0085, 0.0195,\n", " 0.0152, 0.0555, 0.0450, 0.0236, 0.0250, 0.0662, 0.0616, 0.0269, 0.0239],\n", " grad_fn=)" ] }, "execution_count": 235, "metadata": {}, "output_type": "execute_result" } ], "source": [ "F.softmax(logits, 1)[0]" ] }, { "cell_type": "code", "execution_count": 234, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([ 0.0719, 0.0881, 0.0193, 0.0493, 0.0169, 0.0864, 0.0226, 0.0356,\n", " -0.9835, 0.0314, 0.0364, 0.0383, 0.0424, 0.0279, 0.0317, 0.0142,\n", " 0.0085, 0.0195, 0.0152, 0.0555, 0.0450, 0.0236, 0.0250, 0.0662,\n", " 0.0616, 0.0269, 0.0239], grad_fn=)" ] }, "execution_count": 234, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dlogits[0] * n" ] }, { "cell_type": "code", "execution_count": 238, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor(1.3970e-09, grad_fn=)" ] }, "execution_count": 238, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dlogits[0].sum()" ] }, { "cell_type": "code", "execution_count": 239, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 239, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAANkAAAD5CAYAAACqEpBAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAUz0lEQVR4nO2dXYxd5XWG3zXGBjOeMR47UIMHxtjGfzPEQciKRBSlpQ0uqmy4ACUXlS9QnIsgNVJ7YdGLcJlWJVEuKiQoqG6VJqAmgIWsEGS1opWqiB+R+WHs8d/UNlhj4t+xsfHf6sXZg8buWe85853z7cPMvI9kzcxe59t77X3O63PO+629PnN3CCHy0dbqBISY6UhkQmRGIhMiMxKZEJmRyITIjEQmRGZuamSwmW0C8DMAcwD8k7v/mD2+q6vLu7u7o32F465evTrlMWxqYs6cOVM+FjvewMBAOKa3t3fK+2uEZk/JsP2l5J96zqnn1dY29feRa9euTXnMxx9/jJMnT1Y9uWSRmdkcAP8I4M8AHAXwrpntdPePojHd3d146623qsZuuilO5cyZM1W3z5s3Lxzz+eefh7GFCxeGsbNnz4axSJzLly8Px+zcuTOMsRdcaoz9J5Eyhr3g2H9WkShuueWWpGNduXIljDHmz58fxqIcL168OOXjbNmyJYw18nFxI4D97n7Q3S8B+CWA+EhCzFIaEdldAI5M+vtosU0IMYlGRFbtM8v/e/81s21m9p6ZvXfixIkGDifE9KQRkR0FMNnFWAbgkxsf5O4vuPuD7v7g4sWLGzicENOTRkT2LoBVZrbczOYB+A6A+Fu+ELOUZHfR3a+Y2dMA3kLFwn/Z3YdqjMHly5erxph71NXVVXX7uXPnwjHM+Tp//nwYS7H+9+3bl7S/uXPnJo1LcRfXrFkTjhkZGZny/gDuBkY5sjHRa6MWqfZ+5CIy2z/F3m9onszddwHY1cg+hJjpqOJDiMxIZEJkRiITIjMSmRCZkciEyExD7uJUcXdcunSpaozZ0pHVyqxbZsMy65wVsEb7ZGOYLc2KmFnBNDu3aJrho4/Cum2sWLEijA0PD4exlCmIzs7OcMxnn30WxlLt/ej1BsTXmB0rpapf72RCZEYiEyIzEpkQmZHIhMiMRCZEZkp1F4HYnWFOYdRmIPUW/VSnKqU4lI1hDiIbl1I8fPPNN4djjhw5EsYuXLgQxliOUYwVdTO3lT2fq1evDmN79uwJY9FrkbW1iPJg+emdTIjMSGRCZEYiEyIzEpkQmZHIhMiMRCZEZkq18AcHB3HvvfdWjbE+GVGfiVSbO7W3RmT9M3s81aZnvTVY/imddpctWxbGDhw4EMbYeUfXkfVeYTF2XsymZ89n9NywomJm70fonUyIzEhkQmRGIhMiMxKZEJmRyITIjEQmRGYaXWlzFMA4gKsArrj7g+zxfX19dFG8iBTblFVFs8pyNi7q5cEsX2bTs/NKqXAHYnufTSUcPXo0jDELnFXNRzmuW7cuHMOmcVKtf3bHRfS8LViwYMpjGM2YJ/tjd/9DE/YjxIxEHxeFyEyjInMAvzWz981sWzMSEmKm0ejHxYfc/RMzux3A22a2x93fmfyAQnzbAODOO+9s8HBCTD8aeidz90+Kn8cBvIbKYu03PkYrbYpZTbLIzKzdzDomfgfwbQCDzUpMiJlCIx8X7wDwWmF53wTg39z9NzUPGFjJzFaPrHPWlIW1U2a2NGu5HdnSrCp+5cqVYWxoKF6YlO2TWfiRxcys5/b29jDW0dEx5WMBcWv1/fv3h2NS7zxIXWkz2idbiTWlTXcjy9keBPDV1PFCzBZk4QuRGYlMiMxIZEJkRiITIjMSmRCZKb0XfmQ/syrxaAXGJUuWhGNOnjwZxlgDGFZZHlndbIXIwcF46jC1ejzlToHu7u5wDKt+T+m7z2JsuoBNyaSubZDSnIfdHcGmGSL0TiZEZiQyITIjkQmRGYlMiMxIZEJkpnR3MYXI4Tp9+nQ4hrlAK1asCGOsLXVUHMoKdlMLlZnbyvYZFe2OjIyEY1JXLGU5Rs5dSoEtwB3hlBVQgdh5jIqbAX7OEXonEyIzEpkQmZHIhMiMRCZEZiQyITIjkQmRmVItfHcPrfVoBU4AOHToUNXtbPVFVhjK7Gy2z/Hx8arbOzs7wzGsDwbrJcHyZ6SMYzZ9am+NaNyZM2fCMay/CiseZjmygu/oWjGbPnr9smuhdzIhMiORCZEZiUyIzEhkQmRGIhMiMxKZEJmpaeGb2csA/gLAcXfvLbZ1AXgFQA+AUQBPuvupeg4YWaB79+4Nx0SV28y6ZVX4zG5lFn5U7Z3a1jml/wQAzJ8/P4xFljXLg622MzY2FsZSeo2wfij33HNPGGMtzVOncqL8WVU/O+eIet7J/hnAphu2bQew291XAdhd/C2EqEJNkRXrjd3Y+mkLgB3F7zsAPNbctISYOaR+J7vD3Y8BQPHz9ualJMTMIrvxYWbbzOw9M3uP9UIUYqaSKrIxM1sKAMXP49EDJ6+02dXVlXg4IaYvqSLbCWBr8ftWAG80Jx0hZh71WPi/APAtAEvM7CiAHwH4MYBXzewpAIcBPFHvASMrOcXOfuSRR8Ixu3btCmOsDTNr2BLlwSxfNpXAYsxyZ41eIouZ3Q0wOjoaxtjzwmJRjqzSnjUxYtc41cKPrjGz6dl1jKgpMnf/bhB6eMpHE2IWoooPITIjkQmRGYlMiMxIZEJkRiITIjOl98KPKuCZnR1VnTObPsVeBoCFCxeGsci+XbNmTTiGrWLJrOfUBjaR1c2mBNi0RUq/e3Y8ZoGzYzFbnT1np07FN4ekNB1KGaN3MiEyI5EJkRmJTIjMSGRCZEYiEyIzEpkQmSndwo+s2CRrlNjSrGq7o6MjjEX97tk+WZMX2iM9calbVsl+4cKFqtt7e3vDMWyaIdofwPOPpl2YhZ965wHbJ7vjIoXo9cumGPROJkRmJDIhMiORCZEZiUyIzEhkQmSmVHfRzMLC18uXL4fjokJU5hyxFRaZY8ZcoltvvbXqdlbczPbHYsxtXb58eRgbHh6uup05oOzaM5eTFfRG7biZM8qOxYqYU/OP3GI2JnrOtNKmEC1EIhMiMxKZEJmRyITIjEQmRGYkMiEyk7rS5rMAvgfg0+Jhz7h73HCjYHBwECtXrqwaYy2aIws/pWUyALS3t4exc+fOhbHI+mdWPJtmYOOYJcyuVbTqJ5sSYFMQrNcImyaJLHc2hk0JpE6TMOs/2mfq6yoidaVNAPipu28o/tUUmBCzldSVNoUQddLId7KnzazfzF42s0VNy0iIGUaqyJ4HsALABgDHADwXPXDySpvse4YQM5Ukkbn7mLtfdfdrAF4EsJE89ouVNtkXVCFmKkkim1jKtuBxAIPNSUeImUfqSpvfMrMNABzAKIDv13Ow9evXY+fOnVVjzDaN2jAzuz21TXfK6pesnwizrJm939PTE8YOHToUxqIqd9Y/g33CiKYEahE9n+ycWTU9u8bsawhrJR4dj00lpKzOmbrS5ku1xgkhKqjiQ4jMSGRCZEYiEyIzEpkQmZHIhMhMqY10hoaGsHbt2qqxw4cPh+Oi6ndmSzPrlsH2GTXSiZrGANx6Zlbx3r17wxg7t8jOZsdiOTJSKvvZGDaNw6rp2fVg+0xpihPlrzbdQrQQiUyIzEhkQmRGIhMiMxKZEJmRyITIjJV5I2VfX5+//vrrVWPMRo5WbWQ97ZlVzI4VVfwDsVWfskoowO8GYNXqKf3d2d0FzB5n1n/KVAIbw47FiKZWAODUqVNhLOV5u++++6puv3r1Kty9qo+vdzIhMiORCZEZiUyIzEhkQmRGIhMiM6UWCANxIWVKsWnqKpap7aCjHFlvisiNAoCRkZEwxvJnrbOj/FN6XdSK0dUlg0Lrzs7OcExqoTXrQxI500B8TdhrIHrOtmzZEo7RO5kQmZHIhMiMRCZEZiQyITIjkQmRGYlMiMzU06a7G8C/APgjANcAvODuPzOzLgCvAOhBpVX3k+4eV2NW9hUWvrJi38gqTrGyAd7Hg+URjWNTAnv27AljUUttgPemYOcWtQVnBccsj/Hx8TBG+1oE14oVRbPpAjbFw6YnUqZk+vr6wjHR89loj48rAP7a3dcC+DqAH5jZOgDbAex291UAdhd/CyFuoJ6VNo+5+wfF7+MAhgHcBWALgB3Fw3YAeCxTjkJMa6b0nczMegB8DcDvANzh7seAihAB3N707ISYAdQtMjNbAOBXAH7o7menMO6LlTZPntTS02L2UZfIzGwuKgL7ubv/utg8NrEYYPHzeLWxk1fa7OrqakbOQkwraorMKrbJSwCG3f0nk0I7AWwtft8K4I3mpyfE9Kdmjw8z+waA/wIwgIqFDwDPoPK97FUAdwM4DOAJd6efB9va2jyykkdHR8NxkUXLKrOZHcx6WqTYyKmV6sw6Z/tkRFZy6h0LLI+U/h+LFi0Kx5w4cSKMsekaZuEz6z+lTXc0ZvPmzejv768arGelzf8GED0LD9caL8RsRxUfQmRGIhMiMxKZEJmRyITIjEQmRGZKbaSzfv16RG26ly5dGo47cuRI1e1RxTnArVtWCZ7SpptNCTDKbNPNbG6WP2uBzfYZWe6nT59OyoNx2223hbFmt+mOoE2FmnYUIURVJDIhMiORCZEZiUyIzEhkQmRGIhMiM6Va+KyRDiOlIj21apvFUlaxZJXqzbSQa8GuByO1Qj86N2Z1s+ZBrPkRu8ZsXHQXBJsaYnd+hDlMeYQQYkpIZEJkRiITIjMSmRCZkciEyEzpK21GTtzY2Fg45ty5c1W3p/bIYKsvsjbdq1evrrp979694RjmVrJiZFbYmtJbg7mLzNVjThsjep5ZHql9TT799NMw1tPTE8ai1xxzQKMi5kbbdAshGkAiEyIzEpkQmZHIhMiMRCZEZiQyITLTyEqbzwL4HoAJ//QZd99VY19hgfD58+fDcVFRJrOXWWEoK8xlsX379lXdzopGWR5nz8aL47DpiRRYETPLn00XsHH3339/1e39/f3hmNSC6Y6OjjDGpoaic2PnFU3xMNu/nnmyiZU2PzCzDgDvm9nbReyn7v4PdexDiFlLPb3wjwGYWOxv3MwmVtoUQtRBIyttAsDTZtZvZi+bWbxchxCzmEZW2nwewAoAG1B5p3suGPfFSptsaRwhZirJK226+5i7X3X3awBeBLCx2tjJK20uXry4WXkLMW1IXmlzYinbgscBDDY/PSGmP/W4iw8B+EsAA2b2YbHtGQDfNbMNABzAKIDv19qRu4dV4imrGzKrldnB4+PjYayzszOMRW26GWvXrg1jAwMDYYxZ5ywWVbLTKnEyzZBaNT80NDTlY7E7Ftg5L1iwIIwxCz+a1mDTHSk0stImnRMTQlRQxYcQmZHIhMiMRCZEZiQyITIjkQmRmVIb6bh7aI8yizmq3L/77rvDMQcOHAhj7FjsboBomoHZ0lHlPsDvImA2csp0B5vSSGnMU2tclAc7Z9ZYiK3QyZoOsWme6NzYtYpei2qkI0QLkciEyIxEJkRmJDIhMiORCZEZiUyIzJRq4be1tYW9xJlVHNm+IyMjSXn09fWFseHh4TAWWeesl3xqkxpm06c0xWF5sBhr6MPuSojGsekOdndE6rVqb28PY9EdBmy6ILr2LAe9kwmRGYlMiMxIZEJkRiITIjMSmRCZkciEyEypFv61a9eSeolHVmtqIx3Wj501jommElgjF3anAFsGN7UvfDSOXV9m07PlfVnleUpDHxZLXW+ATTNE0wJsueMoD1XhC9FCJDIhMiORCZEZiUyIzEhkQmTGmOsEAGZ2C4B3ANyMihv57+7+IzPrAvAKgB5U2nQ/6e5xswUAbW1tHjk6Bw8eDMdFBbjMFWMtpJlzxwqVI5jzxa4vyyO1VXTkcrH9MUeVkXJuzAlksOeTXX/m/Ea5sBVQozGbN2/GwMBA1Ytfzxl/DuBP3P2rqCyTtMnMvg5gO4Dd7r4KwO7ibyHEDdQUmVc4V/w5t/jnALYA2FFs3wHgsRwJCjHdqXd9sjnFii7HAbzt7r8DcEex1O3Ekre3Z8tSiGlMXSIrFvvbAGAZgI1m1lvvASavtFnr+58QM5EpfQt199MA/hPAJgBjEwsBFj+PB2O+WGmTlZ4IMVOpZ6XNr5jZbcXv8wH8KYA9AHYC2Fo8bCuANzLlKMS0pp4C4aUAdpjZHFRE+aq7v2lm/wPgVTN7CsBhAE/U2lFvby/efPPNqjHWJyPq08B6QrDeDqzoNaUQNbVHBjtn1tOCER3v4sWL4ZgcnzBWrVpVdXu0AifArxWbWmEFvSlt15s9tVLPSpv9AL5WZfsJAA9P+YhCzDJU8SFEZiQyITIjkQmRGYlMiMxIZEJkpmYVflMPZvYpgP8t/lwC4A+lHTxGeVyP8rieevO4x92/Ui1QqsiuO3ClzOrBlhxceSiPEvPQx0UhMiORCZGZVorshRYeezLK43qUx/U0nEfLvpMJMVvQx0UhMtMSkZnZJjPba2b7zaxlvUHMbNTMBszsQzN7r8Tjvmxmx81scNK2LjN728z2FT8XtSiPZ83s4+KafGhmj5aQR7eZ/YeZDZvZkJn9VbG91GtC8mjsmrh7qf8AzAFwAMC9AOYB+D2AdWXnUeQyCmBJC477TQAPABictO3vAWwvft8O4O9alMezAP6m5OuxFMADxe8dAEYArCv7mpA8GromrXgn2whgv7sfdPdLAH6JSlOeWYO7vwPg5A2bS29MFORROu5+zN0/KH4fBzAM4C6UfE1IHg3RCpHdBeDIpL+PogknkogD+K2ZvW9m21qUwwRfpsZET5tZf/FxMvvH1smYWQ8q9y+2tFnTDXkADVyTVois2m24rbI4H3L3BwD8OYAfmNk3W5THl4nnAaxApcfmMQDPlXVgM1sA4FcAfujucYfR8vNo6Jq0QmRHAXRP+nsZgE9akAfc/ZPi53EAr6HyUbZV1NWYKDfuPuaV7mTXALyIkq6Jmc1F5YX9c3f/dbG59GtSLY9Gr0krRPYugFVmttzM5gH4DipNeUrFzNrNrGPidwDfBjDIR2XlS9GYaOJFXfA4SrgmVmky8hKAYXf/yaRQqdckyqPha1KmizTJxXkUFefmAIC/bVEO96LibP4ewFCZeQD4BSofOy6j8s7+FIDFqLQ731f87GpRHv8KYABAPyov8qUl5PENVL4y9AP4sPj3aNnXhOTR0DVRxYcQmVHFhxCZkciEyIxEJkRmJDIhMiORCZEZiUyIzEhkQmRGIhMiM/8HOZ2h8oJBO9wAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.figure(figsize=(4, 4))\n", "plt.imshow(dlogits.detach(), cmap='gray')" ] }, { "cell_type": "code", "execution_count": 269, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "max diff: tensor(4.7684e-07, grad_fn=)\n" ] } ], "source": [ "# Exercise 3: backprop through batchnorm but all in one go\n", "# to complete this challenge look at the mathematical expression of the output of batchnorm,\n", "# take the derivative w.r.t. its input, simplify the expression, and just write it out\n", "\n", "# forward pass\n", "\n", "# before:\n", "# bnmeani = 1/n*hprebn.sum(0, keepdim=True)\n", "# bndiff = hprebn - bnmeani\n", "# bndiff2 = bndiff**2\n", "# bnvar = 1/(n-1)*(bndiff2).sum(0, keepdim=True) # note: Bessel's correction (dividing by n-1, not n)\n", "# bnvar_inv = (bnvar + 1e-5)**-0.5\n", "# bnraw = bndiff * bnvar_inv\n", "# hpreact = bngain * bnraw + bnbias\n", "\n", "# now:\n", "hpreact_fast = bngain * (hprebn - hprebn.mean(0, keepdim=True)) / torch.sqrt(hprebn.var(0, keepdim=True, unbiased=True) + 1e-5) + bnbias\n", "print('max diff:', (hpreact_fast - hpreact).abs().max())" ] }, { "cell_type": "code", "execution_count": 279, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "hprebn | exact: False | approximate: True | maxdiff: 9.313225746154785e-10\n" ] } ], "source": [ "# backward pass\n", "\n", "# before we had:\n", "# dbnraw = bngain * dhpreact\n", "# dbndiff = bnvar_inv * dbnraw\n", "# dbnvar_inv = (bndiff * dbnraw).sum(0, keepdim=True)\n", "# dbnvar = (-0.5*(bnvar + 1e-5)**-1.5) * dbnvar_inv\n", "# dbndiff2 = (1.0/(n-1))*torch.ones_like(bndiff2) * dbnvar\n", "# dbndiff += (2*bndiff) * dbndiff2\n", "# dhprebn = dbndiff.clone()\n", "# dbnmeani = (-dbndiff).sum(0)\n", "# dhprebn += 1.0/n * (torch.ones_like(hprebn) * dbnmeani)\n", "\n", "# calculate dhprebn given dhpreact (i.e. backprop through the batchnorm)\n", "# (you'll also need to use some of the variables from the forward pass up above)\n", "\n", "dhprebn = bngain*bnvar_inv/n * (n*dhpreact - dhpreact.sum(0) - n/(n-1)*bnraw*(dhpreact*bnraw).sum(0))\n", "\n", "cmp('hprebn', dhprebn, hprebn) # I can only get approximate to be true, my maxdiff is 9e-10" ] }, { "cell_type": "code", "execution_count": 278, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(torch.Size([32, 64]),\n", " torch.Size([1, 64]),\n", " torch.Size([1, 64]),\n", " torch.Size([32, 64]),\n", " torch.Size([64]))" ] }, "execution_count": 278, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dhprebn.shape, bngain.shape, bnvar_inv.shape, dbnraw.shape, dbnraw.sum(0).shape" ] }, { "cell_type": "code", "execution_count": 286, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "12297\n", " 0/ 200000: 3.7805\n", " 10000/ 200000: 2.1775\n", " 20000/ 200000: 2.3957\n", " 30000/ 200000: 2.5032\n", " 40000/ 200000: 2.0065\n", " 50000/ 200000: 2.3873\n", " 60000/ 200000: 2.3378\n", " 70000/ 200000: 2.0640\n", " 80000/ 200000: 2.3497\n", " 90000/ 200000: 2.1093\n", " 100000/ 200000: 1.9132\n", " 110000/ 200000: 2.2229\n", " 120000/ 200000: 1.9912\n", " 130000/ 200000: 2.4441\n", " 140000/ 200000: 2.3198\n", " 150000/ 200000: 2.1857\n", " 160000/ 200000: 2.0296\n", " 170000/ 200000: 1.8391\n", " 180000/ 200000: 2.0436\n", " 190000/ 200000: 1.9200\n" ] } ], "source": [ "# Exercise 4: putting it all together!\n", "# Train the MLP neural net with your own backward pass\n", "\n", "# init\n", "n_embd = 10 # the dimensionality of the character embedding vectors\n", "n_hidden = 200 # the number of neurons in the hidden layer of the MLP\n", "\n", "g = torch.Generator().manual_seed(2147483647) # for reproducibility\n", "C = torch.randn((vocab_size, n_embd), generator=g)\n", "# Layer 1\n", "W1 = torch.randn((n_embd * block_size, n_hidden), generator=g) * (5/3)/((n_embd * block_size)**0.5)\n", "b1 = torch.randn(n_hidden, generator=g) * 0.1\n", "# Layer 2\n", "W2 = torch.randn((n_hidden, vocab_size), generator=g) * 0.1\n", "b2 = torch.randn(vocab_size, generator=g) * 0.1\n", "# BatchNorm parameters\n", "bngain = torch.randn((1, n_hidden))*0.1 + 1.0\n", "bnbias = torch.randn((1, n_hidden))*0.1\n", "\n", "parameters = [C, W1, b1, W2, b2, bngain, bnbias]\n", "print(sum(p.nelement() for p in parameters)) # number of parameters in total\n", "for p in parameters:\n", " p.requires_grad = True\n", "\n", "# same optimization as last time\n", "max_steps = 200000\n", "batch_size = 32\n", "n = batch_size # convenience\n", "lossi = []\n", "\n", "# use this context manager for efficiency once your backward pass is written (TODO)\n", "with torch.no_grad():\n", "\n", " # kick off optimization\n", " for i in range(max_steps):\n", "\n", " # minibatch construct\n", " ix = torch.randint(0, Xtr.shape[0], (batch_size,), generator=g)\n", " Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y\n", "\n", " # forward pass\n", " emb = C[Xb] # embed the characters into vectors\n", " embcat = emb.view(emb.shape[0], -1) # concatenate the vectors\n", " # Linear layer\n", " hprebn = embcat @ W1 + b1 # hidden layer pre-activation\n", " # BatchNorm layer\n", " # -------------------------------------------------------------\n", " bnmean = hprebn.mean(0, keepdim=True)\n", " bnvar = hprebn.var(0, keepdim=True, unbiased=True)\n", " bnvar_inv = (bnvar + 1e-5)**-0.5\n", " bnraw = (hprebn - bnmean) * bnvar_inv\n", " hpreact = bngain * bnraw + bnbias\n", " # -------------------------------------------------------------\n", " # Non-linearity\n", " h = torch.tanh(hpreact) # hidden layer\n", " logits = h @ W2 + b2 # output layer\n", " loss = F.cross_entropy(logits, Yb) # loss function\n", "\n", " # backward pass\n", " for p in parameters:\n", " p.grad = None\n", " #loss.backward() # use this for correctness comparisons, delete it later!\n", "\n", " # manual backprop! #swole_doge_meme\n", " # -----------------\n", " dlogits = F.softmax(logits, 1)\n", " dlogits[range(n), Yb] -= 1\n", " dlogits /= n\n", " # 2nd layer backprop\n", " dh = dlogits @ W2.T\n", " dW2 = h.T @ dlogits\n", " db2 = dlogits.sum(0)\n", " # tanh\n", " dhpreact = (1.0 - h**2) * dh\n", " # batchnorm backprop\n", " dbngain = (bnraw * dhpreact).sum(0, keepdim=True)\n", " dbnbias = dhpreact.sum(0, keepdim=True)\n", " dhprebn = bngain*bnvar_inv/n * (n*dhpreact - dhpreact.sum(0) - n/(n-1)*bnraw*(dhpreact*bnraw).sum(0))\n", " # 1st layer\n", " dembcat = dhprebn @ W1.T\n", " dW1 = embcat.T @ dhprebn\n", " db1 = dhprebn.sum(0)\n", " # embedding\n", " demb = dembcat.view(emb.shape)\n", " dC = torch.zeros_like(C)\n", " for k in range(Xb.shape[0]):\n", " for j in range(Xb.shape[1]):\n", " ix = Xb[k,j]\n", " dC[ix] += demb[k,j]\n", " grads = [dC, dW1, db1, dW2, db2, dbngain, dbnbias]\n", " # -----------------\n", "\n", " # update\n", " lr = 0.1 if i < 100000 else 0.01 # step learning rate decay\n", " for p, grad in zip(parameters, grads):\n", " #p.data += -lr * p.grad # old way of cheems doge (using PyTorch grad from .backward())\n", " p.data += -lr * grad # new way of swole doge TODO: enable\n", "\n", " # track stats\n", " if i % 10000 == 0: # print every once in a while\n", " print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}')\n", " lossi.append(loss.log10().item())\n", "\n", " # if i >= 100: # TODO: delete early breaking when you're ready to train the full net\n", " # break" ] }, { "cell_type": "code", "execution_count": 285, "metadata": {}, "outputs": [], "source": [ "# useful for checking your gradients\n", "# for p,g in zip(parameters, grads):\n", "# cmp(str(tuple(p.shape)), g, p)" ] }, { "cell_type": "code", "execution_count": 299, "metadata": {}, "outputs": [], "source": [ "# calibrate the batch norm at the end of training\n", "\n", "with torch.no_grad():\n", " # pass the training set through\n", " emb = C[Xtr]\n", " embcat = emb.view(emb.shape[0], -1)\n", " hpreact = embcat @ W1 + b1\n", " # measure the mean/std over the entire training set\n", " bnmean = hpreact.mean(0, keepdim=True)\n", " bnvar = hpreact.var(0, keepdim=True, unbiased=True)\n" ] }, { "cell_type": "code", "execution_count": 300, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "train 2.070523500442505\n", "val 2.109893560409546\n" ] } ], "source": [ "# evaluate train and val loss\n", "\n", "@torch.no_grad() # this decorator disables gradient tracking\n", "def split_loss(split):\n", " x,y = {\n", " 'train': (Xtr, Ytr),\n", " 'val': (Xdev, Ydev),\n", " 'test': (Xte, Yte),\n", " }[split]\n", " emb = C[x] # (N, block_size, n_embd)\n", " embcat = emb.view(emb.shape[0], -1) # concat into (N, block_size * n_embd)\n", " hpreact = embcat @ W1 + b1\n", " hpreact = bngain * (hpreact - bnmean) * (bnvar + 1e-5)**-0.5 + bnbias\n", " h = torch.tanh(hpreact) # (N, n_hidden)\n", " logits = h @ W2 + b2 # (N, vocab_size)\n", " loss = F.cross_entropy(logits, y)\n", " print(split, loss.item())\n", "\n", "split_loss('train')\n", "split_loss('val')" ] }, { "cell_type": "code", "execution_count": 294, "metadata": {}, "outputs": [], "source": [ "# I achieved:\n", "# train 2.0718822479248047\n", "# val 2.1162495613098145" ] }, { "cell_type": "code", "execution_count": 301, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "carmahzamille.\n", "khi.\n", "mreigeet.\n", "khalaysie.\n", "mahnen.\n", "delynn.\n", "jareen.\n", "nellara.\n", "chaiiv.\n", "kaleigh.\n", "ham.\n", "joce.\n", "quinn.\n", "shoison.\n", "jadiquintero.\n", "dearyxi.\n", "jace.\n", "pinsley.\n", "dae.\n", "iia.\n" ] } ], "source": [ "# sample from the model\n", "g = torch.Generator().manual_seed(2147483647 + 10)\n", "\n", "for _ in range(20):\n", " \n", " out = []\n", " context = [0] * block_size # initialize with all ...\n", " while True:\n", " # ------------\n", " # forward pass:\n", " # Embedding\n", " emb = C[torch.tensor([context])] # (1,block_size,d) \n", " embcat = emb.view(emb.shape[0], -1) # concat into (N, block_size * n_embd)\n", " hpreact = embcat @ W1 + b1\n", " hpreact = bngain * (hpreact - bnmean) * (bnvar + 1e-5)**-0.5 + bnbias\n", " h = torch.tanh(hpreact) # (N, n_hidden)\n", " logits = h @ W2 + b2 # (N, vocab_size)\n", " # ------------\n", " # Sample\n", " probs = F.softmax(logits, dim=1)\n", " ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n", " context = context[1:] + [ix]\n", " out.append(ix)\n", " if ix == 0:\n", " break\n", " \n", " print(''.join(itos[i] for i in out))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 4 }