In the ReStructuredText (*.txt) files, many of the copies of Python code were out of date. Replaced copies of Python code with links to sections of the corresponding files. This dramatically reduced the number of lines of code to maintain. This will help keep documentation in sync with code (but will not guarantee synchronization). Renamed two ReStructuredText files to match corresponding Python files. Modified code files for improved readability. Modified .gitignore to ignore tmp directories.

Author: mspandit

.gitignore

@@ -1,4 +1,5 @@
 code/*.pyc
+code/tmp*
 code/midi
 data/mnist.pkl.gz
 data/mnist_py3k.pkl.gz

code/SdA.py

@@ -57,9 +57,14 @@ class SdA(object):
     the dAs are only used to initialize the weights.
     """
 
-    def __init__(self, numpy_rng, theano_rng=None, n_ins=784,
-                 hidden_layers_sizes=[500, 500], n_outs=10,
-                 corruption_levels=[0.1, 0.1]):
+    def __init__(self, 
+        numpy_rng, 
+        theano_rng = None, 
+        n_ins = 784,
+        hidden_layers_sizes = [500, 500], 
+        n_outs = 10,
+        corruption_levels = [0.1, 0.1]
+    ):
         """ This class is made to support a variable number of layers.
 
         :type numpy_rng: numpy.random.RandomState
@@ -241,9 +246,9 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate):
         (test_set_x, test_set_y) = datasets[2]
 
         # compute number of minibatches for training, validation and testing
-        n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
+        n_valid_batches = valid_set_x.get_value(borrow = True).shape[0]
         n_valid_batches /= batch_size
-        n_test_batches = test_set_x.get_value(borrow=True).shape[0]
+        n_test_batches = test_set_x.get_value(borrow = True).shape[0]
         n_test_batches /= batch_size
 
         index = T.lscalar('index')  # index to a [mini]batch
@@ -252,35 +257,41 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate):
         gparams = T.grad(self.finetune_cost, self.params)
 
         # compute list of fine-tuning updates
-        updates = []
-        for param, gparam in zip(self.params, gparams):
-            updates.append((param, param - gparam * learning_rate))
-
-        train_fn = theano.function(inputs=[index],
-              outputs=self.finetune_cost,
-              updates=updates,
-              givens={
-                self.x: train_set_x[index * batch_size:
-                                    (index + 1) * batch_size],
-                self.y: train_set_y[index * batch_size:
-                                    (index + 1) * batch_size]},
-              name='train')
-
-        test_score_i = theano.function([index], self.errors,
-                 givens={
-                   self.x: test_set_x[index * batch_size:
-                                      (index + 1) * batch_size],
-                   self.y: test_set_y[index * batch_size:
-                                      (index + 1) * batch_size]},
-                      name='test')
-
-        valid_score_i = theano.function([index], self.errors,
-              givens={
-                 self.x: valid_set_x[index * batch_size:
-                                     (index + 1) * batch_size],
-                 self.y: valid_set_y[index * batch_size:
-                                     (index + 1) * batch_size]},
-                      name='valid')
+        updates = [
+            (param, param - gparam * learning_rate)
+            for param, gparam in zip(self.params, gparams)
+        ]
+
+        train_fn = theano.function(
+            inputs = [index],
+            outputs = self.finetune_cost,
+            updates = updates,
+            givens = {
+                self.x: train_set_x[index * batch_size : (index + 1) * batch_size],
+                self.y: train_set_y[index * batch_size : (index + 1) * batch_size]
+            },
+            name = 'train'
+        )
+
+        test_score_i = theano.function(
+            [index], 
+            self.errors,
+            givens = {
+               self.x: test_set_x[index * batch_size : (index + 1) * batch_size],
+               self.y: test_set_y[index * batch_size : (index + 1) * batch_size]
+            },
+            name = 'test'
+        )
+
+        valid_score_i = theano.function(
+            [index], 
+            self.errors,
+            givens = {
+                self.x: valid_set_x[index * batch_size : (index + 1) * batch_size],
+                self.y: valid_set_y[index * batch_size : (index + 1) * batch_size]
+            },
+            name='valid'
+        )
 
         # Create a function that scans the entire validation set
         def valid_score():
@@ -333,9 +344,12 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15,
     numpy_rng = numpy.random.RandomState(89677)
     print '... building the model'
     # construct the stacked denoising autoencoder class
-    sda = SdA(numpy_rng=numpy_rng, n_ins=28 * 28,
-              hidden_layers_sizes=[1000, 1000, 1000],
-              n_outs=10)
+    sda = SdA(
+        numpy_rng = numpy_rng, 
+        n_ins = 28 * 28,
+        hidden_layers_sizes = [1000, 1000, 1000],
+        n_outs = 10
+    )
 
     #########################
     # PRETRAINING THE MODEL #

code/convolutional_mlp.py

@@ -76,22 +76,32 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)):
                    numpy.prod(poolsize))
         # initialize weights with random weights
         W_bound = numpy.sqrt(6. / (fan_in + fan_out))
-        self.W = theano.shared(numpy.asarray(
-            rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
-            dtype=theano.config.floatX),
-                               borrow=True)
+        self.W = theano.shared(
+            numpy.asarray(
+                rng.uniform(low = -W_bound, high = W_bound, size = filter_shape),
+                dtype = theano.config.floatX
+            ),
+            borrow = True
+        )
 
         # the bias is a 1D tensor -- one bias per output feature map
         b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
         self.b = theano.shared(value=b_values, borrow=True)
 
         # convolve input feature maps with filters
-        conv_out = conv.conv2d(input=input, filters=self.W,
-                filter_shape=filter_shape, image_shape=image_shape)
+        conv_out = conv.conv2d(
+            input = input, 
+            filters = self.W,
+            filter_shape = filter_shape, 
+            image_shape = image_shape
+        )
 
         # downsample each feature map individually, using maxpooling
-        pooled_out = downsample.max_pool_2d(input=conv_out,
-                                            ds=poolsize, ignore_border=True)
+        pooled_out = downsample.max_pool_2d(
+            input = conv_out,
+            ds = poolsize, 
+            ignore_border = True
+        )
 
         # add the bias term. Since the bias is a vector (1D array), we first
         # reshape it to a tensor of shape (1,n_filters,1,1). Each bias will
@@ -131,9 +141,9 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200,
     test_set_x, test_set_y = datasets[2]
 
     # compute number of minibatches for training, validation and testing
-    n_train_batches = train_set_x.get_value(borrow=True).shape[0]
-    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
-    n_test_batches = test_set_x.get_value(borrow=True).shape[0]
+    n_train_batches = train_set_x.get_value(borrow = True).shape[0]
+    n_valid_batches = valid_set_x.get_value(borrow = True).shape[0]
+    n_test_batches = test_set_x.get_value(borrow = True).shape[0]
     n_train_batches /= batch_size
     n_valid_batches /= batch_size
     n_test_batches /= batch_size
@@ -159,43 +169,64 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200,
     # filtering reduces the image size to (28-5+1,28-5+1)=(24,24)
     # maxpooling reduces this further to (24/2,24/2) = (12,12)
     # 4D output tensor is thus of shape (batch_size,nkerns[0],12,12)
-    layer0 = LeNetConvPoolLayer(rng, input=layer0_input,
-            image_shape=(batch_size, 1, 28, 28),
-            filter_shape=(nkerns[0], 1, 5, 5), poolsize=(2, 2))
+    layer0 = LeNetConvPoolLayer(
+        rng, 
+        input = layer0_input,
+        image_shape = (batch_size, 1, 28, 28),
+        filter_shape = (nkerns[0], 1, 5, 5),
+        poolsize = (2, 2)
+    )
 
     # Construct the second convolutional pooling layer
     # filtering reduces the image size to (12-5+1,12-5+1)=(8,8)
     # maxpooling reduces this further to (8/2,8/2) = (4,4)
     # 4D output tensor is thus of shape (nkerns[0],nkerns[1],4,4)
-    layer1 = LeNetConvPoolLayer(rng, input=layer0.output,
-            image_shape=(batch_size, nkerns[0], 12, 12),
-            filter_shape=(nkerns[1], nkerns[0], 5, 5), poolsize=(2, 2))
+    layer1 = LeNetConvPoolLayer(
+        rng, 
+        input = layer0.output,
+        image_shape = (batch_size, nkerns[0], 12, 12),
+        filter_shape = (nkerns[1], nkerns[0], 5, 5), 
+        poolsize = (2, 2)
+    )
 
     # the HiddenLayer being fully-connected, it operates on 2D matrices of
     # shape (batch_size,num_pixels) (i.e matrix of rasterized images).
     # This will generate a matrix of shape (20,32*4*4) = (20,512)
     layer2_input = layer1.output.flatten(2)
 
     # construct a fully-connected sigmoidal layer
-    layer2 = HiddenLayer(rng, input=layer2_input, n_in=nkerns[1] * 4 * 4,
-                         n_out=500, activation=T.tanh)
+    layer2 = HiddenLayer(
+        rng, 
+        input = layer2_input, 
+        n_in = nkerns[1] * 4 * 4,
+        n_out = 500, 
+        activation = T.tanh
+    )
 
     # classify the values of the fully-connected sigmoidal layer
-    layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10)
+    layer3 = LogisticRegression(input = layer2.output, n_in = 500, n_out = 10)
 
     # the cost we minimize during training is the NLL of the model
     cost = layer3.negative_log_likelihood(y)
 
     # create a function to compute the mistakes that are made by the model
-    test_model = theano.function([index], layer3.errors(y),
-             givens={
-                x: test_set_x[index * batch_size: (index + 1) * batch_size],
-                y: test_set_y[index * batch_size: (index + 1) * batch_size]})
-
-    validate_model = theano.function([index], layer3.errors(y),
-            givens={
-                x: valid_set_x[index * batch_size: (index + 1) * batch_size],
-                y: valid_set_y[index * batch_size: (index + 1) * batch_size]})
+    test_model = theano.function(
+        [index], 
+        layer3.errors(y),
+        givens = {
+            x: test_set_x[index * batch_size : (index + 1) * batch_size],
+            y: test_set_y[index * batch_size : (index + 1) * batch_size]
+        }
+    )
+
+    validate_model = theano.function(
+        [index], 
+        layer3.errors(y),
+        givens = {
+            x: valid_set_x[index * batch_size : (index + 1) * batch_size],
+            y: valid_set_y[index * batch_size : (index + 1) * batch_size]
+        }
+    )
 
     # create a list of all model parameters to be fit by gradient descent
     params = layer3.params + layer2.params + layer1.params + layer0.params
@@ -208,14 +239,20 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200,
     # manually create an update rule for each model parameter. We thus
     # create the updates list by automatically looping over all
     # (params[i],grads[i]) pairs.
-    updates = []
-    for param_i, grad_i in zip(params, grads):
-        updates.append((param_i, param_i - learning_rate * grad_i))
-
-    train_model = theano.function([index], cost, updates=updates,
-          givens={
-            x: train_set_x[index * batch_size: (index + 1) * batch_size],
-            y: train_set_y[index * batch_size: (index + 1) * batch_size]})
+    updates = [
+        (param_i, param_i - learning_rate * grad_i)
+        for param_i, grad_i in zip(params, grads)
+    ]
+
+    train_model = theano.function(
+        [index], 
+        cost, 
+        updates = updates,
+        givens = {
+            x: train_set_x[index * batch_size : (index + 1) * batch_size],
+            y: train_set_y[index * batch_size : (index + 1) * batch_size]
+        }
+    )
 
     ###############
     # TRAIN MODEL #

code/dA.py

@@ -75,9 +75,17 @@ class dA(object):
 
     """
 
-    def __init__(self, numpy_rng, theano_rng=None, input=None,
-                 n_visible=784, n_hidden=500,
-                 W=None, bhid=None, bvis=None):
+    def __init__(
+        self, 
+        numpy_rng, 
+        theano_rng = None, 
+        input = None,
+        n_visible = 784, 
+        n_hidden = 500,
+        W = None, 
+        bhid = None, 
+        bvis = None
+    ):
         """
         Initialize the dA class by specifying the number of visible units (the
         dimension d of the input ), the number of hidden units ( the dimension
@@ -232,9 +240,10 @@ def get_cost_updates(self, corruption_level, learning_rate):
         # to its parameters
         gparams = T.grad(cost, self.params)
         # generate the list of updates
-        updates = []
-        for param, gparam in zip(self.params, gparams):
-            updates.append((param, param - learning_rate * gparam))
+        updates = [
+            (param, param - learning_rate * gparam)
+            for param, gparam in zip(self.params, gparams)
+        ]
 
         return (cost, updates)
 
@@ -277,15 +286,27 @@ def test_dA(learning_rate=0.1, training_epochs=15,
     rng = numpy.random.RandomState(123)
     theano_rng = RandomStreams(rng.randint(2 ** 30))
 
-    da = dA(numpy_rng=rng, theano_rng=theano_rng, input=x,
-            n_visible=28 * 28, n_hidden=500)
-
-    cost, updates = da.get_cost_updates(corruption_level=0.,
-                                        learning_rate=learning_rate)
-
-    train_da = theano.function([index], cost, updates=updates,
-         givens={x: train_set_x[index * batch_size:
-                                (index + 1) * batch_size]})
+    da = dA(
+        numpy_rng = rng, 
+        theano_rng = theano_rng, 
+        input = x,
+        n_visible = 28 * 28, 
+        n_hidden = 500
+    )
+
+    cost, updates = da.get_cost_updates(
+        corruption_level=0.,
+        learning_rate=learning_rate
+    )
+
+    train_da = theano.function(
+        [index], 
+        cost, 
+        updates = updates,
+        givens = {
+            x: train_set_x[index * batch_size : (index + 1) * batch_size]
+        }
+    )
 
     start_time = time.clock()
 
@@ -322,11 +343,18 @@ def test_dA(learning_rate=0.1, training_epochs=15,
     rng = numpy.random.RandomState(123)
     theano_rng = RandomStreams(rng.randint(2 ** 30))
 
-    da = dA(numpy_rng=rng, theano_rng=theano_rng, input=x,
-            n_visible=28 * 28, n_hidden=500)
-
-    cost, updates = da.get_cost_updates(corruption_level=0.3,
-                                        learning_rate=learning_rate)
+    da = dA(
+        numpy_rng = rng, 
+        theano_rng = theano_rng, 
+        input = x,
+        n_visible = 28 * 28, 
+        n_hidden = 500
+    )
+
+    cost, updates = da.get_cost_updates(
+        corruption_level = 0.3,
+        learning_rate=learning_rate
+    )
 
     train_da = theano.function([index], cost, updates=updates,
          givens={x: train_set_x[index * batch_size: