first commit

Author: pkmital

.gitignore

@@ -0,0 +1,3 @@
+MNIST_data
+*.pyc
+*.pyo

1-basics.py

@@ -0,0 +1,137 @@
+"""Summary of tensorflow basics.
+
+Parag K. Mital, Jan 2016."""
+# %% Import tensorflow and pyplot
+import tensorflow as tf
+import matplotlib.pyplot as plt
+
+# %% tf.Graph represents a collection of tf.Operations
+# You can create operations by writing out equations.
+# By default, there is a graph: tf.get_default_graph()
+# and any new operations are added to this graph.
+# The result of a tf.Operation is a tf.Tensor, which holds
+# the values.
+
+# %% First a tf.Tensor
+n_values = 32
+x = tf.linspace(-3.0, 3.0, n_values)
+
+# %% Construct a tf.Session to execut the graph.
+sess = tf.Session()
+result = sess.run(x)
+
+# %% Alternatively pass a session to the eval fn:
+x.eval(session=sess)
+# x.eval() does not work, as it requires a session!
+
+# %% We can setup an interactive session if we don't
+# want to keep passing the session around:
+sess.close()
+sess = tf.InteractiveSession()
+
+# %% Now this will work!
+x.eval()
+
+# %% Now a tf.Operation
+# We'll use our values from [-3, 3] to create a Gaussian Distribution
+sigma = 1.0
+mean = 0.0
+z = (tf.exp(tf.neg(tf.pow(x - mean, 2.0) /
+                   (2.0 * tf.pow(sigma, 2.0)))) *
+     (1.0 / (sigma * tf.sqrt(2.0 * 3.1415))))
+
+# %% By default, new operations are added to the default Graph
+assert z.graph is tf.get_default_graph()
+
+# %% Execute the graph and plot the result
+plt.plot(z.eval())
+
+# %% We can find out the shape of a tensor like so:
+print(z.get_shape())
+
+# %% Or in a more friendly format
+print(z.get_shape().as_list())
+
+# %% Sometimes we may not know the shape of a tensor
+# until it is computed in the graph.  In that case
+# we should use the tf.shape fn, which will return a
+# Tensor which can be eval'ed, rather than a discrete
+# value of tf.Dimension
+print(tf.shape(z).eval())
+
+# %% We can combine tensors like so:
+print(tf.pack([tf.shape(z), tf.shape(z), [3], [4]]).eval())
+
+# %% Let's multiply the two to get a 2d gaussian
+z_2d = tf.matmul(tf.reshape(z, [n_values, 1]), tf.reshape(z, [1, n_values]))
+
+# %% Execute the graph and store the value that `out` represents in `result`.
+plt.imshow(z_2d.eval())
+
+# %% For fun let's create a gabor patch:
+x = tf.reshape(tf.sin(tf.linspace(-3.0, 3.0, n_values)), [n_values, 1])
+y = tf.reshape(tf.ones_like(x), [1, n_values])
+z = tf.mul(tf.matmul(x, y), z_2d)
+plt.imshow(z.eval())
+
+# %% We can also list all the operations of a graph:
+ops = tf.get_default_graph().get_operations()
+print([op.name for op in ops])
+
+# %% Lets try creating a generic function for computing the same thing:
+def gabor(n_values=32, sigma=1.0, mean=0.0):
+    x = tf.linspace(-3.0, 3.0, n_values)
+    z = (tf.exp(tf.neg(tf.pow(x - mean, 2.0) /
+                       (2.0 * tf.pow(sigma, 2.0)))) *
+         (1.0 / (sigma * tf.sqrt(2.0 * 3.1415))))
+    gauss_kernel = tf.matmul(
+        tf.reshape(z, [n_values, 1]), tf.reshape(z, [1, n_values]))
+    x = tf.reshape(tf.sin(tf.linspace(-3.0, 3.0, n_values)), [n_values, 1])
+    y = tf.reshape(tf.ones_like(x), [1, n_values])
+    gabor_kernel = tf.mul(tf.matmul(x, y), gauss_kernel)
+    return gabor_kernel
+
+# %% Confirm this does something:
+plt.imshow(gabor().eval())
+
+# %% And another function which can convolve
+def convolve(img, W):
+    # The W matrix is only 2D
+    # But conv2d will need a tensor which is 4d:
+    # height x width x n_input x n_output
+    if len(W.get_shape()) == 2:
+        dims = W.get_shape().as_list() + [1, 1]
+        W = tf.reshape(W, dims)
+
+    if len(img.get_shape()) == 2:
+        # num x height x width x channels
+        dims = [1] + img.get_shape().as_list() + [1]
+        img = tf.reshape(img, dims)
+    elif len(img.get_shape()) == 3:
+        dims = [1] + img.get_shape().as_list()
+        img = tf.reshape(img, dims)
+        # if the image is 3 channels, then our convolution
+        # kernel needs to be repeated for each input channel
+        W = tf.concat(2, [W, W, W])
+
+    # Stride is how many values to skip for the dimensions of
+    # num, height, width, channels
+    convolved = tf.nn.conv2d(img, W,
+                             strides=[1, 1, 1, 1], padding='SAME')
+    return convolved
+
+# %% Load up an image:
+from skimage import data
+img = data.astronaut()
+plt.imshow(img)
+print(img.shape)
+
+# %% Now create a placeholder for our graph which can store any input:
+x = tf.placeholder(tf.float32, shape=img.shape)
+
+# %% And a graph which can convolve our image with a gabor
+out = convolve(x, gabor())
+
+# %% Now send the image into the graph and compute the result
+result = tf.squeeze(out).eval(feed_dict={x: img})
+plt.imshow(result)

2-linear_regression.py

@@ -0,0 +1,79 @@
+"""Simple tutorial for using TensorFlow to compute a linear regression.
+
+Parag K. Mital, Jan. 2016"""
+# %% imports
+import numpy as np
+import tensorflow as tf
+import matplotlib.pyplot as plt
+
+
+# %% Let's create some toy data
+plt.ion()
+n_observations = 100
+fig, ax = plt.subplots(1, 1)
+xs = np.linspace(-3, 3, n_observations)
+ys = np.sin(xs) + np.random.uniform(-0.5, 0.5, n_observations)
+ax.scatter(xs, ys)
+fig.show()
+plt.draw()
+
+# %% tf.placeholders for the input and output of the network. Placeholders are
+# variables which we need to fill in when we are ready to compute the graph.
+X = tf.placeholder(tf.float32)
+Y = tf.placeholder(tf.float32)
+
+# %% We will try to optimize min_(W,b) ||(X*w + b) - y||^2
+# The `Variable()` constructor requires an initial value for the variable,
+# which can be a `Tensor` of any type and shape. The initial value defines the
+# type and shape of the variable. After construction, the type and shape of
+# the variable are fixed. The value can be changed using one of the assign
+# methods.
+W = tf.Variable(tf.random_normal([1]), name='weight')
+b = tf.Variable(tf.random_normal([1]), name='bias')
+Y_pred = tf.add(tf.mul(X, W), b)
+
+# %% Loss function will measure the distance between our observations
+# and predictions and average over them.
+cost = tf.reduce_sum(tf.pow(Y_pred - Y, 2)) / (n_observations - 1)
+
+# %% if we wanted to add regularization, we could add other terms to the cost,
+# e.g. ridge regression has a parameter controlling the amount of shrinkage
+# over the norm of activations. the larger the shrinkage, the more robust
+# to collinearity.
+# cost = tf.add(cost, tf.mul(1e-6, tf.global_norm([W])))
+
+# %% Use gradient descent to optimize W,b
+# Performs a single step in the negative gradient
+learning_rate = 0.01
+optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
+
+# %% We create a session to use the graph
+n_epochs = 1000
+with tf.Session() as sess:
+    # Here we tell tensorflow that we want to initialize all
+    # the variables in the graph so we can use them
+    sess.run(tf.initialize_all_variables())
+
+    # Fit all training data
+    prev_training_cost = 0.0
+    for epoch_i in range(n_epochs):
+        for (x, y) in zip(xs, ys):
+            sess.run(optimizer, feed_dict={X: x, Y: y})
+
+        training_cost = sess.run(
+            cost, feed_dict={X: xs, Y: ys})
+        print(training_cost)
+
+        if epoch_i % 20 == 0:
+            ax.plot(xs, Y_pred.eval(
+                feed_dict={X: xs}, session=sess),
+                    'k', alpha=epoch_i / n_epochs)
+            fig.show()
+            plt.draw()
+
+        # Allow the training to quit if we've reached a minimum
+        if np.abs(prev_training_cost - training_cost) < 0.000001:
+            break
+        prev_training_cost = training_cost
+fig.show()
+plt.waitforbuttonpress()

3-polynomial_regression.py

@@ -0,0 +1,78 @@
+"""Simple tutorial for using TensorFlow to compute polynomial regression.
+
+Parag K. Mital, Jan. 2016"""
+# %% Imports
+import numpy as np
+import tensorflow as tf
+import matplotlib.pyplot as plt
+
+
+# %% Let's create some toy data
+plt.ion()
+n_observations = 100
+fig, ax = plt.subplots(1, 1)
+xs = np.linspace(-3, 3, n_observations)
+ys = np.sin(xs) + np.random.uniform(-0.5, 0.5, n_observations)
+ax.scatter(xs, ys)
+fig.show()
+plt.draw()
+
+# %% tf.placeholders for the input and output of the network. Placeholders are
+# variables which we need to fill in when we are ready to compute the graph.
+X = tf.placeholder(tf.float32)
+Y = tf.placeholder(tf.float32)
+
+# %% Instead of a single factor and a bias, we'll create a polynomial function
+# of different polynomial degrees.  We will then learn the influence that each
+# degree of the input (X^0, X^1, X^2, ...) has on the final output (Y).
+Y_pred = tf.Variable(tf.random_normal([1]), name='bias')
+for pow_i in range(1, 5):
+    W = tf.Variable(tf.random_normal([1]), name='weight_%d' % pow_i)
+    Y_pred = tf.add(tf.mul(tf.pow(X, pow_i), W), Y_pred)
+
+# %% Loss function will measure the distance between our observations
+# and predictions and average over them.
+cost = tf.reduce_sum(tf.pow(Y_pred - Y, 2)) / (n_observations - 1)
+
+# %% if we wanted to add regularization, we could add other terms to the cost,
+# e.g. ridge regression has a parameter controlling the amount of shrinkage
+# over the norm of activations. the larger the shrinkage, the more robust
+# to collinearity.
+# cost = tf.add(cost, tf.mul(1e-6, tf.global_norm([W])))
+
+# %% Use gradient descent to optimize W,b
+# Performs a single step in the negative gradient
+learning_rate = 0.01
+optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
+
+# %% We create a session to use the graph
+n_epochs = 1000
+with tf.Session() as sess:
+    # Here we tell tensorflow that we want to initialize all
+    # the variables in the graph so we can use them
+    sess.run(tf.initialize_all_variables())
+
+    # Fit all training data
+    prev_training_cost = 0.0
+    for epoch_i in range(n_epochs):
+        for (x, y) in zip(xs, ys):
+            sess.run(optimizer, feed_dict={X: x, Y: y})
+
+        training_cost = sess.run(
+            cost, feed_dict={X: xs, Y: ys})
+        print(training_cost)
+
+        if epoch_i % 100 == 0:
+            ax.plot(xs, Y_pred.eval(
+                feed_dict={X: xs}, session=sess),
+                    'k', alpha=epoch_i / n_epochs)
+            fig.show()
+            plt.draw()
+
+        # Allow the training to quit if we've reached a minimum
+        if np.abs(prev_training_cost - training_cost) < 0.000001:
+            break
+        prev_training_cost = training_cost
+ax.set_ylim([-3, 3])
+fig.show()
+plt.waitforbuttonpress()