Merge pull request tensorflow#40 from jmchen-g/master

update inception slim.

Author: jmchen-g

inception/inception/slim/BUILD

@@ -101,3 +101,12 @@ py_library(
         ":variables",
     ],
 )
+
+py_test(
+    name = "collections_test",
+    size = "small",
+    srcs = ["collections_test.py"],
+    deps = [
+        ":slim",
+    ],
+)

inception/inception/slim/README.md

@@ -43,7 +43,6 @@
 from __future__ import division
 from __future__ import print_function
 
-
 import tensorflow as tf
 
 from inception.slim import ops
@@ -98,10 +97,10 @@ def inception_v3(inputs,
         # 73 x 73 x 64
         end_points['conv3'] = ops.conv2d(end_points['pool1'], 80, [1, 1],
                                          scope='conv3')
-        # 71 x 71 x 80.
+        # 73 x 73 x 80.
         end_points['conv4'] = ops.conv2d(end_points['conv3'], 192, [3, 3],
                                          scope='conv4')
-        # 69 x 69 x 192.
+        # 71 x 71 x 192.
         end_points['pool2'] = ops.max_pool(end_points['conv4'], [3, 3],
                                            stride=2, scope='pool2')
         # 35 x 35 x 192.
@@ -260,7 +259,10 @@ def inception_v3(inputs,
           aux_logits = ops.fc(aux_logits, num_classes, activation=None,
                               stddev=0.001, restore=restore_logits)
           end_points['aux_logits'] = aux_logits
-        # mixed_8: 17 x 17 x 1280.
+        # mixed_8: 8 x 8 x 1280.
+        # Note that the scope below is not changed to not void previous
+        # checkpoints.
+        # (TODO) Fix the scope when appropriate.
         with tf.variable_scope('mixed_17x17x1280a'):
           with tf.variable_scope('branch3x3'):
             branch3x3 = ops.conv2d(net, 192, [1, 1])
@@ -327,3 +329,28 @@ def inception_v3(inputs,
           end_points['predictions'] = tf.nn.softmax(logits, name='predictions')
       return logits, end_points
 
+
+def inception_v3_parameters(weight_decay=0.00004, stddev=0.1,
+                            batch_norm_decay=0.9997, batch_norm_epsilon=0.001):
+  """Yields the scope with the default parameters for inception_v3.
+
+  Args:
+    weight_decay: the weight decay for weights variables.
+    stddev: standard deviation of the truncated guassian weight distribution.
+    batch_norm_decay: decay for the moving average of batch_norm momentums.
+    batch_norm_epsilon: small float added to variance to avoid dividing by zero.
+
+  Yields:
+    a arg_scope with the parameters needed for inception_v3.
+  """
+  # Set weight_decay for weights in Conv and FC layers.
+  with scopes.arg_scope([ops.conv2d, ops.fc],
+                        weight_decay=weight_decay):
+    # Set stddev, activation and parameters for batch_norm.
+    with scopes.arg_scope([ops.conv2d],
+                          stddev=stddev,
+                          activation=tf.nn.relu,
+                          batch_norm_params={
+                              'decay': batch_norm_decay,
+                              'epsilon': batch_norm_epsilon}) as arg_scope:
+      yield arg_scope

inception/inception/slim/inception_model.py

@@ -17,7 +17,6 @@
 from __future__ import division
 from __future__ import print_function
 
-
 import tensorflow as tf
 
 from inception.slim import inception_model as inception
@@ -55,6 +54,22 @@ def testBuildEndPoints(self):
       self.assertListEqual(pre_pool.get_shape().as_list(),
                            [batch_size, 8, 8, 2048])
 
+  def testVariablesSetDevice(self):
+    batch_size = 5
+    height, width = 299, 299
+    num_classes = 1000
+    with self.test_session():
+      inputs = tf.random_uniform((batch_size, height, width, 3))
+      # Force all Variables to reside on the device.
+      with tf.variable_scope('on_cpu'), tf.device('/cpu:0'):
+        inception.inception_v3(inputs, num_classes)
+      with tf.variable_scope('on_gpu'), tf.device('/gpu:0'):
+        inception.inception_v3(inputs, num_classes)
+      for v in tf.get_collection(tf.GraphKeys.VARIABLES, scope='on_cpu'):
+        self.assertDeviceEqual(v.device, '/cpu:0')
+      for v in tf.get_collection(tf.GraphKeys.VARIABLES, scope='on_gpu'):
+        self.assertDeviceEqual(v.device, '/gpu:0')
+
   def testHalfSizeImages(self):
     batch_size = 5
     height, width = 150, 150

inception/inception/slim/inception_test.py

@@ -26,7 +26,6 @@
 from __future__ import division
 from __future__ import print_function
 
-
 import tensorflow as tf
 
 # In order to gather all losses in a network, the user should use this
@@ -35,6 +34,71 @@
 LOSSES_COLLECTION = '_losses'
 
 
+def l1_regularizer(weight=1.0, scope=None):
+  """Define a L1 regularizer.
+
+  Args:
+    weight: scale the loss by this factor.
+    scope: Optional scope for op_scope.
+
+  Returns:
+    a regularizer function.
+  """
+  def regularizer(tensor):
+    with tf.op_scope([tensor], scope, 'L1Regularizer'):
+      l1_weight = tf.convert_to_tensor(weight,
+                                       dtype=tensor.dtype.base_dtype,
+                                       name='weight')
+      return tf.mul(l1_weight, tf.reduce_sum(tf.abs(tensor)), name='value')
+  return regularizer
+
+
+def l2_regularizer(weight=1.0, scope=None):
+  """Define a L2 regularizer.
+
+  Args:
+    weight: scale the loss by this factor.
+    scope: Optional scope for op_scope.
+
+  Returns:
+    a regularizer function.
+  """
+  def regularizer(tensor):
+    with tf.op_scope([tensor], scope, 'L2Regularizer'):
+      l2_weight = tf.convert_to_tensor(weight,
+                                       dtype=tensor.dtype.base_dtype,
+                                       name='weight')
+      return tf.mul(l2_weight, tf.nn.l2_loss(tensor), name='value')
+  return regularizer
+
+
+def l1_l2_regularizer(weight_l1=1.0, weight_l2=1.0, scope=None):
+  """Define a L1L2 regularizer.
+
+  Args:
+    weight_l1: scale the L1 loss by this factor.
+    weight_l2: scale the L2 loss by this factor.
+    scope: Optional scope for op_scope.
+
+  Returns:
+    a regularizer function.
+  """
+  def regularizer(tensor):
+    with tf.op_scope([tensor], scope, 'L1L2Regularizer'):
+      weight_l1_t = tf.convert_to_tensor(weight_l1,
+                                         dtype=tensor.dtype.base_dtype,
+                                         name='weight_l1')
+      weight_l2_t = tf.convert_to_tensor(weight_l2,
+                                         dtype=tensor.dtype.base_dtype,
+                                         name='weight_l2')
+      reg_l1 = tf.mul(weight_l1_t, tf.reduce_sum(tf.abs(tensor)),
+                      name='value_l1')
+      reg_l2 = tf.mul(weight_l2_t, tf.nn.l2_loss(tensor),
+                      name='value_l2')
+      return tf.add(reg_l1, reg_l2, name='value')
+  return regularizer
+
+
 def l1_loss(tensor, weight=1.0, scope=None):
   """Define a L1Loss, useful for regularize, i.e. lasso.
 

inception/inception/slim/losses.py

@@ -18,7 +18,6 @@
 from __future__ import print_function
 
 
-
 import tensorflow as tf
 
 from inception.slim import losses
@@ -47,6 +46,95 @@ def testL2Loss(self):
       self.assertAlmostEqual(loss.eval(), num_elem * wd / 2, 5)
 
 
+class RegularizersTest(tf.test.TestCase):
+
+  def testL1Regularizer(self):
+    with self.test_session():
+      shape = [5, 5, 5]
+      num_elem = 5 * 5 * 5
+      tensor = tf.constant(1.0, shape=shape)
+      loss = losses.l1_regularizer()(tensor)
+      self.assertEquals(loss.op.name, 'L1Regularizer/value')
+      self.assertAlmostEqual(loss.eval(), num_elem, 5)
+
+  def testL1RegularizerWithScope(self):
+    with self.test_session():
+      shape = [5, 5, 5]
+      num_elem = 5 * 5 * 5
+      tensor = tf.constant(1.0, shape=shape)
+      loss = losses.l1_regularizer(scope='L1')(tensor)
+      self.assertEquals(loss.op.name, 'L1/value')
+      self.assertAlmostEqual(loss.eval(), num_elem, 5)
+
+  def testL1RegularizerWithWeight(self):
+    with self.test_session():
+      shape = [5, 5, 5]
+      num_elem = 5 * 5 * 5
+      tensor = tf.constant(1.0, shape=shape)
+      weight = 0.01
+      loss = losses.l1_regularizer(weight)(tensor)
+      self.assertEquals(loss.op.name, 'L1Regularizer/value')
+      self.assertAlmostEqual(loss.eval(), num_elem * weight, 5)
+
+  def testL2Regularizer(self):
+    with self.test_session():
+      shape = [5, 5, 5]
+      num_elem = 5 * 5 * 5
+      tensor = tf.constant(1.0, shape=shape)
+      loss = losses.l2_regularizer()(tensor)
+      self.assertEquals(loss.op.name, 'L2Regularizer/value')
+      self.assertAlmostEqual(loss.eval(), num_elem / 2, 5)
+
+  def testL2RegularizerWithScope(self):
+    with self.test_session():
+      shape = [5, 5, 5]
+      num_elem = 5 * 5 * 5
+      tensor = tf.constant(1.0, shape=shape)
+      loss = losses.l2_regularizer(scope='L2')(tensor)
+      self.assertEquals(loss.op.name, 'L2/value')
+      self.assertAlmostEqual(loss.eval(), num_elem / 2, 5)
+
+  def testL2RegularizerWithWeight(self):
+    with self.test_session():
+      shape = [5, 5, 5]
+      num_elem = 5 * 5 * 5
+      tensor = tf.constant(1.0, shape=shape)
+      weight = 0.01
+      loss = losses.l2_regularizer(weight)(tensor)
+      self.assertEquals(loss.op.name, 'L2Regularizer/value')
+      self.assertAlmostEqual(loss.eval(), num_elem * weight / 2, 5)
+
+  def testL1L2Regularizer(self):
+    with self.test_session():
+      shape = [5, 5, 5]
+      num_elem = 5 * 5 * 5
+      tensor = tf.constant(1.0, shape=shape)
+      loss = losses.l1_l2_regularizer()(tensor)
+      self.assertEquals(loss.op.name, 'L1L2Regularizer/value')
+      self.assertAlmostEqual(loss.eval(), num_elem + num_elem / 2, 5)
+
+  def testL1L2RegularizerWithScope(self):
+    with self.test_session():
+      shape = [5, 5, 5]
+      num_elem = 5 * 5 * 5
+      tensor = tf.constant(1.0, shape=shape)
+      loss = losses.l1_l2_regularizer(scope='L1L2')(tensor)
+      self.assertEquals(loss.op.name, 'L1L2/value')
+      self.assertAlmostEqual(loss.eval(), num_elem + num_elem / 2, 5)
+
+  def testL1L2RegularizerWithWeights(self):
+    with self.test_session():
+      shape = [5, 5, 5]
+      num_elem = 5 * 5 * 5
+      tensor = tf.constant(1.0, shape=shape)
+      weight_l1 = 0.01
+      weight_l2 = 0.05
+      loss = losses.l1_l2_regularizer(weight_l1, weight_l2)(tensor)
+      self.assertEquals(loss.op.name, 'L1L2Regularizer/value')
+      self.assertAlmostEqual(loss.eval(),
+                             num_elem * weight_l1 + num_elem * weight_l2 / 2, 5)
+
+
 class CrossEntropyLossTest(tf.test.TestCase):
 
   def testCrossEntropyLossAllCorrect(self):