Update sample model submission for new submission processing

examples/keras.py

@@ -2,7 +2,7 @@
 import keras.applications
 from test import test_models
 
-# This is an example implementation for submitting resnet50 as a keras model to brain-score
+# This is an example implementation for submitting vgg-16 as a keras model to brain-score
 # If you use keras, don't forget to add it and its dependencies to the setup.py
 
 
@@ -13,7 +13,7 @@ def get_model_list():
 def get_model(name):
     assert name == 'vgg-16'
     model = keras.applications.vgg16.VGG16()
-    model_preprocessing = keras.applications.resnet50.preprocess_input
+    model_preprocessing = keras.applications.vgg16.preprocess_input
     load_preprocess = lambda image_filepaths: model_preprocessing(load_images(image_filepaths, image_size=224))
     wrapper = KerasWrapper(model, load_preprocess)
     wrapper.image_size = 224
@@ -25,6 +25,16 @@ def get_layers(name):
     return [f'block{i + 1}_pool' for i in range(5)] + ['fc1', 'fc2']
 
 
+def get_bibtex(model_identifier):
+    assert model_identifier == 'vgg-16'
+    return """@InProceedings{Simonyan15,
+              author       = "Karen Simonyan and Andrew Zisserman",
+              title        = "Very Deep Convolutional Networks for Large-Scale Image Recognition",
+              booktitle    = "International Conference on Learning Representations",
+              year         = "2015",
+              url          = "https://arxiv.org/abs/1409.1556", 
+            }"""
+
 if __name__ == '__main__':
     test_models.test_base_models(__name__)
 

examples/pytorch.py

@@ -4,12 +4,15 @@
 from model_tools.activations.pytorch import PytorchWrapper
 from model_tools.activations.pytorch import load_preprocess_images
 
-from test import test_models
-
-
 # This is an example implementation for submitting alexnet as a pytorch model
 # If you use pytorch, don't forget to add it to the setup.py
 
+# Attention: It is important, that the wrapper identifier is unique per model!
+# The results will otherwise be the same due to brain-scores internal result caching mechanism.
+# Please load your pytorch model for usage in CPU. There won't be GPUs available for scoring your model.
+# If the model requires a GPU, contact the brain-score team directly.
+from model_tools.check_submission import check_models
+
 
 def get_model_list():
     return ['alexnet']
@@ -30,5 +33,18 @@ def get_layers(name):
             'classifier.2', 'classifier.5']
 
 
+def get_bibtex(model_identifier):
+    return """@incollection{NIPS2012_4824,
+                title = {ImageNet Classification with Deep Convolutional Neural Networks},
+                author = {Alex Krizhevsky and Sutskever, Ilya and Hinton, Geoffrey E},
+                booktitle = {Advances in Neural Information Processing Systems 25},
+                editor = {F. Pereira and C. J. C. Burges and L. Bottou and K. Q. Weinberger},
+                pages = {1097--1105},
+                year = {2012},
+                publisher = {Curran Associates, Inc.},
+                url = {http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf}
+                }"""
+
+
 if __name__ == '__main__':
-    test_models.test_base_models(__name__)
+    check_models.check_base_models(__name__)

examples/tensorflow.py

@@ -0,0 +1,109 @@
+import logging
+import os
+
+import s3
+
+from model_tools.activations import TensorflowWrapper, TensorflowSlimWrapper
+from model_tools.activations.keras import load_images, KerasWrapper
+import keras.applications
+
+
+# This is an example implementation for submitting resnet50 as a tensorflow SLIM model to brain-score
+# If you use tensorflow, don't forget to add it and its dependencies to the setup.py
+from model_tools.utils import fullname
+
+
+def get_model_list():
+    return ['resnet50']
+
+
+def get_model(name):
+    assert name == 'resnet50'
+    model = TFSlimModel.init('resnet-50_v1', net_name='resnet_v1_50', preprocessing_type='vgg',
+                     image_size=224, labels_offset=0)
+    model_preprocessing = keras.applications.resnet50.preprocess_input
+    load_preprocess = lambda image_filepaths: model_preprocessing(load_images(image_filepaths, image_size=224))
+    wrapper = KerasWrapper(model, load_preprocess)
+    wrapper.image_size = 224
+    return wrapper
+
+
+def get_layers(name):
+    assert name == 'resnet-50'
+    return [f'block{i + 1}_pool' for i in range(5)] + ['fc1', 'fc2']
+
+
+def get_bibtex(model_identifier):
+    return """@article{DBLP:journals/corr/HeZRS15,
+              author    = {Kaiming He and
+                           Xiangyu Zhang and
+                           Shaoqing Ren and
+                           Jian Sun},
+              title     = {Deep Residual Learning for Image Recognition},
+              journal   = {CoRR},
+              volume    = {abs/1512.03385},
+              year      = {2015},
+              url       = {http://arxiv.org/abs/1512.03385},
+              archivePrefix = {arXiv},
+              eprint    = {1512.03385},
+              timestamp = {Wed, 17 Apr 2019 17:23:45 +0200},
+              biburl    = {https://dblp.org/rec/journals/corr/HeZRS15.bib},
+              bibsource = {dblp computer science bibliography, https://dblp.org}
+            }"""
+
+
+class TFSlimModel:
+    @staticmethod
+    def init(identifier, preprocessing_type, image_size, net_name=None, labels_offset=1, batch_size=64,
+             model_ctr_kwargs=None):
+        import tensorflow as tf
+        from nets import nets_factory
+
+        tf.compat.v1.reset_default_graph()
+        placeholder = tf.compat.v1.placeholder(dtype=tf.string, shape=[batch_size])
+        preprocess = TFSlimModel._init_preprocessing(placeholder, preprocessing_type, image_size=image_size)
+
+        net_name = net_name or identifier
+        model_ctr = nets_factory.get_network_fn(net_name, num_classes=labels_offset + 1000, is_training=False)
+        logits, endpoints = model_ctr(preprocess, **(model_ctr_kwargs or {}))
+        if 'Logits' in endpoints:  # unify capitalization
+            endpoints['logits'] = endpoints['Logits']
+            del endpoints['Logits']
+
+        session = tf.compat.v1.Session()
+        TFSlimModel._restore_imagenet_weights(identifier, session)
+        wrapper = TensorflowSlimWrapper(identifier=identifier, endpoints=endpoints, inputs=placeholder, session=session,
+                                        batch_size=batch_size, labels_offset=labels_offset)
+        wrapper.image_size = image_size
+        return wrapper
+
+    @staticmethod
+    def _init_preprocessing(placeholder, preprocessing_type, image_size):
+        import tensorflow as tf
+        from preprocessing import vgg_preprocessing, inception_preprocessing
+        from model_tools.activations.tensorflow import load_image
+        preprocessing_types = {
+            'vgg': lambda image: vgg_preprocessing.preprocess_image(
+                image, image_size, image_size, resize_side_min=image_size),
+            'inception': lambda image: inception_preprocessing.preprocess_for_eval(
+                image, image_size, image_size, central_fraction=None)
+        }
+        assert preprocessing_type in preprocessing_types
+        preprocess_image = preprocessing_types[preprocessing_type]
+        preprocess = lambda image_path: preprocess_image(load_image(image_path))
+        preprocess = tf.map_fn(preprocess, placeholder, dtype=tf.float32)
+        return preprocess
+
+    @staticmethod
+    def _restore_imagenet_weights(name, session):
+        import tensorflow as tf
+        var_list = None
+        if name.startswith('mobilenet'):
+            # Restore using exponential moving average since it produces (1.5-2%) higher accuracy according to
+            # https://github.com/tensorflow/models/blob/a6494752575fad4d95e92698dbfb88eb086d8526/research/slim/nets/mobilenet/mobilenet_example.ipynb
+            ema = tf.train.ExponentialMovingAverage(0.999)
+            var_list = ema.variables_to_restore()
+        restorer = tf.compat.v1.train.Saver(var_list)
+
+        restore_path = '' # TODO restore model weights
+        restorer.restore(session, restore_path)

models/base_models.py

@@ -1,9 +1,10 @@
-from test import test_models
+from model_tools.check_submission import check_models
 
 """
 Template module for a base model submission to brain-score
 """
 
+
 def get_model_list():
     """
     This method defines all submitted model names. It returns a list of model names.
@@ -38,5 +39,15 @@ def get_layers(name):
     """
     return []
 
+
+def get_bibtex(model_identifier):
+    """
+    A method returning the bibtex reference of the requested model as a string.
+    """
+    return ''
+
+
 if __name__ == '__main__':
-    test_models.test_base_models(__name__)
+    # Use this method to ensure the correctness of the  BaeeModel implementations.
+    # It executes a mock run of brain-score benchmarks.
+    check_models.check_base_models(__name__)

models/brain_models.py

@@ -1,11 +1,10 @@
-from brainscore.model_interface import BrainModel
-
-from test import test_models
+from model_tools.check_submission import check_models
 
 """
 Template module for a brain model submission to brain-score
 """
 
+
 def get_model_list():
     """
     This method defines all submitted model names. It returns a list of model names.
@@ -16,7 +15,7 @@ def get_model_list():
     return []
 
 
-def get_model(name):
+def get_model(model_identifier):
     """
     This method fetches an instance of a brain model. The instance has to implement the BrainModel interface in the
     brain-score project(see imports). To get a detailed explanation of how the interface hast to be implemented,
@@ -27,5 +26,14 @@ def get_model(name):
     return
 
 
+def get_bibtex(model_identifier):
+    """
+    A method returning the bibtex reference of the requested model as a string.
+    """
+    return ''
+
+
 if __name__ == '__main__':
-    test_models.test_brain_model(__name__)
+    # Use this method to ensure the correctness of the BrainModel implementations.
+    # It executes a mock run of brain-score benchmarks.
+    check_models.check_brain_models(__name__)

setup.py

@@ -9,6 +9,7 @@
 requirements = [
     "model-tools @ git+https://github.com/brain-score/model-tools",
     "numpy",
+    'xarray==0.12',
     "result_caching @ git+https://github.com/mschrimpf/result_caching"
 ]