Split metrics into two - one for performance, others for speed/memory

RJT1990 · RJT1990 · commit 01132da6fcbc · 2019-10-06T13:15:51.000+01:00
diff --git a/torchbench/image_classification/cifar10.py b/torchbench/image_classification/cifar10.py
@@ -64,7 +64,7 @@ def benchmark(
             num_workers=num_workers,
             pin_memory=True,
         )
-        test_results, run_hash = evaluate_classification(
+        test_results, speed_mem_metrics, run_hash = evaluate_classification(
             model=model,
             test_loader=test_loader,
             model_output_transform=model_output_transform,
@@ -84,6 +84,7 @@ def benchmark(
             config=config,
             dataset=cls.dataset.__name__,
             results=test_results,
+            speed_mem_metrics=speed_mem_metrics,
             pytorch_hub_id=pytorch_hub_url,
             model=paper_model_name,
             model_description=model_description,
diff --git a/torchbench/image_classification/cifar100.py b/torchbench/image_classification/cifar100.py
@@ -64,7 +64,7 @@ def benchmark(
             num_workers=num_workers,
             pin_memory=True,
         )
-        test_results, run_hash = evaluate_classification(
+        test_results, speed_mem_metrics, run_hash = evaluate_classification(
             model=model,
             test_loader=test_loader,
             model_output_transform=model_output_transform,
@@ -84,6 +84,7 @@ def benchmark(
             config=config,
             dataset=cls.dataset.__name__,
             results=test_results,
+            speed_mem_metrics=speed_mem_metrics,
             pytorch_hub_id=pytorch_hub_url,
             model=paper_model_name,
             model_description=model_description,
diff --git a/torchbench/image_classification/imagenet.py b/torchbench/image_classification/imagenet.py
@@ -200,7 +200,7 @@ def benchmark(
             num_workers=num_workers,
             pin_memory=pin_memory,
         )
-        test_results, run_hash = evaluate_classification(
+        test_results, speed_mem_metrics, run_hash = evaluate_classification(
             model=model,
             test_loader=test_loader,
             model_output_transform=model_output_transform,
@@ -220,6 +220,7 @@ def benchmark(
             config=config,
             dataset=cls.dataset.__name__,
             results=test_results,
+            speed_mem_metrics=speed_mem_metrics,
             pytorch_hub_id=pytorch_hub_url,
             model=paper_model_name,
             model_description=model_description,
diff --git a/torchbench/image_classification/mnist.py b/torchbench/image_classification/mnist.py
@@ -63,7 +63,7 @@ def benchmark(
             num_workers=num_workers,
             pin_memory=True,
         )
-        test_results, run_hash = evaluate_classification(
+        test_results, speed_mem_metrics, run_hash = evaluate_classification(
             model=model,
             test_loader=test_loader,
             model_output_transform=model_output_transform,
@@ -83,6 +83,7 @@ def benchmark(
             config=config,
             dataset=cls.dataset.__name__,
             results=test_results,
+            speed_mem_metrics=speed_mem_metrics,
             pytorch_hub_id=pytorch_hub_url,
             model=paper_model_name,
             model_description=model_description,
diff --git a/torchbench/image_classification/stl10.py b/torchbench/image_classification/stl10.py
@@ -63,7 +63,7 @@ def benchmark(
             num_workers=num_workers,
             pin_memory=True,
         )
-        test_results, run_hash = evaluate_classification(
+        test_results, speed_mem_metrics, run_hash = evaluate_classification(
             model=model,
             test_loader=test_loader,
             model_output_transform=model_output_transform,
@@ -83,6 +83,7 @@ def benchmark(
             config=config,
             dataset=cls.dataset.__name__,
             results=test_results,
+            speed_mem_metrics=speed_mem_metrics,
             pytorch_hub_id=pytorch_hub_url,
             model=paper_model_name,
             model_description=model_description,
diff --git a/torchbench/image_classification/svhn.py b/torchbench/image_classification/svhn.py
@@ -66,7 +66,7 @@ def benchmark(
             num_workers=num_workers,
             pin_memory=True,
         )
-        test_results, run_hash = evaluate_classification(
+        test_results, speed_mem_metrics, run_hash = evaluate_classification(
             model=model,
             test_loader=test_loader,
             model_output_transform=model_output_transform,
@@ -86,6 +86,7 @@ def benchmark(
             config=config,
             dataset=cls.dataset.__name__,
             results=test_results,
+            speed_mem_metrics=speed_mem_metrics,
             pytorch_hub_id=pytorch_hub_url,
             model=paper_model_name,
             model_description=model_description,
diff --git a/torchbench/image_classification/utils.py b/torchbench/image_classification/utils.py
@@ -62,11 +62,14 @@ def evaluate_classification(
 
             end = time.time()
 
+    speed_mem_metrics = {
+        'Tasks Per Second': test_loader.batch_size/inference_time.avg,
+        'Memory Allocated': memory_allocated
+    }
+
     return (
         {"Top 1 Accuracy": top1.avg / 100,
-         "Top 5 Accuracy": top5.avg / 100,
-         'Tasks Per Second': test_loader.batch_size/inference_time.avg,
-         'Memory Allocated': memory_allocated},
+         "Top 5 Accuracy": top5.avg / 100}, speed_mem_metrics,
         run_hash,
     )
 
diff --git a/torchbench/object_detection/coco.py b/torchbench/object_detection/coco.py
@@ -217,7 +217,7 @@ def benchmark(
             collate_fn=collate_fn,
         )
         test_loader.no_classes = 91  # Number of classes for COCO Detection
-        test_results, run_hash = evaluate_detection_coco(
+        test_results, speed_mem_metrics, run_hash = evaluate_detection_coco(
             model=model,
             test_loader=test_loader,
             model_output_transform=model_output_transform,
@@ -232,6 +232,7 @@ def benchmark(
             config=config,
             dataset='COCO minival',
             results=test_results,
+            speed_mem_metrics=speed_mem_metrics,
             pytorch_hub_id=pytorch_hub_url,
             model=paper_model_name,
             model_description=model_description,
diff --git a/torchbench/object_detection/utils.py b/torchbench/object_detection/utils.py
@@ -249,7 +249,7 @@ def evaluate_detection_coco(
         'Tasks Per Second': test_loader.batch_size/inference_time.avg,
         'Memory Allocated': memory_allocated}
 
-    return ({**get_coco_metrics(coco_evaluator), **device_metrics}, run_hash)
+    return (get_coco_metrics(coco_evaluator), device_metrics, run_hash)
 
 
 def evaluate_detection_voc(
diff --git a/torchbench/semantic_segmentation/ade20k.py b/torchbench/semantic_segmentation/ade20k.py
@@ -85,7 +85,7 @@ def benchmark(
             collate_fn=collate_fn,
         )
         test_loader.no_classes = 150  # Number of classes for ADE20K
-        test_results, run_hash = evaluate_segmentation(
+        test_results, speed_mem_metrics, run_hash = evaluate_segmentation(
             model=model,
             test_loader=test_loader,
             model_output_transform=model_output_transform,
@@ -100,6 +100,7 @@ def benchmark(
             config=config,
             dataset=cls.dataset.__name__ + " val",
             results=test_results,
+            speed_mem_metrics=speed_mem_metrics,
             pytorch_hub_id=pytorch_hub_url,
             model=paper_model_name,
             model_description=model_description,
diff --git a/torchbench/semantic_segmentation/camvid.py b/torchbench/semantic_segmentation/camvid.py
@@ -85,7 +85,7 @@ def benchmark(
             collate_fn=collate_fn,
         )
         test_loader.no_classes = 12  # Number of classes for CamVid
-        test_results = evaluate_segmentation(
+        test_results, speed_mem_metrics, run_hash = evaluate_segmentation(
             model=model,
             test_loader=test_loader,
             model_output_transform=model_output_transform,
@@ -100,6 +100,7 @@ def benchmark(
             config=config,
             dataset=cls.dataset.__name__,
             results=test_results,
+            speed_mem_metrics=speed_mem_metrics,
             pytorch_hub_id=pytorch_hub_url,
             model=paper_model_name,
             model_description=model_description,
diff --git a/torchbench/semantic_segmentation/cityscapes.py b/torchbench/semantic_segmentation/cityscapes.py
@@ -88,7 +88,7 @@ def benchmark(
             collate_fn=collate_fn,
         )
         test_loader.no_classes = 19  # Number of classes for Cityscapes
-        test_results = evaluate_segmentation(
+        test_results, speed_mem_metrics, run_hash = evaluate_segmentation(
             model=model,
             test_loader=test_loader,
             model_output_transform=model_output_transform,
@@ -103,6 +103,7 @@ def benchmark(
             config=config,
             dataset=cls.dataset.__name__,
             results=test_results,
+            speed_mem_metrics=speed_mem_metrics,
             pytorch_hub_id=pytorch_hub_url,
             model=paper_model_name,
             model_description=model_description,
diff --git a/torchbench/semantic_segmentation/pascalcontext.py b/torchbench/semantic_segmentation/pascalcontext.py
@@ -85,7 +85,7 @@ def benchmark(
             collate_fn=collate_fn,
         )
         test_loader.no_classes = 59  # Number of classes for PASCALContext
-        test_results = evaluate_segmentation(
+        test_results, speed_mem_metrics, run_hash = evaluate_segmentation(
             model=model,
             test_loader=test_loader,
             model_output_transform=model_output_transform,
@@ -100,6 +100,7 @@ def benchmark(
             config=config,
             dataset=cls.dataset.__name__,
             results=test_results,
+            speed_mem_metrics=speed_mem_metrics,
             pytorch_hub_id=pytorch_hub_url,
             model=paper_model_name,
             model_description=model_description,
diff --git a/torchbench/semantic_segmentation/pascalvoc.py b/torchbench/semantic_segmentation/pascalvoc.py
@@ -130,7 +130,7 @@ def benchmark(
             collate_fn=collate_fn,
         )
         test_loader.no_classes = 21  # Number of classes for PASCALVOC
-        test_results, run_hash = evaluate_segmentation(
+        test_results, speed_mem_metrics, run_hash = evaluate_segmentation(
             model=model,
             test_loader=test_loader,
             model_output_transform=model_output_transform,
@@ -144,6 +144,7 @@ def benchmark(
             config=config,
             dataset='PASCAL VOC %s %s' % (dataset_year, "val"),
             results=test_results,
+            speed_mem_metrics=speed_mem_metrics,
             pytorch_hub_id=pytorch_hub_url,
             model=paper_model_name,
             model_description=model_description,
diff --git a/torchbench/semantic_segmentation/utils.py b/torchbench/semantic_segmentation/utils.py
@@ -182,8 +182,11 @@ def evaluate_segmentation(
 
     acc_global, acc, iu = confmat.compute()
 
+    speed_mem_metrics = {
+        'Tasks Per Second': test_loader.batch_size / inference_time.avg,
+        'Memory Allocated': memory_allocated}
+
     return {
         "Accuracy": acc_global.item(),
-        "Mean IOU": iu.mean().item(),
-        'Tasks Per Second': test_loader.batch_size / inference_time.avg,
-        'Memory Allocated': memory_allocated}, run_hash
+        "Mean IOU": iu.mean().item()}, \
+           speed_mem_metrics, run_hash
