Skip to content

Commit fd006c6

Browse files
RJT1990RJT1990
authored
Merge pull request #13 from paperswithcode/inference_speed
Inference speed
2 parents 8ea8295 + 01132da commit fd006c6

File tree

16 files changed

+76
-25
lines changed

16 files changed

+76
-25
lines changed

‎torchbench/image_classification/cifar10.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ def benchmark(
6464
num_workers=num_workers,
6565
pin_memory=True,
6666
)
67-
test_results, run_hash = evaluate_classification(
67+
test_results, speed_mem_metrics, run_hash = evaluate_classification(
6868
model=model,
6969
test_loader=test_loader,
7070
model_output_transform=model_output_transform,
@@ -84,6 +84,7 @@ def benchmark(
8484
config=config,
8585
dataset=cls.dataset.__name__,
8686
results=test_results,
87+
speed_mem_metrics=speed_mem_metrics,
8788
pytorch_hub_id=pytorch_hub_url,
8889
model=paper_model_name,
8990
model_description=model_description,

‎torchbench/image_classification/cifar100.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ def benchmark(
6464
num_workers=num_workers,
6565
pin_memory=True,
6666
)
67-
test_results, run_hash = evaluate_classification(
67+
test_results, speed_mem_metrics, run_hash = evaluate_classification(
6868
model=model,
6969
test_loader=test_loader,
7070
model_output_transform=model_output_transform,
@@ -84,6 +84,7 @@ def benchmark(
8484
config=config,
8585
dataset=cls.dataset.__name__,
8686
results=test_results,
87+
speed_mem_metrics=speed_mem_metrics,
8788
pytorch_hub_id=pytorch_hub_url,
8889
model=paper_model_name,
8990
model_description=model_description,

‎torchbench/image_classification/imagenet.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ def benchmark(
200200
num_workers=num_workers,
201201
pin_memory=pin_memory,
202202
)
203-
test_results, run_hash = evaluate_classification(
203+
test_results, speed_mem_metrics, run_hash = evaluate_classification(
204204
model=model,
205205
test_loader=test_loader,
206206
model_output_transform=model_output_transform,
@@ -220,6 +220,7 @@ def benchmark(
220220
config=config,
221221
dataset=cls.dataset.__name__,
222222
results=test_results,
223+
speed_mem_metrics=speed_mem_metrics,
223224
pytorch_hub_id=pytorch_hub_url,
224225
model=paper_model_name,
225226
model_description=model_description,

‎torchbench/image_classification/mnist.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ def benchmark(
6363
num_workers=num_workers,
6464
pin_memory=True,
6565
)
66-
test_results, run_hash = evaluate_classification(
66+
test_results, speed_mem_metrics, run_hash = evaluate_classification(
6767
model=model,
6868
test_loader=test_loader,
6969
model_output_transform=model_output_transform,
@@ -83,6 +83,7 @@ def benchmark(
8383
config=config,
8484
dataset=cls.dataset.__name__,
8585
results=test_results,
86+
speed_mem_metrics=speed_mem_metrics,
8687
pytorch_hub_id=pytorch_hub_url,
8788
model=paper_model_name,
8889
model_description=model_description,

‎torchbench/image_classification/stl10.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ def benchmark(
6363
num_workers=num_workers,
6464
pin_memory=True,
6565
)
66-
test_results, run_hash = evaluate_classification(
66+
test_results, speed_mem_metrics, run_hash = evaluate_classification(
6767
model=model,
6868
test_loader=test_loader,
6969
model_output_transform=model_output_transform,
@@ -83,6 +83,7 @@ def benchmark(
8383
config=config,
8484
dataset=cls.dataset.__name__,
8585
results=test_results,
86+
speed_mem_metrics=speed_mem_metrics,
8687
pytorch_hub_id=pytorch_hub_url,
8788
model=paper_model_name,
8889
model_description=model_description,

‎torchbench/image_classification/svhn.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ def benchmark(
6666
num_workers=num_workers,
6767
pin_memory=True,
6868
)
69-
test_results, run_hash = evaluate_classification(
69+
test_results, speed_mem_metrics, run_hash = evaluate_classification(
7070
model=model,
7171
test_loader=test_loader,
7272
model_output_transform=model_output_transform,
@@ -86,6 +86,7 @@ def benchmark(
8686
config=config,
8787
dataset=cls.dataset.__name__,
8888
results=test_results,
89+
speed_mem_metrics=speed_mem_metrics,
8990
pytorch_hub_id=pytorch_hub_url,
9091
model=paper_model_name,
9192
model_description=model_description,

‎torchbench/image_classification/utils.py‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import time
22

3+
import numpy as np
34
import tqdm
45
import torch
56
import torchvision
@@ -16,31 +17,31 @@ def evaluate_classification(
1617
send_data_to_device,
1718
device="cuda",
1819
):
19-
batch_time = AverageMeter()
20+
inference_time = AverageMeter()
2021
top1 = AverageMeter()
2122
top5 = AverageMeter()
23+
iterator = tqdm.tqdm(test_loader, desc="Evaluation", mininterval=5)
2224

2325
end = time.time()
2426

25-
iterator = tqdm.tqdm(test_loader, desc="Evaluation", mininterval=5)
26-
2727
with torch.no_grad():
2828
for i, (input, target) in enumerate(iterator):
2929

3030
input, target = send_data_to_device(input, target, device=device)
3131
output = model(input)
3232

33+
inference_time.update(time.time() - end)
34+
3335
if model_output_transform is not None:
3436
output = model_output_transform(output, target, model=model)
3537

3638
check_metric_inputs(output, target, test_loader.dataset, i)
3739
prec1, prec5 = accuracy(output, target, topk=(1, 5))
3840
top1.update(prec1.item(), input.size(0))
3941
top5.update(prec5.item(), input.size(0))
40-
batch_time.update(time.time() - end)
41-
end = time.time()
4242

4343
if i == 0: # for sotabench.com caching of evaluation
44+
memory_allocated = torch.cuda.memory_allocated(device=device)
4445
run_hash = calculate_run_hash([prec1, prec5], output)
4546
# if we are in check model we don't need to go beyond the first
4647
# batch
@@ -59,8 +60,16 @@ def evaluate_classification(
5960
)
6061
return cached_res, run_hash
6162

63+
end = time.time()
64+
65+
speed_mem_metrics = {
66+
'Tasks Per Second': test_loader.batch_size/inference_time.avg,
67+
'Memory Allocated': memory_allocated
68+
}
69+
6270
return (
63-
{"Top 1 Accuracy": top1.avg / 100, "Top 5 Accuracy": top5.avg / 100},
71+
{"Top 1 Accuracy": top1.avg / 100,
72+
"Top 5 Accuracy": top5.avg / 100}, speed_mem_metrics,
6473
run_hash,
6574
)
6675

‎torchbench/object_detection/coco.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ def benchmark(
217217
collate_fn=collate_fn,
218218
)
219219
test_loader.no_classes = 91 # Number of classes for COCO Detection
220-
test_results, run_hash = evaluate_detection_coco(
220+
test_results, speed_mem_metrics, run_hash = evaluate_detection_coco(
221221
model=model,
222222
test_loader=test_loader,
223223
model_output_transform=model_output_transform,
@@ -232,6 +232,7 @@ def benchmark(
232232
config=config,
233233
dataset='COCO minival',
234234
results=test_results,
235+
speed_mem_metrics=speed_mem_metrics,
235236
pytorch_hub_id=pytorch_hub_url,
236237
model=paper_model_name,
237238
model_description=model_description,

‎torchbench/object_detection/utils.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,12 @@
66
import torchvision
77
from sotabenchapi.check import in_check_mode
88
from sotabenchapi.client import Client
9+
import time
910

1011
from pycocotools import mask as coco_mask
1112
from pycocotools.coco import COCO
1213

13-
from torchbench.utils import calculate_run_hash
14+
from torchbench.utils import calculate_run_hash, AverageMeter
1415
from torchbench.datasets import CocoDetection
1516

1617
from .coco_eval import CocoEvaluator
@@ -198,20 +199,28 @@ def evaluate_detection_coco(
198199
iou_types = ['bbox']
199200
coco_evaluator = CocoEvaluator(coco, iou_types)
200201

202+
inference_time = AverageMeter()
203+
201204
iterator = tqdm.tqdm(test_loader, desc="Evaluation", mininterval=5)
202205

206+
end = time.time()
207+
203208
with torch.no_grad():
204209
for i, (input, target) in enumerate(iterator):
205210
input, target = send_data_to_device(input, target, device=device)
206211
original_output = model(input)
212+
213+
inference_time.update(time.time() - end)
214+
207215
output, target = model_output_transform(original_output, target)
216+
208217
result = {
209218
tar["image_id"].item(): out for tar, out in zip(target, output)
210219
}
211220
coco_evaluator.update(result)
212221

213-
214222
if i == 0: # for sotabench.com caching of evaluation
223+
memory_allocated = torch.cuda.memory_allocated(device=device)
215224
run_hash = calculate_run_hash([], original_output)
216225
# if we are in check model we don't need to go beyond the first
217226
# batch
@@ -230,11 +239,17 @@ def evaluate_detection_coco(
230239
)
231240
return cached_res, run_hash
232241

242+
end = time.time()
243+
233244
coco_evaluator.synchronize_between_processes()
234245
coco_evaluator.accumulate()
235246
coco_evaluator.summarize()
236247

237-
return (get_coco_metrics(coco_evaluator), run_hash)
248+
device_metrics = {
249+
'Tasks Per Second': test_loader.batch_size/inference_time.avg,
250+
'Memory Allocated': memory_allocated}
251+
252+
return (get_coco_metrics(coco_evaluator), device_metrics, run_hash)
238253

239254

240255
def evaluate_detection_voc(

‎torchbench/semantic_segmentation/ade20k.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ def benchmark(
8585
collate_fn=collate_fn,
8686
)
8787
test_loader.no_classes = 150 # Number of classes for ADE20K
88-
test_results, run_hash = evaluate_segmentation(
88+
test_results, speed_mem_metrics, run_hash = evaluate_segmentation(
8989
model=model,
9090
test_loader=test_loader,
9191
model_output_transform=model_output_transform,
@@ -100,6 +100,7 @@ def benchmark(
100100
config=config,
101101
dataset=cls.dataset.__name__ + " val",
102102
results=test_results,
103+
speed_mem_metrics=speed_mem_metrics,
103104
pytorch_hub_id=pytorch_hub_url,
104105
model=paper_model_name,
105106
model_description=model_description,

0 commit comments

Comments
 (0)