Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion laya/evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import inspect
import json
import math
import statistics
import time
from dataclasses import dataclass, field
Expand Down Expand Up @@ -290,8 +291,10 @@ def _group_cases(cases: Sequence[Dict[str, Any]], key: str) -> Dict[str, List[Di
def _percentiles(values: Sequence[float]) -> Tuple[float, float]:
"""(median, 95th) using the nearest-rank rule the report has always used for latency."""
ordered = sorted(values)
# Nearest-rank: the ceil(0.95 * n)-th smallest. int(n * 0.95) would land one rank too high
# whenever 0.95 * n is whole -- every sample size that is a multiple of 20 (#680).
return (float(statistics.median(ordered)),
float(ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))]))
float(ordered[math.ceil(len(ordered) * 0.95) - 1]))


def _aggregate(cases: Sequence[Dict[str, Any]], evaluators: Sequence[Evaluator]) -> Dict[str, float]:
Expand Down
13 changes: 13 additions & 0 deletions tests/test_evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,19 @@ def test_batched_latency_is_what_a_request_waited(monkeypatch):
"batching trades request latency for throughput; the report has to say so"


def test_p95_is_nearest_rank_when_the_sample_size_is_a_multiple_of_20():
# 0.95 * 20 = 19 exactly: the 19th smallest, not the 20th (#680 -- int(n * 0.95) landed
# one rank too high whenever the product was whole).
from laya.evals import _percentiles

assert _percentiles([float(i) for i in range(1, 21)])[1] == 19.0
assert _percentiles([float(i) for i in range(1, 41)])[1] == 38.0
# Away from the exact boundary the rank is unchanged, and the ends hold.
assert _percentiles([float(i) for i in range(1, 22)])[1] == 20.0
assert _percentiles([10.0])[1] == 10.0
assert _percentiles([3.0, 7.0])[1] == 7.0


def test_the_throughput_share_keeps_its_own_metric(monkeypatch):
solo, _ = _timed_pair(monkeypatch, None)
batched, _ = _timed_pair(monkeypatch, 8)
Expand Down
Loading