Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Fixed `Metric` ignoring an active `torch.device` context manager on torch 2.3-2.7 ([#3448](https://github.com/Lightning-AI/torchmetrics/pull/3448))


- Fixed `PESQ` metric aborting on a batch containing a sample the backend cannot score, which now returns `nan` for that sample instead of raising ([#3304](https://github.com/Lightning-AI/torchmetrics/issues/3304))


- Fixed `CHRFScore` dropping the reference n-grams of a sentence that matches no reference n-gram, which inflated the corpus-level score ([#3481](https://github.com/Lightning-AI/torchmetrics/pull/3481))


Expand Down
11 changes: 9 additions & 2 deletions src/torchmetrics/audio/pesq.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,11 @@ class PerceptualEvaluationSpeechQuality(Metric):
for a batch. To obtain a PESQ value for each sample, you may use the functional counterpart in
:func:`~torchmetrics.functional.audio.pesq.perceptual_evaluation_speech_quality`.

.. note::
Samples that the ``pesq`` backend cannot score, e.g. a silent reference for which no utterance can be
detected, are excluded from the average instead of aborting the calculation for the whole batch. If no
sample in any of the updates could be scored, ``compute`` returns ``nan``.

Args:
fs: sampling frequency, should be 16000 or 8000 (Hz)
mode: ``'wb'`` (wide-band) or ``'nb'`` (narrow-band)
Expand Down Expand Up @@ -127,8 +132,10 @@ def update(self, preds: Tensor, target: Tensor) -> None:
preds, target, self.fs, self.mode, False, self.n_processes
).to(self.sum_pesq.device)

self.sum_pesq += pesq_batch.sum()
self.total += pesq_batch.numel()
# samples the backend could not score are returned as `nan` and are left out of the average
scored = ~pesq_batch.isnan()
self.sum_pesq += pesq_batch[scored].sum()
self.total += scored.sum()

def compute(self) -> Tensor:
"""Compute metric."""
Expand Down
69 changes: 58 additions & 11 deletions src/torchmetrics/functional/audio/pesq.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,15 @@ def perceptual_evaluation_speech_quality(
n_processes: integer specifying the number of processes to run in parallel for the metric calculation.
Only applies to batches of data and if ``multiprocessing`` package is installed.

.. note::
Samples that the ``pesq`` backend cannot score, e.g. a silent reference for which no utterance can be
detected, are returned as ``nan`` instead of raising an error, so that a single degenerate sample does not
abort the calculation for the rest of the batch. The class-based metric
:class:`~torchmetrics.audio.pesq.PerceptualEvaluationSpeechQuality` excludes such samples from its average.

Returns:
Float tensor with shape ``(...,)`` of PESQ values per sample
Float tensor with shape ``(...,)`` of PESQ values per sample, with ``nan`` for samples the backend
could not score

Raises:
ModuleNotFoundError:
Expand Down Expand Up @@ -89,7 +96,28 @@ def perceptual_evaluation_speech_quality(
def _issubtype_number(x: Any) -> bool:
return np.issubdtype(type(x), np.number)

_filter_error_msg = np.vectorize(_issubtype_number)
_filter_error_msg = np.vectorize(_issubtype_number, otypes=[bool])

# with ``on_error=PesqError.RETURN_VALUES`` the backend reports a failure as one of these negative codes
# instead of raising; all valid PESQ scores are >= -0.5 so the codes cannot collide with a real score
error_codes = [
code
for name, code in vars(pesq_backend.PesqError).items()
if not name.startswith("_") and isinstance(code, int) and code < 0
]

def _errors_to_nan(values: Any) -> np.ndarray:
"""Convert raw backend outputs into float scores, mapping every failure onto ``nan``.

A failure is reported either as an error code (``pesq``/``pesq_batch`` with
``on_error=PesqError.RETURN_VALUES``) or as an exception object (``pesq_batch`` collects exceptions
raised inside its worker processes). Both are replaced by ``nan``, keeping one entry per input sample.

"""
values = np.asarray(values, dtype=object).reshape(-1)
scores = np.where(_filter_error_msg(values), values, np.nan).astype(np.float32)
scores[np.isin(scores, error_codes)] = np.nan
return scores

if fs not in (8000, 16000):
raise ValueError(f"Expected argument `fs` to either be 8000 or 16000 but got {fs}")
Expand All @@ -98,21 +126,40 @@ def _issubtype_number(x: Any) -> bool:
_check_same_shape(preds, target)

if preds.ndim == 1:
pesq_val_np = pesq_backend.pesq(fs, target.detach().cpu().numpy(), preds.detach().cpu().numpy(), mode)
pesq_val = torch.tensor(pesq_val_np)
pesq_val_np = pesq_backend.pesq(
fs=fs,
ref=target.detach().cpu().numpy(),
deg=preds.detach().cpu().numpy(),
mode=mode,
on_error=pesq_backend.PesqError.RETURN_VALUES,
)
pesq_val = torch.tensor(_errors_to_nan(pesq_val_np)[0])
else:
preds_np = preds.reshape(-1, preds.shape[-1]).detach().cpu().numpy()
target_np = target.reshape(-1, preds.shape[-1]).detach().cpu().numpy()

if _MULTIPROCESSING_AVAILABLE and n_processes != 1:
pesq_val_np = pesq_backend.pesq_batch(fs, target_np, preds_np, mode, n_processor=n_processes)
pesq_val_np = np.array(pesq_val_np)
pesq_val_np = pesq_backend.pesq_batch(
fs=fs,
ref=target_np,
deg=preds_np,
mode=mode,
n_processor=n_processes,
on_error=pesq_backend.PesqError.RETURN_VALUES,
)
else:
pesq_val_np = np.empty(shape=(preds_np.shape[0]))
for b in range(preds_np.shape[0]):
pesq_val_np[b] = pesq_backend.pesq(fs, target_np[b, :], preds_np[b, :], mode)
pesq_val = torch.from_numpy(pesq_val_np[_filter_error_msg(pesq_val_np)].astype(np.float32))
pesq_val = pesq_val.reshape(len(pesq_val))
pesq_val_np = [
pesq_backend.pesq(
fs=fs,
ref=target_np[b, :],
deg=preds_np[b, :],
mode=mode,
on_error=pesq_backend.PesqError.RETURN_VALUES,
)
for b in range(preds_np.shape[0])
]
pesq_val = torch.from_numpy(_errors_to_nan(pesq_val_np))
pesq_val = pesq_val.reshape(preds.shape[:-1])

if keep_same_device:
return pesq_val.to(preds.device)
Expand Down
39 changes: 39 additions & 0 deletions tests/unittests/audio/test_pesq.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,45 @@ def test_error_on_different_shape(metric_class=PerceptualEvaluationSpeechQuality
metric(torch.randn(100), torch.randn(50))


@pytest.mark.parametrize("num_processes", [1, 2])
def test_degenerate_sample_does_not_abort_batch(num_processes):
"""Test that a single sample the backend cannot score does not abort the rest of the batch.

A silent reference makes the ``pesq`` backend fail with ``NoUtterancesError``. The batch should still be
scored, with ``nan`` in place of the offending sample so that the returned tensor keeps one value per input
sample.

"""
preds = torch.rand(3, 2100)
target = torch.rand(3, 2100)
target[1] = 0.0 # silent reference -> no utterances can be detected

pesq_val = perceptual_evaluation_speech_quality(preds, target, 8000, "nb", n_processes=num_processes)
assert pesq_val.shape == (3,)
assert not torch.isnan(pesq_val[0])
assert torch.isnan(pesq_val[1])
assert not torch.isnan(pesq_val[2])

# the class metric averages over the samples that could be scored
metric = PerceptualEvaluationSpeechQuality(8000, "nb", n_processes=num_processes)
metric.update(preds, target)
assert metric.total == 2
assert torch.allclose(metric.compute(), pesq_val[[0, 2]].mean())


def test_degenerate_sample_returns_nan_for_single_sample():
"""Test that an unscorable single (non-batched) sample yields ``nan`` instead of raising."""
pesq_val = perceptual_evaluation_speech_quality(torch.rand(2100), torch.zeros(2100), 8000, "nb")
assert pesq_val.ndim == 0
assert torch.isnan(pesq_val)


def test_multidim_input_keeps_shape():
"""Test that the returned tensor has shape ``preds.shape[:-1]``, as documented."""
pesq_val = perceptual_evaluation_speech_quality(torch.rand(2, 3, 2100), torch.rand(2, 3, 2100), 8000, "nb")
assert pesq_val.shape == (2, 3)


def test_on_real_audio():
"""Test that metric works as expected on real audio signals."""
rate, ref = wavfile.read(_SAMPLE_AUDIO_SPEECH)
Expand Down
Loading