Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Reject ambiguous channel layouts instead of inferring the channel axis
Per review: inferring the channel axis with argmin is unsafe. It picks an
arbitrary axis for ndim >= 3, and it silently downmixes 5.1 audio where the
weighting should be the caller's choice.

Keep the documented (channels, samples) layout, which is what torchcodec
returns, and raise when the input does not conform. A channels-last array gets
an error naming the transpose, rather than a transcription of two samples.

Also move the conversion ahead of F.resample. The resampler operates on the last
axis, so a multi-channel array had to be reduced first, and the stride
arithmetic reads shape[0] as samples.
  • Loading branch information
Kayvan-Zahiri authored and Rocketknight1 committed Aug 19, 2026
commit bb6f79e1c835855b1409ab6d466c1da2f3bdc9e3
58 changes: 46 additions & 12 deletions src/transformers/pipelines/automatic_speech_recognition.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,46 @@
from ..models.auto.modeling_auto import MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES


def _to_mono(inputs):
"""
Reduce a waveform to a single channel, or refuse when the layout is ambiguous.

The pipeline reads audio as `(channels, samples)`, which is what torchcodec returns
and what `datasets` returned before 4.0. Anything else is rejected rather than
guessed at: the shape alone cannot distinguish two-channel audio from a two-sample
recording, and averaging the wrong axis produces a silently wrong transcription
instead of an error.
"""
if inputs.ndim == 1:
return inputs

if inputs.ndim != 2:
raise ValueError(
"AutomaticSpeechRecognitionPipeline expects mono audio shaped `(samples,)` or "
f"multi-channel audio shaped `(channels, samples)`, got {inputs.ndim} dimensions "
f"with shape {tuple(inputs.shape)}."
)

num_channels = inputs.shape[0]
if num_channels > 2:
raise ValueError(
"AutomaticSpeechRecognitionPipeline reads axis 0 as channels, and it has size "
f"{num_channels} in the input of shape {tuple(inputs.shape)}. If this array is "
"channels-last `(samples, channels)`, which is what `soundfile.read`, "
"`librosa.load(mono=False)` and `scipy.io.wavfile.read` return, transpose it "
"first. If it genuinely has more than 2 channels, downmix it to mono yourself "
"so the weighting is yours to choose."
)

if num_channels > 1:
logger.warning(
"We expect a single channel audio input for AutomaticSpeechRecognitionPipeline, got "
f"{num_channels} channels in an input of shape {tuple(inputs.shape)}. Taking the mean "
"over the channels for mono conversion."
)
Comment thread
Rocketknight1 marked this conversation as resolved.
Resolved
return inputs.mean(axis=0)


def rescale_stride(stride, ratio):
"""
Rescales the stride values from audio space to tokens/logits space.
Expand Down Expand Up @@ -395,6 +435,11 @@ def preprocess(self, inputs, chunk_length_s=0, stride_length_s=None):
in_sampling_rate = inputs.pop("sampling_rate")
extra = inputs
inputs = _inputs
# Downmix before resampling. `F.resample` operates on the last axis, so a
# multi-channel array has to be reduced first or the resampler works on the
# wrong axis. The stride arithmetic below also reads `shape[0]` as samples.
if isinstance(inputs, (np.ndarray, torch.Tensor)):
inputs = _to_mono(inputs)
if in_sampling_rate != self.feature_extractor.sampling_rate:
if is_torchaudio_available():
from torchaudio import functional as F
Expand Down Expand Up @@ -423,18 +468,7 @@ def preprocess(self, inputs, chunk_length_s=0, stride_length_s=None):
stride = (inputs.shape[0], int(round(stride[0] * ratio)), int(round(stride[1] * ratio)))
if not isinstance(inputs, (np.ndarray, torch.Tensor)):
raise TypeError(f"We expect a numpy ndarray or torch tensor as input, got `{type(inputs)}`")
if inputs.ndim != 1:
# `soundfile.read`, `librosa.load(mono=False)` and `scipy.io.wavfile.read`
# all return channels-last, `(samples, channels)`. Averaging over axis 0
# on that layout averages across *time*, collapsing the waveform to one
# value per channel. Infer the channel axis instead: real audio has far
# more samples than channels, so the shorter axis is the channel axis.
channel_axis = int(np.argmin(inputs.shape))
logger.warning(
f"We expect a single channel audio input for AutomaticSpeechRecognitionPipeline, got shape "
f"{tuple(inputs.shape)}. Taking the mean over axis {channel_axis} for mono conversion."
)
inputs = inputs.mean(axis=channel_axis)
inputs = _to_mono(inputs)

if chunk_length_s:
if stride_length_s is None:
Expand Down
40 changes: 28 additions & 12 deletions tests/pipelines/test_pipelines_automatic_speech_recognition.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
)
from transformers.pipelines import AutomaticSpeechRecognitionPipeline, pipeline
from transformers.pipelines.audio_utils import chunk_bytes_iter, ffmpeg_microphone_live
from transformers.pipelines.automatic_speech_recognition import chunk_iter
from transformers.pipelines.automatic_speech_recognition import _to_mono, chunk_iter
from transformers.testing_utils import (
Expectations,
compare_pipeline_output_to_hub_spec,
Expand Down Expand Up @@ -183,11 +183,9 @@ def test_small_model_pt(self):
_ = speech_recognizer(waveform, return_timestamps="char")

@require_torch
def test_multichannel_mono_conversion_is_layout_agnostic(self):
# `soundfile.read`, `librosa.load(mono=False)` and `scipy.io.wavfile.read` all
# return channels-last `(samples, channels)`. Averaging over a hardcoded axis 0
# collapsed such input to one value per channel, so a 34,000-sample waveform
# became 2 samples and was transcribed as silence with only a warning.
def test_multichannel_mono_conversion(self):
# The pipeline reads audio as `(channels, samples)`, which is what torchcodec
# returns. A 2-channel waveform is averaged down to mono.
speech_recognizer = pipeline(
task="automatic-speech-recognition",
model="facebook/s2t-small-mustc-en-fr-st",
Expand All @@ -196,16 +194,34 @@ def test_multichannel_mono_conversion_is_layout_agnostic(self):
waveform = np.tile(np.arange(1000, dtype=np.float32), 34)
expected = speech_recognizer(waveform)

channels_last = np.stack([waveform, waveform], axis=-1) # (samples, 2)
channels_first = np.stack([waveform, waveform], axis=0) # (2, samples)
self.assertEqual(channels_last.shape, (waveform.size, 2))
self.assertEqual(speech_recognizer(channels_last), expected)
self.assertEqual(speech_recognizer(channels_first), expected)

# a 2-D mono waveform must survive in either orientation too
self.assertEqual(speech_recognizer(waveform[:, None]), expected)
# a single channel expressed as 2-D collapses to the same thing
self.assertEqual(speech_recognizer(waveform[None, :]), expected)

def test_multichannel_ambiguous_layout_raises(self):
# `soundfile.read`, `librosa.load(mono=False)` and `scipy.io.wavfile.read` all
# return channels-last `(samples, channels)`. Averaging axis 0 on that layout
# averages across time, so a 34,000-sample waveform silently became 2 samples
# and was transcribed as silence. The shape alone cannot tell that apart from
# genuine `(channels, samples)` audio, so refuse instead of guessing. See #47886.
waveform = np.tile(np.arange(1000, dtype=np.float32), 34)

channels_last = np.stack([waveform, waveform], axis=-1) # (samples, 2)
with self.assertRaisesRegex(ValueError, "channels-last"):
_to_mono(channels_last)

surround = np.stack([waveform] * 6, axis=0) # (6, samples)
with self.assertRaisesRegex(ValueError, "downmix it to mono yourself"):
_to_mono(surround)

with self.assertRaisesRegex(ValueError, "3 dimensions"):
_to_mono(waveform[None, None, :])

def test_mono_passthrough(self):
waveform = np.tile(np.arange(1000, dtype=np.float32), 34)
self.assertIs(_to_mono(waveform), waveform)

@require_torch
def test_small_model_pt_fp16(self):
speech_recognizer = pipeline(
Expand Down