Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Fix ASR pipeline mono conversion for channels-last audio
The pipeline averaged multi-channel input over a hardcoded axis 0, which
assumes channels-first. soundfile.read, librosa.load(mono=False) and
scipy.io.wavfile.read all return channels-last, so a stereo waveform was
averaged across time and collapsed to one value per channel: 48000 samples
became 2, were padded back to silence, and transcribed as " you" with only
a warning saying the conversion had succeeded.

Infer the channel axis from the shorter dimension instead, and report the
actual shape rather than ndim, which always printed 2 for any 2-D input.
  • Loading branch information
Kayvan-Zahiri authored and Rocketknight1 committed Aug 19, 2026
commit f395a6ae64f89e62d5b2fb31b2d9d36397f05f99
11 changes: 9 additions & 2 deletions src/transformers/pipelines/automatic_speech_recognition.py
Original file line number Diff line number Diff line change
Expand Up @@ -424,10 +424,17 @@ def preprocess(self, inputs, chunk_length_s=0, stride_length_s=None):
if not isinstance(inputs, (np.ndarray, torch.Tensor)):
raise TypeError(f"We expect a numpy ndarray or torch tensor as input, got `{type(inputs)}`")
if inputs.ndim != 1:
# `soundfile.read`, `librosa.load(mono=False)` and `scipy.io.wavfile.read`
# all return channels-last, `(samples, channels)`. Averaging over axis 0
# on that layout averages across *time*, collapsing the waveform to one
# value per channel. Infer the channel axis instead: real audio has far
# more samples than channels, so the shorter axis is the channel axis.
channel_axis = int(np.argmin(inputs.shape))

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This feels extremely dangerous to me! If we want to be robust to either (samples, channels) or (channels, samples) then we should be much more explicit; and probably only do this for dims of 1 or 2 - if you have 5.1 audio then you can handle downmixing yourself 😅

logger.warning(
f"We expect a single channel audio input for AutomaticSpeechRecognitionPipeline, got {inputs.ndim}. Taking the mean of the channels for mono conversion."
f"We expect a single channel audio input for AutomaticSpeechRecognitionPipeline, got shape "
f"{tuple(inputs.shape)}. Taking the mean over axis {channel_axis} for mono conversion."
)
inputs = inputs.mean(axis=0)
inputs = inputs.mean(axis=channel_axis)

if chunk_length_s:
if stride_length_s is None:
Expand Down
24 changes: 24 additions & 0 deletions tests/pipelines/test_pipelines_automatic_speech_recognition.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,30 @@ def test_small_model_pt(self):
):
_ = speech_recognizer(waveform, return_timestamps="char")

@require_torch
def test_multichannel_mono_conversion_is_layout_agnostic(self):
# `soundfile.read`, `librosa.load(mono=False)` and `scipy.io.wavfile.read` all
# return channels-last `(samples, channels)`. Averaging over a hardcoded axis 0
# collapsed such input to one value per channel, so a 34,000-sample waveform
# became 2 samples and was transcribed as silence with only a warning.
speech_recognizer = pipeline(
task="automatic-speech-recognition",
model="facebook/s2t-small-mustc-en-fr-st",
tokenizer="facebook/s2t-small-mustc-en-fr-st",
)
waveform = np.tile(np.arange(1000, dtype=np.float32), 34)
expected = speech_recognizer(waveform)

channels_last = np.stack([waveform, waveform], axis=-1) # (samples, 2)
channels_first = np.stack([waveform, waveform], axis=0) # (2, samples)
self.assertEqual(channels_last.shape, (waveform.size, 2))
self.assertEqual(speech_recognizer(channels_last), expected)
self.assertEqual(speech_recognizer(channels_first), expected)

# a 2-D mono waveform must survive in either orientation too
self.assertEqual(speech_recognizer(waveform[:, None]), expected)
self.assertEqual(speech_recognizer(waveform[None, :]), expected)

@require_torch
def test_small_model_pt_fp16(self):
speech_recognizer = pipeline(
Expand Down