Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
fix(nano): correct punctuation timestamps across VAD merges
Anchor zero-acoustic-mass punctuation at the preceding spoken token end (FunASR #3702). Reviewed patch only.
  • Loading branch information
Bruce-Yii committed Sep 11, 2026
commit 4552e01d695e7309aa29dde95a704294325c9912
8 changes: 7 additions & 1 deletion funasr/models/fun_asr_nano/inference_vllm.py
Original file line number Diff line number Diff line change
Expand Up @@ -694,7 +694,10 @@ def generate(
@torch.no_grad()
def _compute_timestamps(self, encoder_out, encoder_out_lens, text):
"""CTC forced alignment for character-level timestamps."""
from funasr.models.fun_asr_nano.tools.utils import forced_align
from funasr.models.fun_asr_nano.tools.utils import (
anchor_punctuation_timestamps,
forced_align,
)

decoder_out, decoder_out_lens = self.ctc_decoder(encoder_out, encoder_out_lens)
ctc_logits = self.ctc.log_softmax(decoder_out)
Expand All @@ -709,6 +712,9 @@ def _compute_timestamps(self, encoder_out, encoder_out_lens, text):
ts["token"] = self.ctc_tokenizer.decode([ts["token"]])
ts["start_time"] = ts["start_time"] * 6 * 10 / 1000
ts["end_time"] = ts["end_time"] * 6 * 10 / 1000
# See model.py: anchor punctuation at the preceding spoken token's end
# (issue #3702); punctuation has no acoustic extent to align.
anchor_punctuation_timestamps(timestamps)
return timestamps

@classmethod
Expand Down
12 changes: 11 additions & 1 deletion funasr/models/fun_asr_nano/inference_vllm_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,10 @@ def _process_one(self, audio_path, **kwargs):

def _compute_all_timestamps(self, segment_audios, vad_segments, asr_results):
"""Compute CTC timestamps for all segments with VAD offsets."""
from funasr.models.fun_asr_nano.tools.utils import forced_align
from funasr.models.fun_asr_nano.tools.utils import (
anchor_punctuation_timestamps,
forced_align,
)

all_timestamps = []
for seg_audio, vad_seg, text in zip(segment_audios, vad_segments, asr_results):
Expand Down Expand Up @@ -356,6 +359,13 @@ def _compute_all_timestamps(self, segment_audios, vad_segments, asr_results):
ts["token"] = self.asr_engine.ctc_tokenizer.decode([ts["token"]])
ts["start_time"] = ts["start_time"] * 6 * 10 / 1000 + vad_offset_ms / 1000
ts["end_time"] = ts["end_time"] * 6 * 10 / 1000 + vad_offset_ms / 1000
# Same invariant as the offline/vLLM paths (issue #3702), applied
# per independently aligned VAD segment: punctuation carries no
# acoustic extent, so pin sandwiched punctuation at the preceding
# spoken token's end. Segment-local on purpose — speech in a
# later VAD segment must not rewrite this segment's trailing
# punctuation.
anchor_punctuation_timestamps(timestamps)
all_timestamps.extend(timestamps)
except Exception as e:
logger.debug(f"Timestamp failed for segment: {e}")
Expand Down
8 changes: 7 additions & 1 deletion funasr/models/fun_asr_nano/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
from .ctc import CTC
from .checkpoint_utils import disable_incomplete_ctc, normalize_checkpoint_state
from .device_utils import resolve_autocast_device_type
from .tools.utils import forced_align
from .tools.utils import anchor_punctuation_timestamps, forced_align

dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}

Expand Down Expand Up @@ -1069,6 +1069,12 @@ def inference_llm(
timestamp["token"] = self.ctc_tokenizer.decode([timestamp["token"]])
timestamp["start_time"] = timestamp["start_time"] * 6 * 10 / 1000
timestamp["end_time"] = timestamp["end_time"] * 6 * 10 / 1000
# Punctuation tokens have no acoustic realization; without anchoring,
# forced alignment places them at the next sentence's onset inside
# merged multi-sentence windows (issue #3702). Applied after decoding
# (tokens are still integer ids above). ctc_timestamps is left
# untouched: greedy CTC text carries no punctuation to anchor.
anchor_punctuation_timestamps(result["timestamps"])

if ibest_writer is not None:
ibest_writer["text"][key[0]] = response.replace("\n", " ")
Expand Down
66 changes: 65 additions & 1 deletion funasr/models/fun_asr_nano/tools/utils.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from itertools import groupby
import re

import librosa
import soundfile as sf
Expand Down Expand Up @@ -77,4 +78,67 @@ def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0)
)
except Exception:
pass
return items
return items


_PUNCTUATION_TOKEN_RE = re.compile(r"^[^\w\s]+$")
_SPECIAL_TOKEN_RE = re.compile(r"^<[^>]*>$")


def _classify_timestamp_token(token):
"""Classify one decoded timestamp token into spoken/punctuation/special.

Special tokens (``<sil>``-shaped) are their own class: they are neither
spoken content nor punctuation, so they neither trigger anchoring nor
serve as anchors. Non-string tokens (integer ids, if called pre-decode)
fall back to spoken to preserve pass-through behavior.
"""
if not isinstance(token, str) or not token:
return "spoken"
if _SPECIAL_TOKEN_RE.match(token):
return "special"
if _PUNCTUATION_TOKEN_RE.match(token):
Comment thread
LauraGPT marked this conversation as resolved.
Resolved
Comment thread
LauraGPT marked this conversation as resolved.
Resolved
return "punctuation"
return "spoken"


def anchor_punctuation_timestamps(timestamps):
"""Anchor punctuation-only tokens at the preceding spoken token's end.

CTC forced alignment must place every target token — including punctuation
tokens with no acoustic realization — on at least one frame. Inside a
merged multi-sentence window (see FunASR issue #3702) such a token lands
at the next sentence's onset frame, so the current sentence's punctuation
timestamp reads as the next sentence start. Punctuation carries no acoustic
extent, so a punctuation span that is followed by more speech is pinned
(zero-width) at the end of the speech it terminates.

Leading punctuation (no predecessor) and trailing punctuation (no
successor, e.g. single-segment output) are left untouched, as are special
tokens such as ``<sil>``. This preserves existing single-segment behavior.

Args:
timestamps: List of ``{"token": str, "start_time": float,
"end_time": float}`` dicts in time order. Mutated in place.

Returns:
The same list, for convenient chaining.
"""
last_spoken_end = None
total = len(timestamps)
for index, ts in enumerate(timestamps):
kind = _classify_timestamp_token(ts.get("token", ""))
if kind != "punctuation":
if kind == "spoken":
end = ts.get("end_time")
if isinstance(end, (int, float)):
last_spoken_end = end
continue
following_spoken = any(
_classify_timestamp_token(later.get("token", "")) == "spoken"
for later in timestamps[index + 1 : total]
)
if last_spoken_end is not None and following_spoken:
ts["start_time"] = last_spoken_end
ts["end_time"] = last_spoken_end
return timestamps
164 changes: 164 additions & 0 deletions tests/test_fun_asr_nano_punctuation_timestamps.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
"""Punctuation-timestamp anchoring for Fun-ASR-Nano forced alignment.

Issue #3702: with merge_vad, a sentence-final punctuation token (which has no
acoustic realization) is placed by CTC forced alignment at the next sentence's
onset frame, so the current sentence's punctuation timestamp lands on the next
sentence start. Proven with synthetic peaky emissions through the repo's exact
``forced_align`` (see run evidence): in a merged [sent1 + silence + sent2]
window the punctuation lands at the silence end, while the single-sentence
control keeps it at the window end.

These tests pin the post-processing rule: a punctuation-only token that is
followed by more speech is anchored (zero-width) at the preceding spoken
token's end. Punctuation carries no acoustic extent, so anchoring invents no
duration. Leading punctuation (no predecessor) and trailing punctuation (no
successor, e.g. single-segment output) are left untouched, preserving
existing single-segment behavior.
"""

import copy

import pytest

from funasr.models.fun_asr_nano.tools.utils import anchor_punctuation_timestamps


def _ts(token, start, end):
return {"token": token, "start_time": start, "end_time": end}


def test_punctuation_anchored_at_previous_spoken_end():
# Mirrors the proven repro: 好 ends at 19, 。misplaced at [72, 73).
timestamps = [_ts("好", 18, 19), _ts("。", 72, 73), _ts("他", 78, 79)]
result = anchor_punctuation_timestamps(timestamps)
assert result[1] == {"token": "。", "start_time": 19, "end_time": 19}
assert result[0] == {"token": "好", "start_time": 18, "end_time": 19}
assert result[2] == {"token": "他", "start_time": 78, "end_time": 79}


def test_leading_punctuation_without_predecessor_is_kept():
timestamps = [_ts("。", 5, 6), _ts("好", 18, 19)]
before = copy.deepcopy(timestamps)
assert anchor_punctuation_timestamps(timestamps) == before


def test_consecutive_punctuation_share_one_anchor():
timestamps = [_ts("好", 18, 19), _ts(",", 50, 51), _ts("。", 72, 73), _ts("他", 78, 79)]
result = anchor_punctuation_timestamps(timestamps)
assert (result[1]["start_time"], result[1]["end_time"]) == (19, 19)
assert (result[2]["start_time"], result[2]["end_time"]) == (19, 19)
assert (result[3]["start_time"], result[3]["end_time"]) == (78, 79)


def test_spoken_tokens_are_never_rewritten():
timestamps = [_ts("你", 12, 13), _ts("好", 18, 19), _ts("。", 39, 40)]
before = copy.deepcopy(timestamps)
result = anchor_punctuation_timestamps(timestamps)
assert result[0] == before[0]
# Single-sentence control shape: punctuation already at window end is kept.
assert result[2] == before[2]


def test_special_tokens_are_not_treated_as_punctuation():
timestamps = [_ts("<sil>", 0, 5), _ts("好", 18, 19)]
before = copy.deepcopy(timestamps)
assert anchor_punctuation_timestamps(timestamps) == before


def test_punctuation_followed_only_by_special_is_kept():
# No later real speech: the trailing <sil> must not make 。 eligible.
timestamps = [_ts("好", 18, 19), _ts("。", 72, 73), _ts("<sil>", 80, 85)]
before = copy.deepcopy(timestamps)
assert anchor_punctuation_timestamps(timestamps) == before


def test_special_token_is_not_used_as_predecessor_anchor():
# The <sil> between speech and punctuation must not become the anchor.
timestamps = [_ts("好", 18, 19), _ts("<sil>", 30, 35), _ts("。", 72, 73), _ts("他", 78, 79)]
result = anchor_punctuation_timestamps(timestamps)
assert (result[2]["start_time"], result[2]["end_time"]) == (19, 19)
assert (result[1]["start_time"], result[1]["end_time"]) == (30, 35)
assert (result[3]["start_time"], result[3]["end_time"]) == (78, 79)


def test_pipeline_segment_local_anchor_boundary(monkeypatch):
"""Pipeline segment boundary: anchoring applies per VAD segment, never
across segments. segA shows intra-segment anchoring (misplaced 。 before
later speech in the SAME aligned segment anchors); segB shows a
segment-final 。 staying put even though segC contains later speech;
VAD offsets and spoken spans pass through verbatim."""
import types

import numpy as np
import torch

from funasr.models.fun_asr_nano.inference_vllm_pipeline import FunASRNanoVLLMPipeline

id2char = {2: "好", 3: "。", 4: "他", 5: "在", 6: "是"}
spans = iter(
[
[
{"token": 2, "start_time": 18, "end_time": 19},
{"token": 3, "start_time": 72, "end_time": 73},
{"token": 4, "start_time": 78, "end_time": 79},
],
[
{"token": 6, "start_time": 10, "end_time": 12},
{"token": 3, "start_time": 40, "end_time": 41},
],
[{"token": 5, "start_time": 50, "end_time": 56}],
]
)

class StubTok:
def encode(self, text):
return [1] * max(1, len(text))

def decode(self, ids):
return "".join(id2char.get(i, "?") for i in ids)

engine = types.SimpleNamespace(
frontend=object(),
device="cpu",
blank_id=0,
audio_encoder=lambda s, sl: (torch.zeros(1, 10, 8), torch.tensor([10])),
ctc_decoder=lambda e, l: (torch.zeros(1, 10, 8), torch.tensor([10])),
ctc=types.SimpleNamespace(log_softmax=lambda d: d),
ctc_tokenizer=StubTok(),
)
monkeypatch.setattr(
"funasr.utils.load_utils.extract_fbank",
lambda *a, **k: (torch.zeros(1, 4, 80), torch.tensor([[4]])),
)
monkeypatch.setattr(
"funasr.models.fun_asr_nano.tools.utils.forced_align",
lambda *a, **k: next(spans),
)

pipeline = FunASRNanoVLLMPipeline.__new__(FunASRNanoVLLMPipeline)
pipeline.asr_engine = engine
pipeline.device = "cpu"
result = pipeline._compute_all_timestamps(
[
np.zeros(32000, dtype=np.float32),
np.zeros(16000, dtype=np.float32),
np.zeros(16000, dtype=np.float32),
],
[[1000, 3000], [5000, 6000], [9000, 10000]],
["segAtext", "segBtext", "segCtext"],
)

assert [t["token"] for t in result] == ["好", "。", "他", "是", "。", "在"]
# segA (offset 1.0s): intra-segment 。 anchors at 好.end bit-identically.
assert result[1]["start_time"] == result[0]["end_time"]
assert result[1]["end_time"] == result[0]["end_time"]
assert result[0]["end_time"] == pytest.approx(2.14)
assert result[2]["start_time"] == pytest.approx(5.68)
# segB (offset 5.0s): segment-final 。 keeps its aligned span even though
# segC contains later speech — cross-segment speech must not trigger it.
assert result[4]["start_time"] == pytest.approx(7.4)
assert result[4]["end_time"] == pytest.approx(7.46)
assert result[3]["end_time"] == pytest.approx(5.72)
# segC (offset 9.0s) passes through untouched.
assert result[5]["start_time"] == pytest.approx(12.0)
assert result[5]["end_time"] == pytest.approx(12.36)