fix(auto-model): preserve VAD segments on punctuation mismatch (#3434)

Fall back to VAD-aligned string sentence records when a sized punctuation array cannot align with long-audio ASR timestamps.

Verified against both reporter attachments: 103.mp3 recovers 199 sentence records and 2980.mp3 remains byte-for-byte unchanged.

Signed-off-by: LauraGPT <lauragpt@users.noreply.github.com>

Author: LauraGPT

funasr/auto/auto_model.py

@@ -68,6 +68,42 @@ def _join_vad_texts(texts):
     return joined
 
 
+def _vad_segment_sentences(restored_data, vadsegments):
+    """Build readable sentence records directly from VAD-aligned ASR chunks."""
+    sentences = []
+    for result, vadsegment in zip(restored_data, vadsegments):
+        text = re.sub(r"<\|[^|]*\|>", "", str(result.get("text", ""))).strip()
+        if not text:
+            continue
+
+        timestamps = []
+        raw_timestamps = result.get("timestamp")
+        if raw_timestamps is None:
+            raw_timestamps = result.get("timestamps", [])
+        for item in raw_timestamps or []:
+            if isinstance(item, dict):
+                start = item.get("start_time")
+                end = item.get("end_time")
+                if start is None or end is None:
+                    continue
+                timestamps.append([int(float(start) * 1000), int(float(end) * 1000)])
+            elif isinstance(item, (list, tuple)) and len(item) >= 2:
+                timestamps.append([int(item[0]), int(item[1])])
+
+        start = timestamps[0][0] if timestamps else vadsegment[0]
+        end = timestamps[-1][1] if timestamps else vadsegment[1]
+        sentences.append(
+            {
+                "start": start,
+                "end": end,
+                "text": text,
+                "sentence": text,
+                "timestamp": timestamps,
+            }
+        )
+    return sentences
+
+
 def _get_punc_tokens(text, punc_array, punc_model):
     """Return the surface tokens represented by a CT-Transformer punctuation array."""
     try:
@@ -1018,6 +1054,7 @@ def inference_with_vad(self, input, input_len=None, **cfg):
 
             timestamp_text = punc_input_text
             sentence_timestamps = result.get("timestamp", [])
+            punc_alignment_failed = False
             if punc_res is not None:
                 try:
                     punc_length = len(punc_array)
@@ -1038,6 +1075,7 @@ def inference_with_vad(self, input, input_len=None, **cfg):
                         timestamp_text, sentence_timestamps = merged_units
                 if punc_array is not None and punc_length != len(sentence_timestamps):
                     punc_array = None
+                    punc_alignment_failed = True
             surface_sentence_list = None
             if aligned_word_text is not None and punc_array is not None:
                 surface_sentence_list = _timestamp_sentences_from_surface(
@@ -1131,25 +1169,17 @@ def inference_with_vad(self, input, input_len=None, **cfg):
                 if not len(result["text"].strip()):
                     sentence_list = []
                 elif self.punc_model is None and punc_res is None and not sentence_timestamps:
-                    sentence_list = []
-                    for rest, vadsegment in zip(restored_data, vadsegments):
-                        text = str(rest.get("text", "")).strip()
-                        if not text:
-                            continue
-                        sentence_list.append(
-                            {
-                                "start": vadsegment[0],
-                                "end": vadsegment[1],
-                                "text": text,
-                                "sentence": text,
-                                "timestamp": [],
-                            }
-                        )
+                    sentence_list = _vad_segment_sentences(restored_data, vadsegments)
                 elif punc_res is None:
                     logging.warning(
                         "punc_model is required for sentence_timestamp, skipping sentence segmentation."
                     )
                     sentence_list = []
+                elif punc_alignment_failed:
+                    logging.warning(
+                        "punctuation timestamps could not be aligned, falling back to VAD segments."
+                    )
+                    sentence_list = _vad_segment_sentences(restored_data, vadsegments)
                 elif surface_sentence_list is not None:
                     sentence_list = surface_sentence_list
                 else:

tests/test_punc_model_none.py

@@ -274,6 +274,60 @@ def mock_inference(data, *args, **kwargs):
             ],
         )
 
+    @patch("funasr.auto.auto_model.slice_padding_audio_samples")
+    @patch("funasr.auto.auto_model.load_audio_text_image_video")
+    @patch("funasr.auto.auto_model.prepare_data_iterator")
+    def test_sentence_timestamp_falls_back_to_vad_when_punctuation_alignment_fails(
+        self, mock_prep, mock_load, mock_slice
+    ):
+        """A malformed long-audio punctuation array must not collapse all VAD segments."""
+        am = self._make_auto_model(punc_model=MagicMock())
+        tag = "<|zh|><|NEUTRAL|><|Speech|><|woitn|>"
+        results_seq = [
+            [{"key": "test_utt", "value": [[100, 1100], [1300, 2300]]}],
+            [
+                {
+                    "text": f"{tag}第一句",
+                    "timestamp": [[0, 300], [300, 600], [600, 900]],
+                    "words": ["第", "一", "句"],
+                }
+            ],
+            [
+                {
+                    "text": f"{tag}第二句",
+                    "timestamp": [[0, 300], [300, 600], [600, 900]],
+                    "words": ["第", "二", "句"],
+                }
+            ],
+            [{"text": "第一句。第二句。", "punc_array": [3]}],
+        ]
+        am.inference = MagicMock(side_effect=lambda *args, **kwargs: results_seq.pop(0))
+        mock_prep.return_value = (["test_utt"], [np.zeros(40000, dtype=np.float32)])
+        mock_load.return_value = np.zeros(40000, dtype=np.float32)
+        mock_slice.return_value = ([np.zeros(16000, dtype=np.float32)], [16000])
+
+        results = am.inference_with_vad("dummy_input", sentence_timestamp=True)
+
+        self.assertEqual(
+            results[0]["sentence_info"],
+            [
+                {
+                    "start": 100,
+                    "end": 1000,
+                    "text": "第一句",
+                    "sentence": "第一句",
+                    "timestamp": [[100, 400], [400, 700], [700, 1000]],
+                },
+                {
+                    "start": 1300,
+                    "end": 2200,
+                    "text": "第二句",
+                    "sentence": "第二句",
+                    "timestamp": [[1300, 1600], [1600, 1900], [1900, 2200]],
+                },
+            ],
+        )
+
     @patch("funasr.auto.auto_model.distribute_spk")
     @patch("funasr.auto.auto_model.postprocess")
     @patch("funasr.auto.auto_model.sv_chunk")