Generate resource L0 summaries from overview prompt (#2890)

Author: heaoxiang-ai

openviking/storage/queuefs/semantic_dag.py

@@ -812,8 +812,7 @@ async def _overview_task(self, dir_uri: str) -> None:
                     overview = await self._processor._generate_overview(
                         dir_uri, file_summaries, children_abstracts
                     )
-                abstract = self._processor._extract_abstract_from_overview(overview)
-                overview, abstract = self._processor._enforce_size_limits(overview, abstract)
+                overview, abstract = self._processor._normalize_overview_generation(overview)
 
             # Write directly, protected by the outer semantic lock.
             try:

openviking/storage/queuefs/semantic_processor.py

@@ -3,6 +3,7 @@
 """SemanticProcessor: Processes messages from SemanticQueue, generates .abstract.md and .overview.md."""
 
 import asyncio
+import re
 import threading
 from contextlib import nullcontext
 from dataclasses import dataclass, field
@@ -652,11 +653,10 @@ async def _gen(idx: int, file_path: str) -> None:
                 for file_path, summary in zip(file_paths, file_summaries, strict=False)
                 if file_path in paths_to_vectorize and summary is not None
             ]
-            overview = await self._generate_overview(
+            generated_content = await self._generate_overview(
                 dir_uri, completed_summaries, [], llm_sem=llm_sem
             )
-            abstract = self._extract_abstract_from_overview(overview)
-            overview, abstract = self._enforce_size_limits(overview, abstract)
+            overview, abstract = self._normalize_overview_generation(generated_content)
 
             try:
                 wrote_semantics = await self._write_memory_directory_semantics(
@@ -1155,8 +1155,49 @@ async def _generate_single_file_summary(
         else:
             return await self._generate_text_summary(file_path, file_name, llm_sem, ctx=ctx)
 
+    def _replace_index_references(
+        self, generated_content: str, file_index_map: Dict[int, str]
+    ) -> str:
+        def replace_index(match):
+            idx = int(match.group(1))
+            return file_index_map.get(idx, match.group(0))
+
+        return re.sub(r"\[(\d+)\]", replace_index, generated_content)
+
+    def _truncate_generated_text(self, text: str, max_chars: int) -> str:
+        if max_chars <= 0 or len(text) <= max_chars:
+            return text
+
+        if max_chars <= 3:
+            return text[:max_chars]
+
+        first_sentence_end = None
+        last_sentence_end_within_limit = None
+        for sentence_end_match in re.finditer(
+            r"\.(?!\d)(?=\s|$)|[!?](?=\s|$)|[。？！]", text
+        ):
+            sentence_end = sentence_end_match.end()
+            if first_sentence_end is None:
+                first_sentence_end = sentence_end
+            if sentence_end <= max_chars:
+                last_sentence_end_within_limit = sentence_end
+            elif last_sentence_end_within_limit is not None:
+                break
+
+        if last_sentence_end_within_limit is not None:
+            return text[:last_sentence_end_within_limit].strip()
+        if first_sentence_end is not None:
+            return text[:first_sentence_end].strip()
+
+        candidate = text[: max_chars - 3].rstrip()
+        word_boundary = candidate.rfind(" ")
+        if word_boundary > 0:
+            return candidate[:word_boundary].rstrip() + "..."
+
+        return candidate + "..."
+
     def _extract_abstract_from_overview(self, overview_content: str) -> str:
-        """Extract abstract from overview.md."""
+        """Extract an abstract from the Markdown overview brief description."""
         lines = overview_content.split("\n")
 
         # Skip header lines (starting with #)
@@ -1178,13 +1219,19 @@ def _extract_abstract_from_overview(self, overview_content: str) -> str:
 
         return "\n".join(content_lines).strip()
 
+    def _normalize_overview_generation(self, generated_content: str) -> Tuple[str, str]:
+        """Convert raw Markdown overview output into final L1 overview and L0 abstract."""
+        overview = generated_content
+        abstract = self._extract_abstract_from_overview(generated_content)
+        return self._enforce_size_limits(overview, abstract)
+
     def _enforce_size_limits(self, overview: str, abstract: str) -> Tuple[str, str]:
         """Enforce max size limits on overview and abstract."""
         semantic = get_openviking_config().semantic
         if len(overview) > semantic.overview_max_chars:
-            overview = overview[: semantic.overview_max_chars]
+            overview = self._truncate_generated_text(overview, semantic.overview_max_chars)
         if len(abstract) > semantic.abstract_max_chars:
-            abstract = abstract[: semantic.abstract_max_chars - 3] + "..."
+            abstract = self._truncate_generated_text(abstract, semantic.abstract_max_chars)
         return overview, abstract
 
     def _parse_overview_md(self, overview_content: str) -> Dict[str, str]:
@@ -1247,7 +1294,7 @@ async def _generate_overview(
         children_abstracts: List[Dict[str, str]],
         llm_sem: Optional[asyncio.Semaphore] = None,
     ) -> str:
-        """Generate directory's .overview.md (L1).
+        """Generate raw directory overview model output.
 
         For small directories, generates a single overview from all file summaries.
         For large directories that would exceed the prompt budget, splits file
@@ -1260,7 +1307,7 @@ async def _generate_overview(
             children_abstracts: Subdirectory summary list
 
         Returns:
-            Overview content
+            Markdown overview content generated by the model.
         """
 
         config = get_openviking_config()
@@ -1359,9 +1406,8 @@ async def _single_generate_overview(
         output_language: str = "en",
     ) -> str:
         """Generate overview from a single prompt (small directories)."""
-        import re
-
-        vlm = get_openviking_config().vlm
+        config = get_openviking_config()
+        vlm = config.vlm
 
         try:
             prompt = render_prompt(
@@ -1377,12 +1423,7 @@ async def _single_generate_overview(
             with bind_telemetry_stage("resource_summarize"):
                 overview = await vlm.get_completion_async(prompt)
 
-            # Post-process: replace [number] with actual file name
-            def replace_index(match):
-                idx = int(match.group(1))
-                return file_index_map.get(idx, match.group(0))
-
-            overview = re.sub(r"\[(\d+)\]", replace_index, overview)
+            overview = self._replace_index_references(overview, file_index_map)
 
             return overview.strip()
 
@@ -1407,10 +1448,9 @@ async def _batched_generate_overview(
         Splits file summaries into batches, generates a partial overview per
         batch, then merges all partials into a final overview.
         """
-        import re
-
-        vlm = get_openviking_config().vlm
-        semantic = get_openviking_config().semantic
+        config = get_openviking_config()
+        vlm = config.vlm
+        semantic = config.semantic
         batch_size = semantic.overview_batch_size
         dir_name = dir_uri.split("/")[-1]
 
@@ -1459,19 +1499,12 @@ async def _batched_generate_overview(
             )
             batch_prompts.append((batch_idx, prompt, batch_index_map))
 
-        def make_replacer(idx_map):
-            def replacer(match):
-                idx = int(match.group(1))
-                return idx_map.get(idx, match.group(0))
-
-            return replacer
-
         async def _run_batch(batch_idx: int, prompt: str, batch_index_map: Dict[int, str]) -> None:
             try:
                 async with llm_sem:
                     with bind_telemetry_stage("resource_summarize"):
                         partial = await vlm.get_completion_async(prompt)
-                partial = re.sub(r"\[(\d+)\]", make_replacer(batch_index_map), partial)
+                partial = self._replace_index_references(partial, batch_index_map)
                 partial_overviews[batch_idx] = partial.strip()
             except Exception as e:
                 logger.warning(
@@ -1503,6 +1536,7 @@ async def _run_batch(batch_idx: int, prompt: str, batch_index_map: Dict[int, str
             )
             with bind_telemetry_stage("resource_summarize"):
                 overview = await vlm.get_completion_async(prompt)
+            overview = self._replace_index_references(overview, file_index_map)
             return overview.strip()
         except Exception as e:
             logger.error(

tests/storage/test_semantic_dag_incremental.py

@@ -86,11 +86,8 @@ async def _generate_overview(self, dir_uri, file_summaries, children_abstracts):
             lines.append(f"- {name}: {summary}")
         return "\n".join(lines)
 
-    def _extract_abstract_from_overview(self, overview):
-        return "abstract"
-
-    def _enforce_size_limits(self, overview, abstract):
-        return overview, abstract
+    def _normalize_overview_generation(self, overview):
+        return overview, "abstract"
 
     async def _sync_topdown_recursive(
         self, root_uri, target_uri, ctx=None, file_change_status=None, lock=None

tests/storage/test_semantic_dag_skip_files.py

@@ -54,11 +54,8 @@ async def _generate_single_file_summary(self, file_path, llm_sem=None, ctx=None)
     async def _generate_overview(self, dir_uri, file_summaries, children_abstracts):
         return "overview"
 
-    def _extract_abstract_from_overview(self, overview):
-        return "abstract"
-
-    def _enforce_size_limits(self, overview, abstract):
-        return overview, abstract
+    def _normalize_overview_generation(self, overview):
+        return overview, "abstract"
 
     async def _vectorize_directory(
         self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=None

tests/storage/test_semantic_dag_stats.py

@@ -16,7 +16,8 @@ def __init__(self, tree):
         self._tree = tree
         self.writes = []
 
-    async def ls(self, uri, ctx=None):
+    async def ls(self, uri, node_limit=None, ctx=None):
+        del node_limit
         return self._tree.get(uri, [])
 
     async def write_file(self, path, content, ctx=None):
@@ -37,11 +38,8 @@ async def _generate_single_file_summary(self, file_path, llm_sem=None, ctx=None)
     async def _generate_overview(self, dir_uri, file_summaries, children_abstracts):
         return "overview"
 
-    def _extract_abstract_from_overview(self, overview):
-        return "abstract"
-
-    def _enforce_size_limits(self, overview, abstract):
-        return overview, abstract
+    def _normalize_overview_generation(self, overview):
+        return overview, "abstract"
 
     async def _vectorize_directory(
         self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=None

tests/storage/test_semantic_processor_l0_l1.py

@@ -0,0 +1,153 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+from types import SimpleNamespace
+
+from openviking.storage.queuefs import semantic_processor as semantic_processor_module
+from openviking.storage.queuefs.semantic_processor import SemanticProcessor
+
+
+def _patch_semantic_limits(monkeypatch, *, abstract_max_chars=256, overview_max_chars=4000):
+    config = SimpleNamespace(
+        semantic=SimpleNamespace(
+            abstract_max_chars=abstract_max_chars,
+            overview_max_chars=overview_max_chars,
+        )
+    )
+    monkeypatch.setattr(semantic_processor_module, "get_openviking_config", lambda: config)
+
+
+def test_markdown_overview_uses_brief_description_as_abstract(monkeypatch):
+    _patch_semantic_limits(monkeypatch)
+    processor = SemanticProcessor()
+    generated = (
+        "# README\n\n"
+        "This brief description is the retrieval abstract.\n\n"
+        "## Quick Navigation\n\n"
+        "- Read README.md"
+    )
+
+    overview, abstract = processor._normalize_overview_generation(generated)
+
+    assert overview == generated
+    assert abstract == "This brief description is the retrieval abstract."
+
+
+def test_markdown_overview_extracts_multiline_brief_description(monkeypatch):
+    _patch_semantic_limits(monkeypatch)
+    processor = SemanticProcessor()
+    generated = (
+        "# README\n\n"
+        "This is the first abstract line.\n"
+        "This is the second abstract line.\n\n"
+        "## Quick Navigation\n\n"
+        "- Read README.md"
+    )
+
+    overview, abstract = processor._normalize_overview_generation(generated)
+
+    assert overview == generated
+    assert abstract == "This is the first abstract line.\nThis is the second abstract line."
+
+
+def test_index_references_are_replaced_inside_markdown_overview(monkeypatch):
+    _patch_semantic_limits(monkeypatch)
+    processor = SemanticProcessor()
+    generated = "# README\n\nUse [1] to get started."
+
+    replaced = processor._replace_index_references(generated, {1: "README.md"})
+
+    assert replaced == "# README\n\nUse README.md to get started."
+
+
+def test_abstract_truncation_prefers_complete_sentence(monkeypatch):
+    _patch_semantic_limits(monkeypatch, abstract_max_chars=80)
+    processor = SemanticProcessor()
+    abstract = (
+        "This is a complete sentence. "
+        "This second sentence contains onboarding material that would be cut."
+    )
+
+    overview, abstract = processor._enforce_size_limits("# README\n\nBody", abstract)
+
+    assert overview == "# README\n\nBody"
+    assert abstract == "This is a complete sentence."
+
+
+def test_abstract_truncation_keeps_first_sentence_even_over_limit(monkeypatch):
+    _patch_semantic_limits(monkeypatch, abstract_max_chars=80)
+    processor = SemanticProcessor()
+    first_sentence = (
+        "This directory is a timestamped media storage container for a single MP4 video "
+        "file, organized to preserve the exact capture or creation time of its contents."
+    )
+    abstract = f"{first_sentence} This second sentence should be omitted."
+
+    _, abstract = processor._enforce_size_limits("# video\n\nBody", abstract)
+
+    assert abstract == first_sentence
+
+
+def test_overview_truncation_prefers_complete_sentence(monkeypatch):
+    _patch_semantic_limits(monkeypatch, overview_max_chars=45)
+    processor = SemanticProcessor()
+    overview = (
+        "# README\n\n"
+        "This is a complete sentence. "
+        "This second sentence would be cut in the middle."
+    )
+
+    overview, abstract = processor._enforce_size_limits(overview, "abstract")
+
+    assert overview == "# README\n\nThis is a complete sentence."
+    assert abstract == "abstract"
+
+
+def test_overview_truncation_keeps_last_complete_sentence_within_limit(monkeypatch):
+    _patch_semantic_limits(monkeypatch, overview_max_chars=57)
+    processor = SemanticProcessor()
+    overview = (
+        "# README\n\n"
+        "First sentence. "
+        "Second sentence. "
+        "Third sentence should be omitted."
+    )
+
+    overview, abstract = processor._enforce_size_limits(overview, "abstract")
+
+    assert overview == "# README\n\nFirst sentence. Second sentence."
+    assert abstract == "abstract"
+
+
+def test_truncation_keeps_multiple_short_sentences_within_limit(monkeypatch):
+    _patch_semantic_limits(monkeypatch, abstract_max_chars=10)
+    processor = SemanticProcessor()
+
+    _, abstract = processor._enforce_size_limits("# README\n\nBody", "A. B. C. D.E.")
+
+    assert abstract == "A. B. C."
+
+
+def test_abstract_truncation_does_not_treat_decimal_point_as_sentence_end_without_period(
+    monkeypatch,
+):
+    _patch_semantic_limits(monkeypatch, abstract_max_chars=24)
+    processor = SemanticProcessor()
+    abstract = "This covers version 3.14 compatibility checks for onboarding"
+
+    _, abstract = processor._enforce_size_limits("# README\n\nBody", abstract)
+
+    assert abstract == "This covers version..."
+
+
+def test_abstract_truncation_accepts_sentence_period_after_number(monkeypatch):
+    _patch_semantic_limits(monkeypatch, abstract_max_chars=70)
+    processor = SemanticProcessor()
+    abstract = (
+        "This import check was generated at 16:55. "
+        "This second sentence would otherwise be truncated midstream."
+    )
+
+    _, abstract = processor._enforce_size_limits("# README\n\nBody", abstract)
+
+    assert abstract == "This import check was generated at 16:55."