feat(question-tools): add Zhipu GLM-OCR PDF parser and enhance question generation pipeline

- Add Zhipu GLM-OCR PDF parser implementation with layout analysis and image extraction
- Introduce configurable PDF parser selection (zhipu/mineru) in question config
- Increase max parallel questions from 1 to 50 for improved throughput
- Extend LLM request timeouts from 120s to 600s for longer-running operations
- Add comprehensive test suite for PDF parsing, OCR, extraction, and full pipeline flows
- Update exam mimic workflow to dynamically select parser based on configuration
- Enhance API responses with parser type information in status messages
- Export new Zhipu parser from question tools module

Author: TNCNBO

config/main.yaml

@@ -40,8 +40,14 @@ logging:
 question:
   # Refactored: no iteration loops (max_rounds removed)
   rag_query_count: 3
-  max_parallel_questions: 1
+  max_parallel_questions: 50
   rag_mode: naive
+  # PDF parsing configuration
+  pdf_parser: zhipu  # Options: mineru, zhipu
+  zhipu_ocr:
+    model: glm-ocr
+    api_url: https://open.bigmodel.cn/api/paas/v4/layout_parsing
+    timeout: 300
   agents:
     retrieve:
       top_k: 30

src/api/routers/question.py

@@ -30,6 +30,7 @@
 config = load_config_with_main("question_config.yaml", project_root)
 log_dir = config.get("paths", {}).get("user_log_dir") or config.get("logging", {}).get("log_dir")
 logger = get_logger("QuestionAPI", log_dir=log_dir)
+QUESTION_CFG = config.get("question", {})
 
 router = APIRouter()
 
@@ -203,7 +204,10 @@ def close(self):
                     {
                         "type": "status",
                         "stage": "parsing",
-                        "content": "Parsing PDF exam paper (MinerU)...",
+                        "content": (
+                            "Parsing PDF exam paper "
+                            f"({'Zhipu GLM-OCR' if QUESTION_CFG.get('pdf_parser', 'zhipu') == 'zhipu' else 'MinerU'})..."
+                        ),
                     }
                 )
                 logger.info(f"Saved and validated uploaded PDF to: {pdf_path}")
@@ -461,9 +465,7 @@ async def log_pusher():
                             "Please create or select a knowledge base and add documents first."
                         )
                     try:
-                        await websocket.send_json(
-                            {"type": "error", "content": error_content}
-                        )
+                        await websocket.send_json({"type": "error", "content": error_content})
                     except (RuntimeError, WebSocketDisconnect):
                         pass
                     logger.warning(f"Question generation failed: {error_content}")

src/services/llm/cloud_provider.py

@@ -231,7 +231,7 @@ async def _openai_complete(
         if "response_format" in kwargs:
             data["response_format"] = kwargs["response_format"]
 
-        timeout = aiohttp.ClientTimeout(total=120)
+        timeout = aiohttp.ClientTimeout(total=600)
         async with aiohttp.ClientSession(timeout=timeout) as session:
             async with session.post(url, headers=headers, json=data) as resp:
                 if resp.status == 200:
@@ -401,7 +401,7 @@ async def _anthropic_complete(
         "temperature": kwargs.get("temperature", 0.7),
     }
 
-    timeout = aiohttp.ClientTimeout(total=120)
+    timeout = aiohttp.ClientTimeout(total=600)
     async with aiohttp.ClientSession(timeout=timeout) as session:
         async with session.post(url, headers=headers, json=data) as response:
             if response.status != 200:

src/tools/question/__init__.py

@@ -6,10 +6,12 @@
 
 from .exam_mimic import mimic_exam_questions
 from .pdf_parser import parse_pdf_with_mineru
+from .pdf_parser_zhipu import parse_pdf_with_zhipu
 from .question_extractor import extract_questions_from_paper
 
 __all__ = [
     "parse_pdf_with_mineru",
+    "parse_pdf_with_zhipu",
     "extract_questions_from_paper",
     "mimic_exam_questions",
 ]

src/tools/question/exam_mimic.py

@@ -29,6 +29,7 @@
 # Note: AgentCoordinator is imported inside functions to avoid circular import
 from src.services.llm.config import get_llm_config
 from src.tools.question.pdf_parser import parse_pdf_with_mineru
+from src.tools.question.pdf_parser_zhipu import parse_pdf_with_zhipu
 from src.tools.question.question_extractor import extract_questions_from_paper
 
 # Type alias for WebSocket callback
@@ -201,12 +202,20 @@ async def send_progress(event_type: str, data: dict[str, Any]):
     # If a PDF is provided, parse it first
     elif pdf_path:
         # Stage 1: Parsing PDF
+        # Load config to determine which parser to use
+        from src.services.config import load_config_with_main
+        config = load_config_with_main("question_config.yaml", project_root)
+        question_cfg = config.get("question", {})
+        pdf_parser = question_cfg.get("pdf_parser", "zhipu")  # Default to zhipu
+        
+        parser_name = "Zhipu GLM-OCR" if pdf_parser == "zhipu" else "MinerU"
+        
         await send_progress(
             "progress",
-            {"stage": "parsing", "status": "running", "message": "Parsing PDF with MinerU..."},
+            {"stage": "parsing", "status": "running", "message": f"Parsing PDF with {parser_name}..."},
         )
 
-        print("🔄 Step 1: parse the PDF exam")
+        print(f"🔄 Step 1: parse the PDF exam (using {parser_name})")
         print("-" * 80)
 
         # Use provided output_dir or default to mimic_papers
@@ -216,11 +225,15 @@ async def send_progress(event_type: str, data: dict[str, Any]):
             output_base = project_root / "data" / "user" / "question" / "mimic_papers"
         output_base.mkdir(parents=True, exist_ok=True)
 
-        success = parse_pdf_with_mineru(pdf_path=pdf_path, output_base_dir=str(output_base))
+        # Choose parser based on config
+        if pdf_parser == "zhipu":
+            success = parse_pdf_with_zhipu(pdf_path=pdf_path, output_base_dir=str(output_base))
+        else:
+            success = parse_pdf_with_mineru(pdf_path=pdf_path, output_base_dir=str(output_base))
 
         if not success:
-            await send_progress("error", {"content": "Failed to parse PDF with MinerU"})
-            return {"success": False, "error": "Failed to parse PDF"}
+            await send_progress("error", {"content": f"Failed to parse PDF with {parser_name}"})
+            return {"success": False, "error": f"Failed to parse PDF with {parser_name}"}
 
         print()