11import json
22import logging
3+ import re
34from pathlib import Path
45from collections import defaultdict
6+ from difflib import SequenceMatcher
7+ from typing import Optional
58from utils .text_utils import load_caption_map , CONTROL_CHAR_PATTERN
69
10+ logger = logging .getLogger (__name__ )
11+
12+ # ───────────────── Phase 4.7d Commit 1：title 三軸融合（v2 §3.1 §4.1 §5 §6） ─────────────────
13+ # 黑名單（中英 + PDF 匯出器預設值 + 履歷 label）
14+ TITLE_BLACKLIST_EXACT = {
15+ # 章節 label
16+ "contents" , "table of contents" , "目錄" , "目次" ,
17+ "abstract" , "摘要" , "summary" , "executive summary" ,
18+ "introduction" , "前言" , "緒論" , "intro" ,
19+ "references" , "参考文献" , "參考文獻" , "bibliography" ,
20+ "acknowledgements" , "acknowledgments" , "致謝" ,
21+ "appendix" , "附錄" ,
22+ "preface" , "序" ,
23+ # PDF 匯出器預設值
24+ "untitled" , "untitled document" , "untitled1" ,
25+ "document" , "document1" , "document2" ,
26+ "未命名" , "未命名文件" ,
27+ "powerpoint presentation" , "slide 1" , "slide1" ,
28+ "presentation" ,
29+ # 履歷 label
30+ "resume" , "cv" , "curriculum vitae" , "履歷" , "個人簡歷" ,
31+ }
32+ TITLE_BLACKLIST_PATTERN = [
33+ r"^第\s*\d+\s*頁$" ,
34+ r"^page\s*\d+$" ,
35+ r"^chapter\s*\d+$" ,
36+ r"^第\s*\d+\s*章$" ,
37+ r"^slide\s*\d+$" ,
38+ r"^section\s*\d+$" ,
39+ r"^\d+$" ,
40+ r"^[\d.]+$" ,
41+ ]
42+ # 中文 domain token 排除字
43+ _DOM_STOPS_CN = {"的" , "與" , "和" , "之" , "對" , "在" , "及" , "或" }
44+
45+
46+ def _title_in_blacklist (title : str ) -> bool :
47+ """v2 §3.1：title 是否為已知雜訊（章節 label / 頁碼 / 預設值）。"""
48+ if not title :
49+ return False
50+ t = title .strip ().casefold ()
51+ if t in TITLE_BLACKLIST_EXACT :
52+ return True
53+ for pat in TITLE_BLACKLIST_PATTERN :
54+ if re .match (pat , t , re .IGNORECASE ):
55+ return True
56+ return False
57+
58+
59+ def _title_sim (a : str , b : str ) -> bool :
60+ """v2 §6：相似度三段（相等 / 包含 / SequenceMatcher.ratio() >= 0.7）。"""
61+ if not a or not b :
62+ return False
63+ A = re .sub (r"\s+" , "" , a ).casefold ()
64+ B = re .sub (r"\s+" , "" , b ).casefold ()
65+ if A == B :
66+ return True
67+ if A in B or B in A :
68+ return True
69+ return SequenceMatcher (None , A , B ).ratio () >= 0.7
70+
71+
72+ def _tokenize_for_domain (s : str ) -> set :
73+ """v2 §5.1：英文 token + 中文 2/3-gram；去停用詞。"""
74+ if not s :
75+ return set ()
76+ tokens = set ()
77+ for m in re .finditer (r"[A-Za-z][A-Za-z0-9-]*" , s ):
78+ tokens .add (m .group ().casefold ())
79+ for chunk in re .findall (r"[一-鿿]+" , s ):
80+ if len (chunk ) >= 2 :
81+ for i in range (len (chunk ) - 1 ):
82+ tokens .add (chunk [i :i + 2 ])
83+ if len (chunk ) >= 3 :
84+ for i in range (len (chunk ) - 2 ):
85+ tokens .add (chunk [i :i + 3 ])
86+ return tokens - _DOM_STOPS_CN
87+
88+
89+ def _dom_match (value : str , domain : str ) -> bool :
90+ """v2 §5.1：value 與 domain 的 token 交集 ≥ 1。domain 空則一律 False。"""
91+ if not domain :
92+ return False
93+ return len (_tokenize_for_domain (value ) & _tokenize_for_domain (domain )) >= 1
94+
95+
96+ def _resolve_title (
97+ data : dict ,
98+ metadata : Optional [dict ],
99+ doc_type : Optional [str ],
100+ domain : str ,
101+ original_filename : Optional [str ] = None ,
102+ paper_uuid : Optional [str ] = None ,
103+ ) -> tuple :
104+ """三軸融合 title（v2 §4.1 25 狀況決策樹）。
105+
106+ Returns:
107+ (title_en, title_zh, confidence_log)
108+ """
109+ raw_en = (data .get ('title' ) or '' ).strip ()
110+ raw_zh = (data .get ('translated_title' ) or raw_en ).strip ()
111+
112+ m = metadata or {}
113+ m_title = ((m .get ('title' ) or {}).get ('value' ) or '' ).strip ()
114+ m_trans = ((m .get ('translated_title' ) or {}).get ('value' ) or '' ).strip ()
115+ m_source = ((m .get ('title' ) or {}).get ('source' ) or '' )
116+ m_cand = ((m .get ('candidate_name' ) or {}).get ('value' ) or '' ).strip ()
117+
118+ def _fallback ():
119+ fb = (original_filename or '' ).strip ()
120+ if fb .lower ().endswith ('.pdf' ):
121+ fb = fb [:- 4 ].strip ()
122+ fb = fb or (paper_uuid or 'untitled' )
123+ return (fb , fb , 'low (fallback to filename/uuid)' )
124+
125+ # #1 resume 短路（doc_type-specific）
126+ if doc_type == 'resume' and m_cand :
127+ return (m_cand , m_cand , 'high (resume candidate_name)' )
128+
129+ # #2-#3 resume 無 candidate_name → raw 是姓名
130+ if doc_type == 'resume' and not m_cand :
131+ if raw_en and not _title_in_blacklist (raw_en ):
132+ return (raw_en , raw_zh , 'medium (resume raw=name)' )
133+ return _fallback ()
134+
135+ # #4 全空 fallback
136+ if not raw_en and not m_title :
137+ return _fallback ()
138+
139+ # #5-#6 raw 空、metadata 有
140+ if not raw_en and m_title :
141+ if _title_in_blacklist (m_title ):
142+ return _fallback ()
143+ return (m_title , m_trans or m_title , 'high (metadata only)' )
144+
145+ # #7-#8 raw 有、metadata 空
146+ if raw_en and not m_title :
147+ if _title_in_blacklist (raw_en ):
148+ return _fallback ()
149+ return (raw_en , raw_zh , 'medium (raw only)' )
150+
151+ # 兩者皆有
152+ raw_bl = _title_in_blacklist (raw_en )
153+ sim = _title_sim (raw_en , m_title )
154+
155+ # #9-#10 both_agree（雙路確認）
156+ if m_source == 'both_agree' :
157+ return (m_title , m_trans or m_title , 'high (both_agree)' )
158+
159+ # #20 manual（保留）
160+ if m_source == 'manual' :
161+ return (m_title , m_trans or m_title , 'high (manual override)' )
162+
163+ # #11-#13 llm_page1
164+ if m_source == 'llm_page1' :
165+ if sim :
166+ return (m_title , m_trans or m_title , 'high (llm sim raw)' )
167+ if raw_bl :
168+ return (m_title , m_trans or m_title , 'high (raw blacklisted)' )
169+ # #13 都合法但不同 → domain 仲裁（v2 §5.2）
170+ arb_R = _dom_match (raw_en , domain )
171+ arb_M = _dom_match (m_title , domain )
172+ if arb_M and arb_R :
173+ return (m_title , m_trans or m_title , 'medium (dom both match, take M)' )
174+ if arb_M and not arb_R :
175+ return (m_title , m_trans or m_title , 'medium (dom match M)' )
176+ if arb_R and not arb_M :
177+ return (raw_en , raw_zh , 'medium (dom match raw)' )
178+ logger .warning (
179+ f"[md_restore] title 仲裁失敗 raw={ raw_en !r} { m_title !r}
180+ f"domain={ domain !r}
181+ )
182+ return (raw_en , raw_zh , 'low (dom both fail, take raw)' )
183+
184+ # #14-#16 pdf_metadata（baron 核心觀察：99% 是垃圾預設值，raw 勝）
185+ if m_source == 'pdf_metadata' :
186+ if sim :
187+ return (m_title , m_trans or m_title , 'medium (pdf_meta sim raw)' )
188+ if raw_bl :
189+ return (m_title , m_trans or m_title , 'medium (raw blacklisted, only pdf_meta)' )
190+ return (raw_en , raw_zh , 'medium (pdf_meta untrustworthy, take raw)' )
191+
192+ # #17-#19 llm_page1 (conflict with pdf_metadata)
193+ if 'conflict' in m_source .lower ():
194+ if sim :
195+ return (m_title , m_trans or m_title , 'medium (conflict sim raw)' )
196+ if raw_bl :
197+ return (m_title , m_trans or m_title , 'medium (raw blacklisted)' )
198+ arb_R = _dom_match (raw_en , domain )
199+ arb_M = _dom_match (m_title , domain )
200+ if arb_M :
201+ return (m_title , m_trans or m_title , 'medium (conflict, dom match M)' )
202+ if arb_R :
203+ return (raw_en , raw_zh , 'medium (conflict, dom match raw)' )
204+ logger .warning (
205+ f"[md_restore] conflict 仲裁失敗 raw={ raw_en !r} { m_title !r}
206+ )
207+ return (raw_en , raw_zh , 'low (conflict, take raw)' )
208+
209+ # #24 未知 source 兜底
210+ logger .warning (f"[md_restore] 未知 metadata.title.source={ m_source !r} )
211+ return (raw_en , raw_zh , 'low (unknown source)' )
212+
213+
7214class RestoreProcessor :
8215 """恢复处理器, 将提供的json文件还原成中英两篇md文档"""
9216
@@ -214,10 +421,19 @@ def _process_section(self, section, output_path_en, output_path_zh, level=1, vis
214421 self ._process_section (child , output_path_en , output_path_zh , level + 1 , vision_captions )
215422
216423 def process (self , input_path : str , output_path_en : str , output_path_zh : str ,
217- images_info_path : str = None ) -> tuple :
424+ images_info_path : str = None ,
425+ metadata : Optional [dict ] = None ,
426+ doc_type : Optional [str ] = None ,
427+ domain : str = '' ,
428+ original_filename : Optional [str ] = None ,
429+ paper_uuid : Optional [str ] = None ) -> tuple :
218430 """
219431 读取 input.json，恢复成中英文两篇md文档
220432 1. 中文用翻译部分；如果没有翻译则保留英文原文
433+
434+ Phase 4.7d Commit 1：title 改走三軸融合（v2 §4.1 25 狀況決策樹）。
435+ 新增 metadata/doc_type/domain/original_filename/paper_uuid 參數；
436+ 皆有預設值，舊 caller 不傳則退回「只看 data['title']」舊行為。
221437 """
222438 try :
223439 input_path = Path (input_path )
@@ -240,11 +456,22 @@ def process(self, input_path: str, output_path_en: str, output_path_zh: str,
240456 with input_path .open ('r' , encoding = 'utf-8' ) as f :
241457 data = json .load (f )
242458
243- # 处理文档标题
244- title_en = data .get ('title' , '' )
459+ # Phase 4.7d Commit 1：title 三軸融合（v2 §4.1）。舊 caller
460+ # 不傳 metadata/doc_type/domain 時 _resolve_title 退回「raw only」分支、
461+ # 行為與舊版等效。
462+ title_en , title_zh , conf_log = _resolve_title (
463+ data , metadata , doc_type , domain , original_filename , paper_uuid
464+ )
465+ self .logger .info (
466+ f"[md_restore] 三軸融合 title: { conf_log }
467+ )
468+ self .logger .info (
469+ f" raw_en={ data .get ('title' )!r} { title_en !r}
470+ )
471+ self .logger .info (
472+ f" raw_zh={ data .get ('translated_title' )!r} { title_zh !r}
473+ )
245474 self ._write_to_md (output_path_en , f"# { title_en } )
246-
247- title_zh = data .get ('translated_title' , title_en )
248475 self ._write_to_md (output_path_zh , f"# { title_zh } )
249476
250477 # 处理作者信息
0 commit comments