Skip to content

Commit a9290dc

Browse files
fix(docreader): Excel 百分比被读小 100 倍 —— 加开关(默认关) (#4423)
1 parent a7d915a commit a9290dc

6 files changed

Lines changed: 629 additions & 11 deletions

File tree

‎.env.example‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,17 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
425425
# DOCREADER_DOCX_MAX_PAGES=0
426426
# MarkItDown(重型解析器)并发 worker 数。0 关闭限流;默认 1 适合单机/GPU 受限。
427427
# DOCREADER_MARKITDOWN_MAX_WORKERS=1
428+
# ----- XLSX / Excel -----
429+
# 按显示格式物化百分比单元格。Excel 的百分比是显示格式而非存储值:单元格实存 0.6112,
430+
# 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的 61.12% 变成 0.6112
431+
# (100 倍偏差)。开启后百分比单元格变成字面量文本 "61.12%"。默认 false:开启会使单元格
432+
# 由数值变为字符串,改变下游排序/计算看到的内容;关闭时读取行为与升级前完全一致。
433+
# 单次上传可用 parser_engine_overrides.xlsx_percent_as_text 覆盖(true/false)。
434+
# DOCREADER_XLSX_PERCENT_AS_TEXT=false
435+
# 兜底:同一行有 >= N 个显式百分比格式单元格时,把该行漏标格式的 0<|v|<1 数字也按百分比
436+
# 处理(源文件偶有漏刷格式的格子,每命中一次打 INFO 日志)。仅在上一开关开启时生效;
437+
# 0=关闭该兜底(默认)。
438+
# DOCREADER_XLSX_PCT_ROW_HINT_MIN=0
428439
# 出站 HTTP/HTTPS 代理(抓取网页 / 远程图片时用)。
429440
# DOCREADER_EXTERNAL_HTTP_PROXY=
430441
# DOCREADER_EXTERNAL_HTTPS_PROXY=

‎docker-compose.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,16 @@ services:
463463
# ========== 解析通用 ==========
464464
- DOCREADER_DOCX_MAX_PAGES=${DOCREADER_DOCX_MAX_PAGES:-0}
465465
- DOCREADER_MARKITDOWN_MAX_WORKERS=${DOCREADER_MARKITDOWN_MAX_WORKERS:-1}
466+
# ========== XLSX / Excel ==========
467+
# 按显示格式物化百分比单元格:Excel 的百分比是显示格式而非存储值,单元格实存
468+
# 0.6112 靠 "0.00%" 渲染成 61.12%,而 pandas/openpyxl 只读到分数,报表里的
469+
# 61.12% 会变成 0.6112(100 倍偏差)。开启后该单元格变成字面量文本 "61.12%"。
470+
# 默认 false:开启会使单元格由数值变为字符串,改变下游排序/计算看到的内容,
471+
# 存量部署行为保持不变。单次上传可用 parser_engine_overrides.xlsx_percent_as_text 覆盖。
472+
- DOCREADER_XLSX_PERCENT_AS_TEXT=${DOCREADER_XLSX_PERCENT_AS_TEXT:-false}
473+
# 同一行有 >= N 个显式百分比格式单元格时,把该行漏标格式的 0<|v|<1 数字也按百分比
474+
# 处理(源文件偶有漏刷格式的格子)。仅在上一开关开启时生效;0=关闭(默认)。
475+
- DOCREADER_XLSX_PCT_ROW_HINT_MIN=${DOCREADER_XLSX_PCT_ROW_HINT_MIN:-0}
466476
# 出站 HTTP/HTTPS 代理(抓取网页 / 远程图片时)
467477
- DOCREADER_EXTERNAL_HTTP_PROXY=${DOCREADER_EXTERNAL_HTTP_PROXY:-}
468478
- DOCREADER_EXTERNAL_HTTPS_PROXY=${DOCREADER_EXTERNAL_HTTPS_PROXY:-}

‎docreader/parser/excel_parser.py‎

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -74,11 +74,16 @@ def __init__(
7474
file_name: str = "",
7575
file_type: str | None = None,
7676
xlsx_first_row_as_header: Any = False,
77+
xlsx_percent_as_text: Any = None,
7778
**kwargs: Any,
7879
):
7980
super().__init__(file_name=file_name, file_type=file_type, **kwargs)
8081
self.xlsx_first_row_as_header = _parse_bool(xlsx_first_row_as_header)
81-
82+
# Tri-state: None (or an empty value) keeps the deployment default from
83+
# DOCREADER_XLSX_PERCENT_AS_TEXT, an explicit value wins. Same
84+
# precedence as the pdf_force_scanned parser override.
85+
self.xlsx_percent_as_text = _parse_optional_bool(xlsx_percent_as_text)
86+
8287
def parse_into_text(self, content: bytes) -> Document:
8388
"""Parse Excel file bytes into a Document object.
8489
@@ -100,7 +105,11 @@ def parse_into_text(self, content: bytes) -> Document:
100105
source_blocks: List[dict] = []
101106
start, end = 0, 0
102107

103-
excel_file = _open_excel_file(content, file_type=self.file_type)
108+
excel_file = _open_excel_file(
109+
content,
110+
file_type=self.file_type,
111+
percent_as_text=self.xlsx_percent_as_text,
112+
)
104113

105114
# Process each sheet in the Excel file
106115
for excel_sheet_name in excel_file.sheet_names:
@@ -205,15 +214,27 @@ def _parse_bool(value: Any) -> bool:
205214
return str(value).strip().lower() in {"1", "true", "yes", "on"}
206215

207216

208-
def _prepare_xlsx_bytes(data: bytes) -> bytes:
217+
def _parse_optional_bool(value: Any) -> bool | None:
218+
"""Parse a tri-state parser override; None keeps the deployment default."""
219+
if value is None:
220+
return None
221+
if isinstance(value, bool):
222+
return value
223+
text = str(value).strip().lower()
224+
if not text:
225+
return None
226+
return text in {"1", "true", "yes", "y", "on"}
227+
228+
229+
def _prepare_xlsx_bytes(data: bytes, percent_as_text: bool | None = None) -> bytes:
209230
repaired = repair_xlsx_bytes(data)
210231
if repaired is not None:
211232
data = repaired
212233
readable = strip_unreadable_ranges_xlsx(data)
213234
if readable is not None:
214235
data = readable
215236
try:
216-
return fill_merged_cells_xlsx(data)
237+
return fill_merged_cells_xlsx(data, percent_as_text=percent_as_text)
217238
except TypeError:
218239
# A non-conforming styles.xml (empty/malformed fills) makes openpyxl's
219240
# load_workbook raise before pandas ever sees the file (#3637).
@@ -222,10 +243,14 @@ def _prepare_xlsx_bytes(data: bytes) -> bytes:
222243
sanitized = sanitize_xlsx_styles(data)
223244
if sanitized is None:
224245
raise
225-
return fill_merged_cells_xlsx(sanitized)
246+
return fill_merged_cells_xlsx(sanitized, percent_as_text=percent_as_text)
226247

227248

228-
def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFile:
249+
def _open_excel_file(
250+
content: bytes,
251+
file_type: str | None = None,
252+
percent_as_text: bool | None = None,
253+
) -> pd.ExcelFile:
229254
"""Open an Excel workbook with explicit engine selection and fallbacks."""
230255
data = content
231256
converted_via_soffice = False
@@ -254,7 +279,7 @@ def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFi
254279

255280
engine = engine_for_format(ext)
256281
if ext == "xlsx":
257-
data = _prepare_xlsx_bytes(data)
282+
data = _prepare_xlsx_bytes(data, percent_as_text=percent_as_text)
258283
engine = "openpyxl"
259284
try:
260285
return pd.ExcelFile(BytesIO(data), engine=engine)
@@ -269,7 +294,7 @@ def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFi
269294
if repaired is None:
270295
raise
271296
logger.info("Repaired XLSX sharedStrings packaging before parse")
272-
data = _prepare_xlsx_bytes(repaired)
297+
data = _prepare_xlsx_bytes(repaired, percent_as_text=percent_as_text)
273298
continue
274299
except ValueError as exc:
275300
if converted_via_soffice or "cannot be determined" not in str(exc):

‎docreader/parser/xlsx_merge.py‎

Lines changed: 190 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,26 +3,207 @@
33
from __future__ import annotations
44

55
import logging
6+
import os
7+
import re
68
import zipfile
79
from io import BytesIO
810

911
logger = logging.getLogger(__name__)
1012

13+
# --- Percentage number formats (opt-in) --------------------------------------
14+
# Excel keeps a percentage as the stored fraction plus a display format: the
15+
# cell holds 0.61123366 and "0.00%" renders it as 61.12%. openpyxl and pandas
16+
# read only the fraction, so a value the user sees as 61.12% reaches the
17+
# pipeline as 0.6112 — a 100x error that a model will copy verbatim into a
18+
# report. When this switch is on, percentage cells are materialized as the
19+
# literal text the spreadsheet shows ("61.12%") instead.
20+
#
21+
# It is off by default on purpose: the cell changes from a number into a
22+
# string, so anything that sorts, aggregates or computes with it sees different
23+
# input. With it off, every workbook is read exactly as before. A single upload
24+
# can override it through parser_engine_overrides.xlsx_percent_as_text.
25+
PERCENT_AS_TEXT_ENV = "DOCREADER_XLSX_PERCENT_AS_TEXT"
1126

12-
def fill_merged_cells_xlsx(content: bytes) -> bytes:
27+
# Only percentages are materialized. Currency and thousands separators change
28+
# how a number is displayed, not what it means ("1,234,567" and 1234567 are the
29+
# same number), so rewriting them would churn cells for no semantic gain, and
30+
# date formats are already converted to datetimes by openpyxl.
31+
32+
# --- Same-row hint fallback (opt-in, off by default) -------------------------
33+
# Workbooks occasionally leave one cell of a percentage row without a
34+
# percentage format (a real financial statement had 折旧率 0.0592 formatted as
35+
# "0.00_ ;[Red]\-0.00\ "). When a row carries at least this many explicitly
36+
# percentage-formatted cells, an unformatted 0 < |value| < 1 number in that row
37+
# is treated as a percentage too. Only consulted while percent materialization
38+
# is on; 0 disables the fallback.
39+
ROW_HINT_MIN_ENV = "DOCREADER_XLSX_PCT_ROW_HINT_MIN"
40+
_DEFAULT_ROW_HINT_MIN = 0
41+
42+
# Only materialize a known numeric percentage pattern. Excel literals, conditions,
43+
# scaling and arbitrary decorations require a full format renderer; leave those
44+
# values untouched rather than guessing that every '%' means multiply by 100.
45+
_PERCENT_PATTERN = re.compile(
46+
r"(?P<sign>-?)(?P<open>\()?(?P<group>#,##)?0"
47+
r"(?:\.(?P<decimals>0+))?%(?P<close>\))?"
48+
)
49+
_FORMAT_COLOR = re.compile(
50+
r"\[(?:Black|Blue|Cyan|Green|Magenta|Red|White|Yellow|Color[0-9]+)\]", re.I
51+
)
52+
53+
54+
def _env_bool(name: str, default: bool) -> bool:
55+
value = os.environ.get(name)
56+
if value is None or not str(value).strip():
57+
return default
58+
return str(value).strip().lower() in {"1", "true", "yes", "y", "on"}
59+
60+
61+
def _env_int(name: str, default: int) -> int:
62+
try:
63+
return int(str(os.environ.get(name, default)).strip())
64+
except (TypeError, ValueError):
65+
return default
66+
67+
68+
def percent_as_text_default() -> bool:
69+
"""Deployment-wide default for materializing percentage formats.
70+
71+
Read per call (not at import time) so the value can be flipped without a
72+
module reload; a per-upload override still wins over it.
73+
"""
74+
return _env_bool(PERCENT_AS_TEXT_ENV, False)
75+
76+
77+
def _format_percentage(value, number_format: str):
78+
"""Render supported Excel percentage formats, leaving other formats intact."""
79+
if isinstance(value, bool) or not isinstance(value, (int, float)):
80+
return None
81+
if not number_format or "%" not in number_format:
82+
return None
83+
# Quoted/escaped percentages are literal text, not percentage operators.
84+
# Reject unsupported conditions across all sections before selecting one.
85+
cleaned = _FORMAT_COLOR.sub("", number_format)
86+
if any(char in cleaned for char in ('"', "\\", "_", "*", "[", "]")):
87+
return None
88+
sections = cleaned.split(";")
89+
if len(sections) > 4:
90+
return None
91+
section_index = 1 if value < 0 and len(sections) >= 2 else 0
92+
if value == 0 and len(sections) >= 3:
93+
section_index = 2
94+
match = _PERCENT_PATTERN.fullmatch(sections[section_index].strip())
95+
if not match or bool(match["open"]) != bool(match["close"]):
96+
return None
97+
decimals = len(match["decimals"] or "")
98+
# Excel's negative section formats the magnitude; a single section instead
99+
# supplies the minus sign automatically.
100+
amount = abs(value) if section_index == 1 else value
101+
grouping = "," if match["group"] else ""
102+
try:
103+
number = f"{float(amount) * 100:{grouping}.{decimals}f}"
104+
return f"{match['sign']}{match['open'] or ''}{number}%{match['close'] or ''}"
105+
except (TypeError, ValueError, OverflowError):
106+
return None
107+
108+
109+
def _materialize_percentage_cells(ws, row_hint_min: int) -> tuple[int, int]:
110+
"""Rewrite the percentage-formatted numbers of *ws* as their display text.
111+
112+
Must run before merged ranges are filled: only the master cell of a merge
113+
carries a number_format while the covered cells are General, so filling
114+
first would copy a bare fraction into rows 2..n and leave it unconverted —
115+
the same row would then show both "0.55%" and 0.005509433890047721.
116+
117+
Returns ``(materialized, hinted)`` cell counts.
118+
"""
119+
row_hints: dict = {}
120+
if row_hint_min > 0:
121+
# Count the explicit percentage formats before converting anything:
122+
# the conversion below resets number_format to General.
123+
for row in ws.iter_rows():
124+
count = sum(
125+
1
126+
for cell in row
127+
if type(cell.value) in (int, float) # noqa: E721 - excludes bool
128+
and _format_percentage(cell.value, cell.number_format) is not None
129+
)
130+
if count:
131+
row_hints[row[0].row] = count
132+
133+
materialized = 0
134+
hinted = 0
135+
for row in ws.iter_rows():
136+
hint = row_hints.get(row[0].row, 0)
137+
for cell in row:
138+
if type(cell.value) not in (int, float): # noqa: E721 - excludes bool
139+
continue
140+
text = _format_percentage(cell.value, cell.number_format)
141+
if (
142+
text is None
143+
and "%" not in str(cell.number_format or "")
144+
and hint >= row_hint_min > 0
145+
and 0 < abs(cell.value) < 1
146+
):
147+
# Fallback for a cell that lost its percentage format while its
148+
# row kept one. Logged because it rewrites an unformatted value.
149+
text = f"{float(cell.value) * 100:.2f}%"
150+
hinted += 1
151+
logger.info(
152+
"XLSX pre-parse: %s!%s%d has no percentage format but its row "
153+
"does; reading it as %.2f%% (stored value %r)",
154+
ws.title,
155+
cell.column_letter,
156+
cell.row,
157+
float(cell.value) * 100,
158+
cell.value,
159+
)
160+
if text is None:
161+
continue
162+
cell.value = text
163+
cell.number_format = "General"
164+
materialized += 1
165+
return materialized, hinted
166+
167+
168+
def fill_merged_cells_xlsx(
169+
content: bytes, percent_as_text: bool | None = None
170+
) -> bytes:
13171
"""Unmerge ranges and copy the master cell value into every covered cell.
14172
15173
openpyxl only stores values on the top-left cell of a merge; pandas then
16174
sees NaN in the rest. Filling makes row-wise RAG chunks retain context.
175+
176+
``percent_as_text`` also materializes percentage-formatted numbers as the
177+
text the spreadsheet displays ("0.6112" -> "61.12%"), because the stored
178+
fraction is 100x off from what a reader sees. ``None`` (the default) uses
179+
the ``DOCREADER_XLSX_PERCENT_AS_TEXT`` deployment default, which is off, so
180+
the default path is unchanged.
17181
"""
182+
if percent_as_text is None:
183+
percent_as_text = percent_as_text_default()
184+
18185
if not zipfile.is_zipfile(BytesIO(content)):
19186
return content
20187

21188
from openpyxl import load_workbook
22189

23190
wb = load_workbook(BytesIO(content), data_only=True)
24191
changed = False
192+
materialized_cells = 0
193+
hinted_cells = 0
194+
row_hint_min = (
195+
_env_int(ROW_HINT_MIN_ENV, _DEFAULT_ROW_HINT_MIN) if percent_as_text else 0
196+
)
197+
25198
for ws in wb.worksheets:
199+
if percent_as_text:
200+
# Before the merge fill below — see _materialize_percentage_cells.
201+
materialized, hinted = _materialize_percentage_cells(ws, row_hint_min)
202+
materialized_cells += materialized
203+
hinted_cells += hinted
204+
if materialized:
205+
changed = True
206+
26207
if not ws.merged_cells.ranges:
27208
continue
28209
for merge_range in list(ws.merged_cells.ranges):
@@ -52,5 +233,13 @@ def fill_merged_cells_xlsx(content: bytes) -> bytes:
52233

53234
out = BytesIO()
54235
wb.save(out)
236+
if materialized_cells:
237+
logger.info(
238+
"Materialized %d percentage cell(s) in XLSX before parse "
239+
"(%d recovered by the same-row hint fallback, threshold=%d)",
240+
materialized_cells,
241+
hinted_cells,
242+
row_hint_min,
243+
)
55244
logger.info("Filled merged cells in XLSX before parse")
56245
return out.getvalue()

0 commit comments

Comments
 (0)