|
3 | 3 | from __future__ import annotations |
4 | 4 |
|
5 | 5 | import logging |
| 6 | +import os |
| 7 | +import re |
6 | 8 | import zipfile |
7 | 9 | from io import BytesIO |
8 | 10 |
|
9 | 11 | logger = logging.getLogger(__name__) |
10 | 12 |
|
| 13 | +# --- Percentage number formats (opt-in) -------------------------------------- |
| 14 | +# Excel keeps a percentage as the stored fraction plus a display format: the |
| 15 | +# cell holds 0.61123366 and "0.00%" renders it as 61.12%. openpyxl and pandas |
| 16 | +# read only the fraction, so a value the user sees as 61.12% reaches the |
| 17 | +# pipeline as 0.6112 — a 100x error that a model will copy verbatim into a |
| 18 | +# report. When this switch is on, percentage cells are materialized as the |
| 19 | +# literal text the spreadsheet shows ("61.12%") instead. |
| 20 | +# |
| 21 | +# It is off by default on purpose: the cell changes from a number into a |
| 22 | +# string, so anything that sorts, aggregates or computes with it sees different |
| 23 | +# input. With it off, every workbook is read exactly as before. A single upload |
| 24 | +# can override it through parser_engine_overrides.xlsx_percent_as_text. |
| 25 | +PERCENT_AS_TEXT_ENV = "DOCREADER_XLSX_PERCENT_AS_TEXT" |
11 | 26 |
|
12 | | -def fill_merged_cells_xlsx(content: bytes) -> bytes: |
| 27 | +# Only percentages are materialized. Currency and thousands separators change |
| 28 | +# how a number is displayed, not what it means ("1,234,567" and 1234567 are the |
| 29 | +# same number), so rewriting them would churn cells for no semantic gain, and |
| 30 | +# date formats are already converted to datetimes by openpyxl. |
| 31 | + |
| 32 | +# --- Same-row hint fallback (opt-in, off by default) ------------------------- |
| 33 | +# Workbooks occasionally leave one cell of a percentage row without a |
| 34 | +# percentage format (a real financial statement had 折旧率 0.0592 formatted as |
| 35 | +# "0.00_ ;[Red]\-0.00\ "). When a row carries at least this many explicitly |
| 36 | +# percentage-formatted cells, an unformatted 0 < |value| < 1 number in that row |
| 37 | +# is treated as a percentage too. Only consulted while percent materialization |
| 38 | +# is on; 0 disables the fallback. |
| 39 | +ROW_HINT_MIN_ENV = "DOCREADER_XLSX_PCT_ROW_HINT_MIN" |
| 40 | +_DEFAULT_ROW_HINT_MIN = 0 |
| 41 | + |
| 42 | +# Only materialize a known numeric percentage pattern. Excel literals, conditions, |
| 43 | +# scaling and arbitrary decorations require a full format renderer; leave those |
| 44 | +# values untouched rather than guessing that every '%' means multiply by 100. |
| 45 | +_PERCENT_PATTERN = re.compile( |
| 46 | + r"(?P<sign>-?)(?P<open>\()?(?P<group>#,##)?0" |
| 47 | + r"(?:\.(?P<decimals>0+))?%(?P<close>\))?" |
| 48 | +) |
| 49 | +_FORMAT_COLOR = re.compile( |
| 50 | + r"\[(?:Black|Blue|Cyan|Green|Magenta|Red|White|Yellow|Color[0-9]+)\]", re.I |
| 51 | +) |
| 52 | + |
| 53 | + |
| 54 | +def _env_bool(name: str, default: bool) -> bool: |
| 55 | + value = os.environ.get(name) |
| 56 | + if value is None or not str(value).strip(): |
| 57 | + return default |
| 58 | + return str(value).strip().lower() in {"1", "true", "yes", "y", "on"} |
| 59 | + |
| 60 | + |
| 61 | +def _env_int(name: str, default: int) -> int: |
| 62 | + try: |
| 63 | + return int(str(os.environ.get(name, default)).strip()) |
| 64 | + except (TypeError, ValueError): |
| 65 | + return default |
| 66 | + |
| 67 | + |
| 68 | +def percent_as_text_default() -> bool: |
| 69 | + """Deployment-wide default for materializing percentage formats. |
| 70 | +
|
| 71 | + Read per call (not at import time) so the value can be flipped without a |
| 72 | + module reload; a per-upload override still wins over it. |
| 73 | + """ |
| 74 | + return _env_bool(PERCENT_AS_TEXT_ENV, False) |
| 75 | + |
| 76 | + |
| 77 | +def _format_percentage(value, number_format: str): |
| 78 | + """Render supported Excel percentage formats, leaving other formats intact.""" |
| 79 | + if isinstance(value, bool) or not isinstance(value, (int, float)): |
| 80 | + return None |
| 81 | + if not number_format or "%" not in number_format: |
| 82 | + return None |
| 83 | + # Quoted/escaped percentages are literal text, not percentage operators. |
| 84 | + # Reject unsupported conditions across all sections before selecting one. |
| 85 | + cleaned = _FORMAT_COLOR.sub("", number_format) |
| 86 | + if any(char in cleaned for char in ('"', "\\", "_", "*", "[", "]")): |
| 87 | + return None |
| 88 | + sections = cleaned.split(";") |
| 89 | + if len(sections) > 4: |
| 90 | + return None |
| 91 | + section_index = 1 if value < 0 and len(sections) >= 2 else 0 |
| 92 | + if value == 0 and len(sections) >= 3: |
| 93 | + section_index = 2 |
| 94 | + match = _PERCENT_PATTERN.fullmatch(sections[section_index].strip()) |
| 95 | + if not match or bool(match["open"]) != bool(match["close"]): |
| 96 | + return None |
| 97 | + decimals = len(match["decimals"] or "") |
| 98 | + # Excel's negative section formats the magnitude; a single section instead |
| 99 | + # supplies the minus sign automatically. |
| 100 | + amount = abs(value) if section_index == 1 else value |
| 101 | + grouping = "," if match["group"] else "" |
| 102 | + try: |
| 103 | + number = f"{float(amount) * 100:{grouping}.{decimals}f}" |
| 104 | + return f"{match['sign']}{match['open'] or ''}{number}%{match['close'] or ''}" |
| 105 | + except (TypeError, ValueError, OverflowError): |
| 106 | + return None |
| 107 | + |
| 108 | + |
| 109 | +def _materialize_percentage_cells(ws, row_hint_min: int) -> tuple[int, int]: |
| 110 | + """Rewrite the percentage-formatted numbers of *ws* as their display text. |
| 111 | +
|
| 112 | + Must run before merged ranges are filled: only the master cell of a merge |
| 113 | + carries a number_format while the covered cells are General, so filling |
| 114 | + first would copy a bare fraction into rows 2..n and leave it unconverted — |
| 115 | + the same row would then show both "0.55%" and 0.005509433890047721. |
| 116 | +
|
| 117 | + Returns ``(materialized, hinted)`` cell counts. |
| 118 | + """ |
| 119 | + row_hints: dict = {} |
| 120 | + if row_hint_min > 0: |
| 121 | + # Count the explicit percentage formats before converting anything: |
| 122 | + # the conversion below resets number_format to General. |
| 123 | + for row in ws.iter_rows(): |
| 124 | + count = sum( |
| 125 | + 1 |
| 126 | + for cell in row |
| 127 | + if type(cell.value) in (int, float) # noqa: E721 - excludes bool |
| 128 | + and _format_percentage(cell.value, cell.number_format) is not None |
| 129 | + ) |
| 130 | + if count: |
| 131 | + row_hints[row[0].row] = count |
| 132 | + |
| 133 | + materialized = 0 |
| 134 | + hinted = 0 |
| 135 | + for row in ws.iter_rows(): |
| 136 | + hint = row_hints.get(row[0].row, 0) |
| 137 | + for cell in row: |
| 138 | + if type(cell.value) not in (int, float): # noqa: E721 - excludes bool |
| 139 | + continue |
| 140 | + text = _format_percentage(cell.value, cell.number_format) |
| 141 | + if ( |
| 142 | + text is None |
| 143 | + and "%" not in str(cell.number_format or "") |
| 144 | + and hint >= row_hint_min > 0 |
| 145 | + and 0 < abs(cell.value) < 1 |
| 146 | + ): |
| 147 | + # Fallback for a cell that lost its percentage format while its |
| 148 | + # row kept one. Logged because it rewrites an unformatted value. |
| 149 | + text = f"{float(cell.value) * 100:.2f}%" |
| 150 | + hinted += 1 |
| 151 | + logger.info( |
| 152 | + "XLSX pre-parse: %s!%s%d has no percentage format but its row " |
| 153 | + "does; reading it as %.2f%% (stored value %r)", |
| 154 | + ws.title, |
| 155 | + cell.column_letter, |
| 156 | + cell.row, |
| 157 | + float(cell.value) * 100, |
| 158 | + cell.value, |
| 159 | + ) |
| 160 | + if text is None: |
| 161 | + continue |
| 162 | + cell.value = text |
| 163 | + cell.number_format = "General" |
| 164 | + materialized += 1 |
| 165 | + return materialized, hinted |
| 166 | + |
| 167 | + |
| 168 | +def fill_merged_cells_xlsx( |
| 169 | + content: bytes, percent_as_text: bool | None = None |
| 170 | +) -> bytes: |
13 | 171 | """Unmerge ranges and copy the master cell value into every covered cell. |
14 | 172 |
|
15 | 173 | openpyxl only stores values on the top-left cell of a merge; pandas then |
16 | 174 | sees NaN in the rest. Filling makes row-wise RAG chunks retain context. |
| 175 | +
|
| 176 | + ``percent_as_text`` also materializes percentage-formatted numbers as the |
| 177 | + text the spreadsheet displays ("0.6112" -> "61.12%"), because the stored |
| 178 | + fraction is 100x off from what a reader sees. ``None`` (the default) uses |
| 179 | + the ``DOCREADER_XLSX_PERCENT_AS_TEXT`` deployment default, which is off, so |
| 180 | + the default path is unchanged. |
17 | 181 | """ |
| 182 | + if percent_as_text is None: |
| 183 | + percent_as_text = percent_as_text_default() |
| 184 | + |
18 | 185 | if not zipfile.is_zipfile(BytesIO(content)): |
19 | 186 | return content |
20 | 187 |
|
21 | 188 | from openpyxl import load_workbook |
22 | 189 |
|
23 | 190 | wb = load_workbook(BytesIO(content), data_only=True) |
24 | 191 | changed = False |
| 192 | + materialized_cells = 0 |
| 193 | + hinted_cells = 0 |
| 194 | + row_hint_min = ( |
| 195 | + _env_int(ROW_HINT_MIN_ENV, _DEFAULT_ROW_HINT_MIN) if percent_as_text else 0 |
| 196 | + ) |
| 197 | + |
25 | 198 | for ws in wb.worksheets: |
| 199 | + if percent_as_text: |
| 200 | + # Before the merge fill below — see _materialize_percentage_cells. |
| 201 | + materialized, hinted = _materialize_percentage_cells(ws, row_hint_min) |
| 202 | + materialized_cells += materialized |
| 203 | + hinted_cells += hinted |
| 204 | + if materialized: |
| 205 | + changed = True |
| 206 | + |
26 | 207 | if not ws.merged_cells.ranges: |
27 | 208 | continue |
28 | 209 | for merge_range in list(ws.merged_cells.ranges): |
@@ -52,5 +233,13 @@ def fill_merged_cells_xlsx(content: bytes) -> bytes: |
52 | 233 |
|
53 | 234 | out = BytesIO() |
54 | 235 | wb.save(out) |
| 236 | + if materialized_cells: |
| 237 | + logger.info( |
| 238 | + "Materialized %d percentage cell(s) in XLSX before parse " |
| 239 | + "(%d recovered by the same-row hint fallback, threshold=%d)", |
| 240 | + materialized_cells, |
| 241 | + hinted_cells, |
| 242 | + row_hint_min, |
| 243 | + ) |
55 | 244 | logger.info("Filled merged cells in XLSX before parse") |
56 | 245 | return out.getvalue() |
0 commit comments