fix(docreader): keep spreadsheet cells that say N/A, None or NULL

docreader/parser/excel_parser.py

@@ -118,17 +118,16 @@ def parse_into_text(self, content: bytes) -> Document:
                 excel_sheet_name,
                 xlsx_first_row_as_header=self.xlsx_first_row_as_header,
             )
-            # Remove rows where all values are NaN (completely empty rows)
-            df.dropna(how="all", inplace=True)
-
             # Process each row in the DataFrame. The index is the 0-based
-            # sheet row (rows are read from row 1 and dropped rows keep
+            # sheet row (rows are read from row 1 and skipped rows keep
             # their labels), so index + 1 is the row number users see.
             for row_index, row in df.iterrows():
                 page_content = []
-                # Build key-value pairs for non-null values
+                # Build key-value pairs for non-empty values. A blank cell is
+                # read as "" (see _read_sheet_dataframe), so an empty row has
+                # no pairs and is skipped below.
                 for k, v in row.items():
-                    if pd.notna(v) and not _is_image_function(v):
+                    if pd.notna(v) and v != "" and not _is_image_function(v):
                         page_content.append(f"{k}: {v}")
 
                 # Skip rows with no valid content
@@ -180,7 +179,11 @@ def _read_sheet_dataframe(
     # Keep row 1 as data by default for both XLSX and legacy XLS. Users can
     # explicitly restore the historical behavior where row 1 supplies semantic
     # labels for every row.
-    df = excel_file.parse(sheet_name=sheet_name, header=None)
+    # keep_default_na=False: by default pandas reads a cell whose text is
+    # "N/A", "NA", "NULL", "None", "NaN" or "n/a" as missing, and its pair is
+    # dropped, though the cell says "not applicable" or "none". Blank cells are
+    # then read as "", which parse_into_text skips like a missing value.
+    df = excel_file.parse(sheet_name=sheet_name, header=None, keep_default_na=False)
     if xlsx_first_row_as_header and len(df.index) >= 2:
         df.columns = _stable_header_labels(df.iloc[0].tolist())
         return df.iloc[1:].copy()

docreader/tests/test_excel_parser.py

@@ -594,6 +594,67 @@ def test_xlsx_explicit_false_override_keeps_first_row_as_data(self):
         self.assertEqual(chunks[0], "A: Name,B: Age")
         self.assertEqual(chunks[1], "A: Alice,B: 30")
 
+    def test_cells_that_say_na_none_or_null_are_kept(self):
+        # pandas reads these strings as missing values by default, which
+        # dropped the pair although the cell has an answer in it.
+        content = self._workbook_bytes(
+            [
+                ["Patient", "Allergies", "Region", "Result", "Note"],
+                ["P1", "None", "NA", "N/A", "ok"],
+                ["P2", "Penicillin", "EU", "NULL", None],
+                ["P3", "n/a", "NaN", 42, "nan"],
+            ]
+        )
+
+        document = ExcelParser(
+            file_name="patients.xlsx",
+            file_type="xlsx",
+            xlsx_first_row_as_header=True,
+        ).parse_into_text(content)
+
+        self.assertEqual(
+            [chunk.content.strip() for chunk in document.chunks],
+            [
+                "Patient: P1,Allergies: None,Region: NA,Result: N/A,Note: ok",
+                "Patient: P2,Allergies: Penicillin,Region: EU,Result: NULL",
+                "Patient: P3,Allergies: n/a,Region: NaN,Result: 42,Note: nan",
+            ],
+        )
+
+    def test_a_header_cell_that_says_na_keeps_its_label(self):
+        content = self._workbook_bytes([["Item", "NA", "EU"], ["pens", 3, 4]])
+
+        document = ExcelParser(
+            file_name="sales.xlsx",
+            file_type="xlsx",
+            xlsx_first_row_as_header=True,
+        ).parse_into_text(content)
+
+        self.assertEqual(document.chunks[0].content.strip(), "Item: pens,NA: 3,EU: 4")
+
+    def test_blank_cells_and_blank_rows_are_still_skipped(self):
+        content = self._workbook_bytes(
+            [
+                ["Item", "Count"],
+                ["pens", None],
+                [None, None],
+                ["ink", 2],
+            ]
+        )
+
+        document = ExcelParser(file_name="stock.xlsx", file_type="xlsx").parse_into_text(
+            content
+        )
+
+        self.assertEqual(
+            [chunk.content.strip() for chunk in document.chunks],
+            ["A: Item,B: Count", "A: pens", "A: ink,B: 2"],
+        )
+        self.assertEqual(
+            [block["locator"]["row_start"] for block in document.source_blocks],
+            [1, 2, 4],
+        )
+
     def test_parse_phantom_shared_strings_workbook(self):
         document = ExcelParser().parse_into_text(_xlsx_with_phantom_shared_strings())
         self.assertIn("hello", document.content)