Fixes to address PR reviews

Signed-off-by: Maksym Lysak <mly@zurich.ibm.com>

Author: Maksym Lysak

docling/backend/html_backend.py

@@ -1301,13 +1301,21 @@ def _is_rich_table_cell(self, table_cell: Tag) -> bool:
         has_custom_checkbox = any(
             self._is_custom_checkbox_tag(child) for child in children
         )
+        has_line_break = any(child.name == "br" for child in children)
+        direct_block_text_children = [
+            child
+            for child in table_cell.find_all(recursive=False)
+            if isinstance(child, Tag) and child.name in {"p", "div", "li"}
+        ]
         has_nested_form_semantic_id = any(
             self._is_form_semantic_tag(child)
             for child in children
             if isinstance(child, Tag)
         )
         if has_nested_form_semantic_id:
             return True
+        if has_line_break or len(direct_block_text_children) > 1:
+            return True
         if not children:
             content = [
                 item
@@ -1528,6 +1536,7 @@ def _flush_buffer() -> None:
 
         for node in element.contents:
             if isinstance(node, Tag):
+                name = node.name.lower()
                 if form_field := self._consume_form_field_for_tag(node):
                     _flush_buffer()
                     added_refs.extend(
@@ -1539,8 +1548,11 @@ def _flush_buffer() -> None:
                     )
                     continue
                 if self._is_suppressed_tag(node):
+                    if name == "br":
+                        # Keep explicit line breaks as text boundaries even when
+                        # the <br> tag itself has no rendered bbox.
+                        _flush_buffer()
                     continue
-                name = node.name.lower()
                 has_block_descendants = bool(
                     node.find(_BLOCK_TAGS)
                     or node.find("input")
@@ -1610,16 +1622,28 @@ def _flush_buffer() -> None:
                     _flush_buffer()
                     wk4 = self._walk(node, doc)
                     added_refs.extend(wk4)
-                else:
+                elif self._should_buffer_tag_text_inline(node):
                     buffer.extend(
                         self._extract_text_and_hyperlink_recursively(
                             node, find_parent_annotation=True, keep_newlines=False
                         )
                     )
+                else:
+                    _flush_buffer()
+                    wk5 = self._walk(node, doc)
+                    added_refs.extend(wk5)
             elif isinstance(node, NavigableString) and not isinstance(
                 node, PreformattedString
             ):
-                if str(node).strip("\n\r") == "":
+                node_text = str(node)
+                if node_text.strip("\n\r") == "":
+                    parent_tag = node.parent if isinstance(node.parent, Tag) else None
+                    if (
+                        parent_tag is not None
+                        and parent_tag.name in {"td", "th"}
+                        and "\n" in node_text
+                    ):
+                        _flush_buffer()
                     continue
                 else:
                     buffer.extend(
@@ -2675,6 +2699,26 @@ def _get_tag_classes(tag: Tag) -> set[str]:
             return {classes}
         return {str(value) for value in classes if isinstance(value, str)}
 
+    @staticmethod
+    def _has_inline_display_style(tag: Tag) -> bool:
+        style_attr = tag.get("style")
+        if not isinstance(style_attr, str):
+            return False
+        display_match = re.search(r"display\s*:\s*([^;]+)", style_attr, flags=re.I)
+        if display_match is None:
+            return False
+        display_value = display_match.group(1).strip().lower()
+        return display_value.startswith("inline") or display_value == "contents"
+
+    def _should_buffer_tag_text_inline(self, tag: Tag) -> bool:
+        tag_name = tag.name.lower()
+        if tag_name in _INLINE_HTML_TAGS:
+            return True
+        # Treat explicit inline-styled divs like inline wrappers.
+        if tag_name == "div" and self._has_inline_display_style(tag):
+            return True
+        return False
+
     @staticmethod
     def _is_input_checkbox_or_radio_tag(tag: Tag) -> bool:
         if tag.name != "input":
@@ -3610,6 +3654,7 @@ def _extract_form_region(self, form_tag: Tag) -> Optional[_ExtractedFormRegion]:
                     key_prov = values[0].prov
 
             component_tag_obj_ids: set[int] = {id(value.tag) for value in values}
+            component_tag_obj_ids.update(id(tag) for tag in value_tags)
             if key_tag is not None:
                 component_tag_obj_ids.add(id(key_tag))
             if marker is not None:
@@ -3872,6 +3917,7 @@ def _handle_form_container(self, tag: Tag, doc: DoclingDocument) -> list[RefItem
             consumed_tag_obj_ids: set[int] = set()
             fields_by_key_id: dict[str, _ExtractedFormField] = {}
             if form_region is not None:
+                consumed_tag_ids.update(form_region.consumed_tag_ids)
                 for field in form_region.fields:
                     key_tag_id = self._get_html_id(field.key_tag)
                     if not field.values:

tests/data/groundtruth/docling_v2/example_06.html.itxt

@@ -1,11 +1,12 @@
 item-0 at level 0: unspecified: group _root_
-  item-1 at level 1: text: This is a div with text. This is another div with text.
-  item-2 at level 1: text: This is a regular paragraph.
-  item-3 at level 1: text: This is a third div with a new line.
-  item-4 at level 1: section: group details
-    item-5 at level 2: text: Heading for the details element
-    item-6 at level 2: text: Description of the details element.
-  item-7 at level 1: inline: group group
-    item-8 at level 2: text: This is a fourth div with a
-    item-9 at level 2: text: bold
-    item-10 at level 2: text: paragraph.
+  item-1 at level 1: text: This is a div with text.
+  item-2 at level 1: text: This is another div with text.
+  item-3 at level 1: text: This is a regular paragraph.
+  item-4 at level 1: text: This is a third div with a new line.
+  item-5 at level 1: section: group details
+    item-6 at level 2: text: Heading for the details element
+    item-7 at level 2: text: Description of the details element.
+  item-8 at level 1: inline: group group
+    item-9 at level 2: text: This is a fourth div with a
+    item-10 at level 2: text: bold
+    item-11 at level 2: text: paragraph.

tests/data/groundtruth/docling_v2/example_06.html.json

@@ -29,6 +29,9 @@
       {
         "$ref": "#/texts/3"
       },
+      {
+        "$ref": "#/texts/4"
+      },
       {
         "$ref": "#/groups/0"
       },
@@ -48,10 +51,10 @@
       },
       "children": [
         {
-          "$ref": "#/texts/4"
+          "$ref": "#/texts/5"
         },
         {
-          "$ref": "#/texts/5"
+          "$ref": "#/texts/6"
         }
       ],
       "content_layer": "body",
@@ -64,14 +67,14 @@
         "$ref": "#/body"
       },
       "children": [
-        {
-          "$ref": "#/texts/6"
-        },
         {
           "$ref": "#/texts/7"
         },
         {
           "$ref": "#/texts/8"
+        },
+        {
+          "$ref": "#/texts/9"
         }
       ],
       "content_layer": "body",
@@ -101,8 +104,8 @@
       "content_layer": "body",
       "label": "text",
       "prov": [],
-      "orig": "This is a div with text. This is another div with text.",
-      "text": "This is a div with text. This is another div with text."
+      "orig": "This is a div with text.",
+      "text": "This is a div with text."
     },
     {
       "self_ref": "#/texts/2",
@@ -113,11 +116,23 @@
       "content_layer": "body",
       "label": "text",
       "prov": [],
+      "orig": "This is another div with text.",
+      "text": "This is another div with text."
+    },
+    {
+      "self_ref": "#/texts/3",
+      "parent": {
+        "$ref": "#/body"
+      },
+      "children": [],
+      "content_layer": "body",
+      "label": "text",
+      "prov": [],
       "orig": "This is a regular paragraph.",
       "text": "This is a regular paragraph."
     },
     {
-      "self_ref": "#/texts/3",
+      "self_ref": "#/texts/4",
       "parent": {
         "$ref": "#/body"
       },
@@ -129,7 +144,7 @@
       "text": "This is a third div with a new line."
     },
     {
-      "self_ref": "#/texts/4",
+      "self_ref": "#/texts/5",
       "parent": {
         "$ref": "#/groups/0"
       },
@@ -141,7 +156,7 @@
       "text": "Heading for the details element"
     },
     {
-      "self_ref": "#/texts/5",
+      "self_ref": "#/texts/6",
       "parent": {
         "$ref": "#/groups/0"
       },
@@ -153,7 +168,7 @@
       "text": "Description of the details element."
     },
     {
-      "self_ref": "#/texts/6",
+      "self_ref": "#/texts/7",
       "parent": {
         "$ref": "#/groups/1"
       },
@@ -165,7 +180,7 @@
       "text": "This is a fourth div with a"
     },
     {
-      "self_ref": "#/texts/7",
+      "self_ref": "#/texts/8",
       "parent": {
         "$ref": "#/groups/1"
       },
@@ -184,7 +199,7 @@
       }
     },
     {
-      "self_ref": "#/texts/8",
+      "self_ref": "#/texts/9",
       "parent": {
         "$ref": "#/groups/1"
       },

tests/data/groundtruth/docling_v2/example_06.html.md

@@ -1,4 +1,6 @@
-This is a div with text. This is another div with text.
+This is a div with text.
+
+This is another div with text.
 
 This is a regular paragraph.
 

tests/data/groundtruth/docling_v2/html_rich_table_cells.html.itxt

@@ -20,30 +20,33 @@ item-0 at level 0: unspecified: group _root_
           item-19 at level 5: list_item: Build nest on ground
     item-20 at level 2: table with [4x2]
       item-21 at level 3: unspecified: group rich_cell_group_2_0_1
-        item-22 at level 4: text: Aythya (Diving ducks)
-      item-23 at level 3: unspecified: group rich_cell_group_2_0_2
-        item-24 at level 4: text: Lophonetta (Pintail group)
-      item-25 at level 3: unspecified: group rich_cell_group_2_0_3
-        item-26 at level 4: text: Oxyura (Benthic ducks)
-    item-27 at level 2: table with [4x2]
-      item-28 at level 3: unspecified: group rich_cell_group_3_0_1
-        item-29 at level 4: text: Swim
-      item-30 at level 3: unspecified: group rich_cell_group_3_0_1
-        item-31 at level 4: text: Gracefully glide on H
-        item-32 at level 4: text: 2
-        item-33 at level 4: text: O surfaces.
-      item-34 at level 3: unspecified: group rich_cell_group_3_0_2
-        item-35 at level 4: text: Fly
-      item-36 at level 3: unspecified: group rich_cell_group_3_0_3
-        item-37 at level 4: text: Quack
-      item-38 at level 3: unspecified: group rich_cell_group_4_0_3
-        item-39 at level 4: table with [3x2]
-    item-40 at level 2: table with [5x3]
-      item-41 at level 3: unspecified: group rich_cell_group_5_1_1
-        item-42 at level 4: text: View PNG
-      item-43 at level 3: unspecified: group rich_cell_group_5_1_2
-        item-44 at level 4: picture
-          item-44 at level 5: caption: White-headed duck thumbnail
-      item-45 at level 3: unspecified: group rich_cell_group_5_1_3
-        item-46 at level 4: text: View Full-Size Image
-  item-47 at level 1: caption: White-headed duck thumbnail
+        item-22 at level 4: text: Aythya
+        item-23 at level 4: text: (Diving ducks)
+      item-24 at level 3: unspecified: group rich_cell_group_2_0_2
+        item-25 at level 4: text: Lophonetta
+        item-26 at level 4: text: (Pintail group)
+      item-27 at level 3: unspecified: group rich_cell_group_2_0_3
+        item-28 at level 4: text: Oxyura
+        item-29 at level 4: text: (Benthic ducks)
+    item-30 at level 2: table with [4x2]
+      item-31 at level 3: unspecified: group rich_cell_group_3_0_1
+        item-32 at level 4: text: Swim
+      item-33 at level 3: unspecified: group rich_cell_group_3_0_1
+        item-34 at level 4: text: Gracefully glide on H
+        item-35 at level 4: text: 2
+        item-36 at level 4: text: O surfaces.
+      item-37 at level 3: unspecified: group rich_cell_group_3_0_2
+        item-38 at level 4: text: Fly
+      item-39 at level 3: unspecified: group rich_cell_group_3_0_3
+        item-40 at level 4: text: Quack
+      item-41 at level 3: unspecified: group rich_cell_group_4_0_3
+        item-42 at level 4: table with [3x2]
+    item-43 at level 2: table with [5x3]
+      item-44 at level 3: unspecified: group rich_cell_group_5_1_1
+        item-45 at level 4: text: View PNG
+      item-46 at level 3: unspecified: group rich_cell_group_5_1_2
+        item-47 at level 4: picture
+          item-47 at level 5: caption: White-headed duck thumbnail
+      item-48 at level 3: unspecified: group rich_cell_group_5_1_3
+        item-49 at level 4: text: View Full-Size Image
+  item-50 at level 1: caption: White-headed duck thumbnail