Skip to content

Commit e1a0d20

Browse files
maxmilianmaxmilian
committed
feat(markdown): include table image in Markdown export when available
MarkdownTableSerializer previously ignored TableItem.image entirely, only ever emitting the text/HTML grid, unlike MarkdownPictureSerializer which already honors image_mode for pictures. Extract the shared image-emission logic (_serialize_image_part) from MarkdownPictureSerializer into a module-level function usable by both serializers, and move _image_to_base64 up from PictureItem to the shared FloatingItem base so TableItem can call it too. MarkdownTableSerializer now appends the table image alongside the existing text table when item.image is set and image_mode is not PLACEHOLDER. Tables without a generated image (the common case) see no output change regardless of image_mode. Closes #474 Signed-off-by: Max Hsu <maxmilian@gmail.com>
1 parent cf8648b commit e1a0d20

3 files changed

Lines changed: 137 additions & 55 deletions

File tree

‎docling_core/transforms/serializer/markdown.py‎

Lines changed: 49 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -576,11 +576,59 @@ def serialize(
576576
if table_text:
577577
res_parts.append(create_ser_result(text=table_text, span_source=item))
578578

579+
if item.image is not None and params.image_mode != ImageRefMode.PLACEHOLDER:
580+
img_res = _serialize_image_part(
581+
item=item,
582+
doc=doc,
583+
image_mode=params.image_mode,
584+
image_placeholder=params.image_placeholder,
585+
)
586+
if img_res.text:
587+
res_parts.append(img_res)
588+
579589
text_res = "\n\n".join([r.text for r in res_parts])
580590

581591
return create_ser_result(text=text_res, span_source=res_parts)
582592

583593

594+
def _serialize_image_part(
595+
item: FloatingItem,
596+
doc: DoclingDocument,
597+
image_mode: ImageRefMode,
598+
image_placeholder: str,
599+
) -> SerializationResult:
600+
error_response = "<!-- 🖼️❌ Image not available. Please use `PdfPipelineOptions(generate_picture_images=True)` -->"
601+
if image_mode == ImageRefMode.PLACEHOLDER:
602+
text_res = image_placeholder
603+
elif image_mode == ImageRefMode.EMBEDDED:
604+
# short-cut: we already have the image in base64
605+
if isinstance(item.image, ImageRef) and isinstance(item.image.uri, AnyUrl) and item.image.uri.scheme == "data":
606+
text = f"![Image]({item.image.uri})"
607+
text_res = text
608+
else:
609+
# get the item.image._pil or crop it out of the page-image
610+
img = item.get_image(doc=doc)
611+
612+
if img is not None:
613+
imgb64 = item._image_to_base64(img)
614+
text = f"![Image](data:image/png;base64,{imgb64})"
615+
616+
text_res = text
617+
else:
618+
text_res = error_response
619+
elif image_mode == ImageRefMode.REFERENCED:
620+
if not isinstance(item.image, ImageRef) or (
621+
isinstance(item.image.uri, AnyUrl) and item.image.uri.scheme == "data"
622+
):
623+
text_res = image_placeholder
624+
else:
625+
text_res = f"![Image]({item.image.uri!s})"
626+
else:
627+
text_res = image_placeholder
628+
629+
return create_ser_result(text=text_res, span_source=item)
630+
631+
584632
class MarkdownPictureSerializer(BasePictureSerializer):
585633
"""Markdown-specific picture item serializer."""
586634

@@ -614,7 +662,7 @@ def serialize(
614662
if ann_res.text:
615663
res_parts.append(ann_res)
616664

617-
img_res = self._serialize_image_part(
665+
img_res = _serialize_image_part(
618666
item=item,
619667
doc=doc,
620668
image_mode=params.image_mode,
@@ -640,51 +688,6 @@ def serialize(
640688

641689
return create_ser_result(text=text_res, span_source=res_parts)
642690

643-
def _serialize_image_part(
644-
self,
645-
item: PictureItem,
646-
doc: DoclingDocument,
647-
image_mode: ImageRefMode,
648-
image_placeholder: str,
649-
**kwargs: Any,
650-
) -> SerializationResult:
651-
error_response = (
652-
"<!-- 🖼️❌ Image not available. Please use `PdfPipelineOptions(generate_picture_images=True)` -->"
653-
)
654-
if image_mode == ImageRefMode.PLACEHOLDER:
655-
text_res = image_placeholder
656-
elif image_mode == ImageRefMode.EMBEDDED:
657-
# short-cut: we already have the image in base64
658-
if (
659-
isinstance(item.image, ImageRef)
660-
and isinstance(item.image.uri, AnyUrl)
661-
and item.image.uri.scheme == "data"
662-
):
663-
text = f"![Image]({item.image.uri})"
664-
text_res = text
665-
else:
666-
# get the item.image._pil or crop it out of the page-image
667-
img = item.get_image(doc=doc)
668-
669-
if img is not None:
670-
imgb64 = item._image_to_base64(img)
671-
text = f"![Image](data:image/png;base64,{imgb64})"
672-
673-
text_res = text
674-
else:
675-
text_res = error_response
676-
elif image_mode == ImageRefMode.REFERENCED:
677-
if not isinstance(item.image, ImageRef) or (
678-
isinstance(item.image.uri, AnyUrl) and item.image.uri.scheme == "data"
679-
):
680-
text_res = image_placeholder
681-
else:
682-
text_res = f"![Image]({item.image.uri!s})"
683-
else:
684-
text_res = image_placeholder
685-
686-
return create_ser_result(text=text_res, span_source=item)
687-
688691

689692
class MarkdownKeyValueSerializer(BaseKeyValueSerializer):
690693
"""Markdown-specific key-value item serializer."""

‎docling_core/types/doc/document.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2113,6 +2113,15 @@ def get_image(self, doc: "DoclingDocument", prov_index: int = 0) -> Optional[PIL
21132113
return self.image.pil_image
21142114
return super().get_image(doc=doc, prov_index=prov_index)
21152115

2116+
# Convert the image to Base64
2117+
def _image_to_base64(self, pil_image, format="PNG"):
2118+
"""Base64 representation of the image."""
2119+
buffered = BytesIO()
2120+
pil_image.save(buffered, format=format) # Save the image to the byte stream
2121+
img_bytes = buffered.getvalue() # Get the byte data
2122+
img_base64 = base64.b64encode(img_bytes).decode("utf-8") # Encode to Base64 and decode to string
2123+
return img_base64
2124+
21162125

21172126
class CodeItem(FloatingItem, TextItem):
21182127
"""CodeItem."""
@@ -2274,15 +2283,6 @@ def _migrate_annotations_to_meta(self) -> Self:
22742283

22752284
return self
22762285

2277-
# Convert the image to Base64
2278-
def _image_to_base64(self, pil_image, format="PNG"):
2279-
"""Base64 representation of the image."""
2280-
buffered = BytesIO()
2281-
pil_image.save(buffered, format=format) # Save the image to the byte stream
2282-
img_bytes = buffered.getvalue() # Get the byte data
2283-
img_base64 = base64.b64encode(img_bytes).decode("utf-8") # Encode to Base64 and decode to string
2284-
return img_base64
2285-
22862286
@staticmethod
22872287
def _image_to_hexhash(img: Optional[PILImage.Image]) -> Optional[str]:
22882288
"""Hexash from the image."""

‎test/test_serialization.py‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from xml.etree import ElementTree as ET
77

88
import pytest
9+
from PIL import Image
910

1011
from docling_core.transforms.serializer.common import _DEFAULT_LABELS
1112
from docling_core.transforms.serializer.html import (
@@ -32,12 +33,14 @@
3233
DescriptionAnnotation,
3334
EntitiesMetaField,
3435
EntityMention,
36+
ImageRef,
3537
LanguageMetaField,
3638
PictureClassificationMetaField,
3739
PictureClassificationPrediction,
3840
PictureMeta,
3941
RefItem,
4042
RichTableCell,
43+
Size,
4144
SummaryMetaField,
4245
TableCell,
4346
TableData,
@@ -352,6 +355,82 @@ def test_md_single_row_table():
352355
verify(exp_file=exp_file, actual=actual)
353356

354357

358+
def test_md_table_with_image_embedded():
359+
doc = DoclingDocument(name="")
360+
table = doc.add_table(data=TableData(num_rows=1, num_cols=1))
361+
doc.add_table_cell(
362+
table_item=table,
363+
cell=TableCell(
364+
start_row_offset_idx=0,
365+
end_row_offset_idx=1,
366+
start_col_offset_idx=0,
367+
end_col_offset_idx=1,
368+
text="foo",
369+
),
370+
)
371+
table.image = ImageRef.from_pil(image=Image.new("RGB", (2, 2)), dpi=72)
372+
373+
ser = MarkdownDocSerializer(
374+
doc=doc,
375+
params=MarkdownParams(image_mode=ImageRefMode.EMBEDDED),
376+
)
377+
actual = ser.serialize().text
378+
assert "| foo |" in actual
379+
assert "![Image](data:image/png;base64," in actual
380+
381+
382+
def test_md_table_with_image_referenced():
383+
doc = DoclingDocument(name="")
384+
table = doc.add_table(data=TableData(num_rows=1, num_cols=1))
385+
doc.add_table_cell(
386+
table_item=table,
387+
cell=TableCell(
388+
start_row_offset_idx=0,
389+
end_row_offset_idx=1,
390+
start_col_offset_idx=0,
391+
end_col_offset_idx=1,
392+
text="foo",
393+
),
394+
)
395+
table.image = ImageRef(
396+
mimetype="image/png",
397+
dpi=72,
398+
size=Size(width=2, height=2),
399+
uri="table_0.png",
400+
)
401+
402+
ser = MarkdownDocSerializer(
403+
doc=doc,
404+
params=MarkdownParams(image_mode=ImageRefMode.REFERENCED),
405+
)
406+
actual = ser.serialize().text
407+
assert actual == "| foo |\n|-------|\n\n![Image](table_0.png)"
408+
409+
410+
def test_md_table_without_image_unaffected_by_image_mode():
411+
"""A table with no generated image must not gain an image line, even if
412+
image_mode is set — this must remain a strict no-op for the common case."""
413+
doc = DoclingDocument(name="")
414+
table = doc.add_table(data=TableData(num_rows=1, num_cols=1))
415+
doc.add_table_cell(
416+
table_item=table,
417+
cell=TableCell(
418+
start_row_offset_idx=0,
419+
end_row_offset_idx=1,
420+
start_col_offset_idx=0,
421+
end_col_offset_idx=1,
422+
text="foo",
423+
),
424+
)
425+
426+
ser = MarkdownDocSerializer(
427+
doc=doc,
428+
params=MarkdownParams(image_mode=ImageRefMode.EMBEDDED),
429+
)
430+
actual = ser.serialize().text
431+
assert actual == "| foo |\n|-------|"
432+
433+
355434
def test_md_pipe_in_table():
356435
doc = DoclingDocument(name="Pipe in Table")
357436
table = doc.add_table(data=TableData(num_rows=1, num_cols=1))

0 commit comments

Comments
 (0)