Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions tests/baseline_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,53 @@ def test_baseline_article_dominance():
assert all(post.format(i) in result2 for i in range(3))


def test_baseline_article_block_boundaries():
"regression (#896): the article strategy fused the text of adjacent block elements -- a \
minified <article> carries no whitespace between </h1> and its <p>, so text_content() ran \
them together as 'Notice TitleMunicipal...'. Mirrors the block-spacing assertions in \
test_html2txt: block boundaries separate, inline runs inside a block stay joined."
# padded past the one hundred character gate so the article strategy accepts the text
para = "Municipal notice body sentence carrying the real page content here. " * 2
doc = f"<html><body><article><h1>Notice Title</h1><p>{para}</p><p>Hyper<b>link</b></p></article></body></html>"
_, text, _ = baseline(doc)
assert "TitleMunicipal" not in text # block boundary no longer fused
assert "Notice Title Municipal notice body" in text
assert "Hyperlink" in text # inline run inside a block stays joined


_ARTICLE_BODY = "The article body text continues here with enough words to clear the length gate. " * 2


@pytest.mark.parametrize(
"inner",
[
"<h2>Section Heading</h2><p>{body}</p>",
"<ul><li>Section Heading</li><li>{body}</li></ul>",
"<table><tr><td>Section Heading</td><td>{body}</td></tr></table>",
],
ids=["h2_p", "li_li", "td_td"],
)
def test_baseline_article_spacing_covers_all_block_elements(inner):
"regression (#896): the fusion is not specific to the <h1> of the report -- every block \
boundary inside the article ran together, h2/p/li/td alike, so the spacing pass applies to \
the whole _BLOCK_ELEMS set rather than to headings only."
doc = f"<html><body><article>{inner.format(body=_ARTICLE_BODY)}</article></body></html>"
_, text, _ = baseline(doc)
assert "HeadingThe" not in text
assert "Section Heading The article body text" in text


def test_spaced_text_content_does_not_mutate_input():
"the article tier's spacing pass writes .text/.tail, so _spaced_text_content must work \
on a copy -- baseline() shares one tree across all its strategies."
from trafilatura.baseline import _spaced_text_content

elem = html.fromstring("<article><h1>A</h1><p>B</p></article>")
before = html.tostring(elem)
_spaced_text_content(elem)
assert html.tostring(elem) == before


def test_html2txt():
mydoc = "<html><body>Here is the body text</body></html>"
assert html2txt(mydoc) == "Here is the body text"
Expand Down
36 changes: 29 additions & 7 deletions trafilatura/baseline.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import json
import re
from collections.abc import Iterable
from copy import copy
from copy import copy, deepcopy
from html import unescape
from typing import Any

Expand Down Expand Up @@ -198,7 +198,8 @@ def baseline(filecontent: Any) -> tuple[_Element, str, int]:
article_texts = [
text
for elem in tree.xpath(".//article[not(ancestor::article)]")
if len(text := trim(elem.text_content())) > _MIN_CONTENT_LENGTH
# spaced text: without it the text of adjacent elements fuses ("TitleThis body...")
if len(text := trim(_spaced_text_content(elem))) > _MIN_CONTENT_LENGTH
]
if article_texts:
# never None: the longest article passes both its own length gate and the cutoff
Expand Down Expand Up @@ -268,6 +269,29 @@ def baseline(filecontent: Any) -> tuple[_Element, str, int]:
}


def _space_block_boundaries(elem: HtmlElement) -> None:
"""Pad block-element boundaries with spaces so adjacent text runs don't stick
together in text_content() (minified pages carry no whitespace there).

Modifies the element in place -- use _spaced_text_content on a shared tree.
"""
# remove_control_characters guards the .text write against chars lxml rejects
# (short-circuits on printable; str input pre-cleaned)
for block in elem.iter(*_BLOCK_ELEMS):
block.text = f" {remove_control_characters(block.text)}" if block.text else " "
block.tail = f" {remove_control_characters(block.tail)}" if block.tail else " "


def _spaced_text_content(elem: HtmlElement) -> str:
"""Text of an element, with block boundaries spaced so adjacent runs don't fuse.

Works on a copy, so it is safe on a tree shared with other extraction steps.
"""
spaced = deepcopy(elem)
_space_block_boundaries(spaced)
return spaced.text_content()


def html2txt(content: Any, clean: bool = True) -> str:
"""Run basic html2txt on a document.

Expand All @@ -293,9 +317,7 @@ def html2txt(content: Any, clean: bool = True) -> str:
body = tree
if clean:
body = basic_cleaning(body)
# space block boundaries so adjacent runs don't stick (minified pages). remove_control_characters
# guards the .text write against chars lxml rejects (short-circuits on printable; str input pre-cleaned)
for elem in body.iter(*_BLOCK_ELEMS):
elem.text = f" {remove_control_characters(elem.text)}" if elem.text else " "
elem.tail = f" {remove_control_characters(elem.tail)}" if elem.tail else " "
# space block boundaries so adjacent runs don't stick (minified pages); the tree
# here is freshly parsed or already copied, so it can be modified in place
_space_block_boundaries(body)
return " ".join(body.text_content().split())
Loading