fix(extraction): add support for matching elements that use their own namespaces.

* fix(extraction): support MathML, SVG, and other non-XHTML namespace nodes

* Add test cases.

---------

Co-authored-by: bookfere <bookfere@gmail.com>

Author: elxy

lib/element.py

@@ -3,7 +3,8 @@
 import copy
 from typing import Any
 
-from lxml import etree
+from lxml import etree  # type: ignore
+
 from calibre import prepare_string_for_xml as xml_escape  # type: ignore
 
 from .utils import (
@@ -684,7 +685,7 @@ def load_reserve_rules(self, rules=[]):
         # conflicts with the mechanism of merge translation.
         default_rules = (
             'img', 'code', 'br', 'hr', 'sub', 'sup', 'kbd', 'abbr', 'wbr',
-            'var', 'canvas', 'svg', 'script', 'style')
+            'var', 'canvas', 'svg', 'script', 'style', 'math')
         self.reserve_pattern = create_xpath(default_rules + tuple(rules))
 
     def prepare_original(self, elements):

lib/utils.py

@@ -10,8 +10,8 @@
 from subprocess import Popen
 from contextlib import contextmanager
 
-from mechanize import Browser, Request, HTTPError
-from mechanize._response import response_seek_wrapper as Response
+from mechanize import Browser, Request, HTTPError  # type: ignore
+from mechanize._response import response_seek_wrapper as Response  # type: ignore
 
 from calibre import get_proxies  # type: ignore
 from calibre.utils.logging import Log  # type: ignore
@@ -23,8 +23,6 @@
 ns = {'x': 'http://www.w3.org/1999/xhtml'}
 is_test = 'unittest' in sys.modules
 log = Log(level=Log.DEBUG if os.environ.get('CALIBRE_DEBUG') else Log.INFO)
-
-log.debug('Backup original socket: ', id(socket.socket))
 original_socket = socket.socket
 
 
@@ -36,24 +34,32 @@ def sep(char='═', count=38):
     return char * count
 
 
-def css(selector):
+def css(selector: str) -> str | None:
     try:
         return GenericTranslator().css_to_xpath(selector, prefix='self::x:')
     except SelectorError:
         return None
 
 
-def css_to_xpath(selectors):
+def css_to_xpath(selectors: tuple | list) -> list:
     patterns = []
+    simple_tag = re.compile(r'^[A-Za-z][\w-]*$')
     for selector in selectors:
-        if rule := css(selector):
-            patterns.append(rule)
+        rule = css(selector)
+        if rule is None:
+            continue
+        # Add support for matching elements that use their own namespaces.
+        if simple_tag.match(selector):
+            rule = f'({rule} or self::*[local-name()="{selector}"])'
+        patterns.append(rule)
     return patterns
 
 
-def create_xpath(selectors):
+def create_xpath(selectors: tuple | str) -> str | None:
     selectors = (selectors,) if isinstance(selectors, str) else selectors
-    return './/*[%s]' % ' or '.join(css_to_xpath(selectors))
+    if patterns := css_to_xpath(selectors):
+        return './/*[%s]' % ' or '.join(patterns)
+    return None
 
 
 def uid(*args):
@@ -196,6 +202,7 @@ def socks_proxy(host: str, port: int) -> Generator[ModuleType, None, None]:
     """This is a monkey-patch approach to enforce Mechanize to use a SOCKS5
     proxy. The context manager restores the original socket after it exits.
     """
+    log.debug('Backup original socket: ', id(socket.socket))
     # Temporarily remove environment proxies to prevent conflicts with the
     # SOCKS5 proxy, which might otherwise send connections through an HTTP
     # proxy, causing a "General SOCKS server failure" error.

tests/lib/test_element.py

@@ -2,7 +2,7 @@
 import unittest
 from unittest.mock import patch, Mock
 
-from lxml import etree
+from lxml import etree  # type: ignore
 
 from calibre.ebooks.oeb.base import TOC, Metadata  # type: ignore
 
@@ -437,7 +437,7 @@ def test_get_content(self):
         self.assertEqual(
             '<img src="w2.jpg"></img>', self.element.reserve_elements[4])
         self.assertEqual(
-            '<img alt="{\D}" src="w3.jpg"></img>',
+            '<img alt="{\D}" src="w3.jpg"></img>', # type: ignore
             self.element.reserve_elements[5])
         self.assertEqual(
             '<img src="w3.jpg"></img>', self.element.reserve_elements[6])
@@ -740,10 +740,10 @@ def test_add_translation_only_keep_anchor(self):
         self.assertEqual('abc', a.get('href'))
         self.assertEqual('A', a.text)
 
-    def test_add_translation_table(slef):
+    def test_add_translation_table(self):
         pass
 
-    def test_add_translation_table_only(slef):
+    def test_add_translation_table_only(self):
         pass
 
     def test_add_translation_line_break_below(self):
@@ -934,8 +934,6 @@ def test_create_extraction(self):
         self.assertEqual(
             [re.compile(default_rule)],
             self.extraction.filter_patterns)
-        self.assertEqual(
-            ['self::x:pre', 'self::x:code'], self.extraction.ignore_patterns)
 
     def test_get_sorted_pages(self):
         self.assertEqual(
@@ -1278,13 +1276,11 @@ def test_set_column_gap(self):
 
     def test_load_remove_rules(self):
         self.handler.load_remove_rules()
-        self.assertEqual(
-            './/*[self::x:rt or self::x:rp]', self.handler.remove_pattern)
+        self.assertIsNotNone(self.handler.remove_pattern)
 
     def test_load_reserve_rules(self):
         self.handler.load_reserve_rules()
-        self.assertRegex(
-            self.handler.reserve_pattern, r'^\.//\*\[self::x:img.*style\]$')
+        self.assertIsNotNone(self.handler.reserve_pattern)
 
     @patch('calibre_plugins.ebook_translator.lib.element.uid')
     def test_prepare_original(self, mock_uid):
@@ -1312,12 +1308,6 @@ def test_prepare_original(self, mock_uid):
                 self.assertEqual('red', element.original_color)
                 self.assertEqual('green', element.translation_color)
                 self.assertEqual(('percentage', 20), element.column_gap)
-                self.assertEqual(
-                    './/*[self::x:rt or self::x:rp]',
-                    self.handler.remove_pattern)
-                self.assertRegex(
-                    element.reserve_pattern or '',
-                    r'^\.//\*\[self::x:img.*style\]$')
 
     @patch('calibre_plugins.ebook_translator.lib.element.uid')
     def test_prepare_translation_contains_ignored_element(self, mock_uid):
@@ -1544,11 +1534,6 @@ def test_prepare_original_merge_separator(self, mock_uid):
                 self.assertEqual('red', element.original_color)
                 self.assertEqual('green', element.translation_color)
                 self.assertEqual(('percentage', 20), element.column_gap)
-                self.assertEqual(
-                    './/*[self::x:rt or self::x:rp]',
-                    self.handler.remove_pattern)
-                self.assertRegex(
-                    element.reserve_pattern, r'^\.//\*\[self::x:img.*style\]$')
 
     @patch('calibre_plugins.ebook_translator.lib.element.uid')
     def test_prepare_original_merge_separator_multiple(self, mock_uid):

tests/lib/test_utils.py

@@ -2,16 +2,71 @@
 from unittest.mock import patch
 from types import GeneratorType
 
+from lxml import etree  # type: ignore
+
+from ...vendor.cssselect import SelectorError
+
 from ...lib.utils import (
-    css_to_xpath, uid, trim, chunk, group, open_file, request)
+    ns, css, css_to_xpath, create_xpath, uid, trim, chunk, group, open_file,
+    request)
 
 
 module_name = 'calibre_plugins.ebook_translator.lib.utils'
 
 
 class TestUtils(unittest.TestCase):
+    def test_css(self):
+        self.assertEqual("self::x:div[@id = 'id']", css('div#id'))
+        self.assertIsNone(css('div>>p'))  # omit invalid selector
+
     def test_css_to_xpath(self):
-        self.assertEqual(["self::x:*[@id = 'id']"], css_to_xpath(['#id']))
+        self.assertEqual([], css_to_xpath([]))
+        self.assertEqual([], css_to_xpath(['div>>p']))  # omit invalid selector
+        self.assertEqual(["self::x:*[@id = 'test']"], css_to_xpath(['#test']))
+        self.assertEqual(
+            [
+                "self::x:div[@id = 'test']",
+                "(self::x:span or self::*[local-name()=\"span\"])"
+            ],
+            css_to_xpath(["div>>p", 'div#test', 'span']))
+
+    def test_create_xpath(self):
+        pattern = create_xpath(('p', 'math',))
+        self.assertEqual(
+            pattern,
+            './/*[(self::x:p or self::*[local-name()="p"]) or '
+            '(self::x:math or self::*[local-name()="math"])]')
+
+        # A sample code to test XPath pattern matching with MathML elements
+        # that have independent namespaces.
+        xhtml = etree.XML("""<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
+<head><title>Document</title></head>
+<body>
+    <p>Test MathML element</p>
+    <math xmlns="http://www.w3.org/1998/Math/MathML">
+        <munder>
+            <mo>∑</mo>
+            <mi>A</mi>
+        </munder>
+        <munder displaystyle="true">
+            <mo>∑</mo>
+            <mi>A</mi>
+        </munder>
+        <munder>
+            <mo>∑</mo>
+            <mi scriptlevel="0">A</mi>
+        </munder>
+    </math>
+</body>
+</html>""".encode())
+        body = xhtml.find('./x:body', namespaces=ns)
+        elements = body.xpath(pattern, namespaces=ns)
+
+        self.assertEqual(2, len(elements))
+        self.assertEqual('p', etree.QName(elements[0]).localname)
+        self.assertEqual('math', etree.QName(elements[1]).localname)
 
     def test_uid(self):
         self.assertEqual('202cb962ac59075b964b07152d234b70', uid('123'))