docs: Switch from 'breathe' to 'docleaf' for API documentation generation

.github/workflows/doc-build.yml

@@ -85,7 +85,7 @@ jobs:
           DOC_TARGET="html"
         fi
 
-        DOC_TAG=${DOC_TAG} SPHINXOPTS="-q -W -t publish" make -C doc ${DOC_TARGET}
+        DOC_TAG=${DOC_TAG} SPHINXOPTS="-q -W --keep-going -t publish" make -C doc ${DOC_TARGET}
 
     - name: compress-docs
       run: |

doc/_extensions/zephyr/doxyrunner.py

@@ -9,7 +9,7 @@
 ============
 
 This Sphinx plugin can be used to run Doxygen build as part of the Sphinx build
-process. It is meant to be used with other plugins such as ``breathe`` in order
+process. It is meant to be used with other plugins such as ``docleaf`` in order
 to improve the user experience. The principal features offered by this plugin
 are:
 

doc/_extensions/zephyr/warnings_filter.py

@@ -24,7 +24,7 @@
 
 import logging
 import re
-from typing import Dict, Any, List
+from typing import Dict, Any, List, Optional
 
 from sphinx.application import Sphinx
 from sphinx.util.logging import NAMESPACE
@@ -41,6 +41,7 @@ class WarningsFilter(logging.Filter):
         silent: If true, warning is hidden, otherwise it is shown as INFO.
         name: Filter name.
     """
+
     def __init__(self, expressions: List[str], silent: bool, name: str = "") -> None:
         super().__init__(name)
 
@@ -52,7 +53,8 @@ def filter(self, record: logging.LogRecord) -> bool:
             return True
 
         for expression in self._expressions:
-            if re.match(expression, record.msg):
+            # The message isn't always a string so we convert it before regexing as we can only regex strings
+            if expression.match(str(record.msg)):
                 if self._silent:
                     return False
                 else:
@@ -63,6 +65,21 @@ def filter(self, record: logging.LogRecord) -> bool:
         return True
 
 
+class Expression:
+    """
+    Encapsulate a log filter pattern and track if it ever matches a log line.
+    """
+
+    def __init__(self, pattern):
+        self.pattern = pattern
+        self.matched = False
+
+    def match(self, str):
+        matches = bool(re.match(self.pattern, str))
+        self.matched = matches or self.matched
+        return matches
+
+
 def configure(app: Sphinx) -> None:
     """Entry point.
 
@@ -74,8 +91,10 @@ def configure(app: Sphinx) -> None:
     with open(app.config.warnings_filter_config) as f:
         expressions = list()
         for line in f.readlines():
-            if not line.startswith("#"):
-                expressions.append(line.rstrip())
+            if line.strip() and not line.startswith("#"):
+                expressions.append(Expression(line.rstrip()))
+
+    app.env.warnings_filter_expressions = expressions
 
     # install warnings filter to all the Sphinx logger handlers
     filter = WarningsFilter(expressions, app.config.warnings_filter_silent)
@@ -84,11 +103,28 @@ def configure(app: Sphinx) -> None:
         handler.filters.insert(0, filter)
 
 
+def finished(app: Sphinx, exception: Optional[Exception]):
+    """
+    Prints out any patterns that have not matched a log line to allow us to clean up any that are not used.
+    """
+    if exception:
+        # Early exit if there has been an exception as matching data is only
+        # valid for complete builds
+        return
+
+    expressions = app.env.warnings_filter_expressions
+
+    for expression in expressions:
+        if not expression.matched:
+            logging.warning(f"Unused expression: {expression.pattern}")
+
+
 def setup(app: Sphinx) -> Dict[str, Any]:
     app.add_config_value("warnings_filter_config", "", "")
     app.add_config_value("warnings_filter_silent", True, "")
 
     app.connect("builder-inited", configure)
+    app.connect("build-finished", finished)
 
     return {
         "version": __version__,

doc/_static/css/custom.css

@@ -843,13 +843,13 @@ kbd, .kbd,
     background-color: var(--navbar-scrollbar-active-color);
 }
 
-/* Breathe tweaks */
+/* Docleaf tweaks */
 
 .rst-content .section > dl > dd {
     margin-left: 0;
 }
 
-.rst-content p.breathe-sectiondef-title {
+.rst-content p.docleaf-sectiondef-title {
     font-size: 115%;
     color: var(--link-color);
 }

doc/conf.py

@@ -67,7 +67,7 @@
 # -- General configuration ------------------------------------------------
 
 extensions = [
-    "breathe",
+    "docleaf.doxygen",
     "sphinx.ext.todo",
     "sphinx.ext.extlinks",
     "sphinx.ext.autodoc",
@@ -211,16 +211,17 @@
 doxyrunner_fmt_vars = {"ZEPHYR_BASE": str(ZEPHYR_BASE), "ZEPHYR_VERSION": version}
 doxyrunner_outdir_var = "DOXY_OUT"
 
-# -- Options for Breathe plugin -------------------------------------------
+# -- Options for Docleaf plugin -------------------------------------------
 
-breathe_projects = {"Zephyr": str(doxyrunner_outdir / "xml")}
-breathe_default_project = "Zephyr"
-breathe_domain_by_extension = {
+docleaf_projects = {"Zephyr": {"xml": str(doxyrunner_outdir / "xml"), "root": "../"}}
+docleaf_default_project = "Zephyr"
+docleaf_domain_by_extension = {
     "h": "c",
     "c": "c",
 }
-breathe_show_enumvalue_initializer = True
-breathe_default_members = ("members", )
+# Filters out any 'function' or 'variable' members that have 'all caps' names as
+# they are likely unprocessed macro calls
+docleaf_doxygen_skip = ["members:all_caps"]
 
 cpp_id_attributes = [
     "__syscall",

doc/connectivity/bluetooth/api/mesh/blob.rst

@@ -127,4 +127,3 @@ This section contains types and defines common to the BLOB Transfer models.
 
 .. doxygengroup:: bt_mesh_blob
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/blob_cli.rst

@@ -93,4 +93,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_blob_cli
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/blob_srv.rst

@@ -67,4 +67,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_blob_srv
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/dfd_srv.rst

@@ -23,4 +23,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_dfd_srv
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/dfu.rst

@@ -221,8 +221,6 @@ This section lists the types common to the Device Firmware Update mesh models.
 
 .. doxygengroup:: bt_mesh_dfu
    :project: Zephyr
-   :members:
 
 .. doxygengroup:: bt_mesh_dfu_metadata
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/dfu_cli.rst

@@ -11,4 +11,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_dfu_cli
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/dfu_srv.rst

@@ -68,4 +68,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_dfu_srv
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/lcd_cli.rst

@@ -17,4 +17,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_large_comp_data_cli
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/lcd_srv.rst

@@ -27,4 +27,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_large_comp_data_srv
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/od_cli.rst

@@ -26,4 +26,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_od_priv_proxy_cli
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/od_srv.rst

@@ -25,4 +25,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_od_priv_proxy_srv
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/op_agg_cli.rst

@@ -27,4 +27,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_op_agg_cli
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/op_agg_srv.rst

@@ -29,4 +29,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_op_agg_srv
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/priv_beacon_cli.rst

@@ -33,4 +33,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_priv_beacon_cli
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/priv_beacon_srv.rst

@@ -36,4 +36,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_priv_beacon_srv
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/rpr_cli.rst

@@ -129,4 +129,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_rpr_cli
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/rpr_srv.rst

@@ -27,4 +27,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_rpr_srv
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/srpl_cli.rst

@@ -28,4 +28,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_sol_pdu_rpl_cli
    :project: Zephyr
-   :members:

doc/connectivity/bluetooth/api/mesh/srpl_srv.rst

@@ -27,4 +27,3 @@ API reference
 
 .. doxygengroup:: bt_mesh_sol_pdu_rpl_srv
    :project: Zephyr
-   :members:

doc/contribute/documentation/generation.rst

@@ -52,7 +52,7 @@ The project's documentation contains the following items:
       header [shape="rectangle" label="c header\ncomments"]
       xml [shape="rectangle" label="XML"]
       html [shape="rectangle" label="HTML\nweb site"]
-      sphinx[shape="ellipse" label="sphinx +\nbreathe,\ndocutils"]
+      sphinx[shape="ellipse" label="sphinx +\ndocleaf,\ndocutils"]
       images -> sphinx
       rst -> sphinx
       conf -> sphinx
@@ -65,7 +65,7 @@ The project's documentation contains the following items:
 
 
 The reStructuredText files are processed by the Sphinx documentation system,
-and make use of the breathe extension for including the doxygen-generated API
+and make use of the docleaf extension for including the doxygen-generated API
 material.  Additional tools are required to generate the
 documentation locally, as described in the following sections.
 
@@ -226,16 +226,16 @@ build the documentation directly from there:
 Filtering expected warnings
 ***************************
 
-There are some known issues with Sphinx/Breathe that generate Sphinx warnings
+There are some known issues with Sphinx/Docleaf that generate Sphinx warnings
 even though the input is valid C code. While these issues are being considered
 for fixing we have created a Sphinx extension that allows to filter them out
 based on a set of regular expressions. The extension is named
 ``zephyr.warnings_filter`` and it is located at
 ``doc/_extensions/zephyr/warnings_filter.py``. The warnings to be filtered out
 can be added to the ``doc/known-warnings.txt`` file.
 
-The most common warning reported by Sphinx/Breathe is related to duplicate C
-declarations. This warning may be caused by different Sphinx/Breathe issues:
+The most common warning reported by Sphinx/Docleaf is related to duplicate C
+declarations. This warning may be caused by different Sphinx/Docleaf issues:
 
 - Multiple declarations of the same object are not supported
 - Different objects (e.g. a struct and a function) can not share the same name

doc/known-warnings.txt

@@ -1,18 +1,30 @@
 # Each line should contain the regular expression of a known Sphinx warning
 # that should be filtered out
-.*Duplicate C declaration.*\n.*'\.\. c:.*:: dma_config'.*
+
+# Function and (enum or struct) name
 .*Duplicate C declaration.*\n.*'\.\. c:.*:: flash_img_check'.*
-.*Duplicate C declaration.*\n.*'\.\. c:.*:: zsock_fd_set'.*
-.*Duplicate C declaration.*\n.*'\.\. c:.*:: net_if_mcast_monitor'.*
 .*Duplicate C declaration.*\n.*'\.\. c:.*:: fs_statvfs'.*
 .*Duplicate C declaration.*\n.*'\.\. c:.*:: .*dmic_trigger.*'.*
-.*Duplicate C declaration.*\n.*'\.\. c:.*:: uint16_t id'.*
+.*Duplicate C declaration.*\n.*'\.\. c:.*:: dma_config'.*
+.*Duplicate C declaration.*\n.*'\.\. c:.*:: net_if_mcast_monitor'.*
+.*Duplicate C declaration.*\n.*'\.\. c:struct:: bt_ots_init'.*
+
+# Struct and typedef name
+.*Duplicate C declaration.*\n.*'\.\. c:.*:: zsock_fd_set'.*
+
+# Function and extern function
 .*Duplicate C declaration.*\n.*'\.\. c:.*:: .*net_if_ipv4_addr_mask_cmp.*'.*
 .*Duplicate C declaration.*\n.*'\.\. c:.*:: .*net_if_ipv4_is_addr_bcast.*'.*
 .*Duplicate C declaration.*\n.*'\.\. c:.*:: .*net_if_ipv4_addr_lookup.*'.*
 .*Duplicate C declaration.*\n.*'\.\. c:.*:: .*net_if_ipv6_addr_lookup.*'.*
 .*Duplicate C declaration.*\n.*'\.\. c:.*:: .*net_if_ipv6_maddr_lookup.*'.*
+
+# Common field names
 .*Duplicate C declaration.*\n.*'\.\. c:.*:: .*struct in_addr.*'.*
 .*Duplicate C declaration.*\n.*'\.\. c:.*:: .*struct in6_addr.*'.*
 .*Duplicate C declaration.*\n.*'\.\. c:.*:: .*struct net_if.*'.*
-.*Duplicate C declaration.*\n.*'\.\. c:struct:: bt_ots_init'.*
+
+# Clash with field of nested anonymous struct
+.*Duplicate C declaration.*\n.*'\.\. c:member:: enum *bt_mesh_dfd_upload_phase bt_mesh_dfd_srv.phase'.*
+.*Duplicate C declaration.*\n.*'\.\. c:member:: struct *bt_mesh_blob_xfer bt_mesh_dfu_cli.blob'.*
+.*Duplicate C declaration.*\n.*'\.\. c:member:: struct *net_if *\* net_if_mcast_monitor.iface'.

scripts/requirements-doc.txt

@@ -1,6 +1,6 @@
 # DOC: used to generate docs
 
-breathe>=4.34
+docleaf==0.8.0
 sphinx~=5.0,!=5.2.0.post0
 sphinx_rtd_theme~=1.0
 sphinx-tabs