[doc] add ipython3 lexer hook for notebooks with shell/magic cells (ray-project#63515)

## Description

Sphinx falls back to the `python3` lexer on `.ipynb` files that don't
declare `language_info.pygments_lexer`, but `python3` can't tokenise
`!shell` or `%magic` cells. The pygments warning is fatal under
Readthedocs `-W` and breaks the docs build.

This source-read hook injects `pygments_lexer = "ipython3"` into
notebooks matching configurable glob patterns (`ipython3_lexer_patterns`
/ `_exclude_patterns`). Globs cover the in-tree example layouts plus
`_collections/**/*.ipynb` for notebooks fetched at build time by
`sphinx_collections`.

Review feedback from the previous round on ray-project#59984 (early-return pattern,
no broad `except`) is preserved.

## Related issues

Related to ray-project#59984 (closed-stale 2026-02-07).

## Additional information

Verified locally with `make html` against `upstream/master` plus this
commit and a fixture notebook at `_collections/lexer-fixture/test.ipynb`
containing a `!pip install` cell and no `language_info`: build
succeeded, no `WARNING.*lexer` matches in the log, fixture confirmed
source-read.

Today most `_collections/` notebooks are excluded from the build via
`exclude_patterns` in favour of their `.md` counterparts, so the new
`_collections/**/*.ipynb` glob is mostly defensive. The exclude design
is upstream of this PR.

---------

Signed-off-by: Aydin Abiar <aydin@anyscale.com>
Signed-off-by: Neelansh Khare <kharen@uci.edu>

Author: Aydin-ab

doc/source/conf.py

@@ -17,6 +17,7 @@
 from jinja2.filters import FILTERS
 from sphinx.ext.autosummary import generate
 from sphinx.util.inspect import safe_getattr
+from sphinx.util.matching import compile_matchers
 
 DEFAULT_API_GROUP = "Others"
 
@@ -783,6 +784,11 @@ def fix_collections_code_blocks(app, docname, source):
 
     app.connect('source-read', fix_collections_code_blocks)
 
+    app.add_config_value("ipython3_lexer_patterns", [], "env")
+    app.add_config_value("ipython3_lexer_exclude_patterns", [], "env")
+    app.connect("config-inited", _compile_pattern_matchers)
+    app.connect("source-read", apply_ipython3_lexer)
+
 
 redoc = [
     {
@@ -921,3 +927,46 @@ def fix_collections_code_blocks(app, docname, source):
 ), "If ray is already imported, we will not render documentation correctly!"
 
 os.environ["RAY_DOC_BUILD"] = "1"
+
+ipython3_lexer_patterns = [
+    # External templates fetched by sphinx_collections (see #62179) land here at
+    # build time; their notebook JSON has no language_info, so Sphinx defaults
+    # to the python3 lexer and chokes on !pip / %magic cells.
+    "_collections/**/*.ipynb",
+    "ray-overview/examples/**/content/**.ipynb",
+    "serve/tutorials/**/content/**.ipynb",
+    "data/examples/**/content/**.ipynb",
+    "tune/examples/**/content/**.ipynb",
+]
+ipython3_lexer_exclude_patterns = []
+
+
+def _compile_pattern_matchers(app, config):
+    app.ipython3_lexer_patterns = compile_matchers(
+        config.ipython3_lexer_patterns or []
+    )
+    app.ipython3_lexer_exclude_patterns = compile_matchers(
+        config.ipython3_lexer_exclude_patterns or []
+    )
+
+
+def apply_ipython3_lexer(app, docname, source):
+    """Force the ipython3 pygments lexer on notebooks matching
+    ``ipython3_lexer_patterns`` (minus ``ipython3_lexer_exclude_patterns``).
+
+    Sphinx + myst-nb otherwise default to the python3 lexer, which fails on
+    ``!shell`` and ``%magic`` cells and is fatal under Readthedocs ``-W``.
+    """
+    doc_source = app.env.doc2path(docname, base=False)
+    if not doc_source.endswith(".ipynb"):
+        return
+    if any(m(doc_source) for m in app.ipython3_lexer_exclude_patterns):
+        return
+    if not any(m(doc_source) for m in app.ipython3_lexer_patterns):
+        return
+
+    notebook = json.loads(source[0])
+    lang_info = notebook.setdefault("metadata", {}).setdefault("language_info", {})
+    if lang_info.get("pygments_lexer") != "ipython3":
+        lang_info["pygments_lexer"] = "ipython3"
+        source[0] = json.dumps(notebook, ensure_ascii=False)