Docs/pipe 2738 migrate cli documentation

.github/copilot-instructions.md

@@ -3,11 +3,13 @@
 You are an expert Python developer assisting with code refactoring, documentation, code reviews, and git workflows. Apply the following rules based on the context of the user's request.
 
 ## 1. Code Refactoring & Generation Rules
-**Strictly adhere to Python 3.10+ standards and the following style guide:**
+**Strictly adhere to Python 3.12+ standards and the following style guide:**
 
-* **Modern Typing (PEP 604/585):**
+* **Modern Typing (PEP 604/585/695):**
     * Use built-in generics (`list[str]`, `dict[str, int]`) instead of `typing.List` or `typing.Dict`.
     * Use the pipe union syntax (`str | None`) instead of `typing.Union` or `typing.Optional`.
+    * Use the `type` statement for type aliases (PEP 695): `type Vector = list[float]` instead of `Vector = list[float]` or `TypeAlias`.
+    * Use the new generic syntax (PEP 695): `def func[T](x: T) -> T` instead of `TypeVar`-based generics.
     * **Mandatory:** Add type hints to all function arguments and return definitions.
 * **Formatting & Style:**
     * **Line Limit:** Hard wrap at **120 characters**.
@@ -27,7 +29,7 @@ You are an expert Python developer assisting with code refactoring, documentatio
 ## 3. Code Review Guidelines
 **When asked to review code, focus on these priorities:**
 
-1.  **Modernization:** Flag uses of deprecated typing (e.g., `List`, `Union`) and suggest 3.10+ alternatives.
+1.  **Modernization:** Flag uses of deprecated typing (e.g., `List`, `Union`, `TypeVar`, `TypeAlias`) and suggest 3.12+ alternatives.
 2.  **Safety:** specific checks for logging formatting (ensure lazy evaluation).
 3.  **Readability:** Identify lines exceeding 120 chars or complex logic that needs refactoring.
 4.  **Tone:** Be constructive and concise.
@@ -49,7 +51,11 @@ You are an expert Python developer assisting with code refactoring, documentatio
 
 **Bad (Avoid):**
 ```python
-from typing import List, Union
+from typing import List, Union, TypeVar, TypeAlias
+
+Vector: TypeAlias = list[float]
+T = TypeVar('T')
+
 def fetch_data(ids: List[Union[int, str]]) -> dict:
     """
     Fetch data.
@@ -59,4 +65,30 @@ def fetch_data(ids: List[Union[int, str]]) -> dict:
         dict: The result.
     """
     logger.info(f"Fetching {len(ids)}")
-    return {}
+    return {}
+
+def first(items: list[T]) -> T:  # Should use def first[T] syntax instead
+    ...
+    return items[0]
+```
+
+**Good (Preferred):**
+```python
+type Vector = list[float]
+
+def fetch_data(ids: list[int | str]) -> dict:
+    """Fetch data.
+
+    Args:
+        ids: List of IDs.
+
+    Returns:
+        The result.
+    """
+    logger.info('Fetching %s', len(ids))
+    return {}
+
+def first[T](items: list[T]) -> T:
+    ...
+    return items[0]
+```

.github/workflows/test-unit-pixi.yml

@@ -30,16 +30,9 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest]
         pixi_env:
-          - casa666-py310
-          - casa671-py310
           - casa671-py312
           - casa675-py312
           - casa675-py313
-        exclude:
-          # pybdsf not available on cp313 for osx-arm64
-          - os: macos-latest
-            pixi_env: casa675-py313
-
     name: "Unit Testing (${{ matrix.os }}, ${{ matrix.pixi_env }})"
 
     uses: ./.github/workflows/env-setup-run-template-pixi.yml

.readthedocs.yaml

@@ -1,26 +1,33 @@
 version: 2
 
-# Build tools and environment setup
+formats:
+  - pdf
+
 build:
   os: ubuntu-24.04
   tools:
-    python: miniconda-latest # use miniconda-latest before miniforge3 becomes available.
+    python: "latest"
   apt_packages:
-    - git
     - xvfb
     - imagemagick
     - poppler-utils
     - latexmk
     - texlive-latex-recommended
     - texlive-latex-extra
     - texlive-fonts-recommended
-  # Build commands for HTML and PDF
-  commands:
-    - conda env update --file=environment.yml
-    - conda run -n pipeline python -m pip install .[docs]
-    - mkdir -p ~/.casa/data
-    - cd docs && conda run -n pipeline make html_docs
-    - cd docs && conda run -n pipeline make latexpdf
-    - mkdir --parents _readthedocs/html/ _readthedocs/pdf/
-    - cp --recursive docs/_build/html/* _readthedocs/html/
-    - cp --recursive docs/_build/latex/*.pdf _readthedocs/pdf/
+  jobs:
+    create_environment:
+      - asdf plugin add pixi
+      - asdf install pixi latest
+      - asdf global pixi latest
+      - mkdir -p ~/.casa/data
+    install:
+      - pixi install -e docs
+    build:
+      html:
+        - pixi run -e docs --frozen sphinx-build -T -b html docs/source $READTHEDOCS_OUTPUT/html
+      pdf:
+        - pixi run -e docs --frozen sphinx-build -T -b latex docs/source docs/_build/latex
+        - cd docs/_build/latex && latexmk -pdf -interaction=nonstopmode *.tex
+        - mkdir -p $READTHEDOCS_OUTPUT/pdf/
+        - cp docs/_build/latex/*.pdf $READTHEDOCS_OUTPUT/pdf/

docs/Makefile

@@ -4,7 +4,7 @@
 #	export pythonbin=${casa_dir}/bin/python3
 PYTHON := $(if $(pythonbin),$(pythonbin),python3)
 
-SPHINXOPTS    = -v # -j auto
+SPHINXOPTS    = -v -j auto
 SPHINXBUILD   = ${PYTHON} -msphinx
 #SPHINXOPTS    = -v -j auto -b coverage
 SPHINXPROJ    = pipeline_docs
@@ -43,12 +43,18 @@ taskapidoc:
 html_docs:
 	@rm -rf ${SOURCEDIR}/_apidoc
 	# @make apidoc
-	@rm -rf ${BUILDDIR}/html ${BUILDDIR}/doctrees 
+	@rm -rf ${BUILDDIR}/html ${BUILDDIR}/doctrees
 	@rm -rf ${SOURCEDIR}/_autosummary*
 	@rm -rf ${SOURCEDIR}/_automodapi
 	@make html
 	@rm -rf casa-*.log ${SOURCEDIR}/casa-*log
 
+# Incremental build: keeps doctrees and autosummary stubs — much faster for iterative dev.
+# Run html_docs at least once first to generate stubs.
+html_fast:
+	@make html
+	@rm -rf casa-*.log ${SOURCEDIR}/casa-*log
+
 pdf_taskdocs:
 	@rm -rf ${BUILDDIR}/latex ${BUILDDIR}/doctrees
 	@rm -rf ${SOURCEDIR}/_autosummary*

docs/README.rst

@@ -6,9 +6,9 @@ Pipeline
 
 .. important::
 
-  This is a **demonstration** of the new internal documentation framework for the Pipeline development.
-  It is designed to provide *up-to-date and version-controlled* supplementary technical information to complement the official Pipeline documentatios from the `ALMA <https://almascience.nrao.edu/processing/science-pipeline>`_ and `VLA <https://science.nrao.edu/facilities/vla/data-processing>`_ science portals.
-  Pipeline development is a collaborative effort led by `NRAO`_, `ESO`_, and `NAOJ`_.
+  This is the **internal developer documentation** for the Pipeline project.
+  It provides *up-to-date and version-controlled* supplementary technical information to complement the official Pipeline portals from `ALMA <https://almascience.nrao.edu/processing/science-pipeline>`_ and `VLA <https://science.nrao.edu/facilities/vla/data-processing>`_.
+  Pipeline development is a collaborative effort led by `NRAO`_, `ESO`_, and `NAOJ`_, with additional contributions from `MPIfR`_, `NOVA`_, and `UKATC`_ under contract to `ESO`_.
 
 
 Official Repository
@@ -20,8 +20,7 @@ The official public repository is accessible here:
 Highlights
 ----------
 
-`This demo <pipe-docs.readthedocs.io>`_ is automatically generated from `a development branch <https://open-bitbucket.nrao.edu/projects/PIPE/repos/pipeline/browse?at=refs%2Fheads%2FPIPE-1669-run-dev-pipeline-with-modular-casa6>`_ of the official repository and remains under active restructuring
-Some sections are highlighted below:
+This documentation is automatically generated from the main repository and includes:
 
   * `Past Pipeline Releases <https://pipe-docs.readthedocs.io/en/latest/releases.html#>`_
   * `Pipeline Dependencies <https://pipe-docs.readthedocs.io/en/latest/dependencies.html>`_
@@ -32,6 +31,7 @@ Some sections are highlighted below:
 .. _ESO: https://www.eso.org  
 .. _UKATC: https://www.ukatc.stfc.ac.uk  
 .. _MPIfR: https://www.mpifr-bonn.mpg.de  
+.. _NOVA: https://nova-astronomy.nl/  
 .. _NAOJ: https://www.nao.ac.jp  
 
 .. |Docs Pages| image:: https://img.shields.io/github/actions/workflow/status/r-xue/pipeline/build-gh-pages.yml?style=plastic&logo=githubactions&label=docs-pages

docs/source/_extension/cli_function_stubs.py

@@ -0,0 +1,122 @@
+"""Generate autosummary stubs for CLI task functions.
+
+Sphinx autosummary only generates stubs for items listed directly in
+source ``.. autosummary:: :toctree:`` directives.  It does NOT process
+nested ``:toctree:`` inside stubs it just generated, **and** it strips
+directive options (like ``:toctree:`` / ``:nosignatures:``) and extra
+blocks (like ``.. toctree:: :hidden:``) from template-generated stubs.
+
+This extension works around both limitations:
+
+1. Hook ``builder-inited`` with priority 900 (after autosummary at 500)
+   so the module stubs already exist.
+2. Introspect each CLI module to discover public functions.
+3. Generate an individual ``.rst`` stub for every function.
+4. Append a ``.. toctree:: :hidden:`` block to each module stub so that
+   the sidebar shows functions nested under their parent module.
+"""
+
+import importlib
+import inspect
+import logging
+from pathlib import Path
+
+from sphinx.application import Sphinx
+
+logger = logging.getLogger(__name__)
+
+#: CLI modules whose public functions should get individual stubs.
+CLI_MODULES = [
+    'pipeline.h.cli',
+    'pipeline.hif.cli',
+    'pipeline.hifa.cli',
+    'pipeline.hifv.cli',
+    'pipeline.hsd.cli',
+    'pipeline.hsdn.cli',
+]
+
+#: Directory (relative to source) where stubs are written.
+AUTOSUMMARY_DIR = '_autosummary'
+
+#: Template for individual function stubs.
+STUB_TEMPLATE = """\
+{title}
+{underline}
+
+.. currentmodule:: {module}
+
+.. autofunction:: {name}
+"""
+
+#: Marker so we only patch a module stub once.
+_TOCTREE_MARKER = '.. cli_function_stubs toctree'
+
+
+def _discover_functions(mod_name: str) -> list[str]:
+    """Return sorted list of public function names exported by *mod_name*."""
+    try:
+        mod = importlib.import_module(mod_name)
+    except ImportError:
+        logger.warning('cli_function_stubs: could not import %s', mod_name)
+        return []
+
+    names = []
+    for name, obj in inspect.getmembers(mod, inspect.isfunction):
+        if name.startswith('_'):
+            continue
+        func_module = getattr(obj, '__module__', '') or ''
+        if func_module.startswith(mod_name):
+            names.append(name)
+    return sorted(names)
+
+
+def _generate_cli_function_stubs(app: Sphinx) -> None:
+    """Generate function stubs and patch module stubs with a hidden toctree."""
+    srcdir = Path(app.srcdir)
+    outdir = srcdir / AUTOSUMMARY_DIR
+    outdir.mkdir(exist_ok=True)
+
+    overwrite = getattr(app.config, 'autosummary_generate_overwrite', True)
+
+    for mod_name in CLI_MODULES:
+        func_names = _discover_functions(mod_name)
+        if not func_names:
+            continue
+
+        # --- 1. Generate individual function stubs ---
+        for name in func_names:
+            stub_path = outdir / f'{mod_name}.{name}.rst'
+            if stub_path.exists() and not overwrite:
+                continue
+
+            escaped = name.replace('_', r'\_')
+            content = STUB_TEMPLATE.format(
+                title=escaped,
+                underline='=' * len(escaped),
+                module=mod_name,
+                name=name,
+            )
+            stub_path.write_text(content)
+
+        # --- 2. Patch module stub with hidden toctree ---
+        mod_stub = outdir / f'{mod_name}.rst'
+        if not mod_stub.exists():
+            continue
+
+        text = mod_stub.read_text()
+        if _TOCTREE_MARKER in text:
+            continue  # already patched
+
+        toctree_entries = '\n'.join(f'   {mod_name}.{n}' for n in func_names)
+        patch = (
+            f'\n.. {_TOCTREE_MARKER}\n'
+            f'.. toctree::\n'
+            f'   :hidden:\n\n'
+            f'{toctree_entries}\n'
+        )
+        mod_stub.write_text(text + patch)
+
+
+def setup(app: Sphinx) -> dict:
+    app.connect('builder-inited', _generate_cli_function_stubs, priority=900)
+    return {'version': '0.1', 'parallel_read_safe': True}