chore: merge develop into test (pipeline and scripts only, DOCS excluded)

Author: mckeea

.Rprofile

@@ -0,0 +1,30 @@
+# CLMS Technical Library — local PDF preview font setup.
+#
+# Makes the bundled Typst fonts (_meta/theme/typst-fonts/) discoverable so that
+# rendering a DOCS/*.qmd to PDF — e.g. via the RStudio "Render" button — uses the
+# CLMS fonts (Lato, Liberation Sans, JetBrains Mono) without installing anything
+# system-wide. Quarto's typst format has no font-path option, so we set Typst's
+# native TYPST_FONT_PATHS env var here; RStudio runs this file on session start,
+# and the quarto render it launches inherits the variable.
+#
+# Preview only — the CI build (build-docs.sh) does not use this.
+local({
+  font_dir <- normalizePath(
+    file.path(getwd(), "_meta", "theme", "typst-fonts"),
+    mustWork = FALSE
+  )
+  if (dir.exists(font_dir)) {
+    existing <- Sys.getenv("TYPST_FONT_PATHS")
+    paths <- if (nzchar(existing)) {
+      strsplit(existing, .Platform$path.sep, fixed = TRUE)[[1]]
+    } else {
+      character(0)
+    }
+    if (!(font_dir %in% paths)) {
+      Sys.setenv(
+        TYPST_FONT_PATHS = paste(c(font_dir, paths), collapse = .Platform$path.sep)
+      )
+      message("CLMS: TYPST_FONT_PATHS set to bundled fonts for local PDF preview.")
+    }
+  }
+})

.github/non_browsable_doc_map.json

@@ -15,6 +15,16 @@
       "source": "Copernicus_Land_Data_Store_CLDS/Survey_Evaluation_v1.qmd",
       "base": "dr3yejtzxollhygq0lwfgjxf755jvzyly2lemmvpgiqrzf5own2074qpdivug54c",
       "url": "/dr3yejtzxollhygq0lwfgjxf755jvzyly2lemmvpgiqrzf5own2074qpdivug54c.html"
+    },
+    {
+      "source": "CDSE_Migration/CLMS_CDSE_Migration_Status.qmd",
+      "base": "54ab2b07e82f82143780f24e2704ef211394c00216e83839e7c1dc42ba8e3a91",
+      "url": "/54ab2b07e82f82143780f24e2704ef211394c00216e83839e7c1dc42ba8e3a91.html"
+    },
+    {
+      "source": "CDSE_Migration/CLMS_CDSE_Migration_Dashboard.qmd",
+      "base": "a3e44009d6b7375f60a58b11278cbe079d93407249c0c5e1c97f00e12e98c09a",
+      "url": "/a3e44009d6b7375f60a58b11278cbe079d93407249c0c5e1c97f00e12e98c09a.html"
     }
   ]
 }

.github/runners/Dockerfile.quarto-doc-builder

@@ -55,7 +55,7 @@ RUN arch=$(dpkg --print-architecture) && \
 #                 files.upload(file=), see gemini_client.py)
 # tiktoken     -> token counting for the LLM rate limiter
 RUN pip install --no-cache-dir --break-system-packages \
-    panflute PyYAML google-genai==2.6.0 tiktoken==0.13.0 && \
+    panflute>=2.3.1 PyYAML google-genai==2.6.0 tiktoken==0.13.0 && \
     python3 -m pip cache purge 2>/dev/null || true && \
     rm -rf /root/.cache /tmp/* /var/tmp/*
 

.github/scripts/ai/update_versions_and_changelogs.py

@@ -861,6 +861,8 @@ def _process(sub_batch, _i=i):
 
 def calculate_new_version(current_version, bump_type, major_from_filename):
     """Calculate new version based on bump type"""
+    # First published version is 1.0.0, so a _v0 filename floors to major 1.
+    major_from_filename = max(major_from_filename, 1)
     try:
         parts = current_version.split(".")
         major = int(parts[0])
@@ -946,7 +948,7 @@ def initialize_first_release(all_files):
             print(f"[ERROR] {filepath}: {e}")
             continue
 
-        initial_version = f"{major_version}.0.0"
+        initial_version = f"{max(major_version, 1)}.0.0"
 
         update_qmd_version_only(filepath, initial_version)
 
@@ -1072,7 +1074,7 @@ def main():
         file_info[filepath] = {
             "major_version": major_version,
             "current_version": versions_metadata.get(filepath, {}).get(
-                "current_version", f"{major_version}.0.0"
+                "current_version", f"{max(major_version, 1)}.0.0"
             ),
         }
 

.github/scripts/build/apply_cached_intros.py

@@ -15,8 +15,9 @@
 SCRIPTS_ROOT = SCRIPT_DIR.parent
 sys.path.insert(0, str(SCRIPTS_ROOT))
 
-from helpers.qmd_utils import find_qmd_files  # noqa: E402
+from helpers.qmd_utils import find_qmd_files, read_qmd_frontmatter  # noqa: E402
 from helpers.file_updater import apply_all_updates  # noqa: E402
+from helpers.doc_types import element_off  # noqa: E402
 
 ROOT_DIR = (SCRIPT_DIR / "../../..").resolve()
 CACHE_DIR = (ROOT_DIR / ".llm_cache").resolve()
@@ -52,6 +53,20 @@ def main() -> int:
         print(f"[apply_cached_intros] no .qmd files under {input_dir}")
         return 0
 
+    # Skip types that take neither intro nor keywords (dashboard) so nothing is
+    # injected, not even into the origin_DOCS copy. They come as one bundle,
+    # hence the "both off" check — split it if a type ever wants only one.
+    def _wants_intro(qmd: Path) -> bool:
+        fm, _ = read_qmd_frontmatter(qmd)
+        dtype = fm.get("type")
+        return not (element_off(dtype, "keywords") and element_off(dtype, "description"))
+
+    kept = {q for q in qmd_files if _wants_intro(q)}
+    skipped = len(qmd_files) - len(kept)
+    if skipped:
+        print(f"[apply_cached_intros] skipping {skipped} file(s) whose type omits intros/keywords")
+    qmd_files = kept
+
     stats = apply_all_updates(
         qmd_files,
         CACHE_DIR,

.github/scripts/build/build-docs.sh

@@ -31,6 +31,11 @@ step() {
   _STEP_PREV=$now; _STEP_NAME="$*"
 }
 
+# Snapshot the pristine source before the build mutates it, so a local build is
+# easy to undo. After testing, restore your working tree with:
+#   rm -rf DOCS origin_DOCS && mv source_DOCS DOCS
+rm -rf source_DOCS && cp -rp DOCS source_DOCS
+
 # Apply cached intros/keywords before the rename - the cache is keyed by original path.
 echo "Injecting cached intros & keywords (no API)..."
 python3 .github/scripts/build/apply_cached_intros.py DOCS
@@ -73,6 +78,13 @@ python3 ../.github/scripts/build/fill_version.py .
 echo "Balancing table column widths..."
 python3 ../.github/scripts/qmd-tools/fix_table_colwidths.py .
 
+# Promote bare "Table N:"/"Figure N:" captions (left as plain text by the
+# converters) into .tbl-caption divs / image alt text, so they render as styled
+# captions instead of body text. Runs after the grid->pipe conversion above so a
+# caption next to a (now pipe) table is recognised. Build-copy only; idempotent.
+echo "Promoting bare table/figure captions..."
+python3 ../.github/scripts/qmd-tools/promote_bare_captions.py .
+
 # Bake image descriptions into the qmd source now, so the render doesn't re-hash
 # every image once per format (see the script). This was a Lua filter.
 echo "Baking image descriptions into qmd source..."
@@ -83,9 +95,12 @@ python3 ../.github/scripts/build/inject_image_descriptions.py .
 cp _quarto-no-headers.yml _quarto.yml
 
 step "[3/6] Rendering all documents (HTML + Typst PDF + gfm) in one pass..."
-# Render every format in the config - html (site), typst (PDFs), gfm
-# (the .llms.md sidecars). One pass over the files instead of one per format.
+# # Temporary move out of docs before render to avoid Jupyter engine selections crashes
+# echo "	 [BUILD BYPASS] Temporarily moving Ice products out of the build context..."
+# mv products/products_Algorithm_theoretical_basis_document_-_High_Resolution_Ice_products_Europe.qmd ../origin_DOCS/
 quarto_render --no-clean
+# echo "	 [BUILD BYPASS] Restoring Ice products..."
+# mv ../origin_DOCS/products_Algorithm_theoretical_basis_document_-_High_Resolution_Ice_products_Europe.qmd products/
 
 # Back up sitemap.xml and llms.txt - the index.qmd renders below regenerate
 # them, and we want to keep the values from this first render.
@@ -99,8 +114,10 @@ python3 ../.github/scripts/build/generate_index_all.py
 step "[5/6] Rendering index.qmd files..."
 mv _quarto.yml _quarto_not_used.yml
 mv _quarto-index.yml _quarto.yml
+# Serial (-P1): parallel renders raced on shared _site files (sitemap/search/
+# listings) and intermittently failed the whole build with xargs exit 123.
 find ./ -type f -name index.qmd -print0 | \
-  xargs -0 -P4 -I{} \
+  xargs -0 -P1 -I{} \
   bash -c 'quarto render "$1" --profile index --to html --no-clean --quiet 2>&1 | grep -v -e "Unknown meta key .* specified in a metadata Shortcode" -e "^Output created:"; exit ${PIPESTATUS[0]}' _ {}
 mv _quarto.yml _quarto-index.yml
 cp _quarto_not_used.yml _quarto.yml && rm _quarto_not_used.yml
@@ -110,6 +127,10 @@ cp _site/sitemap.xml.bkp _site/sitemap.xml
 rm -f _site/sitemap.xml.bkp
 mv _site/llms.txt.bkp _site/llms.txt
 
+# Drop .llms.md for llms-off types (dashboard), before the llm sitemap so it
+# falls back to the HTML URL.
+python3 ../.github/scripts/build/strip_llms_sidecars.py . _site
+
 # Remove non-browsable links from sitemap.xml and llms.txt
 python3 ../.github/scripts/build/remove_non_browsable.py _site/sitemap.xml
 python3 ../.github/scripts/build/remove_non_browsable.py --format llms _site/llms.txt

.github/scripts/build/clean_llms_md.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Strip residual block-level HTML wrappers from generated .llms.md sidecars.
+
+The gfm writer + the simplify_tables_gfm.lua filter already linearize tables and
+remove styling, but Quarto injects crossref float wrappers (`<div id="tbl-…">` …
+`</div>`) and the occasional layout `<div style="overflow-x:…">` AFTER the pandoc
+filters run, so they can't be removed at filter level. These bare wrapper lines are
+pure noise in the plain-text companion the RAG ingests. This post-render pass removes
+standalone block-level `<div>`/`</div>` lines (inline tags like <sub>/<sup> in prose
+are left intact).
+
+Idempotent. Usage:
+    python3 clean_llms_md.py _site            # walk *.llms.md under a directory
+    python3 clean_llms_md.py a.llms.md b.md    # specific files
+"""
+
+import re
+import sys
+from pathlib import Path
+
+# A line that is ONLY an opening/closing <div …> (optionally indented). Block-level
+# wrapper noise — never matches inline tags mid-prose.
+_BARE_DIV = re.compile(r"^[ \t]*</?div\b[^>]*>[ \t]*$")
+
+
+def clean_text(text: str) -> str:
+    out = [ln for ln in text.split("\n") if not _BARE_DIV.match(ln)]
+    cleaned = "\n".join(out)
+    # collapse the blank-line runs a removed wrapper can leave behind (3+ → 2)
+    cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
+    return cleaned
+
+
+def clean_file(path: Path) -> bool:
+    original = path.read_text(encoding="utf-8")
+    cleaned = clean_text(original)
+    if cleaned != original:
+        path.write_text(cleaned, encoding="utf-8")
+        return True
+    return False
+
+
+def _iter_targets(args):
+    for a in args:
+        p = Path(a)
+        if p.is_dir():
+            yield from p.rglob("*.llms.md")
+        elif p.exists():
+            yield p
+
+
+def main(argv) -> int:
+    if not argv:
+        print(__doc__)
+        return 1
+    changed = 0
+    for path in _iter_targets(argv):
+        if clean_file(path):
+            changed += 1
+    print(f"clean_llms_md: cleaned {changed} file(s)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

.github/scripts/build/fill_version.py

@@ -6,17 +6,22 @@
 version, so we fill it each build, like intros and changelogs.
 
 The value is the tracked version from .llm_cache/versions.json, looked up via the
-original-filename field. No record yet -> {major}.0.0 from the _vN.qmd name. No
+original-filename field. The doc's own `version:` frontmatter is never trusted as
+input - it is output we overwrite. No record yet -> seed one (max(major,1).0.0,
+so a _v0 or a name without _vN starts at 1.0.0) so the doc becomes tracked. No
 bump is computed here.
 """
 
 import argparse
+import json
 import re
 import sys
+from datetime import date
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))  # .github/scripts
 from helpers.json_io import load_json_or_empty
+from helpers.doc_types import element_off
 
 VERSIONS_FILE = ".llm_cache/versions.json"
 
@@ -68,7 +73,8 @@ def main():
     vf = Path(args.versions_file) if args.versions_file else repo_root / VERSIONS_FILE
     versions = load_json_or_empty(vf, label="versions")
 
-    filled = baseline = 0
+    filled = baseline = seeded = 0
+    today = date.today().isoformat()
     for qmd in sorted(Path(args.docs_dir).rglob("*.qmd")):
         if {"_site", ".quarto", "_meta"} & set(qmd.parts):
             continue
@@ -78,32 +84,54 @@ def main():
             continue
         _, end = bounds
 
-        src = fm_value(lines, end, "original-filename")
-        major = major_from_name(src) if src else None
-        if major is None:
-            major = major_from_name(qmd.name)
-        if major is None:
-            print(f"[fill_version] {qmd}: no _vN in filename, skipping")
+        # version-off types keep their own `version:` header, if any; don't touch.
+        if element_off(fm_value(lines, end, "type"), "version"):
             continue
 
+        src = fm_value(lines, end, "original-filename")
+        mj = major_from_name(src) if src else None
+        if mj is None:
+            mj = major_from_name(qmd.name)
+        # First published version is 1.0.0: a _v0 or a name without _vN starts
+        # at 1.0.0, while _v4 etc. keep their filename major.
+        baseline_major = max(mj, 1) if mj is not None else 1
+
+        key = f"DOCS/{src}" if src else None
         tracked = None
-        if src:
-            entry = versions.get(f"DOCS/{src}") or versions.get(src)
+        if key:
+            entry = versions.get(key) or versions.get(src)
             tracked = (entry or {}).get("current_version")
 
-        if tracked and tracked.split(".")[0] == str(major):
+        if tracked and (mj is None or tracked.split(".")[0] == str(mj)):
             version = tracked
         else:
-            version = f"{major}.0.0"
+            version = f"{baseline_major}.0.0"
             baseline += 1
+            # No record yet: seed one so the doc becomes tracked, rather than
+            # re-deriving the fallback every build. Keyed by original-filename.
+            if key and key not in versions:
+                versions[key] = {
+                    "current_version": version,
+                    "last_bump": "initial",
+                    "last_bump_reason": "First release",
+                    "last_release_tag": "initial",
+                    "last_updated": today,
+                    "major_from_filename": baseline_major,
+                }
+                seeded += 1
 
         if set_version(lines, end, version):
             qmd.write_text("".join(lines), encoding="utf-8")
             filled += 1
 
+    if seeded:
+        vf.parent.mkdir(parents=True, exist_ok=True)
+        with vf.open("w", encoding="utf-8") as f:
+            json.dump(versions, f, indent=2, sort_keys=True)
+
     print(
         f"[fill_version] set version on {filled} files "
-        f"({baseline} fell back to {{major}}.0.0)"
+        f"({baseline} used max(major,1).0.0 baseline, {seeded} new cache entries)"
     )