publish-modules.yml

# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
name: publish-modules

on:
  push:
    branches: [dev, main]
  workflow_dispatch:
    inputs:
      bundles:
        description: "Comma-separated bundle names (for example: specfact-backlog,specfact-project). Empty = auto-detect."
        required: false
        default: ""
      dry_run:
        description: "Prepare registry changes but do not push commit"
        required: false
        default: false
        type: boolean

permissions:
  contents: write
  pull-requests: write

concurrency:
  group: publish-modules-${{ github.ref }}
  cancel-in-progress: false

jobs:
  publish:
    runs-on: ubuntu-latest
    env:
      SPECFACT_MODULE_PRIVATE_SIGN_KEY: ${{ secrets.SPECFACT_MODULE_PRIVATE_SIGN_KEY }}
      SPECFACT_MODULE_PRIVATE_SIGN_KEY_PASSPHRASE: ${{ secrets.SPECFACT_MODULE_PRIVATE_SIGN_KEY_PASSPHRASE }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install publish dependencies
        run: |
          python -m pip install --upgrade pip
          # cryptography/cffi: required when the publish step invokes scripts/sign-modules.py
          # to add integrity.signature before packaging (same stack as sign-modules.yml).
          python -m pip install pyyaml packaging cryptography cffi

      - name: Resolve publish bundle set
        id: bundles
        env:
          EVENT_NAME: ${{ github.event_name }}
          DISPATCH_BUNDLES: ${{ github.event.inputs.bundles }}
          BEFORE_SHA: ${{ github.event.before }}
          AFTER_SHA: ${{ github.sha }}
          TARGET_REGISTRY_BASE_REF: ${{ github.event.repository.default_branch }}
        run: |
          python - <<'PY'
          import json
          import os
          import subprocess
          from pathlib import Path

          import yaml

          from publish_bundle_selection import determine_registry_baseline_ref, resolve_publish_bundles

          def normalize_bundle(name: str) -> str:
            cleaned = name.strip()
            if not cleaned:
              return ""
            if not cleaned.startswith("specfact-"):
              cleaned = f"specfact-{cleaned}"
            return cleaned

          def load_registry_versions(payload: dict) -> dict[str, str]:
            modules = payload.get("modules")
            if not isinstance(modules, list):
              return {}
            result: dict[str, str] = {}
            for entry in modules:
              if not isinstance(entry, dict):
                continue
              module_id = str(entry.get("id") or "").strip()
              latest_version = str(entry.get("latest_version") or "").strip()
              if not module_id or not latest_version:
                continue
              bundle_name = module_id.split("/", 1)[-1]
              if not bundle_name.startswith("specfact-"):
                continue
              result[bundle_name] = latest_version
            return result

          def load_manifest_versions(repo_root: Path) -> dict[str, str]:
            result: dict[str, str] = {}
            for manifest_path in sorted((repo_root / "packages").glob("*/module-package.yaml")):
              manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8"))
              if not isinstance(manifest, dict):
                continue
              bundle_name = manifest_path.parent.name
              version = str(manifest.get("version") or "").strip()
              if bundle_name.startswith("specfact-") and version:
                result[bundle_name] = version
            return result

          def load_baseline_registry(repo_root: Path, base_ref: str) -> dict:
            current_branch = os.environ.get("GITHUB_REF_NAME", "").strip()
            normalized = determine_registry_baseline_ref(
              current_branch=current_branch,
              default_branch=(base_ref or "main"),
            )
            if normalized == current_branch:
              return json.loads((repo_root / "registry" / "index.json").read_text(encoding="utf-8"))
            subprocess.run(["git", "fetch", "origin", normalized, "--depth=1"], check=True)
            raw = subprocess.check_output(
              ["git", "show", f"origin/{normalized}:registry/index.json"],
              text=True,
            )
            return json.loads(raw)

          event_name = os.environ.get("EVENT_NAME", "")
          dispatch_bundles = os.environ.get("DISPATCH_BUNDLES", "").strip()
          repo_root = Path(".").resolve()
          bundles: list[str] = []
          changed_paths: list[str] = []

          if event_name == "workflow_dispatch" and dispatch_bundles:
            bundles = [normalize_bundle(part) for part in dispatch_bundles.split(",")]
            bundles = sorted({bundle for bundle in bundles if bundle})
          else:
            before_sha = os.environ.get("BEFORE_SHA", "").strip()
            after_sha = os.environ.get("AFTER_SHA", "").strip() or "HEAD"
            if not before_sha or before_sha == "0000000000000000000000000000000000000000":
              before_sha = "HEAD~1"

            try:
              changed_paths = subprocess.check_output(
                ["git", "diff", "--name-only", before_sha, after_sha, "--", "packages"],
                text=True,
              ).splitlines()
            except subprocess.CalledProcessError:
              changed_paths = []

            manifest_versions = load_manifest_versions(repo_root)
            baseline_registry = load_baseline_registry(repo_root, os.environ.get("TARGET_REGISTRY_BASE_REF", "main"))
            registry_versions = load_registry_versions(baseline_registry)
            selection = resolve_publish_bundles(
              changed_paths=changed_paths,
              manifest_versions=manifest_versions,
              registry_versions=registry_versions,
            )
            bundles = sorted(selection)
            reason_map = {bundle: sorted(reasons) for bundle, reasons in selection.items()}
            print(f"Resolved bundle reasons: {reason_map}")
          if event_name == "workflow_dispatch" and dispatch_bundles:
            reason_map = {bundle: ["manual-dispatch"] for bundle in bundles}

          out_path = Path(os.environ["GITHUB_OUTPUT"])
          out_path.write_text(
            f"bundles_json={json.dumps(bundles)}\ncount={len(bundles)}\nbundle_reasons_json={json.dumps(reason_map)}\n",
            encoding="utf-8",
          )
          print(f"Resolved bundles: {bundles}")
          PY

      - name: Publish changed bundles into registry
        if: steps.bundles.outputs.count != '0'
        env:
          BUNDLES_JSON: ${{ steps.bundles.outputs.bundles_json }}
          BUNDLE_REASONS_JSON: ${{ steps.bundles.outputs.bundle_reasons_json }}
          TARGET_REGISTRY_BASE_REF: ${{ github.event.repository.default_branch }}
        run: |
          python - <<'PY'
          import hashlib
          import json
          import subprocess
          import tarfile
          from pathlib import Path

          import yaml
          from publish_bundle_selection import determine_registry_baseline_ref

          repo_root = Path(".").resolve()
          import os

          bundles = json.loads(os.environ["BUNDLES_JSON"])
          bundle_reasons = json.loads(os.environ.get("BUNDLE_REASONS_JSON", "{}"))
          registry_index_path = repo_root / "registry" / "index.json"
          registry_modules_dir = repo_root / "registry" / "modules"
          registry_signatures_dir = repo_root / "registry" / "signatures"
          registry_modules_dir.mkdir(parents=True, exist_ok=True)
          registry_signatures_dir.mkdir(parents=True, exist_ok=True)

          registry = json.loads(registry_index_path.read_text(encoding="utf-8"))
          modules = registry.get("modules")
          if not isinstance(modules, list):
            raise ValueError("registry/index.json must contain a 'modules' list")

          ignored_dir_names = {".git", "tests", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", "logs"}
          ignored_suffixes = {".pyc", ".pyo"}

          def manifest_has_signature(data: dict) -> bool:
              integrity_obj = data.get("integrity")
              if not isinstance(integrity_obj, dict):
                  return False
              return bool(str(integrity_obj.get("signature") or "").strip())

          signing_key = os.environ.get("SPECFACT_MODULE_PRIVATE_SIGN_KEY", "").strip()

          def sign_manifest_if_unsigned(manifest_path: Path, *, reason: str) -> None:
              if not signing_key:
                  return
              if not manifest_path.is_file():
                  return
              raw = yaml.safe_load(manifest_path.read_text(encoding="utf-8"))
              if not isinstance(raw, dict):
                  return
              if manifest_has_signature(raw):
                  return
              print(f"Signing {manifest_path} ({reason}).", flush=True)
              subprocess.run(
                  [
                      "python",
                      "scripts/sign-modules.py",
                      "--payload-from-filesystem",
                      "--allow-same-version",
                      str(manifest_path),
                  ],
                  cwd=str(repo_root),
                  check=True,
              )

          skipped_bundles: list[str] = []
          current_branch = os.environ.get("GITHUB_REF_NAME", "").strip()
          baseline_ref = determine_registry_baseline_ref(
            current_branch=current_branch,
            default_branch=(os.environ.get("TARGET_REGISTRY_BASE_REF") or "main"),
          )
          baseline_index_path = repo_root / ".publish-registry-baseline.json"

          if baseline_ref == current_branch:
            baseline_index_path.write_text(registry_index_path.read_text(encoding="utf-8"), encoding="utf-8")
          else:
            subprocess.run(["git", "fetch", "origin", baseline_ref, "--depth=1"], check=True)
            raw_baseline = subprocess.check_output(
              ["git", "show", f"origin/{baseline_ref}:registry/index.json"],
              text=True,
            )
            baseline_index_path.write_text(raw_baseline, encoding="utf-8")

          for bundle in bundles:
            reasons = bundle_reasons.get(bundle, [])
            print(f"Processing {bundle} because of: {', '.join(reasons) if reasons else 'unspecified'}")
            # Keep existing monotonic version guard logic.
            publish_check = subprocess.run(
              [
                "python",
                "scripts/publish-module.py",
                "--bundle",
                bundle,
                "--repo-root",
                ".",
                "--registry-index-path",
                str(baseline_index_path),
              ],
              check=False,
              capture_output=True,
              text=True,
            )
            combined_output = "\n".join(
              part.strip() for part in (publish_check.stdout, publish_check.stderr) if part and part.strip()
            )
            if publish_check.returncode != 0:
              if "Bundle version must be greater than registry latest_version" in combined_output:
                print(f"Skipping {bundle}: registry already at manifest version.")
                if combined_output:
                  print(combined_output)
                skipped_bundles.append(bundle)
                continue
              if combined_output:
                print(combined_output)
              raise subprocess.CalledProcessError(
                publish_check.returncode,
                publish_check.args,
                output=publish_check.stdout,
                stderr=publish_check.stderr,
              )

            bundle_dir = repo_root / "packages" / bundle
            manifest_path = bundle_dir / "module-package.yaml"
            if not manifest_path.exists():
              raise FileNotFoundError(f"Missing manifest: {manifest_path}")

            manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8"))
            if not isinstance(manifest, dict):
                raise ValueError(f"Invalid manifest content: {manifest_path}")

            if not manifest_has_signature(manifest):
                sign_manifest_if_unsigned(
                    manifest_path,
                    reason="missing integrity.signature before registry packaging",
                )
                manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8"))
                if not isinstance(manifest, dict):
                    raise ValueError(f"Invalid manifest content after signing: {manifest_path}")
                if signing_key and not manifest_has_signature(manifest):
                    raise ValueError(f"Signing did not produce integrity.signature: {manifest_path}")
                if not signing_key:
                    print(
                        f"::warning::Publishing {bundle} with checksum-only tree manifest "
                        "(SPECFACT_MODULE_PRIVATE_SIGN_KEY unset).",
                        flush=True,
                    )

            module_id = str(manifest.get("name") or f"nold-ai/{bundle}")
            version = str(manifest.get("version") or "").strip()
            if not version:
              raise ValueError(f"Manifest missing version: {manifest_path}")

            artifact_name = f"{bundle}-{version}.tar.gz"
            artifact_path = registry_modules_dir / artifact_name
            signature_path = registry_signatures_dir / f"{bundle}-{version}.tar.sig"

            with tarfile.open(artifact_path, mode="w:gz") as tar:
              for path in sorted(bundle_dir.rglob("*")):
                if not path.is_file():
                  continue
                rel = path.relative_to(bundle_dir)
                if any(part in ignored_dir_names for part in rel.parts):
                  continue
                if path.suffix.lower() in ignored_suffixes:
                  continue
                tar.add(path, arcname=f"{bundle}/{rel.as_posix()}")

            digest = hashlib.sha256(artifact_path.read_bytes()).hexdigest()
            (artifact_path.with_suffix(artifact_path.suffix + ".sha256")).write_text(f"{digest}\n", encoding="utf-8")

            integrity = manifest.get("integrity")
            if isinstance(integrity, dict):
              signature_text = str(integrity.get("signature") or "").strip()
              if signature_text:
                signature_path.write_text(signature_text + "\n", encoding="utf-8")

            entry = next(
              (
                item
                for item in modules
                if isinstance(item, dict) and str(item.get("id") or "").strip() == module_id
              ),
              None,
            )
            if entry is None:
              entry = {"id": module_id}
              modules.append(entry)

            entry["latest_version"] = version
            entry["download_url"] = f"modules/{artifact_name}"
            entry["checksum_sha256"] = digest
            if "tier" in manifest:
              entry["tier"] = manifest["tier"]
            if "publisher" in manifest:
              entry["publisher"] = manifest["publisher"]
            if "bundle_dependencies" in manifest:
              entry["bundle_dependencies"] = manifest["bundle_dependencies"]
            if "description" in manifest:
              entry["description"] = manifest["description"]

            print(f"Published registry artifact for {module_id} v{version}")

          if skipped_bundles:
            print(f"Skipped already-published bundles: {skipped_bundles}")

          for bundle in skipped_bundles:
              sign_manifest_if_unsigned(
                  repo_root / "packages" / bundle / "module-package.yaml",
                  reason="registry version already published; still align git manifest signature",
              )

          for manifest_path in sorted((repo_root / "packages").glob("*/module-package.yaml")):
              sign_manifest_if_unsigned(
                  manifest_path,
                  reason="final pass: ensure no unsigned module-package.yaml remains before commit",
              )

          registry_index_path.write_text(json.dumps(registry, indent=2) + "\n", encoding="utf-8")
          PY

      - name: Commit registry updates via PR branch
        if: steps.bundles.outputs.count != '0'
        env:
          DRY_RUN: ${{ github.event.inputs.dry_run }}
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          BUNDLE_REASONS_JSON: ${{ steps.bundles.outputs.bundle_reasons_json }}
        run: |
          if git diff --quiet -- registry/index.json registry/modules registry/signatures && git diff --quiet -- packages/; then
            echo "No registry or signed package manifest changes to commit."
            exit 0
          fi

          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"

          TARGET_BRANCH="${GITHUB_REF_NAME}"
          PUBLISH_BRANCH="auto/publish-${TARGET_BRANCH}-${GITHUB_RUN_ID}"
          git checkout -b "${PUBLISH_BRANCH}"

          # Registry artifacts plus any module-package.yaml updates from in-workflow signing
          # (otherwise dev→main PRs fail verify-modules-signature --require-signature).
          git add registry/index.json registry/modules registry/signatures
          git add -u packages/
          git commit -m "chore(registry): publish changed modules [skip ci]"

          if [ "${DRY_RUN:-false}" = "true" ]; then
            echo "Dry run enabled; skipping push."
            exit 0
          fi

          git push origin "${PUBLISH_BRANCH}"

          REASON_LINES=$(python - <<'PY'
          import json
          import os

          reason_map = json.loads(os.environ.get("BUNDLE_REASONS_JSON", "{}"))
          lines = []
          for bundle in sorted(reason_map):
            reasons = ", ".join(reason_map[bundle])
            lines.append(f"- `{bundle}`: {reasons}")
          print("\n".join(lines))
          PY
          )

          gh pr create \
            --base "${TARGET_BRANCH}" \
            --head "${PUBLISH_BRANCH}" \
            --title "chore(registry): publish changed modules" \
            --body "Automated registry publish update from workflow run ${GITHUB_RUN_ID}.

          Bundle selection reasons:
          ${REASON_LINES}"