Merge pull request #250 from nold-ai/auto/sign-dev-24969261534 #256
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json | |
| name: publish-modules | |
| on: | |
| push: | |
| branches: [dev, main] | |
| workflow_dispatch: | |
| inputs: | |
| bundles: | |
| description: "Comma-separated bundle names (for example: specfact-backlog,specfact-project). Empty = auto-detect." | |
| required: false | |
| default: "" | |
| dry_run: | |
| description: "Prepare registry changes but do not push commit" | |
| required: false | |
| default: false | |
| type: boolean | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| concurrency: | |
| group: publish-modules-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| publish: | |
| runs-on: ubuntu-latest | |
| env: | |
| SPECFACT_MODULE_PRIVATE_SIGN_KEY: ${{ secrets.SPECFACT_MODULE_PRIVATE_SIGN_KEY }} | |
| SPECFACT_MODULE_PRIVATE_SIGN_KEY_PASSPHRASE: ${{ secrets.SPECFACT_MODULE_PRIVATE_SIGN_KEY_PASSPHRASE }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install publish dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| # cryptography/cffi: required when the publish step invokes scripts/sign-modules.py | |
| # to add integrity.signature before packaging (same stack as sign-modules.yml). | |
| python -m pip install pyyaml packaging cryptography cffi | |
| - name: Resolve publish bundle set | |
| id: bundles | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| DISPATCH_BUNDLES: ${{ github.event.inputs.bundles }} | |
| BEFORE_SHA: ${{ github.event.before }} | |
| AFTER_SHA: ${{ github.sha }} | |
| TARGET_REGISTRY_BASE_REF: ${{ github.event.repository.default_branch }} | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import os | |
| import subprocess | |
| from pathlib import Path | |
| import yaml | |
| from publish_bundle_selection import determine_registry_baseline_ref, resolve_publish_bundles | |
| def normalize_bundle(name: str) -> str: | |
| cleaned = name.strip() | |
| if not cleaned: | |
| return "" | |
| if not cleaned.startswith("specfact-"): | |
| cleaned = f"specfact-{cleaned}" | |
| return cleaned | |
| def load_registry_versions(payload: dict) -> dict[str, str]: | |
| modules = payload.get("modules") | |
| if not isinstance(modules, list): | |
| return {} | |
| result: dict[str, str] = {} | |
| for entry in modules: | |
| if not isinstance(entry, dict): | |
| continue | |
| module_id = str(entry.get("id") or "").strip() | |
| latest_version = str(entry.get("latest_version") or "").strip() | |
| if not module_id or not latest_version: | |
| continue | |
| bundle_name = module_id.split("/", 1)[-1] | |
| if not bundle_name.startswith("specfact-"): | |
| continue | |
| result[bundle_name] = latest_version | |
| return result | |
| def load_manifest_versions(repo_root: Path) -> dict[str, str]: | |
| result: dict[str, str] = {} | |
| for manifest_path in sorted((repo_root / "packages").glob("*/module-package.yaml")): | |
| manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) | |
| if not isinstance(manifest, dict): | |
| continue | |
| bundle_name = manifest_path.parent.name | |
| version = str(manifest.get("version") or "").strip() | |
| if bundle_name.startswith("specfact-") and version: | |
| result[bundle_name] = version | |
| return result | |
| def load_baseline_registry(repo_root: Path, base_ref: str) -> dict: | |
| current_branch = os.environ.get("GITHUB_REF_NAME", "").strip() | |
| normalized = determine_registry_baseline_ref( | |
| current_branch=current_branch, | |
| default_branch=(base_ref or "main"), | |
| ) | |
| if normalized == current_branch: | |
| return json.loads((repo_root / "registry" / "index.json").read_text(encoding="utf-8")) | |
| subprocess.run(["git", "fetch", "origin", normalized, "--depth=1"], check=True) | |
| raw = subprocess.check_output( | |
| ["git", "show", f"origin/{normalized}:registry/index.json"], | |
| text=True, | |
| ) | |
| return json.loads(raw) | |
| event_name = os.environ.get("EVENT_NAME", "") | |
| dispatch_bundles = os.environ.get("DISPATCH_BUNDLES", "").strip() | |
| repo_root = Path(".").resolve() | |
| bundles: list[str] = [] | |
| changed_paths: list[str] = [] | |
| if event_name == "workflow_dispatch" and dispatch_bundles: | |
| bundles = [normalize_bundle(part) for part in dispatch_bundles.split(",")] | |
| bundles = sorted({bundle for bundle in bundles if bundle}) | |
| else: | |
| before_sha = os.environ.get("BEFORE_SHA", "").strip() | |
| after_sha = os.environ.get("AFTER_SHA", "").strip() or "HEAD" | |
| if not before_sha or before_sha == "0000000000000000000000000000000000000000": | |
| before_sha = "HEAD~1" | |
| try: | |
| changed_paths = subprocess.check_output( | |
| ["git", "diff", "--name-only", before_sha, after_sha, "--", "packages"], | |
| text=True, | |
| ).splitlines() | |
| except subprocess.CalledProcessError: | |
| changed_paths = [] | |
| manifest_versions = load_manifest_versions(repo_root) | |
| baseline_registry = load_baseline_registry(repo_root, os.environ.get("TARGET_REGISTRY_BASE_REF", "main")) | |
| registry_versions = load_registry_versions(baseline_registry) | |
| selection = resolve_publish_bundles( | |
| changed_paths=changed_paths, | |
| manifest_versions=manifest_versions, | |
| registry_versions=registry_versions, | |
| ) | |
| bundles = sorted(selection) | |
| reason_map = {bundle: sorted(reasons) for bundle, reasons in selection.items()} | |
| print(f"Resolved bundle reasons: {reason_map}") | |
| if event_name == "workflow_dispatch" and dispatch_bundles: | |
| reason_map = {bundle: ["manual-dispatch"] for bundle in bundles} | |
| out_path = Path(os.environ["GITHUB_OUTPUT"]) | |
| out_path.write_text( | |
| f"bundles_json={json.dumps(bundles)}\ncount={len(bundles)}\nbundle_reasons_json={json.dumps(reason_map)}\n", | |
| encoding="utf-8", | |
| ) | |
| print(f"Resolved bundles: {bundles}") | |
| PY | |
| - name: Publish changed bundles into registry | |
| if: steps.bundles.outputs.count != '0' | |
| env: | |
| BUNDLES_JSON: ${{ steps.bundles.outputs.bundles_json }} | |
| BUNDLE_REASONS_JSON: ${{ steps.bundles.outputs.bundle_reasons_json }} | |
| TARGET_REGISTRY_BASE_REF: ${{ github.event.repository.default_branch }} | |
| run: | | |
| python - <<'PY' | |
| import hashlib | |
| import json | |
| import subprocess | |
| import tarfile | |
| from pathlib import Path | |
| import yaml | |
| from publish_bundle_selection import determine_registry_baseline_ref | |
| repo_root = Path(".").resolve() | |
| import os | |
| bundles = json.loads(os.environ["BUNDLES_JSON"]) | |
| bundle_reasons = json.loads(os.environ.get("BUNDLE_REASONS_JSON", "{}")) | |
| registry_index_path = repo_root / "registry" / "index.json" | |
| registry_modules_dir = repo_root / "registry" / "modules" | |
| registry_signatures_dir = repo_root / "registry" / "signatures" | |
| registry_modules_dir.mkdir(parents=True, exist_ok=True) | |
| registry_signatures_dir.mkdir(parents=True, exist_ok=True) | |
| registry = json.loads(registry_index_path.read_text(encoding="utf-8")) | |
| modules = registry.get("modules") | |
| if not isinstance(modules, list): | |
| raise ValueError("registry/index.json must contain a 'modules' list") | |
| ignored_dir_names = {".git", "tests", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", "logs"} | |
| ignored_suffixes = {".pyc", ".pyo"} | |
| def manifest_has_signature(data: dict) -> bool: | |
| integrity_obj = data.get("integrity") | |
| if not isinstance(integrity_obj, dict): | |
| return False | |
| return bool(str(integrity_obj.get("signature") or "").strip()) | |
| signing_key = os.environ.get("SPECFACT_MODULE_PRIVATE_SIGN_KEY", "").strip() | |
| def sign_manifest_if_unsigned(manifest_path: Path, *, reason: str) -> None: | |
| if not signing_key: | |
| return | |
| if not manifest_path.is_file(): | |
| return | |
| raw = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) | |
| if not isinstance(raw, dict): | |
| return | |
| if manifest_has_signature(raw): | |
| return | |
| print(f"Signing {manifest_path} ({reason}).", flush=True) | |
| subprocess.run( | |
| [ | |
| "python", | |
| "scripts/sign-modules.py", | |
| "--payload-from-filesystem", | |
| "--allow-same-version", | |
| str(manifest_path), | |
| ], | |
| cwd=str(repo_root), | |
| check=True, | |
| ) | |
| skipped_bundles: list[str] = [] | |
| current_branch = os.environ.get("GITHUB_REF_NAME", "").strip() | |
| baseline_ref = determine_registry_baseline_ref( | |
| current_branch=current_branch, | |
| default_branch=(os.environ.get("TARGET_REGISTRY_BASE_REF") or "main"), | |
| ) | |
| baseline_index_path = repo_root / ".publish-registry-baseline.json" | |
| if baseline_ref == current_branch: | |
| baseline_index_path.write_text(registry_index_path.read_text(encoding="utf-8"), encoding="utf-8") | |
| else: | |
| subprocess.run(["git", "fetch", "origin", baseline_ref, "--depth=1"], check=True) | |
| raw_baseline = subprocess.check_output( | |
| ["git", "show", f"origin/{baseline_ref}:registry/index.json"], | |
| text=True, | |
| ) | |
| baseline_index_path.write_text(raw_baseline, encoding="utf-8") | |
| for bundle in bundles: | |
| reasons = bundle_reasons.get(bundle, []) | |
| print(f"Processing {bundle} because of: {', '.join(reasons) if reasons else 'unspecified'}") | |
| # Keep existing monotonic version guard logic. | |
| publish_check = subprocess.run( | |
| [ | |
| "python", | |
| "scripts/publish-module.py", | |
| "--bundle", | |
| bundle, | |
| "--repo-root", | |
| ".", | |
| "--registry-index-path", | |
| str(baseline_index_path), | |
| ], | |
| check=False, | |
| capture_output=True, | |
| text=True, | |
| ) | |
| combined_output = "\n".join( | |
| part.strip() for part in (publish_check.stdout, publish_check.stderr) if part and part.strip() | |
| ) | |
| if publish_check.returncode != 0: | |
| if "Bundle version must be greater than registry latest_version" in combined_output: | |
| print(f"Skipping {bundle}: registry already at manifest version.") | |
| if combined_output: | |
| print(combined_output) | |
| skipped_bundles.append(bundle) | |
| continue | |
| if combined_output: | |
| print(combined_output) | |
| raise subprocess.CalledProcessError( | |
| publish_check.returncode, | |
| publish_check.args, | |
| output=publish_check.stdout, | |
| stderr=publish_check.stderr, | |
| ) | |
| bundle_dir = repo_root / "packages" / bundle | |
| manifest_path = bundle_dir / "module-package.yaml" | |
| if not manifest_path.exists(): | |
| raise FileNotFoundError(f"Missing manifest: {manifest_path}") | |
| manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) | |
| if not isinstance(manifest, dict): | |
| raise ValueError(f"Invalid manifest content: {manifest_path}") | |
| if not manifest_has_signature(manifest): | |
| sign_manifest_if_unsigned( | |
| manifest_path, | |
| reason="missing integrity.signature before registry packaging", | |
| ) | |
| manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) | |
| if not isinstance(manifest, dict): | |
| raise ValueError(f"Invalid manifest content after signing: {manifest_path}") | |
| if signing_key and not manifest_has_signature(manifest): | |
| raise ValueError(f"Signing did not produce integrity.signature: {manifest_path}") | |
| if not signing_key: | |
| print( | |
| f"::warning::Publishing {bundle} with checksum-only tree manifest " | |
| "(SPECFACT_MODULE_PRIVATE_SIGN_KEY unset).", | |
| flush=True, | |
| ) | |
| module_id = str(manifest.get("name") or f"nold-ai/{bundle}") | |
| version = str(manifest.get("version") or "").strip() | |
| if not version: | |
| raise ValueError(f"Manifest missing version: {manifest_path}") | |
| artifact_name = f"{bundle}-{version}.tar.gz" | |
| artifact_path = registry_modules_dir / artifact_name | |
| signature_path = registry_signatures_dir / f"{bundle}-{version}.tar.sig" | |
| with tarfile.open(artifact_path, mode="w:gz") as tar: | |
| for path in sorted(bundle_dir.rglob("*")): | |
| if not path.is_file(): | |
| continue | |
| rel = path.relative_to(bundle_dir) | |
| if any(part in ignored_dir_names for part in rel.parts): | |
| continue | |
| if path.suffix.lower() in ignored_suffixes: | |
| continue | |
| tar.add(path, arcname=f"{bundle}/{rel.as_posix()}") | |
| digest = hashlib.sha256(artifact_path.read_bytes()).hexdigest() | |
| (artifact_path.with_suffix(artifact_path.suffix + ".sha256")).write_text(f"{digest}\n", encoding="utf-8") | |
| integrity = manifest.get("integrity") | |
| if isinstance(integrity, dict): | |
| signature_text = str(integrity.get("signature") or "").strip() | |
| if signature_text: | |
| signature_path.write_text(signature_text + "\n", encoding="utf-8") | |
| entry = next( | |
| ( | |
| item | |
| for item in modules | |
| if isinstance(item, dict) and str(item.get("id") or "").strip() == module_id | |
| ), | |
| None, | |
| ) | |
| if entry is None: | |
| entry = {"id": module_id} | |
| modules.append(entry) | |
| entry["latest_version"] = version | |
| entry["download_url"] = f"modules/{artifact_name}" | |
| entry["checksum_sha256"] = digest | |
| if "tier" in manifest: | |
| entry["tier"] = manifest["tier"] | |
| if "publisher" in manifest: | |
| entry["publisher"] = manifest["publisher"] | |
| if "bundle_dependencies" in manifest: | |
| entry["bundle_dependencies"] = manifest["bundle_dependencies"] | |
| if "description" in manifest: | |
| entry["description"] = manifest["description"] | |
| print(f"Published registry artifact for {module_id} v{version}") | |
| if skipped_bundles: | |
| print(f"Skipped already-published bundles: {skipped_bundles}") | |
| for bundle in skipped_bundles: | |
| sign_manifest_if_unsigned( | |
| repo_root / "packages" / bundle / "module-package.yaml", | |
| reason="registry version already published; still align git manifest signature", | |
| ) | |
| for manifest_path in sorted((repo_root / "packages").glob("*/module-package.yaml")): | |
| sign_manifest_if_unsigned( | |
| manifest_path, | |
| reason="final pass: ensure no unsigned module-package.yaml remains before commit", | |
| ) | |
| registry_index_path.write_text(json.dumps(registry, indent=2) + "\n", encoding="utf-8") | |
| PY | |
| - name: Commit registry updates via PR branch | |
| if: steps.bundles.outputs.count != '0' | |
| env: | |
| DRY_RUN: ${{ github.event.inputs.dry_run }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BUNDLE_REASONS_JSON: ${{ steps.bundles.outputs.bundle_reasons_json }} | |
| run: | | |
| if git diff --quiet -- registry/index.json registry/modules registry/signatures && git diff --quiet -- packages/; then | |
| echo "No registry or signed package manifest changes to commit." | |
| exit 0 | |
| fi | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| TARGET_BRANCH="${GITHUB_REF_NAME}" | |
| PUBLISH_BRANCH="auto/publish-${TARGET_BRANCH}-${GITHUB_RUN_ID}" | |
| git checkout -b "${PUBLISH_BRANCH}" | |
| # Registry artifacts plus any module-package.yaml updates from in-workflow signing | |
| # (otherwise dev→main PRs fail verify-modules-signature --require-signature). | |
| git add registry/index.json registry/modules registry/signatures | |
| git add -u packages/ | |
| git commit -m "chore(registry): publish changed modules [skip ci]" | |
| if [ "${DRY_RUN:-false}" = "true" ]; then | |
| echo "Dry run enabled; skipping push." | |
| exit 0 | |
| fi | |
| git push origin "${PUBLISH_BRANCH}" | |
| REASON_LINES=$(python - <<'PY' | |
| import json | |
| import os | |
| reason_map = json.loads(os.environ.get("BUNDLE_REASONS_JSON", "{}")) | |
| lines = [] | |
| for bundle in sorted(reason_map): | |
| reasons = ", ".join(reason_map[bundle]) | |
| lines.append(f"- `{bundle}`: {reasons}") | |
| print("\n".join(lines)) | |
| PY | |
| ) | |
| gh pr create \ | |
| --base "${TARGET_BRANCH}" \ | |
| --head "${PUBLISH_BRANCH}" \ | |
| --title "chore(registry): publish changed modules" \ | |
| --body "Automated registry publish update from workflow run ${GITHUB_RUN_ID}. | |
| Bundle selection reasons: | |
| ${REASON_LINES}" |