diff --git a/README.md b/README.md index 5f6b24e4..9505de59 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ steps: > [!NOTE] > Default sources below assume the consuming workflow follows the Reponomics Dashboard template repository wiring for tokens and secrets. If you decide to use this action outside of that template, pass explicit `with:` input values. -For `collection-token`, use a [fine-grained personal access token](https://github.com/settings/personal-access-tokens/new?name=COLLECTION_TOKEN&description=Read%20repository%20data%20for%20Reponomics%20Dashboard&expires_in=366&administration=read) with repository `Administration: read` for the owner/repositories being collected. Choose **All repositories** for broad automatic discovery, or **Only selected repositories** for a narrower dashboard. If you choose selected repositories, keep the dashboard configuration within that token's repository access. It does not need Pages or Administration write permissions. +For `collection-token`, use a [fine-grained personal access token](https://github.com/settings/personal-access-tokens/new?name=COLLECTION_TOKEN&description=Read%20repository%20data%20for%20Reponomics%20Dashboard&expires_in=366&administration=read) with repository `Administration: read` for the owner/repositories listed under `collect.repositories`. Keep `collect.repositories` within that token's repository access, and use `publish.repositories` to choose the subset rendered in the README and Pages dashboards. It does not need Pages or Administration write permissions. This action accepts one `collection-token`. Fine-grained personal access tokens are scoped to one GitHub resource owner, so a fine-grained token is the right fit only when the dashboard collects from one user or organization owner. If one dashboard must span multiple owners today, the current single-token fallback is a classic PAT with `repo` scope where the relevant organizations allow it. @@ -175,7 +175,7 @@ gh run download RUN_ID --repo OWNER/REPO --name html-dashboard-plaintext --dir . python3 -m http.server 8000 --directory .reponomics-dashboard ``` -For encrypted dashboards, after unlock, use the dashboard `Export CSV` control to download a canonical ZIP of retained CSV files. Export delivery is browser-local: ciphertext is fetched from a published encrypted asset and decrypted in memory before download. The runtime verifies both encrypted-asset and decrypted-bundle digests before download. Plaintext export data is not uploaded back to GitHub by this path. Export scope is canonical retained history, including repos that are currently excluded from dashboard rendering. +For encrypted dashboards, after unlock, use the dashboard `Export CSV` control to download a canonical ZIP of retained CSV files. Export delivery is browser-local: ciphertext is fetched from a published encrypted asset and decrypted in memory before download. The runtime verifies both encrypted-asset and decrypted-bundle digests before download. Plaintext export data is not uploaded back to GitHub by this path. Export scope is canonical retained history, including repos that are not currently listed in `publish.repositories`. See [CSV Export Architecture Guide](./docs/CSV_EXPORT.md) for implementation details, integrity model boundaries, and payload size-estimation formulas. diff --git a/dashboard_action/runtime/managed_docs/config.example.yaml b/dashboard_action/runtime/managed_docs/config.example.yaml index 1ea51640..50ecf485 100644 --- a/dashboard_action/runtime/managed_docs/config.example.yaml +++ b/dashboard_action/runtime/managed_docs/config.example.yaml @@ -47,33 +47,18 @@ auto_doctor_every_n_days: 0 # REPOSITORY SELECTION ------------------------------------------------------- * # * * * * * * -*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-* * * * * * # -# By default, Reponomics auto-discovers up to 200 eligible repositories visible -# to COLLECTION_TOKEN and fills the tracked set from newest to oldest by creation -# date. +# Reponomics only collects repositories you list explicitly. # -# Use full owner/repo names for explicit entries. +# Use bare repo names for repositories owned by this dashboard repository's +# owner. Use owner/repo names when collecting from another owner. # -max_repos: 200 +collect: + repositories: +# - repo-name +# - other-owner/repo-name # -# If non-empty, ONLY these and NO other repositories will be tracked. -include_only: -# - owner/repo-name -# -# Repositories that must always be tracked when accessible. -include: -# - owner/important-repo -# -# Repositories that must never be tracked automatically. -exclude: -# - owner/noisy-repo -# -# Fill remaining slots with other eligible repositories. -include_others: true -# -# Allow repositories created after the initial automatic-selection baseline to -# enter the automatic pool. -include_new: false -# -# Allow private repositories in the automatic pool when COLLECTION_TOKEN can see -# them. -include_private: true +# The published dashboard is intentionally narrow. It must be a subset of +# collect.repositories and can contain at most 8 repositories. +publish: + repositories: +# - repo-name diff --git a/dashboard_action/runtime/managed_docs/configuration.md b/dashboard_action/runtime/managed_docs/configuration.md index 0e4543fa..55cf061a 100644 --- a/dashboard_action/runtime/managed_docs/configuration.md +++ b/dashboard_action/runtime/managed_docs/configuration.md @@ -33,19 +33,11 @@ The setup fields at the top of `config.yaml` represent important user preference - `auto_doctor_every_n_days`: integer from `0` to `30`; `0` disables automatic doctor diagnostics. When set from `1` to `30`, collect-and-publish runs check the auto-doctor marker and run doctor when at least that many UTC days have elapsed since the last successful auto-doctor. -- `max_repos`: positive integer; caps how many repositories the dashboard tracks. +- `collect.repositories`: required list of repositories to collect. Entries may be bare repository names such as `api`, which resolve to the dashboard repository owner, or full names such as `other-owner/api`. Reponomics does not auto-discover or add repositories by default. -- `include_only`: list of `owner/repo` names; when non-empty, track only these repositories and ignore automatic discovery. +- `publish.repositories`: required list of repositories to render in the README and Pages dashboards. Every entry must also be present in `collect.repositories`, and the list can contain at most 8 repositories. -- `include`: list of `owner/repo` names; always include these repositories when the collection token can access them. - -- `exclude`: list of `owner/repo` names; never include these repositories through automatic selection. - -- `include_others`: boolean; when `true`, fill remaining `max_repos` slots from automatically discovered eligible repositories. - -- `include_new`: boolean; when `true`, allow repositories created after the initial automatic-selection baseline into the automatic pool. - -- `include_private`: boolean; when `true`, allow private repositories into the automatic pool when the collection token can access them. +`collect.repositories` is usually append-mostly: add a repository when you want Reponomics to start keeping history for it. To change what appears in dashboards, edit `publish.repositories`; removing a repository from `publish.repositories` does not stop collection. ## Constraints diff --git a/dashboard_action/runtime/managed_docs/repository-guide.md b/dashboard_action/runtime/managed_docs/repository-guide.md index 3f77fe75..8832e7ea 100644 --- a/dashboard_action/runtime/managed_docs/repository-guide.md +++ b/dashboard_action/runtime/managed_docs/repository-guide.md @@ -35,7 +35,7 @@ Common privacy, storage, export, and trust-boundary questions are answered in th For release, dependency, vendored-asset, and generated-artifact verification materials, see [Provenance And Verification Materials](provenance.md). -`COLLECTION_TOKEN` is only for repository data collection, including GitHub traffic data. Create it as a [fine-grained personal access token](https://github.com/settings/personal-access-tokens/new?name=COLLECTION_TOKEN&description=Read%20repository%20data%20for%20Reponomics%20Dashboard&expires_in=366&administration=read), choose the owner whose repositories should be collected, and keep the prefilled repository permission `Administration: read`. Choose **All repositories** for broad automatic discovery, or **Only selected repositories** if you want to limit collection to specific repositories. If you choose selected repositories, keep `config.yaml` within that token's repository access. The setup workflow uses the repository-scoped `GITHUB_TOKEN` to commit workflow enablement changes, and the collect workflow uses the repository-scoped `GITHUB_TOKEN` with job-level `actions: write` for same-repository artifact cleanup after a successful upload, so the collection token does not need repository, Pages, Actions, or Administration write permissions. Ideally, we will have one, limited-scope token responsible for any queries outside of the dashboard repo, and all other operations will be done by the repo's own `GITHUB_TOKEN`. This minimizes the scope of the collection token, which for many users will have access to lots of repositories. +`COLLECTION_TOKEN` is only for repository data collection, including GitHub traffic data. Create it as a [fine-grained personal access token](https://github.com/settings/personal-access-tokens/new?name=COLLECTION_TOKEN&description=Read%20repository%20data%20for%20Reponomics%20Dashboard&expires_in=366&administration=read), choose the owner whose repositories should be collected, and keep the prefilled repository permission `Administration: read`. Reponomics only collects repositories listed in `collect.repositories`; keep that list within the token's repository access. Use `publish.repositories` to choose the subset, up to 8 repositories, rendered in the README and Pages dashboards. The setup workflow uses the repository-scoped `GITHUB_TOKEN` to commit workflow enablement changes, and the collect workflow uses the repository-scoped `GITHUB_TOKEN` with job-level `actions: write` for same-repository artifact cleanup after a successful upload, so the collection token does not need repository, Pages, Actions, or Administration write permissions. Ideally, we will have one, limited-scope token responsible for any queries outside of the dashboard repo, and all other operations will be done by the repo's own `GITHUB_TOKEN`. This minimizes the scope of the collection token, which for many users will have access to lots of repositories. This template currently supports one collection credential. Fine-grained personal access tokens are scoped to one GitHub resource owner. If one dashboard needs to track repositories under multiple users or organizations, the fine-grained token flow is not the right fit for the current single-token setup. Use a classic PAT with `repo` scope where the relevant organizations allow it. Classic PATs are broader and can access repositories your GitHub account can access. diff --git a/dashboard_action/runtime/scripts/collect.py b/dashboard_action/runtime/scripts/collect.py index 74f76d0e..22e4e27e 100644 --- a/dashboard_action/runtime/scripts/collect.py +++ b/dashboard_action/runtime/scripts/collect.py @@ -67,14 +67,11 @@ secondary_retry_window as _http_secondary_retry_window, ) from collect_modules.repositories import ( - build_auto_candidates as _repositories_build_auto_candidates, current_repository as _repositories_current_repository, discover_repositories as _repositories_discover_repositories, is_trackable_repo as _repositories_is_trackable_repo, resolve_named_repos as _repositories_resolve_named_repos, resolve_repositories as _repositories_resolve_repositories, - selection_state as _repositories_selection_state, - sort_auto_candidates as _repositories_sort_auto_candidates, ) from collect_modules.runner import CollectionDependencies, run_collection from collect_modules.status import ( @@ -270,30 +267,8 @@ def discover_repositories(headers: Headers) -> list[RepoMetadata]: _is_trackable_repo = _repositories_is_trackable_repo -_selection_state = _repositories_selection_state _current_repository = _repositories_current_repository _resolve_named_repos = _repositories_resolve_named_repos -_sort_auto_candidates = _repositories_sort_auto_candidates - - -def _build_auto_candidates( - eligible: dict[str, RepoMetadata], - excluded: set[str], - selected_names: set[str], - current_repository: str, - include_private: bool, - include_new: bool, - auto_seeded_at: str, -) -> list[RepoMetadata]: - return _repositories_build_auto_candidates( - eligible, - excluded, - selected_names, - current_repository, - include_private, - include_new, - auto_seeded_at, - ) def resolve_repositories( diff --git a/dashboard_action/runtime/scripts/collect_modules/repositories.py b/dashboard_action/runtime/scripts/collect_modules/repositories.py index 6ad3861d..2906bb48 100644 --- a/dashboard_action/runtime/scripts/collect_modules/repositories.py +++ b/dashboard_action/runtime/scripts/collect_modules/repositories.py @@ -2,22 +2,16 @@ from collect_modules.repository_pages import discover_repositories from collect_modules.repository_selection import ( - build_auto_candidates, current_repository, is_trackable_repo, resolve_named_repos, resolve_repositories, - selection_state, - sort_auto_candidates, ) __all__ = [ - "build_auto_candidates", "current_repository", "discover_repositories", "is_trackable_repo", "resolve_named_repos", "resolve_repositories", - "selection_state", - "sort_auto_candidates", ] diff --git a/dashboard_action/runtime/scripts/collect_modules/repository_selection.py b/dashboard_action/runtime/scripts/collect_modules/repository_selection.py index 430936dd..1bb40a42 100644 --- a/dashboard_action/runtime/scripts/collect_modules/repository_selection.py +++ b/dashboard_action/runtime/scripts/collect_modules/repository_selection.py @@ -1,11 +1,10 @@ -"""Repository eligibility and stable auto-selection.""" +"""Repository eligibility and explicit repository selection.""" from __future__ import annotations import os import sys from collections.abc import Callable -from datetime import datetime, timezone from typing import Any from collect_modules.constants import CURRENT_REPOSITORY_ENV_KEYS @@ -28,17 +27,6 @@ def is_trackable_repo(repo: RepoMetadata, *, allow_pull: bool = False) -> bool: ) -def selection_state(manifest: dict[str, Any]) -> dict[str, str]: - """Return the persisted automatic-selection state.""" - state = manifest.get("selection_state") - if not isinstance(state, dict): - state = {} - manifest["selection_state"] = state - state.setdefault("auto_seeded_at", "") - state.setdefault("auto_cutoff_created_at", "") - return state - - def current_repository() -> str: """Return the repository running the collector when available.""" for env_key in CURRENT_REPOSITORY_ENV_KEYS: @@ -57,23 +45,24 @@ def resolve_repositories( use_github_app_collection_token: Callable[[], bool], current_repository: Callable[[], str], ) -> tuple[list[str], dict[str, Any], dict[str, RepoMetadata]]: - """Resolve the tracked repo set from explicit config plus stable auto-fill.""" + """Resolve the tracked repo set from the explicit collection registry.""" discovered = discover_repositories(headers) eligible = _eligible_repositories( discovered, allow_pull=use_github_app_collection_token(), ) - - if config["include_only"]: - return _resolve_include_only(config, manifest, discovered, eligible) - - return _resolve_mixed_selection( - config, - manifest, - discovered, + resolved_repos, missing_repos = resolve_named_repos( + config["collect_repositories"], eligible, - current_repository=current_repository(), ) + if missing_repos: + _warn_missing_collect_repos(missing_repos) + resolved = [repo["full_name"] for repo in resolved_repos] + if not resolved: + print("Error: no collect.repositories entries resolved to eligible repositories.") + sys.exit(1) + _print_selection_summary(discovered, eligible, resolved) + return resolved, manifest, metadata_for_resolved(resolved, eligible) def _eligible_repositories( @@ -89,50 +78,6 @@ def _eligible_repositories( return eligible -def _resolve_include_only( - config: dict[str, Any], - manifest: dict[str, Any], - discovered: list[RepoMetadata], - eligible: dict[str, RepoMetadata], -) -> tuple[list[str], dict[str, Any], dict[str, RepoMetadata]]: - include_only_repos, missing_include_only = resolve_named_repos( - config["include_only"], - eligible, - ) - if missing_include_only: - _warn_missing_repos("include_only", missing_include_only) - resolved = [repo["full_name"] for repo in include_only_repos[: config["max_repos"]]] - if not resolved: - print("Error: no eligible repositories remain in 'include_only'.") - sys.exit(1) - print( - "Repository discovery: " - + f"{len(discovered)} accessible, {len(eligible)} eligible after filters, " - + f"tracking {len(resolved)} from include_only." - ) - return resolved, manifest, metadata_for_resolved(resolved, eligible) - - -def _resolve_mixed_selection( - config: dict[str, Any], - manifest: dict[str, Any], - discovered: list[RepoMetadata], - eligible: dict[str, RepoMetadata], - *, - current_repository: str, -) -> tuple[list[str], dict[str, Any], dict[str, RepoMetadata]]: - include_repos, missing_include = resolve_named_repos(config["include"], eligible) - if missing_include: - _warn_missing_repos("include", missing_include) - - resolved = [repo["full_name"] for repo in include_repos] - explicit_count = len(resolved) - auto_count = _add_auto_selection(config, manifest, eligible, resolved, current_repository) - _print_selection_summary(discovered, eligible, resolved, explicit_count, auto_count) - _exit_if_empty_selection(resolved) - return resolved, manifest, metadata_for_resolved(resolved, eligible) - - def resolve_named_repos( repo_names: list[str], eligible: dict[str, RepoMetadata], @@ -155,125 +100,25 @@ def resolve_named_repos( return resolved, missing -def _warn_missing_repos(config_key: str, missing: list[str]) -> None: +def _warn_missing_collect_repos(missing: list[str]) -> None: print( - f"Warning: some configured {config_key} repos were not eligible " + "Warning: some configured collect.repositories repos were not eligible " + "for tracking (missing access, archived, forked, disabled, or " + "no push access): " + ", ".join(missing) ) -def _add_auto_selection( - config: dict[str, Any], - manifest: dict[str, Any], - eligible: dict[str, RepoMetadata], - resolved: list[str], - current_repository: str, -) -> int: - state = selection_state(manifest) - if not config["include_others"] or len(resolved) >= config["max_repos"]: - state["auto_cutoff_created_at"] = "" - return 0 - if not state["auto_seeded_at"]: - state["auto_seeded_at"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - selected_auto = build_auto_candidates( - eligible=eligible, - excluded=set(config["exclude"]), - selected_names=set(resolved), - current_repository=current_repository, - include_private=config["include_private"], - include_new=config["include_new"], - auto_seeded_at=state["auto_seeded_at"], - )[: config["max_repos"] - len(resolved)] - resolved.extend(repo["full_name"] for repo in selected_auto) - state["auto_cutoff_created_at"] = ( - selected_auto[-1].get("created_at") or "" if selected_auto else "" - ) - return len(selected_auto) - - -def build_auto_candidates( - eligible: dict[str, RepoMetadata], - excluded: set[str], - selected_names: set[str], - current_repository: str, - include_private: bool, - include_new: bool, - auto_seeded_at: str, -) -> list[RepoMetadata]: - """Return automatic candidates after applying explicit selection rules.""" - candidates = [ - repo - for repo_name, repo in eligible.items() - if _is_auto_candidate( - repo_name, - repo, - excluded, - selected_names, - current_repository, - include_private, - include_new, - auto_seeded_at, - ) - ] - return sort_auto_candidates(candidates) - - -def _is_auto_candidate( - repo_name: str, - repo: RepoMetadata, - excluded: set[str], - selected_names: set[str], - current_repository: str, - include_private: bool, - include_new: bool, - auto_seeded_at: str, -) -> bool: - if repo_name in selected_names or repo_name in excluded: - return False - if current_repository and repo_name == current_repository: - return False - if not include_private and repo.get("private", False): - return False - return not ( - auto_seeded_at - and not include_new - and (repo.get("created_at") or "") > auto_seeded_at - ) - - -def sort_auto_candidates(repos: list[RepoMetadata]) -> list[RepoMetadata]: - """Sort automatic candidates by creation date descending, then name.""" - repos = sorted(repos, key=lambda repo: repo.get("full_name") or "") - return sorted(repos, key=lambda repo: repo.get("created_at") or "", reverse=True) - - def _print_selection_summary( discovered: list[RepoMetadata], eligible: dict[str, RepoMetadata], resolved: list[str], - explicit_count: int, - auto_count: int, ) -> None: print( "Repository discovery: " + f"{len(discovered)} accessible, {len(eligible)} eligible after filters, " - + f"tracking {len(resolved)} " - + f"({explicit_count} explicit, {auto_count} automatic)." - ) - - -def _exit_if_empty_selection(resolved: list[str]) -> None: - if resolved: - return - print("Error: no eligible repositories found for traffic collection.") - print( - "Check your config or token access. Explicit includes must be " - + "accessible, and automatic tracking excludes forks, archived repos, " - + "disabled repos, and repos without push access." + + f"tracking {len(resolved)} from collect.repositories." ) - sys.exit(1) def metadata_for_resolved( diff --git a/dashboard_action/runtime/scripts/load_data.py b/dashboard_action/runtime/scripts/load_data.py index 524e83ff..80708370 100644 --- a/dashboard_action/runtime/scripts/load_data.py +++ b/dashboard_action/runtime/scripts/load_data.py @@ -65,7 +65,7 @@ "_bool_or_none", "_content_label", "_counter_snapshot", - "_filter_excluded_rows", + "_filter_published_rows", "_growth_insight_candidates", "_int_or_none", "_latest_date_from_rows", @@ -111,7 +111,7 @@ def load_daily(data_dir=None): rows = storage.read_csv(os.path.join(data_path, "traffic-daily.csv")) if not rows: rows = storage.read_csv(os.path.join(data_path, "traffic-log.csv")) - return _filter_excluded_rows(rows) + return _filter_published_rows(rows) def load_referrers(data_dir=None): @@ -146,17 +146,17 @@ def load_traffic_coverage(data_dir=None): def _load_csv(filename, data_dir=None): data_path = data_dir or storage.DATA_DIR - return _filter_excluded_rows(storage.read_csv(os.path.join(data_path, filename))) + return _filter_published_rows(storage.read_csv(os.path.join(data_path, filename))) -def _excluded_repos(): - """Return excluded repos from config, ignoring missing config files.""" - return set(load_repo_config().get("exclude_repos", [])) +def _published_repos(): + """Return repos selected for rendering, ignoring missing config files.""" + return set(load_repo_config().get("publish_repositories", [])) -def _filter_excluded_rows(rows): - """Hide excluded repos from rendered outputs while retaining artifact history.""" - excluded = _excluded_repos() - if not excluded: +def _filter_published_rows(rows): + """Limit rendered outputs to publish.repositories while retaining history.""" + published = _published_repos() + if not published: return rows - return [row for row in rows if row.get("repo") not in excluded] + return [row for row in rows if not row.get("repo") or row.get("repo") in published] diff --git a/dashboard_action/runtime/scripts/render_dashboard_support/assets/static/dashboard/state.js b/dashboard_action/runtime/scripts/render_dashboard_support/assets/static/dashboard/state.js index f7f0a8f1..f999b134 100644 --- a/dashboard_action/runtime/scripts/render_dashboard_support/assets/static/dashboard/state.js +++ b/dashboard_action/runtime/scripts/render_dashboard_support/assets/static/dashboard/state.js @@ -52,7 +52,7 @@ export function installState(context) { const SERIES_METRIC_KEYS = ['views', 'uniques', 'clones', 'clone_uniques', 'stars_delta', 'subscribers_delta', 'forks_delta']; const WINDOW_PRESETS = ['7', '14', '30', '90', 'all']; const DEFAULT_WINDOW = '14'; - const MAX_DISPLAY_REPOS = 20; + const MAX_DISPLAY_REPOS = 8; const MAX_COMPARE_REPOS = 8; const CHUNK_FAILURE_LABELS = { missing: 'Missing chunk', diff --git a/dashboard_action/runtime/scripts/repo_config.py b/dashboard_action/runtime/scripts/repo_config.py index 01ddf849..588fd621 100644 --- a/dashboard_action/runtime/scripts/repo_config.py +++ b/dashboard_action/runtime/scripts/repo_config.py @@ -10,13 +10,29 @@ CONFIG_PATH = "config.yaml" -DEFAULT_MAX_REPOS = 200 +DEFAULT_MAX_COLLECT_REPOS = 100 +MAX_PUBLISH_REPOS = 8 MAX_OWNER_LENGTH = 39 MAX_REPO_NAME_LENGTH = 100 MAX_FULL_NAME_LENGTH = MAX_OWNER_LENGTH + 1 + MAX_REPO_NAME_LENGTH OWNER_RE = r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?" REPO_NAME_RE = r"[A-Za-z0-9_.-]{1,100}" +REPO_NAME_ONLY_RE = re.compile(rf"^{REPO_NAME_RE}$") FULL_REPO_NAME_RE = re.compile(rf"^({OWNER_RE})/({REPO_NAME_RE})$") +CURRENT_REPOSITORY_ENV_KEYS = ("GITHUB_REPOSITORY", "GH_REPO") +REMOVED_SELECTION_KEYS = frozenset( + { + "include_only", + "include", + "repos", + "exclude", + "exclude_repos", + "include_others", + "include_new", + "include_private", + "max_repos", + } +) def load_repo_config(config_path: str = CONFIG_PATH) -> dict[str, Any]: @@ -30,63 +46,100 @@ def load_repo_config(config_path: str = CONFIG_PATH) -> dict[str, Any]: if not isinstance(config, dict): raise ValueError(f"'{config_path}' must contain a YAML mapping.") - include_only = _normalize_repo_list( - config_path, - "include_only", - config.get("include_only"), - ) - include = _normalize_repo_list( - config_path, - "include", - config.get("include", config.get("repos")), - ) - exclude = _normalize_repo_list( + removed_keys = sorted(REMOVED_SELECTION_KEYS.intersection(config)) + if removed_keys: + raise ValueError( + f"'{config_path}' uses removed repository-selection key(s): " + + ", ".join(removed_keys) + + ". Use collect.repositories and publish.repositories." + ) + + default_owner = _current_repository_owner() + collect_repos = _normalize_nested_repo_list( config_path, - "exclude", - config.get("exclude", config.get("exclude_repos")), + "collect", + config.get("collect"), + "repositories", + default_owner=default_owner, ) - max_repos = _normalize_positive_int( + publish_repos = _normalize_nested_repo_list( config_path, - "max_repos", - config.get("max_repos", DEFAULT_MAX_REPOS), + "publish", + config.get("publish"), + "repositories", + default_owner=default_owner, ) - if len(include_only) > max_repos: + if not collect_repos: + raise ValueError( + f"'{config_path}' key 'collect.repositories' must contain at least " + + "one repository." + ) + if not publish_repos: + raise ValueError( + f"'{config_path}' key 'publish.repositories' must contain at least " + + "one repository." + ) + if len(collect_repos) > DEFAULT_MAX_COLLECT_REPOS: raise ValueError( - f"'{config_path}' key 'include_only' contains {len(include_only)} " + - f"repositories but 'max_repos' is {max_repos}." + f"'{config_path}' key 'collect.repositories' contains " + + f"{len(collect_repos)} repositories but the beta cap is " + + f"{DEFAULT_MAX_COLLECT_REPOS}." ) - if len(include) > max_repos: + if len(publish_repos) > MAX_PUBLISH_REPOS: raise ValueError( - f"'{config_path}' key 'include' contains {len(include)} " + - f"repositories but 'max_repos' is {max_repos}." + f"'{config_path}' key 'publish.repositories' contains " + + f"{len(publish_repos)} repositories but the dashboard cap is " + + f"{MAX_PUBLISH_REPOS}." + ) + + collect_set = set(collect_repos) + missing_from_collect = [repo for repo in publish_repos if repo not in collect_set] + if missing_from_collect: + raise ValueError( + f"'{config_path}' key 'publish.repositories' includes repos that " + + "are not listed in 'collect.repositories': " + + ", ".join(missing_from_collect) ) return { - "max_repos": max_repos, - "include_only": include_only, - "include": include, - "exclude": exclude, - "include_others": _normalize_bool( - config_path, - "include_others", - config.get("include_others", True), - ), - "include_new": _normalize_bool( - config_path, - "include_new", - config.get("include_new", False), - ), - "include_private": _normalize_bool( - config_path, - "include_private", - config.get("include_private", True), - ), + "max_collect_repos": DEFAULT_MAX_COLLECT_REPOS, + "max_publish_repos": MAX_PUBLISH_REPOS, + "collect_repositories": collect_repos, + "publish_repositories": publish_repos, } -def _normalize_repo_list(config_path: str, key: str, value) -> list[str]: - """Normalize a repo list and validate owner/repo formatting.""" +def _normalize_nested_repo_list( + config_path: str, + parent_key: str, + parent_value: Any, + child_key: str, + *, + default_owner: str, +) -> list[str]: + """Normalize a repository list from a nested config mapping.""" + full_key = f"{parent_key}.{child_key}" + if not isinstance(parent_value, dict): + raise ValueError(f"'{config_path}' key '{parent_key}' must be a mapping.") + if any(not isinstance(key, str) for key in parent_value): + raise ValueError(f"'{config_path}' key '{parent_key}' must contain string keys.") + return _normalize_repo_list( + config_path, + full_key, + parent_value.get(child_key), + default_owner=default_owner, + ) + + +def _normalize_repo_list( + config_path: str, + key: str, + value: Any, + *, + default_owner: str, +) -> list[str]: + """Normalize repo entries and validate GitHub repository formatting.""" value = value or [] if not isinstance(value, list): raise ValueError(f"'{config_path}' key '{key}' must be a list.") @@ -102,13 +155,41 @@ def _normalize_repo_list(config_path: str, key: str, value) -> list[str]: repo = raw_repo.strip() if not repo: continue - _validate_repo_full_name(config_path, key, repo) - if repo not in seen: - normalized.append(repo) - seen.add(repo) + normalized_repo = _normalize_repo_name( + config_path, + key, + repo, + default_owner=default_owner, + ) + if normalized_repo not in seen: + normalized.append(normalized_repo) + seen.add(normalized_repo) return normalized +def _normalize_repo_name( + config_path: str, + key: str, + repo: str, + *, + default_owner: str, +) -> str: + """Return a full owner/repo name for a configured repository entry.""" + if "/" in repo: + _validate_repo_full_name(config_path, key, repo) + return repo + _validate_repo_short_name(config_path, key, repo) + if not default_owner: + raise ValueError( + f"invalid repository entry {repo!r} under '{key}' in {config_path}; " + + "bare repository names require GITHUB_REPOSITORY or GH_REPO so " + + "the dashboard repository owner can be inferred." + ) + full_name = f"{default_owner}/{repo}" + _validate_repo_full_name(config_path, key, full_name) + return full_name + + def _validate_repo_full_name(config_path: str, key: str, repo: str) -> None: """Validate a GitHub owner/repository full name.""" if len(repo) > MAX_FULL_NAME_LENGTH: @@ -136,32 +217,46 @@ def _validate_repo_full_name(config_path: str, key: str, repo: str) -> None: ) -def _normalize_bool(config_path: str, key: str, value: Any) -> bool: - """Validate a YAML boolean setting.""" - if isinstance(value, bool): - return value - raise ValueError( - f"'{config_path}' key '{key}' must be true or false, got {value!r}." - ) - - -def _normalize_positive_int(config_path: str, key: str, value: Any) -> int: - """Validate a positive integer setting.""" - if isinstance(value, bool) or not isinstance(value, int) or value <= 0: +def _validate_repo_short_name(config_path: str, key: str, repo_name: str) -> None: + """Validate a bare GitHub repository name.""" + if len(repo_name) > MAX_REPO_NAME_LENGTH: + raise ValueError( + f"invalid repository entry {repo_name!r} under '{key}' in {config_path}; " + + f"repository names must be at most {MAX_REPO_NAME_LENGTH} characters." + ) + if not REPO_NAME_ONLY_RE.fullmatch(repo_name): + raise ValueError( + f"invalid repository entry {repo_name!r} under '{key}' in {config_path}; " + + "use 'repo' or 'owner/repo' with names containing only ASCII " + + "letters, digits, '.', '-', or '_'." + ) + if repo_name in {".", ".."}: + raise ValueError( + f"invalid repository entry {repo_name!r} under '{key}' in {config_path}; " + + "repository name cannot be '.' or '..'." + ) + if repo_name.lower().endswith((".git", ".wiki")): raise ValueError( - f"'{config_path}' key '{key}' must be a positive integer, " + - f"got {value!r}." + f"invalid repository entry {repo_name!r} under '{key}' in {config_path}; " + + "repository name cannot end with '.git' or '.wiki'." ) - return value + + +def _current_repository_owner() -> str: + """Return the owner of the dashboard repository from the workflow env.""" + for env_key in CURRENT_REPOSITORY_ENV_KEYS: + full_name = (os.environ.get(env_key) or "").strip() + if "/" in full_name: + owner = full_name.split("/", 1)[0].strip() + if owner: + return owner + return "" def _default_config() -> dict[str, Any]: return { - "max_repos": DEFAULT_MAX_REPOS, - "include_only": [], - "include": [], - "exclude": [], - "include_others": True, - "include_new": False, - "include_private": True, + "max_collect_repos": DEFAULT_MAX_COLLECT_REPOS, + "max_publish_repos": MAX_PUBLISH_REPOS, + "collect_repositories": [], + "publish_repositories": [], } diff --git a/dashboard_action/runtime/scripts/storage.py b/dashboard_action/runtime/scripts/storage.py index bfcae43a..ea3e37a9 100644 --- a/dashboard_action/runtime/scripts/storage.py +++ b/dashboard_action/runtime/scripts/storage.py @@ -704,10 +704,6 @@ def _default_manifest(): "last_updated": "", "retention_days": RETENTION_DAYS, "files": list(CSV_REGISTRY.keys()), - "selection_state": { - "auto_seeded_at": "", - "auto_cutoff_created_at": "", - }, } if DATA_MODE in VALID_DATA_MODES: manifest["data_mode"] = DATA_MODE diff --git a/docs/CSV_EXPORT.md b/docs/CSV_EXPORT.md index 3644ab6d..f04b917d 100644 --- a/docs/CSV_EXPORT.md +++ b/docs/CSV_EXPORT.md @@ -26,7 +26,7 @@ The export bundle is built from canonical retained files: - every CSV file registered in `storage.CSV_REGISTRY` - `manifest.json` -This includes repos currently excluded from dashboard rendering, because export is for retained-data portability and parity. +This includes repos not currently listed in `publish.repositories`, because export is for retained-data portability and parity. For a relationship-oriented view of these retained CSV files, see [Retained Data ERD](./RETAINED_DATA_ERD.md). diff --git a/docs/adr/004-browser-local-csv-export-delivery.md b/docs/adr/004-browser-local-csv-export-delivery.md index 92cee733..2c797174 100644 --- a/docs/adr/004-browser-local-csv-export-delivery.md +++ b/docs/adr/004-browser-local-csv-export-delivery.md @@ -120,9 +120,9 @@ avoiding plaintext workflow artifacts. ## Implementation Qualifications - Export scope defaults to canonical retained data fidelity. The bundle includes - the full retained canonical file set (including repos currently excluded from - dashboard rendering) unless a future decision explicitly introduces scoped - export modes. + the full retained canonical file set, including repositories not currently + listed in `publish.repositories`, unless a future decision explicitly + introduces scoped export modes. - Offline/local-file export is best-effort. If browser origin rules block asset fetch (for example `file://` behavior), the runtime must fail closed with a clear user-facing error and no partial plaintext output. @@ -134,8 +134,9 @@ avoiding plaintext workflow artifacts. 1. Should export include excluded repos from retained canonical history, or only currently visible dashboard scope? Answer: canonical retained scope. Export includes the full canonical retained - fileset (including excluded repos) to preserve portability and deterministic - parity with retained artifacts. + fileset, including repositories not currently listed in + `publish.repositories`, to preserve portability and deterministic parity + with retained artifacts. 2. Is offline `file://` export support mandatory or best-effort? Answer: best-effort. Hosted Pages and local HTTP serving are supported paths. diff --git a/docs/adr/016-collection-scale-cost-and-lazy-dashboard-data.md b/docs/adr/016-collection-scale-cost-and-lazy-dashboard-data.md index cfb1a3df..b8df6768 100644 --- a/docs/adr/016-collection-scale-cost-and-lazy-dashboard-data.md +++ b/docs/adr/016-collection-scale-cost-and-lazy-dashboard-data.md @@ -104,6 +104,10 @@ Repository configuration should shift from "choose the small set to protect" to still be able to exclude repositories, pin repositories for display, and set a lower cost ceiling. +Superseded by ADR 029 for repository selection and dashboard publication: +Reponomics now uses explicit `collect.repositories` and `publish.repositories` +lists instead of broad automatic collection with exclusions. + ### Schedule The default schedule should be once per day. diff --git a/docs/adr/029-explicit-collection-and-publish-repository-lists.md b/docs/adr/029-explicit-collection-and-publish-repository-lists.md new file mode 100644 index 00000000..c2c54e4f --- /dev/null +++ b/docs/adr/029-explicit-collection-and-publish-repository-lists.md @@ -0,0 +1,281 @@ +# ADR 029: Explicit Collection And Publish Repository Lists + +Date: 2026-06-26 + +## Status + +Accepted. Implemented on 2026-06-27. + +The implementation treats syntax and structural config errors as fail-fast, +while warning and skipping individual `collect.repositories` entries that are +inaccessible or ineligible so one repository access failure does not abort the +whole collection run. The beta collection cap is hard-coded at 100 +repositories, and the publish cap is hard-coded at 8 repositories. + +## Context + +Reponomics is preparing for beta users. The current repository-selection model +still reflects an earlier product direction: reduce user anxiety by discovering +eligible repositories, collecting broadly, and letting the dashboard display a +bounded subset. That model is technically feasible, but it weakens several +beta priorities. + +Broad automatic selection makes the configuration harder to explain. Users must +understand `include`, `include_only`, `exclude`, `include_others`, automatic +eligibility, and default fill behavior before they can predict which +repositories will be collected. It also creates support and trust questions: +when the configured token can see many repositories, automatic collection can +look like hidden product policy rather than explicit user intent. + +The visual surface has a different constraint from the retained data store. A +dashboard can technically retain and process data for many repositories, but a +multi-repository chart is not useful when too many series are shown at once. +Eight repositories is a reasonable maximum for a published comparison surface; +ten or more is already visually crowded for line charts, legends, labels, hover +states, and similar colors. + +The README dashboard also matters. Unlike the Pages dashboard, README output +cannot rely on browser-only controls, tabs, or a dynamic repository picker. If +Pages supports multiple client-side repository sets while README shows only one +static set, the product becomes asymmetric across its two publication targets. + +Reponomics also has a distinctive trust boundary. It is not a hosted analytics +service. Repository data, retained artifacts, dashboard output, and any +dashboard interaction stay in the user's repository boundary unless the user +chooses to disclose something. This product position is stronger when the +configured repository set is explicit, handwritten, and predictable. + +The explicit model is also intended to increase engagement with the dashboard +repository itself. If Reponomics passively collects everything visible to a +token, users can treat the dashboard as ambient storage they might check later. +Hand-maintained lists create a small maintenance ritual: when a user creates, +adopts, or stops caring about a repository, they return to the dashboard repo +and decide whether that project should be remembered and whether it deserves +published attention. Users can still collect more repositories than they +actively publish, but the collection registry remains a conscious portfolio +rather than a background sweep. + +There are currently no external production users. Breaking changes are +acceptable when they simplify the beta product, but they should be documented +clearly because they affect collection continuity, dashboard shape, and user +expectations. + +## Decision Drivers + +- Make collection consent explicit and inspectable in `config.yaml`. +- Avoid hidden repository discovery or automatic fill behavior. +- Preserve broad enough collection for beta users without making broad + collection the default. +- Keep the published dashboard visually bounded. +- Keep README and Pages dashboard semantics aligned. +- Avoid a dashboard repository picker in the current beta plan. +- Make publication curation low-risk while making collection removal an + intentional choice. +- Encourage users to revisit the dashboard repository when repository ownership + or attention changes. +- Reinforce the trust-minimized product positioning. + +## Decision + +Adopt two explicit repository lists: + +1. `collect.repositories` +2. `publish.repositories` + +Both lists are handwritten by the user. Reponomics does not auto-discover, +auto-fill, or choose repositories by default. + +Example: + +```yaml +collect: + repositories: + - api + - web + - docs + - sdk + - other-owner/infra + +publish: + repositories: + - api + - web + - docs +``` + +Repository entries may use either a bare repository name or a full +`owner/repo` name. Bare names are normalized to the owner of the dashboard +repository. Full names are available when the user wants to collect or publish +repositories from another owner. + +### Collection List + +`collect.repositories` is the collection registry. Reponomics collects new +observations only for repositories listed there. + +Policy: + +- `collect.repositories` is required. +- Entries must be explicit repository names, either as `repo` or `owner/repo`. +- No repositories are added implicitly. +- No automatic discovery fills remaining slots. +- No `include_others` style behavior remains in the beta model. +- The beta implementation should cap this list at 100 repositories. + +The product guidance should treat this list as append-mostly: + +> Add repositories to `collect.repositories` when you want Reponomics to start +> keeping history for them. You usually do not need to remove repositories from +> this list. To change what appears in the dashboard, edit +> `publish.repositories`. + +Removing a repository from `collect.repositories` means the user has +intentionally decided to stop collecting new observations for that repository. +It does not imply deletion of retained historical data. Data deletion and +retention purging are separate concerns and are not introduced by this ADR. + +### Publish List + +`publish.repositories` is the dashboard surface. Reponomics publishes only the +repositories listed there. + +Policy: + +- `publish.repositories` is required. +- Entries must be explicit repository names, either as `repo` or `owner/repo`. +- Every published repository must also appear in `collect.repositories`. +- The list is capped at 8 repositories. +- Reponomics does not choose a default publish list. +- Changing the publish list is the supported way to look at a different subset + of collected data. + +Publishing a different subset requires editing `publish.repositories` and +running publish again. That is acceptable for beta because it keeps both Pages +and README output aligned and avoids a browser-only repository picker. + +### Dashboard Behavior + +The generated dashboard should be served with no more repositories than the +published surface can reasonably display. + +Current product intent: + +- no repository picker in the published dashboard; +- no client-side tabs for alternate repository sets in the beta surface; +- no hidden extra repository universe exposed to Pages but not README; +- comparison charts and repository tables operate on `publish.repositories`; +- README and Pages dashboards render the same repository set. + +The dashboard may still support focusing or comparing within the published +list, but the published list itself is controlled by configuration and publish +time. + +### Relationship To Retained Data + +Retained historical data may contain repositories that are no longer published +and may contain repositories that were previously collected but later removed +from `collect.repositories`. + +That retained history is not automatically deleted by removing a repository +from either list. Export and future retention behavior should continue to treat +canonical retained data as the durable local record unless an explicit, +versioned deletion or purge feature is introduced. + +## Consequences + +### Positive + +- Configuration becomes easier to explain: collect these, publish these. +- Users can predict exactly which repositories Reponomics will collect. +- The trust boundary is clearer because Reponomics does not infer or discover + additional repositories. +- Dashboard rendering is bounded by product semantics, not by an after-the-fact + UI cap. +- README and Pages dashboards remain symmetric. +- Users can change dashboard attention by editing `publish.repositories` + without stopping collection. +- The collection list becomes a deliberate repository history registry rather + than a frequently churned display list. + +### Negative + +- Users must manually add new repositories they want collected. +- Users must manually maintain the published subset. +- Switching dashboard subsets requires a publish run. +- The product gives up the convenience of passive automatic coverage. +- Users with very large portfolios may need to curate collection explicitly + instead of relying on discovery. + +These tradeoffs are intentional for beta. The friction asks users to make +explicit choices about what they want Reponomics to remember and what they want +the dashboard to watch closely. It also makes the dashboard repository a place +the user comes back to when their project set changes, instead of a passive +collector they can ignore indefinitely. + +## Rejected Alternatives + +### Automatic Discovery With Exclusions + +Continue discovering eligible repositories and let users exclude repositories +they do not want. + +Rejected for beta because it makes collection less explicit, keeps the config +model complex, and risks weakening the privacy/trust positioning. + +### Broad Collection With Dashboard Picker + +Collect a large repository set and expose a picker in the Pages dashboard so +users can choose up to 8 repositories at runtime. + +Rejected for the current beta plan because README output cannot provide an +equivalent interaction. It would also keep a larger UI and rendering path than +the beta needs. + +### Named Repository Sets In One Static Dashboard + +Let users define multiple named sets and render tabs for them in Pages. + +Rejected for the current beta plan because it creates a Pages-only experience +unless the README dashboard gains a different static representation. It may be +revisited after beta if users strongly need saved dashboard subsets. + +### One Narrow Repository List + +Use a single list for both collection and publication. + +Rejected because removing a repository from the dashboard would also stop +future collection. That turns ordinary dashboard curation into a decision about +ending an ephemeral data stream. + +## Compatibility And Migration Notes + +This ADR supersedes the repository-selection direction in ADR 016 where it +recommends broad automatic collection with exclusions, display pinning, and +budget-limited fill behavior. + +The implementation rejects the old repository-selection keys: + +- `include_only` +- `include` +- `exclude` +- `include_others` +- `include_new` +- `include_private` +- `max_repos` + +The runtime fails with a precise message rather than silently choosing +repositories from removed keys. Because this change is pre-public and pre-beta, +the implementation does not carry a compatibility parser for old configs. + +Release notes must call out the breaking configuration change before beta users +are invited. + +## Resolved Questions + +- The beta implementation hard-codes `collect.repositories` at 100. +- Export includes full canonical retained history, including repositories not + currently listed in `publish.repositories`. +- Removing a repository from `collect.repositories` does not currently warn + when retained data for that repository already exists. +- Publish runs do not currently report repositories present in retained data + but absent from both current lists. diff --git a/scripts/build_demo_repo.py b/scripts/build_demo_repo.py index edf6264a..bdb78baa 100644 --- a/scripts/build_demo_repo.py +++ b/scripts/build_demo_repo.py @@ -133,7 +133,9 @@ def _validate_public_demo_names(dataset: dict[str, Any]) -> None: raise DemoBuildError("Demo dataset must define featured_repositories.") for item in featured: if not isinstance(item, str) or not item.startswith(f"{DEMO_OWNER}/{DEMO_REPO_PREFIX}"): - raise DemoBuildError("Demo featured repositories must use reponomics-demo/demo-* names.") + raise DemoBuildError( + "Demo featured repositories must use reponomics-demo/demo-* names." + ) for raw in dataset["repositories"]: if not isinstance(raw, dict): raise DemoBuildError("Each demo repository entry must be a mapping.") @@ -157,7 +159,9 @@ def _assert_no_demo_brand_risk_terms(output_dir: Path) -> None: for term in DEMO_DATASET_DENYLIST: if term in content: relative = path.relative_to(output_dir) - raise DemoBuildError(f"Generated demo output contains brand-risk term {term}: {relative}") + raise DemoBuildError( + f"Generated demo output contains brand-risk term {term}: {relative}" + ) def _repo_specs(dataset: dict[str, Any]) -> list[RepoSpec]: @@ -239,7 +243,9 @@ def _aggregate_views(rows: list[dict[str, str]]) -> dict[str, int]: return totals -def _referrer_rows(specs: list[RepoSpec], daily_rows: list[dict[str, str]], as_of: date) -> list[dict[str, str]]: +def _referrer_rows( + specs: list[RepoSpec], daily_rows: list[dict[str, str]], as_of: date +) -> list[dict[str, str]]: totals = _aggregate_views(daily_rows) referrers = [ ("github.com", 0.34), @@ -269,7 +275,9 @@ def _referrer_rows(specs: list[RepoSpec], daily_rows: list[dict[str, str]], as_o return rows -def _path_rows(specs: list[RepoSpec], daily_rows: list[dict[str, str]], as_of: date) -> list[dict[str, str]]: +def _path_rows( + specs: list[RepoSpec], daily_rows: list[dict[str, str]], as_of: date +) -> list[dict[str, str]]: totals = _aggregate_views(daily_rows) templates = [ ("", "Repository overview", 0.40), @@ -300,10 +308,7 @@ def _path_rows(specs: list[RepoSpec], daily_rows: list[dict[str, str]], as_of: d def _metric_rows(specs: list[RepoSpec], daily_rows: list[dict[str, str]]) -> list[dict[str, str]]: - by_repo_day = { - (row["repo"], row["ts"]): int(row["views_count"]) - for row in daily_rows - } + by_repo_day = {(row["repo"], row["ts"]): int(row["views_count"]) for row in daily_rows} days = sorted({row["ts"] for row in daily_rows}) rows: list[dict[str, str]] = [] for spec in specs: @@ -333,7 +338,9 @@ def _metric_rows(specs: list[RepoSpec], daily_rows: list[dict[str, str]]) -> lis "visibility": "public", "default_branch": "main", "has_pages": ( - "True" if spec.name in {"demo-docs", "demo-website", "demo-status-page"} else "False" + "True" + if spec.name in {"demo-docs", "demo-website", "demo-status-page"} + else "False" ), "has_discussions": "True", "archived": "True" if spec.shape == "declining" else "False", @@ -413,11 +420,23 @@ def _materialize_data(output_dir: Path, dataset: dict[str, Any], as_of: date) -> [{key: row[key] for key in storage.SNAPSHOT_FIELDS} for row in daily_rows], storage.SNAPSHOT_FIELDS, ) - _write_csv(data_dir / "traffic-referrers.csv", _referrer_rows(specs, daily_rows, as_of), storage.REFERRER_FIELDS) - _write_csv(data_dir / "traffic-paths.csv", _path_rows(specs, daily_rows, as_of), storage.PATH_FIELDS) - _write_csv(data_dir / "repo-metrics.csv", _metric_rows(specs, daily_rows), storage.REPO_METRIC_FIELDS) + _write_csv( + data_dir / "traffic-referrers.csv", + _referrer_rows(specs, daily_rows, as_of), + storage.REFERRER_FIELDS, + ) + _write_csv( + data_dir / "traffic-paths.csv", _path_rows(specs, daily_rows, as_of), storage.PATH_FIELDS + ) + _write_csv( + data_dir / "repo-metrics.csv", _metric_rows(specs, daily_rows), storage.REPO_METRIC_FIELDS + ) _write_csv(data_dir / "collection-status.csv", status_rows, storage.COLLECTION_STATUS_FIELDS) - _write_csv(data_dir / "collection-days.csv", traffic_reporting.collection_day_rows(status_rows), storage.COLLECTION_DAY_FIELDS) + _write_csv( + data_dir / "collection-days.csv", + traffic_reporting.collection_day_rows(status_rows), + storage.COLLECTION_DAY_FIELDS, + ) _write_csv( data_dir / "traffic-coverage.csv", traffic_reporting.traffic_coverage_rows(daily_rows, status_rows), @@ -427,10 +446,6 @@ def _materialize_data(output_dir: Path, dataset: dict[str, Any], as_of: date) -> manifest = storage._default_manifest() manifest["created_at"] = f"{start.isoformat()}T12:00:00Z" manifest["last_updated"] = f"{as_of.isoformat()}T12:00:00Z" - manifest["selection_state"] = { - "auto_seeded_at": f"{start.isoformat()}T12:00:00Z", - "auto_cutoff_created_at": "2025-01-01T00:00:00Z", - } (data_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") @@ -571,7 +586,9 @@ def _sha256_file(path: Path) -> str: return digest.hexdigest() -def _write_encrypted_seed_artifact(data_dir: Path, seed_output_dir: Path, dataset: dict[str, Any]) -> Path: +def _write_encrypted_seed_artifact( + data_dir: Path, seed_output_dir: Path, dataset: dict[str, Any] +) -> Path: shutil.rmtree(seed_output_dir, ignore_errors=True) seed_path = seed_output_dir / DEMO_SEED_ARTIFACT_PATH with _temporary_env({"REPONOMICS_DEMO_DASHBOARD_KEY": str(dataset["demo_key"])}): @@ -597,7 +614,10 @@ def _assert_no_committed_html_dashboard_outputs(output_dir: Path) -> None: raise DemoBuildError(f"Generated demo publish tree must not include {relative_path}.") -def _write_demo_config(output_dir: Path) -> None: +def _write_demo_config(output_dir: Path, dataset: dict[str, Any]) -> None: + owner = str(dataset["owner"]) + collect_repositories = [f"{owner}/{repo['name']}" for repo in dataset.get("repositories", [])] + publish_repositories = list(dataset.get("featured_repositories", []))[:8] payload = { "i_have_read_the_readme": True, "data_mode": "encrypted", @@ -605,12 +625,12 @@ def _write_demo_config(output_dir: Path) -> None: "publish_readme_dashboard": False, "artifact_retention_days": 90, "use_github_app": False, - "include_only": [], - "exclude": [], - "max_repos": 200, - "include_others": True, - "include_new": False, - "include_private": True, + "collect": { + "repositories": collect_repositories, + }, + "publish": { + "repositories": publish_repositories, + }, } (output_dir / "config.yaml").write_text( yaml.safe_dump(payload, sort_keys=False), @@ -638,7 +658,9 @@ def _write_demo_provenance( "dataset_revision": dataset["dataset_revision"], "as_of": as_of.isoformat(), "synthetic_data": True, - "public_demo_key_sha256": hashlib.sha256(str(dataset["demo_key"]).encode("utf-8")).hexdigest(), + "public_demo_key_sha256": hashlib.sha256( + str(dataset["demo_key"]).encode("utf-8") + ).hexdigest(), "payload_digest": { "algorithm": "sha256", "format": template_provenance.TREE_MANIFEST_FORMAT, @@ -667,7 +689,9 @@ def _assert_csv_headers(data_dir: Path) -> None: _assert_csv_header(data_dir / filename, fields) -def build_demo(output_dir: Path, dataset_path: Path, as_of: date, seed_output_dir: Path = DEMO_SEED_DIR) -> None: +def build_demo( + output_dir: Path, dataset_path: Path, as_of: date, seed_output_dir: Path = DEMO_SEED_DIR +) -> None: if not TEMPLATE_DIR.exists(): raise DemoBuildError("dist/template does not exist; run make build-template first.") dataset = _load_dataset(dataset_path) @@ -675,8 +699,10 @@ def build_demo(output_dir: Path, dataset_path: Path, as_of: date, seed_output_di shutil.copytree(TEMPLATE_DIR, output_dir) _materialize_data(output_dir, dataset, as_of) _assert_csv_headers(output_dir / "data") - seed_artifact_path = _write_encrypted_seed_artifact(output_dir / "data", seed_output_dir, dataset) - _write_demo_config(output_dir) + seed_artifact_path = _write_encrypted_seed_artifact( + output_dir / "data", seed_output_dir, dataset + ) + _write_demo_config(output_dir, dataset) _write_demo_workflow(output_dir, str(dataset["demo_key"])) _render_demo_readme(output_dir, dataset) _prune_retained_data_from_publish_tree(output_dir) @@ -703,7 +729,16 @@ def _load_encrypted_seed(seed_path: Path) -> dict[str, Any]: raise DemoBuildError(f"Encrypted demo seed artifact is invalid JSON: {seed_path}") from exc if not isinstance(payload, dict): raise DemoBuildError(f"Encrypted demo seed artifact must be a JSON object: {seed_path}") - required = {"version", "created_at", "kdf", "iterations", "algorithm", "salt", "iv", "ciphertext"} + required = { + "version", + "created_at", + "kdf", + "iterations", + "algorithm", + "salt", + "iv", + "ciphertext", + } missing = sorted(required - set(payload)) if missing: raise DemoBuildError(f"Encrypted demo seed artifact is missing keys: {missing}") @@ -712,7 +747,9 @@ def _load_encrypted_seed(seed_path: Path) -> dict[str, Any]: return payload -def verify_demo(output_dir: Path, dataset_path: Path = DATASET_PATH, seed_output_dir: Path = DEMO_SEED_DIR) -> None: +def verify_demo( + output_dir: Path, dataset_path: Path = DATASET_PATH, seed_output_dir: Path = DEMO_SEED_DIR +) -> None: dataset = _load_dataset(dataset_path) required = [ output_dir / "README.md", @@ -765,7 +802,9 @@ def verify_demo(output_dir: Path, dataset_path: Path = DATASET_PATH, seed_output if not isinstance(payload_digest, dict): raise DemoBuildError("Demo provenance payload_digest must be a mapping.") if payload_digest.get("digest") != expected_digest.digest: - raise DemoBuildError("Demo provenance payload digest does not match generated publish tree.") + raise DemoBuildError( + "Demo provenance payload digest does not match generated publish tree." + ) if payload_digest.get("file_count") != expected_digest.file_count: raise DemoBuildError("Demo provenance file_count does not match generated publish tree.") if payload_digest.get("byte_count") != expected_digest.byte_count: @@ -780,9 +819,13 @@ def verify_demo(output_dir: Path, dataset_path: Path = DATASET_PATH, seed_output if seed_evidence.get("target_artifact_name") != DEMO_SEED_DASHBOARD_DATA_ARTIFACT_NAME: raise DemoBuildError("Demo provenance retained_data_seed target artifact name is wrong.") if seed_evidence.get("sha256") != _sha256_file(seed_path): - raise DemoBuildError("Demo provenance retained_data_seed digest does not match seed artifact.") + raise DemoBuildError( + "Demo provenance retained_data_seed digest does not match seed artifact." + ) if seed_evidence.get("byte_count") != seed_path.stat().st_size: - raise DemoBuildError("Demo provenance retained_data_seed byte_count does not match seed artifact.") + raise DemoBuildError( + "Demo provenance retained_data_seed byte_count does not match seed artifact." + ) print(f"Verified demo repository at {output_dir}") @@ -806,7 +849,9 @@ def main() -> None: if args.verify_only: verify_demo(args.output, args.dataset, seed_output_dir=args.seed_output) else: - build_demo(args.output, args.dataset, _parse_as_of(args.as_of), seed_output_dir=args.seed_output) + build_demo( + args.output, args.dataset, _parse_as_of(args.as_of), seed_output_dir=args.seed_output + ) if __name__ == "__main__": diff --git a/scripts/template_consumer_e2e.py b/scripts/template_consumer_e2e.py index df922cd6..5c99c3f1 100644 --- a/scripts/template_consumer_e2e.py +++ b/scripts/template_consumer_e2e.py @@ -18,8 +18,10 @@ try: from scripts.repo_paths import find_repo_root + from scripts import dashboard_scenarios except ModuleNotFoundError: # pragma: no cover - direct script execution from repo_paths import find_repo_root # type: ignore[import-not-found,no-redef] + import dashboard_scenarios # type: ignore[import-not-found,no-redef] ROOT = find_repo_root(Path(__file__)) @@ -235,11 +237,13 @@ def _write_setup_config( publish_readme_dashboard: bool = False, artifact_retention_days: int = 90, use_github_app: bool = False, + repositories: list[str] | None = None, ) -> None: config_path = consumer_dir / "config.yaml" payload = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} if not isinstance(payload, dict): raise TemplateConsumerE2EError("Generated config.yaml must contain a mapping") + repo_names = repositories or ["demo/alpha", "demo/beta"] payload.update( { "i_have_read_the_readme": True, @@ -248,11 +252,22 @@ def _write_setup_config( "publish_readme_dashboard": publish_readme_dashboard, "artifact_retention_days": artifact_retention_days, "use_github_app": use_github_app, + "collect": {"repositories": repo_names}, + "publish": {"repositories": repo_names[:8]}, } ) config_path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") +def _scenario_repositories(profile: ConsumerProfile) -> list[str]: + scenario = next( + item + for item in dashboard_scenarios.build_scenarios() + if item.key == profile.scenario + ) + return sorted({row["repo"] for row in scenario.daily_rows if row.get("repo")}) + + def _init_consumer_git_repo(consumer_dir: Path, remote_dir: Path) -> None: _run(["git", "init", "-b", "main"], cwd=consumer_dir) _run(["git", "config", "user.name", "template-consumer-e2e"], cwd=consumer_dir) @@ -522,6 +537,7 @@ def run_e2e( data_mode=profile.data_mode, publish_pages_dashboard=profile.expected_publish_pages, publish_readme_dashboard=profile.generate_readme, + repositories=_scenario_repositories(profile), ) _init_consumer_git_repo(consumer_dir, remote_dir) _invoke_action_runtime( diff --git a/template/README.template.md b/template/README.template.md index fdd38140..1f5422b7 100644 --- a/template/README.template.md +++ b/template/README.template.md @@ -42,20 +42,14 @@ artifact_retention_days: 90 # integer between 14-90 auto_doctor_every_n_days: 0 # 0 disables; otherwise integer between 1-30 -max_repos: 200 +collect: + repositories: + # - repo-name + # - other-owner/repo-name -include_only: - # - owner/repo-name - -include: - # - owner/important-repo - -exclude: - # - owner/noisy-repo - -include_others: true -include_new: false -include_private: true +publish: + repositories: + # - repo-name # Advanced: set to `true` if you wish to use your own GitHub App installation token. use_github_app: false @@ -63,11 +57,11 @@ use_github_app: false The template starts with `artifact_retention_days: 90`, `use_github_app: false`, and `auto_doctor_every_n_days: 0`; these are validated by setup and workflow runs. Set `auto_doctor_every_n_days` to `1` through `30` to check the marker and run doctor as part of the collect-and-publish cadence when that many UTC days have elapsed since the last successful auto-doctor. -If `include_only` is non-empty, Reponomics tracks exactly those repositories and ignores the automatic pool. For more detail, see [Dashboard repository documentation](docs/reponomics/repository-guide.md). +Add repositories to `collect.repositories` when you want Reponomics to keep history for them. Add up to 8 of those same repositories to `publish.repositories` when you want them shown in the README and Pages dashboards. For more detail, see [Dashboard repository documentation](docs/reponomics/repository-guide.md). ### Token Scope And Repository Owners -Repository entries use full `owner/repo` names because a dashboard can be configured against repositories owned by users or organizations. The token you choose still controls which owners can actually be collected. +Repository entries can use bare names such as `api` for repositories owned by the dashboard repository owner. Use full `owner/repo` names when a dashboard is configured against repositories owned by another user or organization. The token you choose still controls which owners can actually be collected. Fine-grained personal access tokens are scoped to one GitHub resource owner. If your dashboard only tracks repositories under one user or one organization, a fine-grained token with repository `Administration: read` is the preferred path. diff --git a/tests/fixtures/collection_quality_preview/config.yaml b/tests/fixtures/collection_quality_preview/config.yaml index 1c3e8678..4a9f88ec 100644 --- a/tests/fixtures/collection_quality_preview/config.yaml +++ b/tests/fixtures/collection_quality_preview/config.yaml @@ -5,11 +5,11 @@ publish_readme_dashboard: false artifact_retention_days: 90 use_github_app: false -include_only: - - demo/alpha - - demo/beta -exclude: [] -max_repos: 50 -include_others: true -include_new: false -include_private: true +collect: + repositories: + - demo/alpha + - demo/beta +publish: + repositories: + - demo/alpha + - demo/beta diff --git a/tests/fixtures/collection_quality_preview/data/manifest.json b/tests/fixtures/collection_quality_preview/data/manifest.json index bfe2cba5..0b010709 100644 --- a/tests/fixtures/collection_quality_preview/data/manifest.json +++ b/tests/fixtures/collection_quality_preview/data/manifest.json @@ -11,9 +11,5 @@ "traffic-paths.csv", "repo-metrics.csv", "collection-status.csv" - ], - "selection_state": { - "auto_seeded_at": "", - "auto_cutoff_created_at": "" - } + ] } diff --git a/tests/fixtures/compat_v2/config.yaml b/tests/fixtures/compat_v2/config.yaml index c2f9ef57..bf08c22d 100644 --- a/tests/fixtures/compat_v2/config.yaml +++ b/tests/fixtures/compat_v2/config.yaml @@ -1,4 +1,6 @@ -repos: - - demo/reponomics -exclude_repos: [] -max_repos: 5 +collect: + repositories: + - demo/reponomics +publish: + repositories: + - demo/reponomics diff --git a/tests/js/dashboard-modules.test.mjs b/tests/js/dashboard-modules.test.mjs index b47b510b..aa1a2628 100644 --- a/tests/js/dashboard-modules.test.mjs +++ b/tests/js/dashboard-modules.test.mjs @@ -211,7 +211,7 @@ test('format helpers install independently of app lifecycle', () => { test('series helpers preserve selected-window and growth aggregation contracts', () => { const context = { - MAX_DISPLAY_REPOS: 20, + MAX_DISPLAY_REPOS: 8, SERIES_METRIC_KEYS: [ 'views', 'uniques', diff --git a/tests/runner/test_runner_auth_config_modes.py b/tests/runner/test_runner_auth_config_modes.py index a464cebb..caa245b3 100644 --- a/tests/runner/test_runner_auth_config_modes.py +++ b/tests/runner/test_runner_auth_config_modes.py @@ -382,7 +382,16 @@ def test_runtime_config_requires_setup_fields( tmp_path: Path, ) -> None: config_path = tmp_path / "config.yaml" - config_path.write_text("max_repos: 200\n", encoding="utf-8") + config_path.write_text( + """collect: + repositories: + - demo/reponomics +publish: + repositories: + - demo/reponomics +""", + encoding="utf-8", + ) monkeypatch.setenv("GITHUB_EVENT_REPOSITORY_PRIVATE", "true") monkeypatch.setenv("REPONOMICS_CONFIG_PATH", str(config_path)) diff --git a/tests/runner/test_runner_collect_http_selection.py b/tests/runner/test_runner_collect_http_selection.py index 30927440..4d0b13e5 100644 --- a/tests/runner/test_runner_collect_http_selection.py +++ b/tests/runner/test_runner_collect_http_selection.py @@ -465,69 +465,26 @@ def test_discover_repositories_rejects_malformed_app_responses( run.collect_mod.discover_repositories({}) -def test_resolve_repositories_applies_stable_auto_selection( +def test_resolve_repositories_tracks_only_explicit_collect_repositories( monkeypatch: pytest.MonkeyPatch, ) -> None: discovered: list[run.collect_mod.RepoMetadata] = [ { "full_name": "demo/manual", - "created_at": "2025-01-01T00:00:00Z", - "permissions": {"push": True}, - }, - { - "full_name": "demo/current", - "created_at": "2025-02-01T00:00:00Z", "permissions": {"push": True}, }, { - "full_name": "demo/new", - "created_at": "2026-02-01T00:00:00Z", - "permissions": {"push": True}, - }, - { - "full_name": "demo/private", - "created_at": "2025-03-01T00:00:00Z", - "private": True, - "permissions": {"push": True}, - }, - { - "full_name": "demo/old-a", - "created_at": "2025-04-01T00:00:00Z", - "permissions": {"push": True}, - }, - { - "full_name": "demo/old-b", - "created_at": "2025-03-01T00:00:00Z", - "permissions": {"push": True}, - }, - { - "full_name": "demo/fork", - "created_at": "2025-05-01T00:00:00Z", - "fork": True, + "full_name": "demo/other", "permissions": {"push": True}, }, ] - def fake_discover(_headers: run.collect_mod.Headers) -> list[run.collect_mod.RepoMetadata]: - return discovered - - monkeypatch.setenv("GITHUB_REPOSITORY", "demo/current") - monkeypatch.setattr(run.collect_mod, "discover_repositories", fake_discover) + monkeypatch.setattr(run.collect_mod, "discover_repositories", lambda _headers: discovered) config: dict[str, Any] = { - "include_only": [], - "include": ["demo/manual", "demo/missing"], - "exclude": [], - "max_repos": 3, - "include_others": True, - "include_private": False, - "include_new": False, - } - manifest: dict[str, Any] = { - "selection_state": { - "auto_seeded_at": "2026-01-01T00:00:00Z", - "auto_cutoff_created_at": "", - } + "collect_repositories": ["demo/manual"], + "publish_repositories": ["demo/manual"], } + manifest: dict[str, Any] = {} resolved, updated_manifest, metadata = run.collect_mod.resolve_repositories( {}, @@ -535,12 +492,9 @@ def fake_discover(_headers: run.collect_mod.Headers) -> list[run.collect_mod.Rep manifest, ) - assert resolved == ["demo/manual", "demo/old-a", "demo/old-b"] - assert sorted(metadata) == sorted(resolved) - assert updated_manifest["selection_state"] == { - "auto_seeded_at": "2026-01-01T00:00:00Z", - "auto_cutoff_created_at": "2025-03-01T00:00:00Z", - } + assert resolved == ["demo/manual"] + assert updated_manifest == manifest + assert sorted(metadata) == ["demo/manual"] def test_resolve_repositories_accepts_pull_only_permissions_for_github_app( @@ -557,17 +511,10 @@ def test_resolve_repositories_accepts_pull_only_permissions_for_github_app( monkeypatch.setenv("REPONOMICS_USE_GITHUB_APP", "true") monkeypatch.setattr(run.collect_mod, "discover_repositories", lambda _headers: discovered) config: dict[str, Any] = { - "include_only": [], - "include": [], - "exclude": [], - "max_repos": 5, - "include_others": True, - "include_private": True, - "include_new": True, - } - manifest: dict[str, Any] = { - "selection_state": {"auto_seeded_at": "", "auto_cutoff_created_at": ""} + "collect_repositories": ["demo/read-only"], + "publish_repositories": ["demo/read-only"], } + manifest: dict[str, Any] = {} resolved, _updated_manifest, metadata = run.collect_mod.resolve_repositories( {}, config, manifest @@ -577,7 +524,7 @@ def test_resolve_repositories_accepts_pull_only_permissions_for_github_app( assert sorted(metadata) == ["demo/read-only"] -def test_resolve_repositories_include_only_warns_and_exits_when_empty( +def test_resolve_repositories_warns_and_skips_ineligible_collect_repos( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: @@ -586,65 +533,29 @@ def test_resolve_repositories_include_only_warns_and_exits_when_empty( "full_name": "demo/fork", "permissions": {"push": True}, "fork": True, - } - ] - config: dict[str, Any] = { - "include_only": ["demo/fork", "demo/missing"], - "include": [], - "exclude": [], - "max_repos": 5, - "include_others": False, - "include_private": True, - "include_new": True, - } - - monkeypatch.setattr(run.collect_mod, "discover_repositories", lambda _headers: discovered) - - with pytest.raises(SystemExit): - run.collect_mod.resolve_repositories({}, config, {}) - - output = capsys.readouterr().out - assert "include_only repos were not eligible" in output - assert "no eligible repositories remain in 'include_only'" in output - - -def test_resolve_repositories_clears_auto_cutoff_when_auto_fill_disabled( - monkeypatch: pytest.MonkeyPatch, -) -> None: - discovered = [ + }, { - "full_name": "demo/manual", - "permissions": {"admin": True}, - "created_at": "2025-01-01T00:00:00Z", - } + "full_name": "demo/ok", + "permissions": {"push": True}, + }, ] config: dict[str, Any] = { - "include_only": [], - "include": ["demo/manual", "demo/manual"], - "exclude": [], - "max_repos": 5, - "include_others": False, - "include_private": True, - "include_new": True, - } - manifest: dict[str, Any] = { - "selection_state": { - "auto_seeded_at": "2026-01-01T00:00:00Z", - "auto_cutoff_created_at": "2025-01-01T00:00:00Z", - } + "collect_repositories": ["demo/fork", "demo/missing", "demo/ok"], + "publish_repositories": ["demo/ok"], } monkeypatch.setattr(run.collect_mod, "discover_repositories", lambda _headers: discovered) - resolved, updated_manifest, metadata = run.collect_mod.resolve_repositories( - {}, - config, - manifest, + resolved, _updated_manifest, metadata = run.collect_mod.resolve_repositories( + {}, config, {} ) - assert resolved == ["demo/manual"] - assert sorted(metadata) == ["demo/manual"] - assert updated_manifest["selection_state"]["auto_cutoff_created_at"] == "" + assert resolved == ["demo/ok"] + assert sorted(metadata) == ["demo/ok"] + output = capsys.readouterr().out + assert "collect.repositories repos were not eligible" in output + assert "demo/fork" in output + assert "demo/missing" in output def test_resolve_repositories_exits_when_no_repos_are_eligible( @@ -652,13 +563,8 @@ def test_resolve_repositories_exits_when_no_repos_are_eligible( capsys: pytest.CaptureFixture[str], ) -> None: config: dict[str, Any] = { - "include_only": [], - "include": [], - "exclude": [], - "max_repos": 5, - "include_others": False, - "include_private": True, - "include_new": True, + "collect_repositories": ["demo/missing"], + "publish_repositories": ["demo/missing"], } monkeypatch.setattr(run.collect_mod, "discover_repositories", lambda _headers: []) @@ -666,4 +572,4 @@ def test_resolve_repositories_exits_when_no_repos_are_eligible( with pytest.raises(SystemExit): run.collect_mod.resolve_repositories({}, config, {}) - assert "no eligible repositories found" in capsys.readouterr().out + assert "no collect.repositories entries resolved" in capsys.readouterr().out diff --git a/tests/runner/test_runner_collect_orchestration.py b/tests/runner/test_runner_collect_orchestration.py index 317d0d7f..ab8eb469 100644 --- a/tests/runner/test_runner_collect_orchestration.py +++ b/tests/runner/test_runner_collect_orchestration.py @@ -17,13 +17,32 @@ ) +SINGLE_REPO_CONFIG = """collect: + repositories: + - demo/reponomics +publish: + repositories: + - demo/reponomics +""" + +TWO_REPO_CONFIG = """collect: + repositories: + - demo/one + - demo/two +publish: + repositories: + - demo/one + - demo/two +""" + + def test_collect_appends_context_rows_for_selected_repo( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: monkeypatch.chdir(tmp_path) config_path = tmp_path / "config.yaml" - config_path.write_text("include_only:\n - demo/reponomics\n", encoding="utf-8") + config_path.write_text(SINGLE_REPO_CONFIG, encoding="utf-8") config = _config(tmp_path, config_path=config_path) discovered = [ { @@ -242,7 +261,7 @@ def test_collect_context_failure_records_warning_without_failing_collection( ) -> None: monkeypatch.chdir(tmp_path) config_path = tmp_path / "config.yaml" - config_path.write_text("include_only:\n - demo/reponomics\n", encoding="utf-8") + config_path.write_text(SINGLE_REPO_CONFIG, encoding="utf-8") summary_path = tmp_path / "summary.md" config = _config(tmp_path, config_path=config_path) discovered = [ @@ -312,7 +331,7 @@ def test_collect_records_pending_statistics_endpoint_without_warning( ) -> None: monkeypatch.chdir(tmp_path) config_path = tmp_path / "config.yaml" - config_path.write_text("include_only:\n - demo/reponomics\n", encoding="utf-8") + config_path.write_text(SINGLE_REPO_CONFIG, encoding="utf-8") summary_path = tmp_path / "summary.md" config = _config(tmp_path, config_path=config_path) discovered = [ @@ -412,10 +431,7 @@ def test_collect_requests_one_detail_per_selected_repo( ) -> None: monkeypatch.chdir(tmp_path) config_path = tmp_path / "config.yaml" - config_path.write_text( - "include_only:\n - demo/one\n - demo/two\nmax_repos: 2\n", - encoding="utf-8", - ) + config_path.write_text(TWO_REPO_CONFIG, encoding="utf-8") config = _config(tmp_path, config_path=config_path) discovered = [ { @@ -504,7 +520,7 @@ def test_detail_failure_falls_back_without_losing_traffic( ) -> None: monkeypatch.chdir(tmp_path) config_path = tmp_path / "config.yaml" - config_path.write_text("include_only:\n - demo/reponomics\n", encoding="utf-8") + config_path.write_text(SINGLE_REPO_CONFIG, encoding="utf-8") config = _config(tmp_path, config_path=config_path) discovered = [ { @@ -576,7 +592,7 @@ def test_community_profile_failure_records_warning_without_losing_metrics( ) -> None: monkeypatch.chdir(tmp_path) config_path = tmp_path / "config.yaml" - config_path.write_text("include_only:\n - demo/reponomics\n", encoding="utf-8") + config_path.write_text(SINGLE_REPO_CONFIG, encoding="utf-8") summary_path = tmp_path / "summary.md" config = _config(tmp_path, config_path=config_path) discovered = [ @@ -636,7 +652,7 @@ def test_collect_records_skipped_unavailable_repo_status( ) -> None: monkeypatch.chdir(tmp_path) config_path = tmp_path / "config.yaml" - config_path.write_text("include_only:\n - demo/reponomics\n", encoding="utf-8") + config_path.write_text(SINGLE_REPO_CONFIG, encoding="utf-8") config = _config(tmp_path, config_path=config_path) discovered = [ { @@ -692,7 +708,7 @@ def test_collect_secondary_rate_limit_aborts_with_status_and_summary( ) -> None: monkeypatch.chdir(tmp_path) config_path = tmp_path / "config.yaml" - config_path.write_text("include_only:\n - demo/reponomics\n", encoding="utf-8") + config_path.write_text(SINGLE_REPO_CONFIG, encoding="utf-8") summary_path = tmp_path / "summary.md" config = _config(tmp_path, config_path=config_path) discovered = [ @@ -750,7 +766,7 @@ def test_collect_records_generic_collection_errors_and_exits( ) -> None: monkeypatch.chdir(tmp_path) config_path = tmp_path / "config.yaml" - config_path.write_text("include_only:\n - demo/reponomics\n", encoding="utf-8") + config_path.write_text(SINGLE_REPO_CONFIG, encoding="utf-8") summary_path = tmp_path / "summary.md" config = _config(tmp_path, config_path=config_path) discovered = [ diff --git a/tests/runner/test_runner_publish_dashboard.py b/tests/runner/test_runner_publish_dashboard.py index 7259ec4b..bda7f664 100644 --- a/tests/runner/test_runner_publish_dashboard.py +++ b/tests/runner/test_runner_publish_dashboard.py @@ -22,11 +22,14 @@ _dashboard_json, _decode_plaintext_dashboard_data, _decrypt_encrypted_dashboard_data, + _daily_row, + _metric_row, _parse_dashboard_html, _published_runtime_text, _published_script_sources, _seed_log, _seed_scenario, + _write_csv, ) @@ -60,6 +63,80 @@ def test_publish_large_corpus_writes_one_encrypted_chunk_per_repo( assert "reponomics-scale/repo-001" not in dashboard +def test_publish_can_switch_publish_repositories_without_recollection( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.chdir(tmp_path) + config = _config(tmp_path, mode="publish", generate_readme=False) + repos = ["demo/alpha", "demo/beta", "demo/gamma"] + _write_csv( + config.data_dir / "traffic-daily.csv", + run.storage.DAILY_FIELDS, + [ + _daily_row("demo/alpha", "2026-05-01", 30, 10, 3), + _daily_row("demo/beta", "2026-05-01", 20, 8, 2), + _daily_row("demo/gamma", "2026-05-01", 10, 4, 1), + ], + ) + _write_csv( + config.data_dir / "repo-metrics.csv", + run.storage.REPO_METRIC_FIELDS, + [ + _metric_row("demo/alpha", "2026-05-01", 30, 5, 3), + _metric_row("demo/beta", "2026-05-01", 20, 4, 2), + _metric_row("demo/gamma", "2026-05-01", 10, 3, 1), + ], + ) + + def write_config(publish_repos: list[str]) -> None: + collect_yaml = "\n".join(f" - {repo}" for repo in repos) + publish_yaml = "\n".join(f" - {repo}" for repo in publish_repos) + config.config_path.write_text( + "\n".join( + [ + "collect:", + " repositories:", + collect_yaml, + "publish:", + " repositories:", + publish_yaml, + "", + ] + ), + encoding="utf-8", + ) + + def published_repo_names() -> tuple[list[str], dict[str, dict[str, object]]]: + dashboard = config.pages_index_path.read_text(encoding="utf-8") + encrypted_data = _dashboard_json( + config.pages_index_path, + dashboard, + "encrypted-dashboard-data", + "encrypted-dashboard-data.json", + ) + summary, chunks = _decrypt_encrypted_dashboard_data(encrypted_data) + return [repo["name"] for repo in summary["repos"]], chunks + + write_config(["demo/alpha", "demo/beta"]) + run.validate_config(config) + run.run_publish(config, restore_artifact=False) + + first_repo_names, first_chunks = published_repo_names() + assert first_repo_names == ["demo/alpha", "demo/beta"] + assert {chunk["repo"] for chunk in first_chunks.values()} == {"demo/alpha", "demo/beta"} + + write_config(["demo/gamma"]) + run.validate_config(config) + run.run_publish(config, restore_artifact=False) + + second_repo_names, second_chunks = published_repo_names() + assert second_repo_names == ["demo/gamma"] + assert {chunk["repo"] for chunk in second_chunks.values()} == {"demo/gamma"} + retained_daily = (config.data_dir / "traffic-daily.csv").read_text(encoding="utf-8") + assert retained_daily.count("demo/") == 3 + + def test_publish_skips_readme_when_generate_readme_is_false( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/runner/test_runner_repo_config.py b/tests/runner/test_runner_repo_config.py index 48fb5f11..9f746120 100644 --- a/tests/runner/test_runner_repo_config.py +++ b/tests/runner/test_runner_repo_config.py @@ -7,18 +7,58 @@ from dashboard_action import run -def test_repo_config_accepts_max_length_repository_full_name(tmp_path: Path) -> None: +def _write_config(path: Path, body: str) -> None: + path.write_text(body, encoding="utf-8") + + +def test_repo_config_normalizes_bare_names_to_dashboard_owner( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config_path = tmp_path / "config.yaml" + monkeypatch.setenv("GITHUB_REPOSITORY", "demo/dashboard") + _write_config( + config_path, + """ +collect: + repositories: + - api + - other-owner/sdk +publish: + repositories: + - api +""", + ) + + config = run.repo_config.load_repo_config(str(config_path)) + + assert config["collect_repositories"] == ["demo/api", "other-owner/sdk"] + assert config["publish_repositories"] == ["demo/api"] + assert config["max_collect_repos"] == 100 + assert config["max_publish_repos"] == 8 + + +def test_repo_config_accepts_max_length_full_name(tmp_path: Path) -> None: config_path = tmp_path / "config.yaml" owner = "o" * 39 repo_name = "r" * 100 - config_path.write_text( - f"include_only:\n - {owner}/{repo_name}\nmax_repos: 1\n", - encoding="utf-8", + repo = f"{owner}/{repo_name}" + _write_config( + config_path, + f""" +collect: + repositories: + - {repo} +publish: + repositories: + - {repo} +""", ) config = run.repo_config.load_repo_config(str(config_path)) - assert config["include_only"] == [f"{owner}/{repo_name}"] + assert config["collect_repositories"] == [repo] + assert config["publish_repositories"] == [repo] @pytest.mark.parametrize( @@ -38,23 +78,114 @@ def test_repo_config_accepts_max_length_repository_full_name(tmp_path: Path) -> "owner/repo\nEVIL=1", ], ) -def test_repo_config_rejects_invalid_repository_full_names( +def test_repo_config_rejects_invalid_repository_names( tmp_path: Path, repo_name: str, ) -> None: config_path = tmp_path / "config.yaml" - config_path.write_text( - f"include_only:\n - {repo_name!r}\nmax_repos: 1\n", - encoding="utf-8", + _write_config( + config_path, + f""" +collect: + repositories: + - {repo_name!r} +publish: + repositories: + - owner/valid +""", ) with pytest.raises(ValueError, match="invalid repository entry"): run.repo_config.load_repo_config(str(config_path)) +def test_repo_config_rejects_bare_names_without_dashboard_owner(tmp_path: Path) -> None: + config_path = tmp_path / "config.yaml" + _write_config( + config_path, + """ +collect: + repositories: + - api +publish: + repositories: + - api +""", + ) + + with pytest.raises(ValueError, match="bare repository names require"): + run.repo_config.load_repo_config(str(config_path)) + + def test_repo_config_rejects_non_string_repository_entries(tmp_path: Path) -> None: config_path = tmp_path / "config.yaml" - config_path.write_text("include_only:\n - 123\nmax_repos: 1\n", encoding="utf-8") + _write_config( + config_path, + """ +collect: + repositories: + - 123 +publish: + repositories: + - owner/repo +""", + ) with pytest.raises(ValueError, match="repository entries must be strings"): run.repo_config.load_repo_config(str(config_path)) + + +def test_repo_config_requires_publish_subset_of_collect(tmp_path: Path) -> None: + config_path = tmp_path / "config.yaml" + _write_config( + config_path, + """ +collect: + repositories: + - owner/api +publish: + repositories: + - owner/web +""", + ) + + with pytest.raises(ValueError, match="not listed in 'collect.repositories'"): + run.repo_config.load_repo_config(str(config_path)) + + +def test_repo_config_rejects_publish_list_over_dashboard_cap(tmp_path: Path) -> None: + config_path = tmp_path / "config.yaml" + repos = "\n".join(f" - owner/repo-{idx}" for idx in range(9)) + _write_config( + config_path, + f""" +collect: + repositories: +{repos} +publish: + repositories: +{repos} +""", + ) + + with pytest.raises(ValueError, match="dashboard cap is 8"): + run.repo_config.load_repo_config(str(config_path)) + + +def test_repo_config_rejects_removed_selection_keys(tmp_path: Path) -> None: + config_path = tmp_path / "config.yaml" + _write_config( + config_path, + """ +collect: + repositories: + - owner/api +publish: + repositories: + - owner/api +include_others: true +""", + ) + + with pytest.raises(ValueError, match="removed repository-selection key"): + run.repo_config.load_repo_config(str(config_path)) diff --git a/tests/test_demo_repository.py b/tests/test_demo_repository.py index b103ba32..55064ec0 100644 --- a/tests/test_demo_repository.py +++ b/tests/test_demo_repository.py @@ -1,6 +1,8 @@ from __future__ import annotations +import json import sys +from datetime import date from pathlib import Path import pytest @@ -160,6 +162,17 @@ def test_demo_dataset_has_single_owner_and_expected_size() -> None: assert all("/demo-" in item for item in dataset["featured_repositories"]) +def test_demo_seed_manifest_does_not_include_selection_state(tmp_path: Path) -> None: + dataset = build_demo_repo._load_dataset(ROOT / "demo" / "dataset.yml") + + build_demo_repo._materialize_data(tmp_path, dataset, date(2026, 6, 27)) + + manifest = json.loads((tmp_path / "data" / "manifest.json").read_text(encoding="utf-8")) + assert "selection_state" not in manifest + assert "auto_seeded_at" not in json.dumps(manifest) + assert manifest["schema_version"] == "4" + + def test_demo_dataset_rejects_ambiguous_public_brand_names(tmp_path: Path) -> None: dataset = { "schema_version": 1, @@ -242,7 +255,9 @@ def test_demo_publish_tree_allows_readme_svg_assets_only(tmp_path: Path) -> None (assets / "chart.umd.min.js").write_text("window.Chart = {}", encoding="utf-8") with pytest.raises(build_demo_repo.DemoBuildError, match="docs/assets/chart\\.umd\\.min\\.js"): build_demo_repo._assert_no_committed_html_dashboard_outputs(output) - with pytest.raises(publish_demo_repo.DemoPublishError, match="docs/assets/chart\\.umd\\.min\\.js"): + with pytest.raises( + publish_demo_repo.DemoPublishError, match="docs/assets/chart\\.umd\\.min\\.js" + ): publish_demo_repo._assert_publish_tree_shape(output) @@ -266,7 +281,9 @@ def test_demo_publisher_rejects_gitignored_publish_files(tmp_path: Path) -> None (output / "ignored.txt").write_text("intended generated file\n", encoding="utf-8") expected_files = publish_demo_repo._output_files(output) - with pytest.raises(publish_demo_repo.DemoPublishError, match="missing from git add -A: ignored.txt"): + with pytest.raises( + publish_demo_repo.DemoPublishError, match="missing from git add -A: ignored.txt" + ): publish_demo_repo._assert_git_add_stages_publish_tree(output, "main", expected_files) @@ -288,15 +305,23 @@ def test_source_demo_publish_workflow_is_manual_or_scheduled_and_repo_scoped() - assert "environment" not in build_job assert build_job["permissions"] == {"contents": "read"} build_step_names = [step["name"] for step in build_job["steps"]] - assert build_step_names.index("Resolve publication source") < build_step_names.index("Checkout source") - resolve_step = next(step for step in build_job["steps"] if step["name"] == "Resolve publication source") + assert build_step_names.index("Resolve publication source") < build_step_names.index( + "Checkout source" + ) + resolve_step = next( + step for step in build_job["steps"] if step["name"] == "Resolve publication source" + ) assert "DEMO_DAILY_SOURCE_REF" in resolve_step["run"] assert "demo-stable" in resolve_step["run"] - build_step = next(step for step in build_job["steps"] if step["name"] == "Build and verify generated demo") + build_step = next( + step for step in build_job["steps"] if step["name"] == "Build and verify generated demo" + ) assert "make build-demo" not in build_step["run"] assert "make verify-demo" in build_step["run"] assert any(step["name"] == "Upload generated demo artifact" for step in build_job["steps"]) - assert any(step["name"] == "Upload encrypted demo dashboard data seed" for step in build_job["steps"]) + assert any( + step["name"] == "Upload encrypted demo dashboard data seed" for step in build_job["steps"] + ) publish_job = workflow["jobs"]["publish-demo"] assert publish_job["needs"] == "build-demo-artifact" @@ -307,7 +332,9 @@ def test_source_demo_publish_workflow_is_manual_or_scheduled_and_repo_scoped() - assert step_names.index("Validate downloaded demo artifact") < step_names.index( "Create demo publication app token" ) - validation_step = next(step for step in steps if step["name"] == "Validate downloaded demo artifact") + validation_step = next( + step for step in steps if step["name"] == "Validate downloaded demo artifact" + ) validation_script = validation_step["run"] assert "grep -q 'docs/assets/hero-stats.svg' demo-repo/README.md" in validation_script assert "test -f demo-repo/docs/assets/hero-stats.svg" in validation_script @@ -321,7 +348,9 @@ def test_source_demo_publish_workflow_is_manual_or_scheduled_and_repo_scoped() - assert token_step["with"]["permission-contents"] == "write" assert token_step["with"]["permission-workflows"] == "write" assert token_step["with"]["permission-actions"] == "write" - publish_step = next(step for step in steps if step["name"] == "Publish generated demo repository") + publish_step = next( + step for step in steps if step["name"] == "Publish generated demo repository" + ) assert "make publish-demo" not in publish_step["run"] assert "git push" in publish_step["run"] assert "seed-and-publish-demo-dashboard.yml/dispatches" in publish_step["run"] diff --git a/tests/test_load_data_unit.py b/tests/test_load_data_unit.py index 1cf022f2..ee7b7147 100644 --- a/tests/test_load_data_unit.py +++ b/tests/test_load_data_unit.py @@ -63,7 +63,7 @@ def _status_row( } -def test_load_daily_falls_back_to_legacy_log_and_filters_excluded_rows( +def test_load_daily_falls_back_to_legacy_log_and_filters_to_published_rows( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: @@ -82,7 +82,7 @@ def read_csv(path: str) -> list[dict[str, str]]: monkeypatch.setattr( load_data, "load_repo_config", - lambda: {"exclude_repos": ["demo/private"]}, + lambda: {"publish_repositories": ["demo/public"]}, ) assert load_data.load_daily(str(tmp_path)) == [ @@ -91,16 +91,26 @@ def read_csv(path: str) -> list[dict[str, str]]: assert calls == ["traffic-daily.csv", "traffic-log.csv"] -def test_csv_loaders_read_expected_files_and_preserve_rows_without_exclusions( +def test_csv_loaders_read_expected_files_and_filter_to_published_repos( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: rows_by_file = { - "traffic-referrers.csv": [{"repo": "demo/app", "referrer": "github.com"}], - "traffic-paths.csv": [{"repo": "demo/app", "path": "/demo/app"}], - "repo-metrics.csv": [_metric_row("demo/app", "2026-05-01", 8, 2, 1)], + "traffic-referrers.csv": [ + {"repo": "demo/app", "referrer": "github.com"}, + {"repo": "demo/hidden", "referrer": "example.com"}, + ], + "traffic-paths.csv": [ + {"repo": "demo/app", "path": "/demo/app"}, + {"repo": "demo/hidden", "path": "/demo/hidden"}, + ], + "repo-metrics.csv": [ + _metric_row("demo/app", "2026-05-01", 8, 2, 1), + _metric_row("demo/hidden", "2026-05-01", 10, 3, 2), + ], "collection-status.csv": [ - _status_row("demo/app", "2026-05-01T12:00:00Z", "ok_with_data") + _status_row("demo/app", "2026-05-01T12:00:00Z", "ok_with_data"), + _status_row("demo/hidden", "2026-05-01T12:00:00Z", "ok_with_data"), ], "collection-days.csv": [{"ts": "2026-05-01", "status": "healthy"}], "traffic-coverage.csv": [ @@ -108,6 +118,11 @@ def test_csv_loaders_read_expected_files_and_preserve_rows_without_exclusions( "repo": "demo/app", "ts": "2026-05-01", "coverage_state": "reported", + }, + { + "repo": "demo/hidden", + "ts": "2026-05-01", + "coverage_state": "reported", } ], } @@ -116,19 +131,33 @@ def read_csv(path: str) -> list[dict[str, str]]: return rows_by_file[os.path.basename(path)] monkeypatch.setattr(load_data.storage, "read_csv", read_csv) - monkeypatch.setattr(load_data, "load_repo_config", lambda: {}) + monkeypatch.setattr( + load_data, + "load_repo_config", + lambda: {"publish_repositories": ["demo/app"]}, + ) - assert load_data.load_referrers(str(tmp_path)) is rows_by_file["traffic-referrers.csv"] - assert load_data.load_paths(str(tmp_path)) is rows_by_file["traffic-paths.csv"] - assert load_data.load_repo_metrics(str(tmp_path)) is rows_by_file["repo-metrics.csv"] - assert load_data.load_collection_status(str(tmp_path)) is rows_by_file[ - "collection-status.csv" + assert load_data.load_referrers(str(tmp_path)) == [ + {"repo": "demo/app", "referrer": "github.com"} + ] + assert load_data.load_paths(str(tmp_path)) == [ + {"repo": "demo/app", "path": "/demo/app"} + ] + assert load_data.load_repo_metrics(str(tmp_path)) == [ + _metric_row("demo/app", "2026-05-01", 8, 2, 1) + ] + assert load_data.load_collection_status(str(tmp_path)) == [ + _status_row("demo/app", "2026-05-01T12:00:00Z", "ok_with_data") ] - assert load_data.load_collection_days(str(tmp_path)) is rows_by_file[ - "collection-days.csv" + assert load_data.load_collection_days(str(tmp_path)) == [ + {"ts": "2026-05-01", "status": "healthy"} ] - assert load_data.load_traffic_coverage(str(tmp_path)) is rows_by_file[ - "traffic-coverage.csv" + assert load_data.load_traffic_coverage(str(tmp_path)) == [ + { + "repo": "demo/app", + "ts": "2026-05-01", + "coverage_state": "reported", + } ]