From eb9f916fb5f5d11df9df0198fe06e596a148d325 Mon Sep 17 00:00:00 2001 From: sirutBuasai Date: Tue, 10 Feb 2026 15:32:45 -0800 Subject: [PATCH 1/7] refactor support policy grouping Signed-off-by: sirutBuasai --- .kiro/steering/docs.md | 16 +- .../pytorch-inference-arm64/2.6-cpu-ec2.yml | 2 +- .../2.6-cpu-sagemaker.yml | 2 +- .../pytorch-inference-arm64/2.6-gpu-ec2.yml | 2 +- .../data/pytorch-inference/2.6-cpu-ec2.yml | 2 +- .../pytorch-inference/2.6-cpu-sagemaker.yml | 2 +- .../data/pytorch-inference/2.6-gpu-ec2.yml | 2 +- .../pytorch-inference/2.6-gpu-sagemaker.yml | 2 +- docs/src/generate.py | 141 ++++++++++++------ docs/src/image_config.py | 19 ++- 10 files changed, 134 insertions(+), 56 deletions(-) diff --git a/.kiro/steering/docs.md b/.kiro/steering/docs.md index 3543c14dfa17..362ba275da96 100644 --- a/.kiro/steering/docs.md +++ b/.kiro/steering/docs.md @@ -376,18 +376,22 @@ table_order: ### Support Policy Consolidation -The `framework_groups` configuration consolidates support policy rows by framework. Repositories in the same group are combined into a single row using the framework name (e.g., "PyTorch"). +The `framework_groups` configuration consolidates support policy rows by framework. The generation follows a 3-step flow: -**Version Display:** +**Validation (at load time):** `load_repository_images` validates that images sharing the same full version within a single repository have identical GA/EOP dates. Raises `ValueError` if not (this is a data bug). -- Images with the same major.minor version (e.g., `2.6.0` and `2.6.1`) are consolidated into a single row displayed as `2.6` if they have identical GA/EOP dates -- If patch versions have different GA/EOP dates, each is displayed separately with full version (e.g., `2.6.0`, `2.6.1`) and a warning is logged +**Step 1 — Group by full version:** All images in a framework group are grouped by full version (e.g., `2.6.0`), deduplicated per repository (one representative image per repo since intra-repo consistency is guaranteed). + +**Step 2 — Cross-repo agreement check:** For each full version, check if all repositories agree on GA/EOP dates: + +- If all repositories agree → one entry using the framework group name (e.g., "PyTorch") +- If repositories disagree → warning logged, each repository gets its own row using its individual display name (e.g., "PyTorch Inference") + +**Step 3 — Major.minor collapse:** Non-split entries are grouped by major.minor. If all full versions within a major.minor share the same dates, they collapse into a single row displayed as the major.minor (e.g., `2.6`). Collapse is skipped for any major.minor that has split (per-repo) rows. **Requirements:** -- All repositories in a group that have a given full version (X.Y.Z) must have identical GA/EOP dates - Missing versions in some repositories are allowed (only present repos are consolidated) -- A `ValueError` is raised if dates differ within a group for the same full version To add a new framework group, add an entry to `framework_groups` with the framework name as key and list of repositories as value. diff --git a/docs/src/data/pytorch-inference-arm64/2.6-cpu-ec2.yml b/docs/src/data/pytorch-inference-arm64/2.6-cpu-ec2.yml index 8c060f9e5c5c..3e239699a0b5 100644 --- a/docs/src/data/pytorch-inference-arm64/2.6-cpu-ec2.yml +++ b/docs/src/data/pytorch-inference-arm64/2.6-cpu-ec2.yml @@ -5,7 +5,7 @@ python: py312 os: ubuntu22.04 platform: ec2 ga: "2025-01-29" -eop: "2026-01-29" +eop: "2026-06-29" public_registry: true tags: diff --git a/docs/src/data/pytorch-inference-arm64/2.6-cpu-sagemaker.yml b/docs/src/data/pytorch-inference-arm64/2.6-cpu-sagemaker.yml index 65ed7e6c969f..27c542df8af5 100644 --- a/docs/src/data/pytorch-inference-arm64/2.6-cpu-sagemaker.yml +++ b/docs/src/data/pytorch-inference-arm64/2.6-cpu-sagemaker.yml @@ -5,7 +5,7 @@ python: py312 os: ubuntu22.04 platform: sagemaker ga: "2025-01-29" -eop: "2026-01-29" +eop: "2026-06-29" public_registry: true tags: diff --git a/docs/src/data/pytorch-inference-arm64/2.6-gpu-ec2.yml b/docs/src/data/pytorch-inference-arm64/2.6-gpu-ec2.yml index 13e389b5a673..30affba75467 100644 --- a/docs/src/data/pytorch-inference-arm64/2.6-gpu-ec2.yml +++ b/docs/src/data/pytorch-inference-arm64/2.6-gpu-ec2.yml @@ -6,7 +6,7 @@ cuda: cu124 os: ubuntu22.04 platform: ec2 ga: "2025-01-29" -eop: "2026-01-29" +eop: "2026-06-29" public_registry: true tags: diff --git a/docs/src/data/pytorch-inference/2.6-cpu-ec2.yml b/docs/src/data/pytorch-inference/2.6-cpu-ec2.yml index 7ec72c03c643..0fe5703804ff 100644 --- a/docs/src/data/pytorch-inference/2.6-cpu-ec2.yml +++ b/docs/src/data/pytorch-inference/2.6-cpu-ec2.yml @@ -5,7 +5,7 @@ python: py312 os: ubuntu22.04 platform: ec2 ga: "2025-01-29" -eop: "2026-01-29" +eop: "2026-06-29" public_registry: true tags: diff --git a/docs/src/data/pytorch-inference/2.6-cpu-sagemaker.yml b/docs/src/data/pytorch-inference/2.6-cpu-sagemaker.yml index 19c00459c0d6..c8fee9df7d23 100644 --- a/docs/src/data/pytorch-inference/2.6-cpu-sagemaker.yml +++ b/docs/src/data/pytorch-inference/2.6-cpu-sagemaker.yml @@ -5,7 +5,7 @@ python: py312 os: ubuntu22.04 platform: sagemaker ga: "2025-01-29" -eop: "2026-01-29" +eop: "2026-06-29" public_registry: true tags: diff --git a/docs/src/data/pytorch-inference/2.6-gpu-ec2.yml b/docs/src/data/pytorch-inference/2.6-gpu-ec2.yml index 53f9da718b13..fc69de32a457 100644 --- a/docs/src/data/pytorch-inference/2.6-gpu-ec2.yml +++ b/docs/src/data/pytorch-inference/2.6-gpu-ec2.yml @@ -6,7 +6,7 @@ cuda: cu124 os: ubuntu22.04 platform: ec2 ga: "2025-01-29" -eop: "2026-01-29" +eop: "2026-06-29" public_registry: true tags: diff --git a/docs/src/data/pytorch-inference/2.6-gpu-sagemaker.yml b/docs/src/data/pytorch-inference/2.6-gpu-sagemaker.yml index fc8b62cff29a..164220f69134 100644 --- a/docs/src/data/pytorch-inference/2.6-gpu-sagemaker.yml +++ b/docs/src/data/pytorch-inference/2.6-gpu-sagemaker.yml @@ -6,7 +6,7 @@ cuda: cu124 os: ubuntu22.04 platform: sagemaker ga: "2025-01-29" -eop: "2026-01-29" +eop: "2026-06-29" public_registry: true tags: diff --git a/docs/src/generate.py b/docs/src/generate.py index 8077aa6e22f6..5a76bd39c5f7 100644 --- a/docs/src/generate.py +++ b/docs/src/generate.py @@ -14,6 +14,7 @@ import logging from pathlib import Path +from pprint import pformat import sorter as sorter_module from constants import ( @@ -176,6 +177,55 @@ def generate_release_notes(dry_run: bool = False) -> None: LOGGER.info("Generated release notes") +def _collapse_minor_versions( + entries: list[tuple[ImageConfig, dict[str, str]]], +) -> list[tuple[ImageConfig, dict[str, str]]]: + """Collapse patch versions (e.g., A.B.C, A.B.D) into major.minor (A.B) when all share identical dates. + + Skips any major.minor that has split (per-repo) rows, since mixing collapsed and split rows + under the same major.minor would be confusing. + + Args: + entries: List of (image, overrides) tuples. Split rows have "framework_group" in overrides. + + Returns: + New list with collapsible groups replaced by a single major.minor entry. + """ + uncollapsible: set[str] = set() + collapsible_groups: dict[str, list[int]] = {} + for idx, (img, overrides) in enumerate(entries): + version_obj = parse_version(img.version) + mm = f"{version_obj.major}.{version_obj.minor}" + if "framework_group" in overrides: + # Find major.minors that have split rows by repository — these cannot be collapsed + # Collapsing split rows will create ambiguity between patch versions + uncollapsible.add(mm) + else: + collapsible_groups.setdefault(mm, []).append(idx) + + # Collapse: if all entries in a major.minor group share dates, keep one with major.minor display + for mm, indices in collapsible_groups.items(): + if mm in uncollapsible: + continue + group_imgs = [entries[idx][0] for idx in indices] + ref_img = group_imgs[0] + if all(cmp_img.ga == ref_img.ga and cmp_img.eop == ref_img.eop for cmp_img in group_imgs): + entries[indices[0]] = (ref_img, {"version": mm}) + for idx in indices[1:]: + entries[idx] = None # mark duplicates for removal + else: + # Log images that have different ga/eop dates + for cmp_img in group_imgs[1:]: + if cmp_img.ga != ref_img.ga or cmp_img.eop != ref_img.eop: + LOGGER.warning( + f"Cannot collapse {mm}: " + f"{ref_img._repository} {ref_img.version} ({ref_img.ga}, {ref_img.eop}) vs " + f"{cmp_img._repository} {cmp_img.version} ({cmp_img.ga}, {cmp_img.eop})" + ) + + return [e for e in entries if e is not None] + + def generate_support_policy(dry_run: bool = False) -> str: """Generate support_policy.md from image configs with GA/EOP dates.""" output_path = REFERENCE_DIR / "support_policy.md" @@ -197,55 +247,62 @@ def generate_support_policy(dry_run: bool = False) -> str: if not images: continue - # Group by major.minor, then decide display format based on date consistency - major_minor_groups: dict[str, list[ImageConfig]] = {} + # Step 1: Group by full version, deduplicate per repo + # Result: {"2.6": [pt-training img, pt-inference img, ...], "2.7": [...]} + version_entries: dict[str, list[ImageConfig]] = {} for img in images: - v = parse_version(img.version) - major_minor = f"{v.major}.{v.minor}" - major_minor_groups.setdefault(major_minor, []).append(img) - - version_map: dict[str, ImageConfig] = {} - for major_minor, group in major_minor_groups.items(): - # Check if all images in group have same ga/eop - first = group[0] - all_same_dates = all(img.ga == first.ga and img.eop == first.eop for img in group) - - if all_same_dates: - # Consolidate to major.minor display - version_map[major_minor] = first + bucket = version_entries.setdefault(img.version, []) + if not any(existing._repository == img._repository for existing in bucket): + bucket.append(img) + LOGGER.debug( + "[%s] Step 1 - versions:\n%s", + framework_group, + pformat({v: [i._repository for i in imgs] for v, imgs in version_entries.items()}), + ) + + # Step 2: Cross-repo agreement check + entries: list[tuple[ImageConfig, dict[str, str]]] = [] + seen_versions: set[str] = set() + for full_ver, repo_imgs in version_entries.items(): + seen_versions.add(full_ver) + ref_img = repo_imgs[0] + if all( + cmp_img.ga == ref_img.ga and cmp_img.eop == ref_img.eop for cmp_img in repo_imgs + ): + entries.append((ref_img, {"version": full_ver})) else: - # Keep full versions, warn about inconsistency - versions_info = ", ".join(f"{img.version} ({img.ga}, {img.eop})" for img in group) LOGGER.warning( - f"Different GA/EOP dates for {framework_group} patch versions: {versions_info}" + f"GA/EOP mismatch in {framework_group} {full_ver} across repositories. " + f"Splitting into individual repository rows." ) - # Keep each patch version as separate row with full version display - for img in group: - existing = version_map.get(img.version) - # Error if same full version (e.g., X.Y.Z) has different dates across images - if existing and (existing.ga != img.ga or existing.eop != img.eop): - raise ValueError( - f"Inconsistent dates for {framework_group} {img.version}: \n" - f"\tExisting: {existing._repository}-{existing.version}-{existing.accelerator}-{existing.platform}\n" - f"\tImage: {img._repository}-{img.version}-{img.accelerator}-{img.platform}\n" - f"\t({existing.ga}, {existing.eop}) vs ({img.ga}, {img.eop})" - ) - # Deduplicate same full version with same dates - version_map[img.version] = img + for img in repo_imgs: + entries.append( + (img, {"version": full_ver, "framework_group": img.display_repository}) + ) + LOGGER.debug( + "[%s] Step 2 - entries:\n%s", + framework_group, + pformat([(img._repository, overrides) for img, overrides in entries]), + ) + + # Step 3: Collapse patch versions into major.minor where possible + # Result: entries with "version" collapsed (e.g., "2.7.0" -> "2.7"), split rows unchanged + entries = _collapse_minor_versions(entries) + LOGGER.debug( + "[%s] Step 3 - collapsed:\n%s", + framework_group, + pformat([(img._repository, overrides) for img, overrides in entries]), + ) # Merge legacy entries for this framework for legacy_img in legacy_data.get(framework_group, []): - if legacy_img.version not in version_map: - version_map[legacy_img.version] = legacy_img + if legacy_img.version not in seen_versions: + entries.append((legacy_img, {"version": legacy_img.version})) # Sort by version descending within this framework group - # Key is the display version (major.minor if consolidated, full version otherwise) - sorted_keys = sorted( - version_map.keys(), key=lambda k: parse_version(version_map[k].version), reverse=True - ) - for key in sorted_keys: - img = version_map[key] - (supported if img.is_supported else unsupported).append((img, key)) + entries.sort(key=lambda e: parse_version(e[0].version), reverse=True) + for img, overrides in entries: + (supported if img.is_supported else unsupported).append((img, overrides)) # Build tables table_config = load_table_config("extra/support_policy") @@ -253,10 +310,10 @@ def generate_support_policy(dry_run: bool = False) -> str: headers = [col["header"] for col in columns] supported_table = render_table( - headers, [build_image_row(img, columns, {"version": ver}) for img, ver in supported] + headers, [build_image_row(img, columns, overrides) for img, overrides in supported] ) unsupported_table = render_table( - headers, [build_image_row(img, columns, {"version": ver}) for img, ver in unsupported] + headers, [build_image_row(img, columns, overrides) for img, overrides in unsupported] ) # Render template diff --git a/docs/src/image_config.py b/docs/src/image_config.py index ed5cdf0e4150..6868227b019f 100644 --- a/docs/src/image_config.py +++ b/docs/src/image_config.py @@ -185,7 +185,24 @@ def load_repository_images(repository: str) -> list[ImageConfig]: repo_dir = DATA_DIR / repository if not repo_dir.exists(): return [] - return [ImageConfig.from_yaml(f, repository) for f in sorted(repo_dir.glob("*.yml"))] + images = [ImageConfig.from_yaml(f, repository) for f in sorted(repo_dir.glob("*.yml"))] + + # Validate: images in the same repository sharing same full version must have identical GA/EOP dates + date_by_version: dict[str, ImageConfig] = {} + for img in images: + if not img.has_support_dates: + continue + if img.version in date_by_version: + ref = date_by_version[img.version] + if ref.ga != img.ga or ref.eop != img.eop: + raise ValueError( + f"Inconsistent dates within {repository} for version {img.version}: " + f"({ref.ga}, {ref.eop}) vs ({img.ga}, {img.eop})" + ) + else: + date_by_version[img.version] = img + + return images def load_images_by_framework_group( From b3e02bfd23a6fc8e92fd24c9a963108c3f1366fb Mon Sep 17 00:00:00 2001 From: sirutBuasai Date: Tue, 10 Feb 2026 15:57:32 -0800 Subject: [PATCH 2/7] fix test Signed-off-by: sirutBuasai --- test/docs/test_generate.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/docs/test_generate.py b/test/docs/test_generate.py index 06267aa39a3e..4ed089bb05a6 100644 --- a/test/docs/test_generate.py +++ b/test/docs/test_generate.py @@ -119,7 +119,9 @@ def test_consistent_dates(self, mock_paths, mock_repo_images): indirect=True, ids=["inconsistent_eop", "inconsistent_ga", "both_inconsistent"], ) - def test_inconsistent_dates_raises(self, mock_paths, mock_repo_images): - """Test that inconsistent dates across repos in same framework group raise ValueError.""" - with pytest.raises(ValueError, match="Inconsistent dates"): - generate_support_policy(dry_run=True) + def test_inconsistent_dates_splits_rows(self, mock_paths, mock_repo_images): + """Test that inconsistent dates across repos split into individual repository rows.""" + content = generate_support_policy(dry_run=True) + assert "Repo A" in content + assert "Repo B" in content + assert "Test Group" not in content From 73bfccbaac896e7373ff7a84d2320ac953371ad0 Mon Sep 17 00:00:00 2001 From: sirutBuasai Date: Tue, 10 Feb 2026 16:47:29 -0800 Subject: [PATCH 3/7] add subgroup Signed-off-by: sirutBuasai --- .kiro/steering/docs.md | 28 +++++++------ docs/src/generate.py | 85 ++++++++++++++++++++++++++++------------ docs/src/global.yml | 23 +++++++---- docs/src/image_config.py | 12 ++++-- docs/src/utils.py | 24 ++++++++---- 5 files changed, 118 insertions(+), 54 deletions(-) diff --git a/.kiro/steering/docs.md b/.kiro/steering/docs.md index 362ba275da96..16cbd3971e6e 100644 --- a/.kiro/steering/docs.md +++ b/.kiro/steering/docs.md @@ -76,9 +76,9 @@ docs/src/ - `constants.py` - Path constants, global variables, `GLOBAL_CONFIG`, `SITE_URL`, `README_PATH`, and `RELEASE_NOTES_REQUIRED_FIELDS` - `sorter.py` - Sorting tiebreaker functions: `platform_sorter`, `accelerator_sorter`, `repository_sorter` -- `utils.py` - Utility functions: `load_yaml()`, `load_table_config()`, `load_jinja2()`, `render_table()`, `write_output()`, `parse_version()`, `clone_git_repository()`, `build_ecr_uri()`, `build_public_ecr_uri()`, `get_framework_order()` +- `utils.py` - Utility functions: `load_yaml()`, `load_table_config()`, `load_jinja2()`, `render_table()`, `write_output()`, `parse_version()`, `clone_git_repository()`, `build_ecr_uri()`, `build_public_ecr_uri()`, `get_framework_order()`, `flatten_group_repos()` - `image_config.py` - `ImageConfig` class, image loaders (`load_repository_images`, `load_legacy_images`, `load_images_by_framework_group`), `sort_by_version`, `get_latest_image_uri`, `build_image_row`, `check_public_registry` -- `generate.py` - `generate_index()`, `generate_support_policy()`, `generate_available_images()`, `generate_release_notes()`, `generate_all()` +- `generate.py` - `generate_index()`, `generate_support_policy()`, `generate_available_images()`, `generate_release_notes()`, `generate_all()`, helpers: `_split_by_subgroup()`, `_collapse_minor_versions()` - `macros.py` - MkDocs macros plugin integration - `hooks.py` - MkDocs hooks entry point @@ -356,16 +356,19 @@ display_names: known_issues: "Known Issues" # Framework groups for support policy consolidation (lowercase keys) +# Flat list: repos split directly to per-repo rows on mismatch +# Nested dict: repos consolidate by sub-group first on mismatch framework_groups: pytorch: - - pytorch-training - - pytorch-inference - - pytorch-training-arm64 - - pytorch-inference-arm64 - tensorflow: - - tensorflow-training - - tensorflow-inference - - tensorflow-inference-arm64 + pytorch-training: + - pytorch-training + - pytorch-training-arm64 + pytorch-inference: + - pytorch-inference + - pytorch-inference-arm64 + vllm: + - vllm + - vllm-arm64 # Table order (controls order in available_images.md and support_policy.md) table_order: @@ -385,7 +388,8 @@ The `framework_groups` configuration consolidates support policy rows by framewo **Step 2 — Cross-repo agreement check:** For each full version, check if all repositories agree on GA/EOP dates: - If all repositories agree → one entry using the framework group name (e.g., "PyTorch") -- If repositories disagree → warning logged, each repository gets its own row using its individual display name (e.g., "PyTorch Inference") +- If repositories disagree and sub-groups are configured (dict format) → consolidate by sub-group first. If all repos in a sub-group agree, one row with sub-group display name (e.g., "PyTorch Inference"). If sub-group also disagrees, per-repo rows. +- If repositories disagree and no sub-groups (list format) → each repository gets its own row using its individual display name **Step 3 — Major.minor collapse:** Non-split entries are grouped by major.minor. If all full versions within a major.minor share the same dates, they collapse into a single row displayed as the major.minor (e.g., `2.6`). Collapse is skipped for any major.minor that has split (per-repo) rows. @@ -393,7 +397,7 @@ The `framework_groups` configuration consolidates support policy rows by framewo - Missing versions in some repositories are allowed (only present repos are consolidated) -To add a new framework group, add an entry to `framework_groups` with the framework name as key and list of repositories as value. +To add a new framework group, add an entry to `framework_groups` with the framework name as key and list of repositories as value. Use nested dict format if you need intermediate sub-group consolidation when dates differ. ### Reordering Tables and Columns diff --git a/docs/src/generate.py b/docs/src/generate.py index 5a76bd39c5f7..2fa3f49e5a36 100644 --- a/docs/src/generate.py +++ b/docs/src/generate.py @@ -33,6 +33,7 @@ ImageConfig, build_image_row, check_public_registry, + dates_agree, load_images_by_framework_group, load_legacy_images, load_repository_images, @@ -40,6 +41,7 @@ ) from jinja2 import Template from utils import ( + build_repo_map, get_framework_order, load_jinja2, load_table_config, @@ -177,6 +179,52 @@ def generate_release_notes(dry_run: bool = False) -> None: LOGGER.info("Generated release notes") +def _split_by_subgroup( + framework_group: str, + full_ver: str, + repo_imgs: list[ImageConfig], +) -> list[tuple[ImageConfig, dict[str, str]]]: + """Split mismatched images by sub-group, falling back to per-repo if sub-group also disagrees. + + When a framework group has nested sub-groups (dict format in global.yml), images are first + grouped by sub-group. If all images in a sub-group agree on dates, they consolidate into one + row with the sub-group display name. Otherwise, each repo gets its own row. + + For flat groups (list format), falls back directly to per-repo rows. + """ + group_config = GLOBAL_CONFIG.get("framework_groups", {}).get(framework_group, []) + entries: list[tuple[ImageConfig, dict[str, str]]] = [] + + # Flat group — no sub-groups, split directly to per-repo + if not isinstance(group_config, dict): + for img in repo_imgs: + entries.append((img, {"version": full_ver, "framework_group": img.display_repository})) + return entries + + # Nested group + repo_to_subgroup = build_repo_map(group_config) + + # Group images by sub-group + subgroup_imgs: dict[str, list[ImageConfig]] = {} + for img in repo_imgs: + subgroup_name = repo_to_subgroup.get(img._repository, img._repository) + subgroup_imgs.setdefault(subgroup_name, []).append(img) + + # Try to consolidate within each sub-group + for subgroup_name, subgroup_images in subgroup_imgs.items(): + ref_img = subgroup_images[0] + if dates_agree(subgroup_images): + display_name = GLOBAL_CONFIG.get("display_names", {}).get(subgroup_name, subgroup_name) + entries.append((ref_img, {"version": full_ver, "framework_group": display_name})) + else: + for img in subgroup_images: + entries.append( + (img, {"version": full_ver, "framework_group": img.display_repository}) + ) + + return entries + + def _collapse_minor_versions( entries: list[tuple[ImageConfig, dict[str, str]]], ) -> list[tuple[ImageConfig, dict[str, str]]]: @@ -209,19 +257,14 @@ def _collapse_minor_versions( continue group_imgs = [entries[idx][0] for idx in indices] ref_img = group_imgs[0] - if all(cmp_img.ga == ref_img.ga and cmp_img.eop == ref_img.eop for cmp_img in group_imgs): + if dates_agree(group_imgs): entries[indices[0]] = (ref_img, {"version": mm}) for idx in indices[1:]: entries[idx] = None # mark duplicates for removal else: - # Log images that have different ga/eop dates - for cmp_img in group_imgs[1:]: - if cmp_img.ga != ref_img.ga or cmp_img.eop != ref_img.eop: - LOGGER.warning( - f"Cannot collapse {mm}: " - f"{ref_img._repository} {ref_img.version} ({ref_img.ga}, {ref_img.eop}) vs " - f"{cmp_img._repository} {cmp_img.version} ({cmp_img.ga}, {cmp_img.eop})" - ) + LOGGER.warning( + f"Cannot collapse {ref_img._repository} {mm}. Please check images GA/EOP dates within this framework." + ) return [e for e in entries if e is not None] @@ -255,9 +298,8 @@ def generate_support_policy(dry_run: bool = False) -> str: if not any(existing._repository == img._repository for existing in bucket): bucket.append(img) LOGGER.debug( - "[%s] Step 1 - versions:\n%s", - framework_group, - pformat({v: [i._repository for i in imgs] for v, imgs in version_entries.items()}), + f"[{framework_group}] Step 1 - versions:\n" + f"{pformat({v: [i._repository for i in imgs] for v, imgs in version_entries.items()})}" ) # Step 2: Cross-repo agreement check @@ -266,32 +308,25 @@ def generate_support_policy(dry_run: bool = False) -> str: for full_ver, repo_imgs in version_entries.items(): seen_versions.add(full_ver) ref_img = repo_imgs[0] - if all( - cmp_img.ga == ref_img.ga and cmp_img.eop == ref_img.eop for cmp_img in repo_imgs - ): + if dates_agree(repo_imgs): entries.append((ref_img, {"version": full_ver})) else: LOGGER.warning( f"GA/EOP mismatch in {framework_group} {full_ver} across repositories. " f"Splitting into individual repository rows." ) - for img in repo_imgs: - entries.append( - (img, {"version": full_ver, "framework_group": img.display_repository}) - ) + entries.extend(_split_by_subgroup(framework_group, full_ver, repo_imgs)) LOGGER.debug( - "[%s] Step 2 - entries:\n%s", - framework_group, - pformat([(img._repository, overrides) for img, overrides in entries]), + f"[{framework_group}] Step 2 - entries:\n" + f"{pformat([overrides for _, overrides in entries])}" ) # Step 3: Collapse patch versions into major.minor where possible # Result: entries with "version" collapsed (e.g., "2.7.0" -> "2.7"), split rows unchanged entries = _collapse_minor_versions(entries) LOGGER.debug( - "[%s] Step 3 - collapsed:\n%s", - framework_group, - pformat([(img._repository, overrides) for img, overrides in entries]), + f"[{framework_group}] Step 3 - collapsed:\n" + f"{pformat([(img._repository, overrides) for img, overrides in entries])}" ) # Merge legacy entries for this framework diff --git a/docs/src/global.yml b/docs/src/global.yml index 27190efd455f..ccdff3d605ed 100644 --- a/docs/src/global.yml +++ b/docs/src/global.yml @@ -113,17 +113,26 @@ display_names: # Repositories in the same group with matching GA/EOP dates for a version # are consolidated into a single row. Use lowercase keys. # Display names are defined in the display_names section. +# +# Supports two formats: +# Flat list: group: [repo-a, repo-b] +# Nested dict: group: { sub-group-a: [repo-a, repo-a-arm64], sub-group-b: [repo-b] } +# Nested sub-groups enable intermediate consolidation when the full group disagrees. # ============================================================================= framework_groups: pytorch: - - pytorch-training - - pytorch-inference - - pytorch-training-arm64 - - pytorch-inference-arm64 + pytorch-training: + - pytorch-training + - pytorch-training-arm64 + pytorch-inference: + - pytorch-inference + - pytorch-inference-arm64 tensorflow: - - tensorflow-training - - tensorflow-inference - - tensorflow-inference-arm64 + tensorflow-training: + - tensorflow-training + tensorflow-inference: + - tensorflow-inference + - tensorflow-inference-arm64 vllm: - vllm - vllm-arm64 diff --git a/docs/src/image_config.py b/docs/src/image_config.py index 6868227b019f..332e32cead28 100644 --- a/docs/src/image_config.py +++ b/docs/src/image_config.py @@ -18,7 +18,7 @@ from typing import Any from constants import DATA_DIR, GLOBAL_CONFIG, LEGACY_DIR, RELEASE_NOTES_REQUIRED_FIELDS -from utils import build_ecr_uri, build_public_ecr_uri, load_yaml, parse_version +from utils import build_ecr_uri, build_public_ecr_uri, flatten_group_repos, load_yaml, parse_version LOGGER = logging.getLogger(__name__) @@ -60,8 +60,8 @@ def repository(self) -> str: @property def framework_group(self) -> str: """Framework group key (or repository if not in a group).""" - for group_key, repos in GLOBAL_CONFIG.get("framework_groups", {}).items(): - if self._repository in repos: + for group_key, group_config in GLOBAL_CONFIG.get("framework_groups", {}).items(): + if self._repository in flatten_group_repos(group_config): return group_key return self._repository @@ -167,6 +167,12 @@ def get_display(self, field: str) -> str: return str(value) if value is not None else "-" +def dates_agree(images: list[ImageConfig]) -> bool: + """Check if all images share the same GA and EOP dates.""" + ref = images[0] + return all(img.ga == ref.ga and img.eop == ref.eop for img in images) + + def build_image_row(img: ImageConfig, columns: list[dict], overrides: dict = None) -> list[str]: """Build a table row from an ImageConfig using column definitions. diff --git a/docs/src/utils.py b/docs/src/utils.py index 611ebd06e4df..dcdd01acf0fa 100644 --- a/docs/src/utils.py +++ b/docs/src/utils.py @@ -94,16 +94,26 @@ def clone_git_repository(git_repository: str, target_dir: str | Path) -> None: subprocess.run(["git", "clone", "--depth", "1", git_repository, target_dir], check=True) +def flatten_group_repos(group_config: list | dict) -> list[str]: + """Return all repository names from a framework_groups entry (flat list or nested dict).""" + if isinstance(group_config, list): + return group_config + return [repo for sub_repos in group_config.values() for repo in sub_repos] + + +def build_repo_map(groups: dict) -> dict[str, str]: + """Build reverse mapping from repository name to its parent group key.""" + repo_map = {} + for group_name, group_config in groups.items(): + for repo in flatten_group_repos(group_config): + repo_map[repo] = group_name + return repo_map + + def get_framework_order() -> list[str]: """Derive framework order from table_order, collapsing framework groups.""" table_order = GLOBAL_CONFIG.get("table_order", []) - framework_groups = GLOBAL_CONFIG.get("framework_groups", {}) - - # Build reverse mapping: repo -> framework group - repo_to_group = {} - for group, repos in framework_groups.items(): - for repo in repos: - repo_to_group[repo] = group + repo_to_group = build_repo_map(GLOBAL_CONFIG.get("framework_groups", {})) seen = set() result = [] From 9135cb7ae948a49868ad018bff3823a02e99ad80 Mon Sep 17 00:00:00 2001 From: sirutBuasai Date: Tue, 10 Feb 2026 17:14:45 -0800 Subject: [PATCH 4/7] consolidate sorting Signed-off-by: sirutBuasai --- .kiro/steering/docs.md | 10 +++--- docs/src/generate.py | 75 +++++++++++++++++++----------------------- 2 files changed, 39 insertions(+), 46 deletions(-) diff --git a/.kiro/steering/docs.md b/.kiro/steering/docs.md index 16cbd3971e6e..fdca4eb31506 100644 --- a/.kiro/steering/docs.md +++ b/.kiro/steering/docs.md @@ -78,7 +78,7 @@ docs/src/ - `sorter.py` - Sorting tiebreaker functions: `platform_sorter`, `accelerator_sorter`, `repository_sorter` - `utils.py` - Utility functions: `load_yaml()`, `load_table_config()`, `load_jinja2()`, `render_table()`, `write_output()`, `parse_version()`, `clone_git_repository()`, `build_ecr_uri()`, `build_public_ecr_uri()`, `get_framework_order()`, `flatten_group_repos()` - `image_config.py` - `ImageConfig` class, image loaders (`load_repository_images`, `load_legacy_images`, `load_images_by_framework_group`), `sort_by_version`, `get_latest_image_uri`, `build_image_row`, `check_public_registry` -- `generate.py` - `generate_index()`, `generate_support_policy()`, `generate_available_images()`, `generate_release_notes()`, `generate_all()`, helpers: `_split_by_subgroup()`, `_collapse_minor_versions()` +- `generate.py` - `generate_index()`, `generate_support_policy()`, `generate_available_images()`, `generate_release_notes()`, `generate_all()`, helpers: `_consolidate_framework_version()`, `_collapse_minor_versions()` - `macros.py` - MkDocs macros plugin integration - `hooks.py` - MkDocs hooks entry point @@ -385,11 +385,11 @@ The `framework_groups` configuration consolidates support policy rows by framewo **Step 1 — Group by full version:** All images in a framework group are grouped by full version (e.g., `2.6.0`), deduplicated per repository (one representative image per repo since intra-repo consistency is guaranteed). -**Step 2 — Cross-repo agreement check:** For each full version, check if all repositories agree on GA/EOP dates: +**Step 2 — Hierarchical consolidation** via `_consolidate_framework_version()`: For each full version, tries three levels of date agreement, stopping at the first that succeeds: -- If all repositories agree → one entry using the framework group name (e.g., "PyTorch") -- If repositories disagree and sub-groups are configured (dict format) → consolidate by sub-group first. If all repos in a sub-group agree, one row with sub-group display name (e.g., "PyTorch Inference"). If sub-group also disagrees, per-repo rows. -- If repositories disagree and no sub-groups (list format) → each repository gets its own row using its individual display name +1. Framework group — all repos agree → single row (e.g., "PyTorch") +1. Sub-group — repos within a sub-group agree → one row per sub-group (e.g., "PyTorch Inference"). Only applies to nested dict groups in `framework_groups`. +1. Per-repo — no agreement → one row per repository using its individual display name **Step 3 — Major.minor collapse:** Non-split entries are grouped by major.minor. If all full versions within a major.minor share the same dates, they collapse into a single row displayed as the major.minor (e.g., `2.6`). Collapse is skipped for any major.minor that has split (per-repo) rows. diff --git a/docs/src/generate.py b/docs/src/generate.py index 2fa3f49e5a36..7e57feb9ab81 100644 --- a/docs/src/generate.py +++ b/docs/src/generate.py @@ -179,48 +179,53 @@ def generate_release_notes(dry_run: bool = False) -> None: LOGGER.info("Generated release notes") -def _split_by_subgroup( +def _consolidate_framework_version( framework_group: str, full_ver: str, repo_imgs: list[ImageConfig], ) -> list[tuple[ImageConfig, dict[str, str]]]: - """Split mismatched images by sub-group, falling back to per-repo if sub-group also disagrees. + """Consolidate images for a single framework version using hierarchical date agreement. - When a framework group has nested sub-groups (dict format in global.yml), images are first - grouped by sub-group. If all images in a sub-group agree on dates, they consolidate into one - row with the sub-group display name. Otherwise, each repo gets its own row. - - For flat groups (list format), falls back directly to per-repo rows. + Tries three levels of consolidation, stopping at the first that succeeds: + 1. Framework group — all repos agree → single row + 2. Sub-group — repos within a sub-group agree → one row per sub-group (nested groups only) + 3. Per-repo — no agreement → one row per repository """ + # All repos agree → single framework-level row (Level 1 consolidation) + if dates_agree(repo_imgs): + return [(repo_imgs[0], {"version": full_ver})] + + LOGGER.warning( + f"GA/EOP mismatch in {framework_group} {full_ver} across repositories. " + f"Splitting into sub-group/repository rows." + ) + group_config = GLOBAL_CONFIG.get("framework_groups", {}).get(framework_group, []) - entries: list[tuple[ImageConfig, dict[str, str]]] = [] - # Flat group — no sub-groups, split directly to per-repo + # Flat group — no sub-groups, fall back directly to per-repo rows (Level 3 consolidation) if not isinstance(group_config, dict): - for img in repo_imgs: - entries.append((img, {"version": full_ver, "framework_group": img.display_repository})) - return entries + return [ + (img, {"version": full_ver, "framework_group": img.display_repository}) + for img in repo_imgs + ] - # Nested group + # Nested group — try sub-group consolidation (Level 2 consolidation), per-repo fallback (Level 3 consolidation) repo_to_subgroup = build_repo_map(group_config) - - # Group images by sub-group subgroup_imgs: dict[str, list[ImageConfig]] = {} for img in repo_imgs: subgroup_name = repo_to_subgroup.get(img._repository, img._repository) subgroup_imgs.setdefault(subgroup_name, []).append(img) - # Try to consolidate within each sub-group - for subgroup_name, subgroup_images in subgroup_imgs.items(): - ref_img = subgroup_images[0] - if dates_agree(subgroup_images): + entries: list[tuple[ImageConfig, dict[str, str]]] = [] + for subgroup_name, images in subgroup_imgs.items(): + if dates_agree(images): display_name = GLOBAL_CONFIG.get("display_names", {}).get(subgroup_name, subgroup_name) - entries.append((ref_img, {"version": full_ver, "framework_group": display_name})) + entries.append((images[0], {"version": full_ver, "framework_group": display_name})) else: - for img in subgroup_images: - entries.append( - (img, {"version": full_ver, "framework_group": img.display_repository}) - ) + entries.extend( + (img, {"version": full_ver, "framework_group": img.display_repository}) + for img in images + ) return entries @@ -263,7 +268,8 @@ def _collapse_minor_versions( entries[idx] = None # mark duplicates for removal else: LOGGER.warning( - f"Cannot collapse {ref_img._repository} {mm}. Please check images GA/EOP dates within this framework." + f"Cannot collapse {ref_img._repository} {mm}. " + f"Please confirm images GA/EOP dates within this framework are intentional." ) return [e for e in entries if e is not None] @@ -291,7 +297,6 @@ def generate_support_policy(dry_run: bool = False) -> str: continue # Step 1: Group by full version, deduplicate per repo - # Result: {"2.6": [pt-training img, pt-inference img, ...], "2.7": [...]} version_entries: dict[str, list[ImageConfig]] = {} for img in images: bucket = version_entries.setdefault(img.version, []) @@ -302,27 +307,16 @@ def generate_support_policy(dry_run: bool = False) -> str: f"{pformat({v: [i._repository for i in imgs] for v, imgs in version_entries.items()})}" ) - # Step 2: Cross-repo agreement check + # Step 2: Consolidate across repos (framework → sub-group → per-repo fallback) entries: list[tuple[ImageConfig, dict[str, str]]] = [] - seen_versions: set[str] = set() for full_ver, repo_imgs in version_entries.items(): - seen_versions.add(full_ver) - ref_img = repo_imgs[0] - if dates_agree(repo_imgs): - entries.append((ref_img, {"version": full_ver})) - else: - LOGGER.warning( - f"GA/EOP mismatch in {framework_group} {full_ver} across repositories. " - f"Splitting into individual repository rows." - ) - entries.extend(_split_by_subgroup(framework_group, full_ver, repo_imgs)) + entries.extend(_consolidate_framework_version(framework_group, full_ver, repo_imgs)) LOGGER.debug( f"[{framework_group}] Step 2 - entries:\n" f"{pformat([overrides for _, overrides in entries])}" ) # Step 3: Collapse patch versions into major.minor where possible - # Result: entries with "version" collapsed (e.g., "2.7.0" -> "2.7"), split rows unchanged entries = _collapse_minor_versions(entries) LOGGER.debug( f"[{framework_group}] Step 3 - collapsed:\n" @@ -331,8 +325,7 @@ def generate_support_policy(dry_run: bool = False) -> str: # Merge legacy entries for this framework for legacy_img in legacy_data.get(framework_group, []): - if legacy_img.version not in seen_versions: - entries.append((legacy_img, {"version": legacy_img.version})) + entries.append((legacy_img, {"version": legacy_img.version})) # Sort by version descending within this framework group entries.sort(key=lambda e: parse_version(e[0].version), reverse=True) From 8305f132dc444b7251543dc6171f806a69d8179f Mon Sep 17 00:00:00 2001 From: sirutBuasai Date: Tue, 10 Feb 2026 17:16:16 -0800 Subject: [PATCH 5/7] remove debug Signed-off-by: sirutBuasai --- docs/src/generate.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/docs/src/generate.py b/docs/src/generate.py index 7e57feb9ab81..94072439ee78 100644 --- a/docs/src/generate.py +++ b/docs/src/generate.py @@ -14,7 +14,6 @@ import logging from pathlib import Path -from pprint import pformat import sorter as sorter_module from constants import ( @@ -302,26 +301,14 @@ def generate_support_policy(dry_run: bool = False) -> str: bucket = version_entries.setdefault(img.version, []) if not any(existing._repository == img._repository for existing in bucket): bucket.append(img) - LOGGER.debug( - f"[{framework_group}] Step 1 - versions:\n" - f"{pformat({v: [i._repository for i in imgs] for v, imgs in version_entries.items()})}" - ) # Step 2: Consolidate across repos (framework → sub-group → per-repo fallback) entries: list[tuple[ImageConfig, dict[str, str]]] = [] for full_ver, repo_imgs in version_entries.items(): entries.extend(_consolidate_framework_version(framework_group, full_ver, repo_imgs)) - LOGGER.debug( - f"[{framework_group}] Step 2 - entries:\n" - f"{pformat([overrides for _, overrides in entries])}" - ) # Step 3: Collapse patch versions into major.minor where possible entries = _collapse_minor_versions(entries) - LOGGER.debug( - f"[{framework_group}] Step 3 - collapsed:\n" - f"{pformat([(img._repository, overrides) for img, overrides in entries])}" - ) # Merge legacy entries for this framework for legacy_img in legacy_data.get(framework_group, []): From f33cac223c99c91eba6a715eb7f0b309f9fd5d6c Mon Sep 17 00:00:00 2001 From: sirutBuasai Date: Tue, 10 Feb 2026 17:20:17 -0800 Subject: [PATCH 6/7] add debug Signed-off-by: sirutBuasai --- docs/src/generate.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/src/generate.py b/docs/src/generate.py index 94072439ee78..7e57feb9ab81 100644 --- a/docs/src/generate.py +++ b/docs/src/generate.py @@ -14,6 +14,7 @@ import logging from pathlib import Path +from pprint import pformat import sorter as sorter_module from constants import ( @@ -301,14 +302,26 @@ def generate_support_policy(dry_run: bool = False) -> str: bucket = version_entries.setdefault(img.version, []) if not any(existing._repository == img._repository for existing in bucket): bucket.append(img) + LOGGER.debug( + f"[{framework_group}] Step 1 - versions:\n" + f"{pformat({v: [i._repository for i in imgs] for v, imgs in version_entries.items()})}" + ) # Step 2: Consolidate across repos (framework → sub-group → per-repo fallback) entries: list[tuple[ImageConfig, dict[str, str]]] = [] for full_ver, repo_imgs in version_entries.items(): entries.extend(_consolidate_framework_version(framework_group, full_ver, repo_imgs)) + LOGGER.debug( + f"[{framework_group}] Step 2 - entries:\n" + f"{pformat([overrides for _, overrides in entries])}" + ) # Step 3: Collapse patch versions into major.minor where possible entries = _collapse_minor_versions(entries) + LOGGER.debug( + f"[{framework_group}] Step 3 - collapsed:\n" + f"{pformat([(img._repository, overrides) for img, overrides in entries])}" + ) # Merge legacy entries for this framework for legacy_img in legacy_data.get(framework_group, []): From 09b369eb0728a8279b0b0df9bbfe65d154a6f57f Mon Sep 17 00:00:00 2001 From: sirutBuasai Date: Tue, 10 Feb 2026 17:20:44 -0800 Subject: [PATCH 7/7] remove debug Signed-off-by: sirutBuasai --- docs/src/generate.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/docs/src/generate.py b/docs/src/generate.py index 7e57feb9ab81..d8513c55f17e 100644 --- a/docs/src/generate.py +++ b/docs/src/generate.py @@ -1,4 +1,4 @@ -# Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +## Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -14,7 +14,6 @@ import logging from pathlib import Path -from pprint import pformat import sorter as sorter_module from constants import ( @@ -302,26 +301,14 @@ def generate_support_policy(dry_run: bool = False) -> str: bucket = version_entries.setdefault(img.version, []) if not any(existing._repository == img._repository for existing in bucket): bucket.append(img) - LOGGER.debug( - f"[{framework_group}] Step 1 - versions:\n" - f"{pformat({v: [i._repository for i in imgs] for v, imgs in version_entries.items()})}" - ) # Step 2: Consolidate across repos (framework → sub-group → per-repo fallback) entries: list[tuple[ImageConfig, dict[str, str]]] = [] for full_ver, repo_imgs in version_entries.items(): entries.extend(_consolidate_framework_version(framework_group, full_ver, repo_imgs)) - LOGGER.debug( - f"[{framework_group}] Step 2 - entries:\n" - f"{pformat([overrides for _, overrides in entries])}" - ) # Step 3: Collapse patch versions into major.minor where possible entries = _collapse_minor_versions(entries) - LOGGER.debug( - f"[{framework_group}] Step 3 - collapsed:\n" - f"{pformat([(img._repository, overrides) for img, overrides in entries])}" - ) # Merge legacy entries for this framework for legacy_img in legacy_data.get(framework_group, []):