[CI] Remove the unused ECR cache lookup from pipeline generation

README.md

@@ -83,7 +83,7 @@ A Python tool (`buildkite/pipeline_generator/`) that reads step definitions from
 - **AMD mirroring**: Steps can define `mirror.amd` to automatically create parallel AMD test runs
 - **Source file dependencies**: Steps can specify which source files they depend on for intelligent test filtering
 - **Block steps**: Optional tests are gated behind manual approval blocks
-- **Variable injection**: Automatically injects registry URLs, cache tags, and image references into step commands
+- **Variable injection**: Automatically injects registry URLs and image references into step commands
 
 #### Jinja2 Template (AMD CI)
 

buildkite/pipeline_generator/buildkite_step.py

@@ -25,11 +25,7 @@
     is_amd_gpu_device,
 )
 from step import Step
-from utils_lib.docker_utils import (
-    get_image,
-    get_ecr_cache_registry,
-    get_torch_nightly_image,
-)
+from utils_lib.docker_utils import get_image, get_torch_nightly_image
 from global_config import get_global_config
 from plugin.k8s_plugin import get_k8s_plugin
 from plugin.docker_plugin import get_docker_plugin
@@ -344,7 +340,6 @@ def _get_variables_to_inject() -> Dict[str, str]:
     if global_config["name"] != "vllm_ci":
         return {}
 
-    cache_from_tag, cache_to_tag = get_ecr_cache_registry()
     registries = global_config["registries"]
     repositories = global_config["repositories"]
     repo = (
@@ -369,8 +364,6 @@ def _get_variables_to_inject() -> Dict[str, str]:
         "$REPO": repo,
         "$BUILDKITE_COMMIT": "$$BUILDKITE_COMMIT",
         "$BRANCH": global_config["branch"],
-        "$CACHE_FROM": cache_from_tag,
-        "$CACHE_TO": cache_to_tag,
         "$IMAGE_TAG": image_tag,
         "$IMAGE_TAG_LATEST": image_tag_latest,
         "$IMAGE_TAG_TORCH_NIGHTLY": get_torch_nightly_image(),

buildkite/pipeline_generator/utils_lib/docker_utils.py

@@ -1,7 +1,3 @@
-import subprocess
-import os
-import re
-from typing import Tuple
 from global_config import get_global_config
 
 
@@ -37,82 +33,3 @@ def get_torch_nightly_image() -> str:
         return f"{registries}/{repositories['main']}:{commit}-torch-nightly"
     else:
         return f"{registries}/{repositories['premerge']}:{commit}-torch-nightly"
-
-
-def _clean_docker_tag(tag: str) -> str:
-    # Only allows alphanumeric, dashes and underscores for Docker tags, and replaces others with '-'
-    return re.sub(r"[^a-zA-Z0-9_.-]", "-", tag or "")
-
-
-def _docker_manifest_exists(image_tag: str) -> bool:
-    try:
-        subprocess.run(
-            ["docker", "manifest", "inspect", image_tag],
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
-            check=True,
-        )
-        return True
-    except subprocess.CalledProcessError:
-        return False
-
-
-def get_ecr_cache_registry() -> Tuple[str, str]:
-    global_config = get_global_config()
-    branch = global_config["branch"]
-    test_cache_ecr = "936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-test-cache"
-    postmerge_cache_ecr = (
-        "936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-postmerge-cache"
-    )
-    cache_from_tag, cache_to_tag = None, None
-    # Authenticate Docker to AWS ECR
-    login_cmd = ["aws", "ecr", "get-login-password", "--region", "us-east-1"]
-    try:
-        proc = subprocess.Popen(login_cmd, stdout=subprocess.PIPE)
-        subprocess.run(
-            [
-                "docker",
-                "login",
-                "--username",
-                "AWS",
-                "--password-stdin",
-                "936637512419.dkr.ecr.us-east-1.amazonaws.com",
-            ],
-            stdin=proc.stdout,
-            check=True,
-        )
-        proc.stdout.close()
-        proc.wait()
-    except Exception as e:
-        raise RuntimeError(f"Failed to authenticate with AWS ECR: {e}")
-
-    if global_config["pull_request"]:  # PR build
-        cache_to_tag = f"{test_cache_ecr}:pr-{global_config['pull_request']}"
-        if _docker_manifest_exists(cache_to_tag):  # use PR cache if exists
-            cache_from_tag = cache_to_tag
-        elif (
-            os.getenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH") != "main"
-        ):  # use base branch cache if exists
-            clean_base = _clean_docker_tag(
-                os.getenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH")
-            )
-            if _docker_manifest_exists(f"{test_cache_ecr}:{clean_base}"):
-                cache_from_tag = f"{test_cache_ecr}:{clean_base}"
-            else:  # fall back to postmerge cache ecr if base branch cache does not exist
-                cache_from_tag = f"{postmerge_cache_ecr}:latest"
-        else:
-            cache_from_tag = f"{postmerge_cache_ecr}:latest"
-    else:  # non-PR build
-        if branch == "main":  # postmerge
-            cache_to_tag = f"{postmerge_cache_ecr}:latest"
-            cache_from_tag = f"{postmerge_cache_ecr}:latest"
-        else:
-            clean_branch = _clean_docker_tag(branch)
-            cache_to_tag = f"{test_cache_ecr}:{clean_branch}"
-            if _docker_manifest_exists(f"{test_cache_ecr}:{clean_branch}"):
-                cache_from_tag = f"{test_cache_ecr}:{clean_branch}"
-            else:
-                cache_from_tag = f"{postmerge_cache_ecr}:latest"
-    if not cache_from_tag or not cache_to_tag:
-        raise RuntimeError("Failed to get ECR cache tags")
-    return cache_from_tag, cache_to_tag

buildkite/tests/pipeline_generator/conftest.py

@@ -29,11 +29,6 @@ def fake_global_config(monkeypatch):
     }
     monkeypatch.setattr(step_module, "get_global_config", lambda: config)
     monkeypatch.setattr(buildkite_step, "get_global_config", lambda: config)
-    monkeypatch.setattr(
-        buildkite_step,
-        "get_ecr_cache_registry",
-        lambda: ("cache-from", "cache-to"),
-    )
     monkeypatch.setattr(
         buildkite_step,
         "get_image",

buildkite/tests/pipeline_generator/test_step.py

@@ -698,6 +698,13 @@ def test_image_tag_matches_get_image_and_latest_suppressed_on_nightly(
     assert vars_["$IMAGE_TAG_LATEST"] is None
 
 
+def test_variable_injection_omits_cache_tags_owned_by_image_build():
+    vars_ = buildkite_step._get_variables_to_inject()
+
+    assert "$CACHE_FROM" not in vars_
+    assert "$CACHE_TO" not in vars_
+
+
 def test_timeout_in_minutes_propagates_to_command_step():
     step = Step(
         label="Timed test",