p

Signed-off-by: Kevin H. Luu <khluu000@gmail.com>

Author: khluu

buildkite/pipeline_generator/buildkite_step.py

@@ -20,32 +20,12 @@ class BuildkiteCommandStep(BaseModel):
     env: Optional[Dict[str, str]] = None
     parallelism: Optional[int] = None
 
-    def to_yaml(self):
-        return {
-            "label": self.label,
-            "group": self.group,
-            "commands": self.commands,
-            "depends_on": self.depends_on,
-            "soft_fail": self.soft_fail,
-            "retry": self.retry,
-            "plugins": self.plugins,
-            "env": self.env,
-            "retry": self.retry,
-            "parallelism": self.parallelism
-        }
 
 class BuildkiteBlockStep(BaseModel):
     block: str
     depends_on: Optional[Union[str, List[str]]] = None
     key: Optional[str] = None
 
-    def to_yaml(self):
-        return {
-            "block": self.block,
-            "depends_on": self.depends_on,
-            "key": self.key
-        }
-
 class BuildkiteGroupStep(BaseModel):
     group: str
     steps: List[Union[BuildkiteCommandStep, BuildkiteBlockStep]]
@@ -57,29 +37,27 @@ def _get_step_plugin(step: Step):
     else:
         return {"docker#v5.2.0": get_docker_plugin(step, get_image(step.no_gpu))}
 
+_GPU_TO_QUEUE = {
+    GPUType.A100.value: AgentQueue.A100_QUEUE,
+    GPUType.H100.value: AgentQueue.MITHRIL_H100_POOL,
+    GPUType.H200.value: AgentQueue.SKYLAB_H200,
+    GPUType.B200.value: AgentQueue.B200,
+}
+
+
 def get_agent_queue(step: Step):
     branch = get_global_config()["branch"]
     if step.label.startswith(":docker:"):
-        if branch == "main":
-            return AgentQueue.CPU_QUEUE_POSTMERGE_US_EAST_1
-        else:
-            return AgentQueue.CPU_QUEUE_PREMERGE_US_EAST_1
-    elif step.label == "Documentation Build":
+        return AgentQueue.CPU_QUEUE_POSTMERGE_US_EAST_1 if branch == "main" else AgentQueue.CPU_QUEUE_PREMERGE_US_EAST_1
+    if step.label == "Documentation Build":
         return AgentQueue.SMALL_CPU_QUEUE_PREMERGE
-    elif step.no_gpu:
+    if step.no_gpu:
         return AgentQueue.CPU_QUEUE_PREMERGE_US_EAST_1
-    elif step.gpu == GPUType.A100:
-        return AgentQueue.A100_QUEUE
-    elif step.gpu == GPUType.H100:
-        return AgentQueue.MITHRIL_H100_POOL
-    elif step.gpu == GPUType.H200:
-        return AgentQueue.SKYLAB_H200
-    elif step.gpu == GPUType.B200:
-        return AgentQueue.B200
-    elif step.num_gpus == 2 or step.num_gpus == 4:
+    if step.gpu in _GPU_TO_QUEUE:
+        return _GPU_TO_QUEUE[step.gpu]
+    if step.num_gpus in (2, 4):
         return AgentQueue.GPU_4_QUEUE
-    else:
-        return AgentQueue.GPU_1_QUEUE
+    return AgentQueue.GPU_1_QUEUE
 
 def _get_variables_to_inject() -> Dict[str, str]:
     global_config = get_global_config()
@@ -189,12 +167,15 @@ def _step_should_run(step: Step, list_file_diff: List[str]) -> bool:
         return False
     if global_config["run_all"]:
         return True
-    if step.source_file_dependencies:
-        for source_file in step.source_file_dependencies:
-            for diff_file in list_file_diff:
-                if source_file in diff_file:
-                    return True
-    return True
+    # If no dependencies specified, always run
+    if not step.source_file_dependencies:
+        return True
+    # Only run if at least one dependency matches a changed file
+    for source_file in step.source_file_dependencies:
+        for diff_file in list_file_diff:
+            if source_file in diff_file:
+                return True
+    return False
 
 def _generate_step_key(step_label: str) -> str:
     return (

buildkite/pipeline_generator/global_config.py

@@ -37,6 +37,10 @@ def init_global_config(pipeline_config_path: str):
     list_file_diff = get_list_file_diff(branch, merge_base_commit)
     pr_labels = get_pr_labels(pull_request)
 
+    run_all_patterns = pipeline_config.get("run_all_patterns") or []
+    run_all_exclude_patterns = pipeline_config.get("run_all_exclude_patterns") or []
+    run_all = _should_run_all(pr_labels, list_file_diff, run_all_patterns, run_all_exclude_patterns)
+
     config = GlobalConfig(
         name=pipeline_config["name"],
         job_dirs=pipeline_config["job_dirs"],
@@ -46,14 +50,14 @@ def init_global_config(pipeline_config_path: str):
         commit=os.getenv("BUILDKITE_COMMIT"),
         pull_request=pull_request,
         docs_only_disable=os.getenv("DOCS_ONLY_DISABLE", "0"),
-        run_all_patterns=pipeline_config.get("run_all_patterns", None),
-        run_all_exclude_patterns=pipeline_config.get("run_all_exclude_patterns", None),
+        run_all_patterns=run_all_patterns or None,
+        run_all_exclude_patterns=run_all_exclude_patterns or None,
         nightly=os.getenv("NIGHTLY", "0"),
-        run_all=_should_run_all(pr_labels, list_file_diff, pipeline_config.get("run_all_patterns", None), pipeline_config.get("run_all_exclude_patterns", None)),
+        run_all=run_all,
         merge_base_commit=merge_base_commit,
         list_file_diff=list_file_diff,
         fail_fast=_should_fail_fast(pr_labels),
-        use_precompiled=_should_use_precompiled(_should_run_all(pr_labels, list_file_diff, pipeline_config.get("run_all_patterns", None), pipeline_config.get("run_all_exclude_patterns", None)), merge_base_commit),
+        use_precompiled=_should_use_precompiled(run_all, merge_base_commit),
     )
 
 def get_global_config():

buildkite/pipeline_generator/pipeline_generator_helper.py

@@ -61,9 +61,9 @@
 
 def get_docker_plugin(step: Step, image: str):
     plugin = None
-    if step.gpu == GPUType.H200:
+    if step.gpu == GPUType.H200.value:
         plugin = copy.deepcopy(h200_plugin_template)
-    elif step.gpu == GPUType.B200:
+    elif step.gpu == GPUType.B200.value:
         plugin = copy.deepcopy(b200_plugin_template)
     else:
         plugin = copy.deepcopy(docker_plugin_template)