Intelligent test filtering if only test files are modified (#184)

Signed-off-by: Reza Barazesh <rezabarazesh@meta.com>

Author: rzabarazesh

buildkite/test-template-ci.j2

@@ -17,6 +17,19 @@
 {% set hf_home_fsx = "/fsx/hf_cache" %}
 {% set list_file_diff = list_file_diff | split("|") %}
 
+{# Intelligent test targeting: Detect when only test files changed and collect them #}
+{%- set tests_acc = namespace(only_tests=true, any=false, changed=[]) %}
+{%- for file in list_file_diff %}
+{%- if file[:6] == 'tests/' and '/test_' in file and file[-3:] == '.py' %}
+{%- set tests_acc.any = true %}
+{%- set tests_acc.changed = tests_acc.changed + [file[6:]] %}
+{%- else %}
+{%- set tests_acc.only_tests = false %}
+{%- endif %}
+{%- endfor %}
+{%- set tests_only = (tests_acc.only_tests and tests_acc.any) %}
+{%- set changed_tests = tests_acc.changed %}
+
 {% macro add_pytest_coverage(cmd, coverage_file) %}
 {% if "pytest " in cmd %}
 COVERAGE_FILE={{ coverage_file }} {{ cmd | replace("pytest ", "pytest --cov=vllm --cov-report=xml --cov-append --durations=0 ") }} || true
@@ -26,10 +39,46 @@ COVERAGE_FILE={{ coverage_file }} {{ cmd | replace("pytest ", "pytest --cov=vllm
 {% endmacro %}
 
 {% macro add_docker_pytest_coverage(step, cov_enabled) %}
-{% if cov_enabled %}
+{# Compute coverage file id #}
 {% set step_length = step.label | length %}
 {% set step_first = step.label | first | default("x") %}
 {% set coverage_file = ".coverage." + step_length ~ "_" ~ step_first %}
+
+{# Intelligent test targeting: Build matched test targets for this step when only tests changed #}
+{%- set match_ns = namespace(targets=[]) %}
+{%- if tests_only and step.source_file_dependencies %}
+{%- for dep in step.source_file_dependencies %}
+{%- if dep[:6] == 'tests/' %}
+{%- set dep_rel = dep[6:] %}
+{# Handle deps that already end with '/' (e.g., tests/benchmarks/) #}
+{%- if dep_rel[-1:] == '/' %}
+{%- set dep_dir_prefix = dep_rel %}
+{%- set dep_file_name = dep_rel[:-1] ~ '.py' %}
+{%- else %}
+{%- set dep_dir_prefix = dep_rel ~ '/' %}
+{%- set dep_file_name = dep_rel ~ '.py' %}
+{%- endif %}
+{%- for t in changed_tests %}
+{# Check if t starts with dep_dir_prefix (for directories) or equals dep_file_name (for files) #}
+{%- set prefix_len = dep_dir_prefix | length %}
+{%- set t_prefix = t[:prefix_len] %}
+{%- set cond1 = (t | length >= prefix_len and t_prefix == dep_dir_prefix) %}
+{%- set cond2 = (t == dep_file_name) %}
+{%- if cond1 or cond2 %}
+{%- set match_ns.targets = match_ns.targets + [t] %}
+{%- endif %}
+{%- endfor %}
+{%- endif %}
+{%- endfor %}
+{%- endif %}
+{%- set matched_targets = match_ns.targets %}
+
+{# If we have matched targets, run only those specific tests #}
+{% if matched_targets | length > 0 %}
+pytest -v -s {{ matched_targets | join(' ') }}
+{% else %}
+{# Default behavior: preserve original commands with optional coverage injection #}
+{% if cov_enabled %}
 {% set ns = namespace(has_pytest=false) %}
 {% if step.command %}
 {% if "pytest " in step.command %}{% set ns.has_pytest = true %}{% endif %}
@@ -42,6 +91,7 @@ COVERAGE_FILE={{ coverage_file }} {{ cmd | replace("pytest ", "pytest --cov=vllm
 {% else %}
 {{ step.command or (step.commands | join(' && ')) | safe }}
 {% endif %}
+{% endif %}
 {% endmacro %}
 
 {% macro render_cuda_config(step, image, default_working_dir, hf_home_fsx, hf_home, branch) %}