Cherry-pick [2.7] Enhancements on CI install_requirements (#4249) (#4283)

Make integration tests' install_requirements more robust

<!--- Put an `x` in all the boxes that apply, and remove the not
applicable items -->
- [x] Non-breaking change (fix or new feature that would not break
existing functionality).
- [ ] Breaking change (fix or new feature that would cause existing
functionality to change).
- [ ] New tests added to cover the changes.
- [ ] Quick tests passed locally by running `./runtest.sh`.
- [ ] In-line docstrings updated.
- [ ] Documentation updated.

---------

Fixes # .

### Description

A few sentences describing the changes proposed in this pull request.

### Types of changes
<!--- Put an `x` in all the boxes that apply, and remove the not
applicable items -->
- [x] Non-breaking change (fix or new feature that would not break
existing functionality).
- [ ] Breaking change (fix or new feature that would cause existing
functionality to change).
- [ ] New tests added to cover the changes.
- [ ] Quick tests passed locally by running `./runtest.sh`.
- [ ] In-line docstrings updated.
- [ ] Documentation updated.

---------

Co-authored-by: Peter Cnudde <pcnudde@nvidia.com>
Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

Author: 3 people

tests/integration_test/install_requirements.py

@@ -22,8 +22,11 @@
 """
 
 import argparse
+import os
+import re
 import subprocess
 import sys
+import tempfile
 
 
 def main():
@@ -42,25 +45,43 @@ def main():
     with open(args.requirements_file) as f:
         lines = f.readlines()
 
-    packages = []
+    filtered = []
     for line in lines:
         stripped = line.strip()
         if not stripped or stripped.startswith("#"):
             continue
-        pkg_lower = stripped.lower()
-        if any(pkg_lower.startswith(ex) for ex in excluded):
-            continue
-        packages.append(stripped)
+        if excluded:
+            # Extract the package specifier before any inline comment for exclude matching
+            spec = re.split(r"\s+#", stripped)[0].strip()
+            if any(spec.lower().startswith(ex) for ex in excluded):
+                continue
+        filtered.append(line)
 
-    if not packages:
+    if not filtered:
         print("No packages to install.")
         return
 
-    cmd = [sys.executable, "-m", "pip", "install"] + packages
-    if args.quiet:
-        cmd.append("--quiet")
+    req_dir = os.path.dirname(os.path.abspath(args.requirements_file))
+    fd, tmp_path = tempfile.mkstemp(suffix=".txt", dir=req_dir)
+    result = 1
+    try:
+        try:
+            with os.fdopen(fd, "w") as tmp:
+                tmp.writelines(filtered)
+        except Exception:
+            os.close(fd)
+            raise
+        cmd = [sys.executable, "-m", "pip", "install", "-r", tmp_path]
+        if args.quiet:
+            cmd.append("--quiet")
+        result = subprocess.call(cmd)
+    finally:
+        try:
+            os.unlink(tmp_path)
+        except FileNotFoundError:
+            pass
 
-    sys.exit(subprocess.call(cmd))
+    sys.exit(result)
 
 
 if __name__ == "__main__":

tests/integration_test/src/provision_site_launcher.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+import shlex
 import shutil
 import tempfile
 import time
@@ -34,14 +35,16 @@
 
 
 def _start_site(site_properties: SiteProperties):
-    process = run_command_in_subprocess(f"bash {os.path.join(site_properties.root_dir, 'startup', 'start.sh')}")
+    process = run_command_in_subprocess(
+        f"bash {shlex.quote(os.path.join(site_properties.root_dir, 'startup', 'start.sh'))}"
+    )
     print(f"Starting {site_properties.name} ...")
     site_properties.process = process
 
 
 def _stop_site(site_properties: SiteProperties):
     run_command_in_subprocess(
-        f"bash {os.path.join(site_properties.root_dir, 'startup', 'stop_fl.sh')}", stdin_data=b"y\n"
+        f"bash {shlex.quote(os.path.join(site_properties.root_dir, 'startup', 'stop_fl.sh'))}", stdin_data=b"y\n"
     )
     print(f"Stopping {site_properties.name} ...")
 

tests/integration_test/src/utils.py

@@ -66,15 +66,17 @@ def run_command_in_subprocess(command, stdin_data=None):
     new_env = os.environ.copy()
     python_path = ":".join(sys.path)[1:]  # strip leading colon
     new_env["PYTHONPATH"] = python_path
+    tokens = [os.path.expandvars(os.path.expanduser(t)) for t in shlex.split(command)]
     process = subprocess.Popen(
-        shlex.split(command),
+        tokens,
         stdin=subprocess.PIPE if stdin_data else None,
         preexec_fn=os.setsid,
         env=new_env,
     )
     if stdin_data:
-        process.stdin.write(stdin_data)
-        process.stdin.close()
+        # communicate() writes stdin and waits for the process to exit.
+        # stdout/stderr are inherited from the parent process (not captured).
+        process.communicate(input=stdin_data)
     return process