Install matter-yamltests files in matter namespace (project-chip#38534)

* Install matter-yamltests files in matter namespace

* Update imports in tests

* Update yamltests imports

* pytests: Include py_matter_idl in venv

Include matter.idl when pytest deps are requested.
Add pytest deps to venv used to run chip-repl tests.

Signed-off-by: Maciej Grela <[email protected]>

* Update tests requirements

* Install testing requirements in python-venv

* Add tests requirements to cache key

* Install matter-yamltests in cert-bins docker image

* Update matter-yamltests sources

---------

Signed-off-by: Maciej Grela <[email protected]>
Co-authored-by: Maciej Grela <[email protected]>

Author: arkq

.github/actions/bootstrap-cache/action.yaml

@@ -11,7 +11,7 @@ runs:
         attempt_limit: 3
         attempt_delay: 2000
         with: |
-          key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'scripts/py_matter_idl/*', 'third_party/pigweed/**', '/etc/lsb-release') }}
+          key: ${{ runner.os }}-env-${{ hashFiles('scripts/setup/*', 'scripts/tests/requirements.txt', 'scripts/py_matter_idl/*', 'third_party/pigweed/**', '/etc/lsb-release') }}
           path: |
               .environment
               build_overrides/pigweed_environment.gni

.github/actions/bootstrap/action.yaml

@@ -25,7 +25,7 @@ runs:
         # In addition to the various setup files, the work directory matters as well,
         # because the bootstrapped Pigweed environment contains absolute paths.
         echo "Calculating bootstrap cache key for '$PWD'"
-        FILES_HASH="${{ hashFiles('scripts/setup/*', 'scripts/py_matter_idl/*', 'third_party/pigweed/**') }}"
+        FILES_HASH="${{ hashFiles('scripts/setup/*', 'scripts/tests/requirements.txt', 'scripts/py_matter_idl/*', 'third_party/pigweed/**') }}"
         case "$RUNNER_OS" in
           macOS) OS_HASH="$(sw_vers | shasum -a 256 | cut -d' ' -f1)";;
           *)     OS_HASH="$(shasum -a 256 /etc/lsb-release | cut -d' ' -f1)";;

.github/workflows/tests.yaml

@@ -233,7 +233,11 @@ jobs:
                     "
             - name: Build Apps
               run: |
-                  scripts/run_in_build_env.sh './scripts/build_python.sh --install_virtual_env out/venv'
+                  ./scripts/run_in_build_env.sh \
+                     "./scripts/build_python.sh \
+                        --install_virtual_env out/venv \
+                        --include_pytest_deps yes \
+                     "
                   ./scripts/run_in_build_env.sh \
                      "./scripts/build/build_examples.py \
                         --target linux-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT} \

BUILD.gn

@@ -76,23 +76,23 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
     }
   }
 
-  # Matter's in-tree pw_python_package or pw_python_distribution targets.
-  _matter_python_packages = [
-    "//examples/chef",
-    "//examples/common/pigweed/rpc_console/py:chip_rpc",
-    "//integrations/mobly:chip_mobly",
-    "//scripts/py_matter_yamltests:matter_yamltests",
-    "//src/python_testing/matter_testing_infrastructure:chip-testing",
-  ]
-
   pw_python_venv("matter_build_venv") {
     path = "$root_build_dir/python-venv"
     constraints = pw_build_PIP_CONSTRAINTS
     requirements = pw_build_PIP_REQUIREMENTS
 
+    # Extra packages required for running tests.
+    requirements += [ "//scripts/tests/requirements.txt" ]
+
     # Packages available to import within GN's build venv.
-    source_packages = _matter_python_packages +
-                      [ "$dir_pw_env_setup:core_pigweed_python_packages" ]
+    # Matter's in-tree pw_python_package or pw_python_distribution targets.
+    source_packages = [
+      "//examples/chef",
+      "//examples/common/pigweed/rpc_console/py:chip_rpc",
+      "//integrations/mobly:chip_mobly",
+      "//src/python_testing/matter_testing_infrastructure:chip-testing",
+      "$dir_pw_env_setup:core_pigweed_python_packages",
+    ]
   }
 
   pw_python_pip_install("pip_install_matter_packages") {
@@ -122,6 +122,7 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
     if (chip_enable_python_modules) {
       deps = [
         "${chip_root}/scripts:matter_yamltests_distribution.wheel",
+        "${chip_root}/scripts/py_matter_yamltests:matter-yamltests.wheel",
         "${chip_root}/src/controller/python:chip-repl",
         "${chip_root}/src/python_testing/matter_testing_infrastructure:chip-testing.wheel",
       ]
@@ -252,8 +253,8 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
         "//:fake_platform_tests",
         "//examples/chef:chef.tests",
         "//scripts/build:build_examples.tests",
-        "//scripts/py_matter_idl:matter_idl.tests",
-        "//scripts/py_matter_yamltests:matter_yamltests.tests",
+        "//scripts/py_matter_idl:matter-idl.tests",
+        "//scripts/py_matter_yamltests:matter-yamltests.tests",
         "//src:tests_run",
         "//src/python_testing/matter_testing_infrastructure:chip-testing.tests",
 

integrations/docker/images/chip-cert-bins/Dockerfile

@@ -312,9 +312,6 @@ COPY --from=chip-build-cert-bins /root/connectedhomeip/out/python_env python_env
 COPY --from=chip-build-cert-bins /root/connectedhomeip/src/python_testing python_testing/scripts/sdk
 COPY --from=chip-build-cert-bins /root/connectedhomeip/data_model python_testing/data_model
 
-COPY --from=chip-build-cert-bins /root/connectedhomeip/scripts/tests/requirements.txt /tmp/requirements.txt
-RUN pip install --break-system-packages -r /tmp/requirements.txt && rm /tmp/requirements.txt
-
 COPY --from=chip-build-cert-bins /root/connectedhomeip/src/python_testing/requirements.txt /tmp/requirements.txt
 RUN pip install --break-system-packages -r /tmp/requirements.txt && rm /tmp/requirements.txt
 
@@ -333,6 +330,8 @@ RUN set -x && DEBIAN_FRONTEND=noninteractive apt-get update; apt-get install -fy
 RUN apt-get remove -y python3-gi
 
 RUN pip install --break-system-packages --no-cache-dir \
+    python_lib/python/obj/scripts/py_matter_idl/matter-idl._build_wheel/matter_idl-*.whl \
+    python_lib/python/obj/scripts/py_matter_yamltests/matter-yamltests._build_wheel/matter_yamltests-*.whl \
     python_lib/obj/src/python_testing/matter_testing_infrastructure/chip-testing._build_wheel/chip_testing-*.whl \
     python_lib/controller/python/chip*.whl
 

scripts/BUILD.gn

@@ -23,7 +23,7 @@ import("$dir_pw_build/python_dist.gni")
 #   out/obj/scripts/matter_yamltests_distribution/  <- source files here
 #   out/obj/scripts/matter_yamltests_distribution._build_wheel/matter_yamltests-0.0.1-py3-none-any.whl
 pw_python_distribution("matter_yamltests_distribution") {
-  packages = [ "${chip_root}/scripts/py_matter_yamltests:matter_yamltests" ]
+  packages = [ "${chip_root}/scripts/py_matter_yamltests:matter-yamltests" ]
   generate_setup_cfg = {
     name = "matter_yamltests"
     version = "0.0.1"

scripts/build_python.sh

@@ -187,8 +187,12 @@ WHEEL=("$OUTPUT_ROOT"/controller/python/chip*.whl)
 WHEEL+=("$OUTPUT_ROOT"/obj/src/python_testing/matter_testing_infrastructure/chip-testing._build_wheel/chip_testing*.whl)
 
 if [ "$install_pytest_requirements" = "yes" ]; then
-    # Add the matter_yamltests_distribution wheel
-    WHEEL+=("$OUTPUT_ROOT"/obj/scripts/matter_yamltests_distribution._build_wheel/matter_yamltests-*.whl)
+    # Add wheels with YAML testing support.
+    WHEEL+=(
+        # Add matter-idl as well as matter-yamltests depends on it.
+        "$OUTPUT_ROOT"/python/obj/scripts/py_matter_idl/matter-idl._build_wheel/matter_idl-*.whl
+        "$OUTPUT_ROOT"/python/obj/scripts/py_matter_yamltests/matter-yamltests._build_wheel/matter_yamltests-*.whl
+    )
 fi
 
 if [ -n "$extra_packages" ]; then

scripts/py_matter_idl/BUILD.gn

@@ -20,7 +20,7 @@ import("$dir_pw_build/python.gni")
 
 import("${chip_root}/scripts/py_matter_idl/files.gni")
 
-pw_python_package("matter_idl") {
+pw_python_package("matter-idl") {
   setup = [
     "setup.py",
     "setup.cfg",

scripts/py_matter_yamltests/BUILD.gn

@@ -18,38 +18,48 @@ import("//build_overrides/chip.gni")
 import("//build_overrides/pigweed.gni")
 import("$dir_pw_build/python.gni")
 
-pw_python_package("matter_yamltests") {
+pw_python_package("matter-yamltests") {
   setup = [
     "setup.py",
     "setup.cfg",
     "pyproject.toml",
   ]
 
   sources = [
-    "matter_yamltests/__init__.py",
-    "matter_yamltests/adapter.py",
-    "matter_yamltests/constraints.py",
-    "matter_yamltests/definitions.py",
-    "matter_yamltests/errors.py",
-    "matter_yamltests/fixes.py",
-    "matter_yamltests/hooks.py",
-    "matter_yamltests/parser.py",
-    "matter_yamltests/parser_builder.py",
-    "matter_yamltests/parser_config.py",
-    "matter_yamltests/pics_checker.py",
-    "matter_yamltests/pseudo_clusters/__init__.py",
-    "matter_yamltests/pseudo_clusters/clusters/delay_commands.py",
-    "matter_yamltests/pseudo_clusters/clusters/log_commands.py",
-    "matter_yamltests/pseudo_clusters/clusters/system_commands.py",
-    "matter_yamltests/pseudo_clusters/pseudo_cluster.py",
-    "matter_yamltests/pseudo_clusters/pseudo_clusters.py",
-    "matter_yamltests/py.typed",
-    "matter_yamltests/runner.py",
-    "matter_yamltests/websocket_runner.py",
-    "matter_yamltests/yaml_loader.py",
+    "matter/yamltests/__init__.py",
+    "matter/yamltests/adapter.py",
+    "matter/yamltests/constraints.py",
+    "matter/yamltests/definitions.py",
+    "matter/yamltests/errors.py",
+    "matter/yamltests/fixes.py",
+    "matter/yamltests/hooks.py",
+    "matter/yamltests/parser.py",
+    "matter/yamltests/parser_builder.py",
+    "matter/yamltests/parser_config.py",
+    "matter/yamltests/pics_checker.py",
+    "matter/yamltests/pseudo_clusters/__init__.py",
+    "matter/yamltests/pseudo_clusters/clusters/__init__.py",
+    "matter/yamltests/pseudo_clusters/clusters/accessory_server_bridge.py",
+    "matter/yamltests/pseudo_clusters/clusters/commissioner_commands.py",
+    "matter/yamltests/pseudo_clusters/clusters/delay_commands.py",
+    "matter/yamltests/pseudo_clusters/clusters/discovery_commands.py",
+    "matter/yamltests/pseudo_clusters/clusters/equality_commands.py",
+    "matter/yamltests/pseudo_clusters/clusters/log_commands.py",
+    "matter/yamltests/pseudo_clusters/clusters/system_commands.py",
+    "matter/yamltests/pseudo_clusters/clusters/webrtc.py",
+    "matter/yamltests/pseudo_clusters/pseudo_cluster.py",
+    "matter/yamltests/pseudo_clusters/pseudo_clusters.py",
+    "matter/yamltests/runner.py",
+    "matter/yamltests/websocket_runner.py",
+    "matter/yamltests/yaml_loader.py",
   ]
 
-  python_deps = [ "${chip_root}/scripts/py_matter_idl:matter_idl" ]
+  inputs = [
+    # Marker file to indicate to mypy that this package is type-annotated.
+    "matter/yamltests/py.typed",
+  ]
+
+  python_deps = [ "${chip_root}/scripts/py_matter_idl:matter-idl" ]
 
   tests = [
     "test_spec_definitions.py",

scripts/py_matter_yamltests/generate_pseudo_cluster_doc_tables.py

@@ -15,7 +15,7 @@
 import os
 import xml.etree.ElementTree as ElementTree
 
-from matter_yamltests.pseudo_clusters.pseudo_clusters import get_default_pseudo_clusters
+from matter.yamltests.pseudo_clusters.pseudo_clusters import get_default_pseudo_clusters
 
 SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
 WARNING = ("<!---\n"