[TO REVIEW][nrf noup][ZAP] Imporove west zap commands

* Now we can obtain the recommended zap version from the
`scripts/setup/zap.version` file instead of reading it from
the `zap_execution.py` file. Sometimes the nightly version is
recommended whereas in the python file there is no `-nightly`
postfix.

* If the sandbox is not configured on a Linux PC, an annoying
issue with the sandbox occurs after updating to the newest ZAP
tool.
To resolve it, ask the  user if they want to add permissions
automatically.

* `update_zcl_in_zap` function must be called only while
the specific clusters are known.

* Added the new zap-update command to synchronize the .zap and
zcl.json files with the new Matter Data Model from SDK.

Author: ArekBalysNordic

scripts/west/zap_append.py

@@ -12,19 +12,20 @@
 from zap_common import DEFAULT_MATTER_PATH, DEFAULT_MATTER_TYPES_RELATIVE_PATH, DEFAULT_ZCL_JSON_RELATIVE_PATH
 
 
-def add_custom_attributes_from_xml(xml_file: Path, zcl_data: dict):
+def add_custom_attributes_from_xml(xml_file: Path, zcl_data: dict, matter_path: Path = DEFAULT_MATTER_PATH):
     """
     Parse the cluster XML file and add attributes with custom types to
     attributeAccessInterfaceAttributes in zcl_data.
 
     Args:
         cluster_xml_path: Path to the cluster XML file
         zcl_data: The loaded zcl.json data dictionary
+        matter_path: Path to the Matter directory
     """
 
     # Step 1: Load all type names from chip-types.xml into a list
     types = []
-    tree = ET.parse(DEFAULT_MATTER_PATH / DEFAULT_MATTER_TYPES_RELATIVE_PATH)
+    tree = ET.parse(matter_path / DEFAULT_MATTER_TYPES_RELATIVE_PATH)
     root = tree.getroot()
 
     for type_element in root.findall('.//type'):
@@ -82,7 +83,7 @@ def add_custom_attributes_from_xml(xml_file: Path, zcl_data: dict):
     return modified
 
 
-def add_cluster_to_zcl(zcl_base: Path, cluster_xml_paths: list, output: Path):
+def add_cluster_to_zcl(zcl_base: Path, cluster_xml_paths: list, output: Path, matter_path: Path = DEFAULT_MATTER_PATH):
     """
     Add the cluster to the ZCL file.
     """
@@ -135,7 +136,7 @@ def add_cluster_to_zcl(zcl_base: Path, cluster_xml_paths: list, output: Path):
             log.dbg(f"Successfully added {file}")
 
         # Add custom attributes from the XML file to the ZCL file
-        add_custom_attributes_from_xml(Path(cluster), zcl_json)
+        add_custom_attributes_from_xml(Path(cluster), zcl_json, matter_path)
 
     # If output file is not provided, we will edit the existing ZCL file
     file_to_write = output if output else zcl_base
@@ -163,7 +164,7 @@ def do_add_parser(self, parser_adder) -> argparse.ArgumentParser:
                             help=f"An absolute path to the Matter directory. If not set the path with be set to the {DEFAULT_MATTER_PATH}")
         parser.add_argument("-o", "--output", type=Path,
                             help=f"Output path to store the generated zcl.json file. If not provided the path will be set to the base zcl.json file (MATTER/{DEFAULT_ZCL_JSON_RELATIVE_PATH}).")
-        parser.add_argument("new_clusters", nargs='+',
+        parser.add_argument("--clusters", nargs='+',
                             help="Paths to the XML files that contain the custom cluster definitions")
         return parser
 
@@ -173,9 +174,9 @@ def do_run(self, args, unknown_args) -> None:
         if not args.output:
             args.output = args.matter.joinpath(DEFAULT_ZCL_JSON_RELATIVE_PATH)
 
-        for cluster in args.new_clusters:
+        for cluster in args.clusters:
             if not Path(cluster).exists():
                 log.err(f"No such cluster file: {cluster}")
                 return
 
-        add_cluster_to_zcl(args.base.absolute(), args.new_clusters, args.output.absolute())
+        add_cluster_to_zcl(args.base.absolute(), args.clusters, args.output.absolute())

scripts/west/zap_common.py

@@ -159,7 +159,7 @@ def post_process_generated_files(output_path: Path):
                 continue
 
 
-def synchronize_zcl_with_base(zcl_json: Path):
+def synchronize_zcl_with_base(zcl_json: Path, matter_path: Path = DEFAULT_MATTER_PATH):
     """
     Synchronizes a zcl.json file with the base/default zcl.json from Matter SDK.
 
@@ -170,15 +170,15 @@ def synchronize_zcl_with_base(zcl_json: Path):
 
     print(f"Synchronizing {zcl_json} with base zcl.json")
 
-    base_zcl_path = DEFAULT_MATTER_PATH / DEFAULT_ZCL_JSON_RELATIVE_PATH
+    base_zcl_path = matter_path / DEFAULT_ZCL_JSON_RELATIVE_PATH
     with open(base_zcl_path, 'r') as f:
         base_zcl_data = json.load(f)
 
     with open(zcl_json, 'r') as f:
         target_zcl_data = json.load(f)
 
     modified = False
-    fields_to_skip = {'xmlRoot'}  # Fields to skip in comparison
+    fields_to_skip = {'xmlRoot', 'manufacturersXml'}  # Fields to skip in comparison
 
     for key, value in base_zcl_data.items():
         if key in fields_to_skip:
@@ -188,6 +188,10 @@ def synchronize_zcl_with_base(zcl_json: Path):
             target_zcl_data[key] = value
             modified = True
             print(f"Added missing field: '{key}'")
+        if value != target_zcl_data[key]:
+            target_zcl_data[key] = value
+            modified = True
+            print(f"Updated field: '{key}'")
 
     if modified:
         with open(zcl_json, 'w') as f:
@@ -199,9 +203,23 @@ def synchronize_zcl_with_base(zcl_json: Path):
     print("Done")
 
 
+def fix_sandbox_permissions(e: subprocess.CalledProcessError) -> None:
+    """
+    Fix sandbox permissions if needed.
+    """
+    if e.returncode == -5:
+        log.inf("\nThe wrong sandbox permissions are used. Do you wanto to add the permissions to the sandbox?")
+        answer = input("y/n: ")
+        if answer == "y":
+            subprocess.check_call(['sudo', 'chown', 'root', str(
+                self.get_install_path() / 'chrome-sandbox')])
+            subprocess.check_call(['sudo', 'chmod', '4755', str(self.get_install_path() / 'chrome-sandbox')])
+            log.inf("Permissions added to the sandbox")
+
+
 class ZapInstaller:
     INSTALL_DIR = Path('.zap-install')
-    ZAP_URL_PATTERN = 'https://github.com/project-chip/zap/releases/download/v%04d.%02d.%02d/%s.zip'
+    ZAP_URL_PATTERN = 'https://github.com/project-chip/zap/releases/download/%s/%s.zip'
 
     def __init__(self, matter_path: Path):
         self.matter_path = matter_path
@@ -261,22 +279,20 @@ def get_zap_cli_path(self) -> Path:
         """
         return self.install_path / self.zap_cli_exe
 
-    def get_recommended_version(self) -> Tuple[int, int, int]:
+    def get_recommended_version(self) -> str:
         """
         Returns ZAP package recommended version as a tuple of integers.
 
-        Parses zap_execution.py script from Matter SDK to determine the minimum
-        required ZAP package version.
+        Reads the version from zap.version file in Matter SDK.
+        Expected format: v{YEAR}.{MONTH}.{DAY}[-suffix]
+        Example: v2025.09.23-nightly
         """
-        RE_MIN_ZAP_VERSION = r'MIN_ZAP_VERSION\s*=\s*\'(\d+)\.(\d+)\.(\d+)'
-        zap_execution_path = self.matter_path / 'scripts/tools/zap/zap_execution.py'
+        zap_version_path = self.matter_path / 'scripts/setup/zap.version'
 
-        with open(zap_execution_path, 'r') as f:
-            if match := re.search(RE_MIN_ZAP_VERSION, f.read()):
-                return tuple(int(group) for group in match.groups())
-            raise RuntimeError(f'Failed to find MIN_ZAP_VERSION in {zap_execution_path}')
+        with open(zap_version_path, 'r') as f:
+            return f.read().strip()
 
-    def get_current_version(self) -> Tuple[int, int, int]:
+    def get_current_version(self) -> str:
         """
         Returns ZAP package current version as a tuple of integers.
 
@@ -292,7 +308,7 @@ def get_current_version(self) -> Tuple[int, int, int]:
 
         RE_VERSION = r'Version:\s*(\d+)\.(\d+)\.(\d+)'
         if match := re.search(RE_VERSION, output):
-            return tuple(int(group) for group in match.groups())
+            return match.group(1) + '.' + match.group(2) + '.' + match.group(3)
 
         raise RuntimeError("Failed to find version in ZAP output")
 
@@ -301,7 +317,7 @@ def install_zap(self, version: Tuple[int, int, int]) -> None:
         Downloads and unpacks selected ZAP package version.
         """
         with tempfile.TemporaryDirectory() as temp_dir:
-            url = ZapInstaller.ZAP_URL_PATTERN % (*version, self.package)
+            url = ZapInstaller.ZAP_URL_PATTERN % (version, self.package)
             log.inf(f'Downloading {url}...')
             zip_file_path = str(Path(temp_dir).joinpath(f'{self.package}.zip'))
 
@@ -343,15 +359,24 @@ def update_zap_if_needed(self) -> None:
         recommended_version = self.get_recommended_version()
         current_version = self.get_current_version()
 
-        log.inf(f'ZAP installation directory: {self.install_path}')
+        # Extract version without prefix and suffix for comparison
+        # recommended_version format: v2025.09.23-nightly or v2025.09.23
+        # current_version format: 2025.9.23
+        recommended_version_clean = recommended_version
+        if match := re.search(r'v?(\d+\.\d+\.\d+)', recommended_version):
+            year, month, day = match.group(1).split('.')
+            recommended_version_clean = f"{int(year)}.{int(month)}.{int(day)}"
+
+            log.inf(f'ZAP installation directory: {self.install_path}')
+            log.inf(f"Current ZAP version: {current_version}")
 
-        if current_version:
-            verdict = 'up to date' if current_version == recommended_version else 'outdated'
-            log.inf('Found ZAP {}.{}.{} ({})'.format(*current_version, verdict))
+            if current_version:
+                verdict = 'up to date' if current_version == recommended_version_clean else 'outdated'
+                log.inf('Found ZAP {} ({})'.format(current_version, verdict))
 
-        if current_version != recommended_version:
-            log.inf('Installing ZAP {}.{}.{}'.format(*recommended_version))
-            self.install_zap(recommended_version)
+            if current_version != recommended_version_clean:
+                log.inf('Installing ZAP {}'.format(recommended_version))
+                self.install_zap(recommended_version)
 
     @staticmethod
     def set_exec_permission(path: Path) -> None:

scripts/west/zap_generate.py

@@ -15,6 +15,7 @@
 from west.commands import CommandError, WestCommand
 from zap_common import (DEFAULT_MATTER_PATH, ZapInstaller, existing_dir_path, existing_file_path, find_zap,
                         post_process_generated_files, synchronize_zcl_with_base, update_zcl_in_zap)
+from zap_update import ZapUpdate
 
 # fmt: off
 scripts_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -106,6 +107,23 @@ def do_run(self, args, unknown_args):
                         if zcl_file and not zcl_file.exists():
                             raise CommandError(f"ZCL file {zcl_file} does not exist")
                         full = zap.get('full', False)
+                        clusters = None
+
+                        if full:
+                            clusters = zap.get('clusters', [])
+                            if not clusters:
+                                raise CommandError("Clusters are not set in the yaml file for the full build")
+
+                        # Prepare arguments for ZapUpdate
+                        zap_update_args = argparse.Namespace(
+                            zap_file=zap_file,
+                            zcl_json=zcl_file,
+                            clusters=clusters,
+                            matter_path=args.matter_path
+                        )
+
+                        # Run zap_update to update and synchronize zap/zcl with clusters using the current Matter SDK
+                        ZapUpdate().do_run(zap_update_args, [])
 
                         zap_entry = ZapFile(name=name, zap_file=zap_file, full=full, zcl_file=zcl_file)
                         zap_files.append(zap_entry)
@@ -161,10 +179,6 @@ def do_run(self, args, unknown_args):
             # Generate .matter file
             self.check_call(self.build_command(zap.zap_file, output_path))
 
-            # Update the zcl in zap file if needed
-            # We need to do this in case the zap gui was not called before.
-            update_zcl_in_zap(zap.zap_file, zcl_file, app_templates_path)
-
             if args.full:
                 # Full build is about generating an apropertiate Matter data model files in a specific directory layout.
                 # Currently, we must align to the following directory layout:
@@ -200,10 +214,9 @@ def do_run(self, args, unknown_args):
                 zap_output_dir = output_path / 'app-common' / 'zap-generated'
                 codegen_output_dir = output_path / 'clusters'
 
-                # Synchronize the zcl.json file with the base zcl.json file
-                # We need to do this to update the zcl.json file with the new clusters and attributes.
-                # It may be helpful if the Matter SDK was updated.
-                synchronize_zcl_with_base(zcl_file)
+                # Update the zcl in zap file if needed
+                # We need to do this in case the zap gui was not called before.
+                update_zcl_in_zap(zap.zap_file, zcl_file, app_templates_path)
 
                 # Temporarily change directory to matter_path so JinjaCodegenTarget and ZAPGenerateTarget can find their scripts
                 original_cwd = os.getcwd()

scripts/west/zap_gui.py

@@ -3,14 +3,18 @@
 # SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
 
 import argparse
+import os
+import pwd
+import grp
 from pathlib import Path
+import subprocess
 
 from textwrap import dedent
 
 from west.commands import WestCommand
 from west import log
 
-from zap_common import existing_file_path, existing_dir_path, find_zap, ZapInstaller, DEFAULT_MATTER_PATH, DEFAULT_ZCL_JSON_RELATIVE_PATH, DEFAULT_APP_TEMPLATES_RELATIVE_PATH, update_zcl_in_zap
+from zap_common import existing_file_path, existing_dir_path, find_zap, ZapInstaller, DEFAULT_MATTER_PATH, DEFAULT_ZCL_JSON_RELATIVE_PATH, DEFAULT_APP_TEMPLATES_RELATIVE_PATH, update_zcl_in_zap, synchronize_zcl_with_base, fix_sandbox_permissions
 from zap_append import add_cluster_to_zcl
 
 
@@ -46,9 +50,8 @@ def do_add_parser(self, parser_adder):
 
     def do_run(self, args, unknown_args):
         default_zcl_path = args.matter_path.joinpath(DEFAULT_ZCL_JSON_RELATIVE_PATH)
-
-        zap_file_path = args.zap_file or find_zap()
         zcl_json_path = Path(args.zcl_json).absolute() if args.zcl_json else default_zcl_path
+        zap_file_path = args.zap_file or find_zap()
 
         if zap_file_path is None:
             log.err("ZAP file not found!")
@@ -58,8 +61,10 @@ def do_run(self, args, unknown_args):
             # If the user provided the clusters and the zcl.json file provided by -j argument does not exist
             # we will create a new zcl.json file according to the base zcl.json file in default_zcl_path.
             # If the provided zcl.json file exists, we will use it as a base and update with a new cluster.
-            base_zcl = zcl_json_path if zcl_json_path.exists() else default_zcl_path
-            add_cluster_to_zcl(base_zcl, args.clusters, zcl_json_path)
+
+            # Add the new cluster to the zcl.json file
+            add_cluster_to_zcl(zcl_json_path, args.clusters, zcl_json_path, args.matter_path)
+
         elif not zcl_json_path.exists():
             # If clusters are not provided, but user provided a zcl.json file we need to check whether the file exists.
             log.err(f"ZCL file not found: {zcl_json_path}")
@@ -89,4 +94,8 @@ def do_run(self, args, unknown_args):
             cmd += ["--stateDirectory", args.cache.absolute()]
         else:
             cmd += ["--tempState"]
-        self.check_call([str(x) for x in cmd])
+
+        try:
+            self.check_call([str(x) for x in cmd])
+        except subprocess.CalledProcessError as e:
+            fix_sandbox_permissions(e)

scripts/west/zap_update.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2025 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
+
+import argparse
+import os
+import pwd
+import grp
+from pathlib import Path
+import subprocess
+
+from textwrap import dedent
+
+from west.commands import WestCommand
+from west import log
+
+from zap_common import existing_file_path, existing_dir_path, find_zap, DEFAULT_MATTER_PATH, DEFAULT_ZCL_JSON_RELATIVE_PATH, DEFAULT_APP_TEMPLATES_RELATIVE_PATH, synchronize_zcl_with_base, ZapInstaller, fix_sandbox_permissions
+from zap_append import add_cluster_to_zcl
+
+
+class ZapUpdate(WestCommand):
+    def __init__(self):
+        super().__init__(
+            'zap-update',
+            'Update the ZAP file with the new clusters',
+            dedent('''
+            Update the ZAP file with the new clusters'''))
+
+    def do_add_parser(self, parser_adder):
+        parser = parser_adder.add_parser(self.name,
+                                         help=self.help,
+                                         formatter_class=argparse.RawDescriptionHelpFormatter,
+                                         description=self.description)
+        parser.add_argument('-z', '--zap-file', type=existing_file_path,
+                            help='Path to data model configuration file (*.zap)')
+        parser.add_argument('-j', '--zcl-json', type=str,
+                            help='Path to data model definition file (zcl.json). If new clusters are added using --clusters, the new zcl.json file will be created and used.')
+        parser.add_argument('-c', '--clusters', nargs='+',
+                            help="Paths to the XML files that contain the external cluster definitions")
+        parser.add_argument('-m', '--matter-path', type=existing_dir_path,
+                            default=DEFAULT_MATTER_PATH, help=f'Path to Matter SDK. Default is set to {DEFAULT_MATTER_PATH}')
+        return parser
+
+    def do_run(self, args, unknown_args):
+
+        zap_file_path = args.zap_file or find_zap()
+
+        if zap_file_path is None:
+            log.err("ZAP file not found!")
+            return
+
+        zcl_file_path = args.zcl_json or args.matter_path.joinpath(DEFAULT_ZCL_JSON_RELATIVE_PATH).absolute()
+        if not zcl_file_path.exists():
+            raise CommandError(f"ZCL file not found: {zcl_file_path}")
+
+        app_templates_path = args.matter_path.joinpath(DEFAULT_APP_TEMPLATES_RELATIVE_PATH)
+
+        zap_installer = ZapInstaller(args.matter_path)
+        zap_installer.update_zap_if_needed()
+
+        # Update the .zap file with with the all changes from the Matter SDK
+        cmd = [zap_installer.get_zap_cli_path()]
+        cmd += ["convert"]
+        cmd += [zap_file_path]
+        cmd += ["--zcl", zcl_file_path]
+        cmd += ["--gen", app_templates_path]
+        cmd += ["--out", zap_file_path]
+        cmd += ["--tempState"]
+
+        try:
+            self.check_call([str(x) for x in cmd])
+        except subprocess.CalledProcessError as e:
+            fix_sandbox_permissions(e)
+
+        # zcl.json file was provided, so synchronize it with the Matter SDK
+        if args.zcl_json:
+            if not args.clusters:
+                raise CommandError("Clusters are not provided, so the zcl.json file cannot be synchronized")
+            synchronize_zcl_with_base(zcl_file_path, args.matter_path)
+
+            # Add the new clusters again to the zcl.json file
+            add_cluster_to_zcl(zcl_file_path, args.clusters, zcl_file_path, args.matter_path)