[Backport v4.2-branch] scripts: west_commands: packages: Execute pip install in new command

doc/develop/beyond-GSG.rst

@@ -136,16 +136,39 @@ Keeping Zephyr updated
 To update the Zephyr project source code, you need to get the latest
 changes via ``git``. Afterwards, run ``west update`` as mentioned in
 the previous paragraph.
-Additionally, in the case of updated or added Python dependencies, running
-``west packages pip --install`` will make sure these are up-to-date.
+Additionally, check for updated or added Python dependencies.
 
-.. code-block:: console
+.. tabs::
 
-   # replace zephyrproject with the path you gave west init
-   cd zephyrproject/zephyr
-   git pull
-   west update
-   west packages pip --install
+   .. group-tab:: Linux/macOS
+
+      .. code-block:: console
+
+         # replace zephyrproject with the path you gave west init
+         cd zephyrproject/zephyr
+         git pull
+         west update
+         west packages pip --install
+
+   .. group-tab:: Windows
+
+      .. tabs::
+
+         .. code-tab:: bat
+
+            :: replace zephyrproject with the path you gave west init
+            cd zephyrproject\zephyr
+            git pull
+            west update
+            cmd /c scripts\utils\west-packages-pip-install.cmd
+
+         .. code-tab:: powershell
+
+            # replace zephyrproject with the path you gave west init
+            cd zephyrproject\zephyr
+            git pull
+            west update
+            python -m pip install @((west packages pip) -split ' ')
 
 Export Zephyr CMake package
 ***************************

doc/develop/getting_started/index.rst

@@ -267,6 +267,10 @@ chosen. You'll also install Zephyr's additional Python dependencies in a
 
             west packages pip --install
 
+         .. note::
+
+            This could downgrade or upgrade west itself.
+
    .. group-tab:: macOS
 
       #. Create a new virtual environment:
@@ -319,6 +323,10 @@ chosen. You'll also install Zephyr's additional Python dependencies in a
 
             west packages pip --install
 
+         .. note::
+
+            This could downgrade or upgrade west itself.
+
    .. group-tab:: Windows
 
       #. Open a ``cmd.exe`` or PowerShell terminal window **as a regular user**
@@ -383,9 +391,19 @@ chosen. You'll also install Zephyr's additional Python dependencies in a
       #. The Zephyr west extension command, ``west packages`` can be used to install Python
          dependencies.
 
-         .. code-block:: bat
+         .. tabs::
 
-            west packages pip --install
+            .. code-tab:: bat
+
+               cmd /c scripts\utils\west-packages-pip-install.cmd
+
+            .. code-tab:: powershell
+
+               python -m pip install @((west packages pip) -split ' ')
+
+         .. note::
+
+            This could downgrade or upgrade west itself.
 
 Install the Zephyr SDK
 **********************

scripts/utils/west-packages-pip-install.cmd

@@ -0,0 +1,27 @@
+:: SPDX-FileCopyrightText: Copyright The Zephyr Project Contributors
+:: SPDX-License-Identifier: Apache-2.0
+
+@echo off
+rem Collect packages from west and install them with a single pip call.
+setlocal enabledelayedexpansion
+
+set "PACKAGES="
+
+for /f "usebackq delims=" %%p in (`west packages pip`) do (
+	if defined PACKAGES (
+		set "PACKAGES=!PACKAGES! %%p"
+	) else (
+		set "PACKAGES=%%p"
+	)
+)
+
+if not defined PACKAGES (
+	echo west packages pip returned no packages to install.
+	exit /b 0
+)
+
+echo Installing packages with: python.exe -m pip install %PACKAGES%
+python.exe -m pip install %PACKAGES%
+set "RESULT=%ERRORLEVEL%"
+
+endlocal & exit /b %RESULT%

scripts/west_commands/packages.py

@@ -4,13 +4,15 @@
 
 import argparse
 import os
+import platform
 import subprocess
 import sys
 import textwrap
 from itertools import chain
-from pathlib import Path
+from pathlib import Path, PureWindowsPath
 
 from west.commands import WestCommand
+from west.util import quote_sh_list
 from zephyr_ext_common import ZEPHYR_BASE
 
 sys.path.append(os.fspath(Path(__file__).parent.parent))
@@ -157,11 +159,38 @@ def do_run_pip(self, args, manager_args):
                 self.die("Running pip install outside of a virtual environment")
 
             if len(requirements) > 0:
-                subprocess.check_call(
-                    [sys.executable, "-m", "pip", "install"]
-                    + list(chain.from_iterable([("-r", r) for r in requirements]))
-                    + manager_args
+                cmd = [sys.executable, "-m", "pip", "install"]
+                cmd += chain.from_iterable([("-r", str(r)) for r in requirements])
+                cmd += manager_args
+                self.dbg(quote_sh_list(cmd))
+
+                # Use os.execv to execute a new program, replacing the current west process,
+                # this unloads all python modules first and allows for pip to update packages safely
+                if platform.system() != 'Windows':
+                    os.execv(cmd[0], cmd)
+
+                # Only reachable on Windows systems
+                # Windows does not really support os.execv:
+                # https://github.com/python/cpython/issues/63323
+                # https://github.com/python/cpython/issues/101191
+                # Warn the users about permission errors as those reported in:
+                # https://github.com/zephyrproject-rtos/zephyr/issues/100296
+                cmdscript = (
+                    PureWindowsPath(__file__).parents[1] / "utils" / "west-packages-pip-install.cmd"
                 )
+                self.wrn(
+                    "Updating packages on Windows with 'west packages pip --install', that are "
+                    "currently in use by west, results in permission errors. Leaving your "
+                    "environment with conflicting package versions. Recommended is to start with "
+                    "a new environment in that case.\n\n"
+                    "To avoid this using powershell run the following command instead:\n"
+                    f"{sys.executable} -m pip install @((west packages pip) -split ' ')\n\n"
+                    "Using cmd.exe execute the helper script:\n"
+                    f"cmd /c {cmdscript}\n\n"
+                    "Running 'west packages pip --install -- --dry-run' can provide information "
+                    "without actually updating the environment."
+                )
+                subprocess.check_call(cmd)
             else:
                 self.inf("Nothing to install")
             return