python3: do not rely on hostdeps python3-pip nor python3-dev; deploy pip via get-pip.py

- python3-pip implies a very old setuptools (which is system-wide and takes precedence)
- python3-dev implies python3-distutils (which is long deprecated)
- get-pip.py allows us to version pip in requirements.txt just like everything else
- in the end this fixes the conundrum with pylibfdt / dtschema on Jammy
- and, finally, the setuptools we specify in requirements.txt will be actually used

Author: rpardini

lib/functions/general/python-tools.sh

@@ -46,18 +46,6 @@ function prepare_python_and_pip() {
 	# Construct the version string (e.g., "python3.12")
 	python3_version_string="python$python3_version_majorminor"
 
-	# Check actual pip3 version
-	#   Note: we don't use "/usr/bin/pip3" at all, since it's commonly missing. instead "python -m pip"
-	#   The hostdep package python3-pip is still required, and other crazy might impact this.
-	#   We might need to install our own pip if it gets bad enough.
-	declare pip3_version
-	pip3_version="$("${python3_binary_path}" -m pip --version)"
-
-	# get the pip3 version number only (eg, "21.2.4" from "pip 21.2.4 from /usr/lib/python3/dist-packages/pip (python 3.9)")
-	declare pip3_version_number
-	pip3_version_number="$(echo "${pip3_version}" | cut -d' ' -f2)" # @TODO: brittle. how to do this better?
-	display_alert "pip3 version" "${pip3_version_number}: '${pip3_version}'" "info"
-
 	# Hash the contents of the dependencies array + the Python version + the release
 	declare python3_pip_dependencies_path
 	declare python3_pip_dependencies_hash
@@ -66,8 +54,15 @@ function prepare_python_and_pip() {
 	# Check for the existence of requirements.txt, fail if not found
 	[[ ! -f "${python3_pip_dependencies_path}" ]] && exit_with_error "Python Pip requirements.txt file not found at path: ${python3_pip_dependencies_path}"
 
+	# We will install our own pip; we don't want to rely on the host's pip version, as that implies old setuptools etc.
+	# Parse the pip version from the requirements.txt file; use grep to find the line starting with "pip == "
+	# Example line: "pip == 25.0.1          # pip is the package installer for Python" so get rid of comments
+	declare pip3_version_number="undetermined"
+	pip3_version_number=$(grep -E "^pip[[:space:]]*==" "${python3_pip_dependencies_path}" | cut -d'=' -f3 | cut -d'#' -f 1 | tr -d '[:space:]')
+	display_alert "pip3 version" "${pip3_version_number}" "info"
+
 	# Calculate the hash for the Pip requirements
-	python3_pip_dependencies_hash="$(echo "${HOSTRELEASE}" "${python3_version}" "${pip3_version}" "$(cat "${python3_pip_dependencies_path}")" | sha256sum | cut -d' ' -f1)"
+	python3_pip_dependencies_hash="$(echo "${HOSTRELEASE}" "${python3_version}" "${pip3_version_number}" "$(cat "${python3_pip_dependencies_path}")" | sha256sum | cut -d' ' -f1)"
 
 	declare non_cache_dir="/armbian-pip"
 	declare python_pip_cache="${SRC}/cache/pip"
@@ -83,16 +78,8 @@ function prepare_python_and_pip() {
 		fi
 	fi
 
-	declare -a pip3_extra_args=("--no-warn-script-location" "--user")
-	# if pip 23+, add "--break-system-packages" to pip3 invocations.
-	# See See PEP 668 -- System-wide package management with pip
-	# but the fact is that we're _not_ managing system-wide, instead --user
-	if linux-version compare "${pip3_version_number}" ge "23.0"; then
-		pip3_extra_args+=("--break-system-packages")
-	fi
-	if linux-version compare "${pip3_version_number}" ge "22.1"; then
-		pip3_extra_args+=("--root-user-action=ignore")
-	fi
+	# we run as root, but with --user; --break-system-packages is required due to PEP 668 (no system packages are installed here anyway)
+	declare -a pip3_extra_args=("--no-warn-script-location" "--user" "--root-user-action=ignore" "--break-system-packages")
 
 	declare python_hash_base="${python_pip_cache}/pip_pkg_hash"
 	declare python_hash_file="${python_hash_base}_${python3_pip_dependencies_hash}"
@@ -110,11 +97,13 @@ function prepare_python_and_pip() {
 		[REQUIREMENTS_PATH]="${python3_pip_dependencies_path}"
 		[VERSION]="${python3_version}"
 		[VERSION_STRING]="${python3_version_string}"
-		[PIP_VERSION]="${pip3_version}"
+		[PIP_VERSION]="${pip3_version_number}"
+		[GET_PIP_BIN]="${PYTHON3_INFO[USERBASE]}/bin/get-pip-${pip3_version_number}.py"
 	)
 
 	# declare a readonly global array for ENV vars to invoke python3 with
 	declare -r -g -a PYTHON3_VARS=(
+		"PYTHONPATH=/does/not/exist/armbian/uses/user/packages/only"
 		"PYTHONUSERBASE=${PYTHON3_INFO[USERBASE]}"
 		"PYTHONUNBUFFERED=yes"
 		"PYTHONPYCACHEPREFIX=${PYTHON3_INFO[PYCACHEPREFIX]}"
@@ -129,6 +118,18 @@ function prepare_python_and_pip() {
 		# remove the old hashes matching base, don't leave junk behind
 		run_host_command_logged rm -fv "${python_hash_base}*"
 
+		# If get-pip.py is not present, download it, using curl.
+		if [[ ! -f "${PYTHON3_INFO[GET_PIP_BIN]}" ]]; then
+			display_alert "Downloading get-pip.py" "from https://bootstrap.pypa.io/get-pip.py" "info"
+			run_host_command_logged curl -sSL -o "${PYTHON3_INFO[GET_PIP_BIN]}" "https://bootstrap.pypa.io/get-pip.py"
+		fi
+
+		# Install pip, using get-pip.py; that bootstraps pip using an embedded, temporary, pip contained in get-pip.py
+		display_alert "Installing pip using get-pip.py" "${pip3_version_number}" "info"
+		run_host_command_logged env -i "${PYTHON3_VARS[@]@Q}" "${PYTHON3_INFO[BIN]}" "${PYTHON3_INFO[GET_PIP_BIN]}" "${pip3_extra_args[@]}" "pip==${pip3_version_number}"
+
+		# Install the dependencies
+		display_alert "Installing Python dependencies" "from ${python3_pip_dependencies_path}" "info"
 		run_host_command_logged env -i "${PYTHON3_VARS[@]@Q}" "${PYTHON3_INFO[BIN]}" -m pip install "${pip3_extra_args[@]}" -r "${python3_pip_dependencies_path}"
 
 		# Create the hash file

lib/functions/host/prepare-host.sh

@@ -219,18 +219,12 @@ function adaptative_prepare_host_dependencies() {
 
 	### Python3 -- required for Armbian's Python tooling, and also for more recent u-boot builds. Needs 3.9+; ffi-dev is needed for some Python packages when the wheel is not prebuilt
 	### 'python3-setuptools' and 'python3-pyelftools' moved to requirements.txt to make sure build hosts use the same/latest versions of these tools.
-	host_dependencies+=("python3-dev" "python3-pip" "libffi-dev")
+	### 'python3-dev' depends on distutils, so instead depend on libpython3-dev which doesn't.
+	host_dependencies+=("python3" "libpython3-dev" "libffi-dev")
 
 	# Needed for some u-boot's, lest "tools/mkeficapsule.c:21:10: fatal error: gnutls/gnutls.h"
 	host_dependencies+=("libgnutls28-dev")
 
-	# Noble/Trixie and later releases do not carry "python3-distutils" https://docs.python.org/3.10/whatsnew/3.10.html#distutils-deprecated
-	if [[ "$host_release" =~ ^(trixie|sid|noble|wilma)$ ]]; then
-		display_alert "python3-distutils not available on host release '${host_release}'" "distutils was deprecated with Python 3.12" "debug"
-	else
-		host_dependencies+=("python3-distutils")
-	fi
-
 	### Python2 -- required for some older u-boot builds
 	# Debian newer than 'bookworm' and Ubuntu newer than 'lunar'/'mantic' does not carry python2 anymore; in this case some u-boot's might fail to build.
 	# Last versions to support python2 were Debian 'bullseye' and Ubuntu 'jammy'

requirements.txt

@@ -5,6 +5,7 @@
 # Always use a fixed version, this is important for correct hashing.
 # Dependabot will keep these versions up to date.
 
+pip == 25.0.1          # pip is the package installer for Python
 setuptools == 75.8.0   # for building Python packages
 pyelftools == 0.31     # for building U-Boot
 unidiff == 0.7.5       # for parsing unified diff