Skip to content

Commit d1623dd

Browse files
trxcllnttrxcllnt
authored
Build shared libpython in pip base images (#495)
This should fix the libucxx [link error](https://github.com/rapidsai/ucxx/actions/runs/14598480223/job/40961661967#step:9:2494) that cropped up: ``` /usr/bin/ld: /usr/local/python/3.12.10/lib/libpython3.12.a(import.o): relocation R_X86_64_TPOFF32 against hidden symbol `pkgcontext' can not be used when making a shared object ```
1 parent a99e78c commit d1623dd

14 files changed

+28
-27
lines changed

‎features/src/cccl-dev/devcontainer-feature.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "NVIDIA CCCL development utilities",
33
"id": "cccl-dev",
4-
"version": "25.6.1",
4+
"version": "25.6.2",
55
"description": "A feature to install NVIDIA CCCL development utilities",
66
"options": {
77
"litVersion": {

‎features/src/cccl-dev/install.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#! /usr/bin/env bash
2-
set -ex
2+
set -e
33

44
LIT_VERSION="${LITVERSION:-latest}";
55

‎features/src/openmpi/devcontainer-feature.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "OpenMPI",
33
"id": "openmpi",
4-
"version": "25.6.1",
4+
"version": "25.6.2",
55
"description": "A feature to install OpenMPI with optional CUDA and UCX support",
66
"options": {
77
"version": {

‎features/src/openmpi/install.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#! /usr/bin/env bash
2-
set -ex
2+
set -e
33

44
OPENMPI_VERSION="${VERSION:-system}";
55

‎features/src/rapids-build-utils/devcontainer-feature.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "NVIDIA RAPIDS devcontainer build utilities",
33
"id": "rapids-build-utils",
4-
"version": "25.6.2",
4+
"version": "25.6.3",
55
"description": "A feature to install the RAPIDS devcontainer build utilities",
66
"containerEnv": {
77
"BASH_ENV": "/etc/bash.bash_env"

‎features/src/rapids-build-utils/install.sh

+7-7
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
99

1010
PKGS=(bc jq pigz sudo wget gettext-base bash-completion ca-certificates);
1111

12-
if ! command -v /usr/bin/python3 >/dev/null 2>&1; then
12+
if ! command -v python3 >/dev/null 2>&1; then
1313
PKGS+=(python3 python3-pip);
14-
elif ! /usr/bin/python3 -m pip >/dev/null 2>&1; then
14+
elif ! python3 -m pip >/dev/null 2>&1; then
1515
PKGS+=(python3-pip);
1616
fi
1717

@@ -43,9 +43,9 @@ if [[ "${DISTRIB_RELEASE}" > "22.04" ]]; then
4343
fi
4444
fi
4545

46-
/usr/bin/python3 -m pip install "${_PIP_INSTALL_ARGS[@]}" "${_PIP_UPGRADE_ARGS[@]}" pip;
46+
python3 -m pip install "${_PIP_INSTALL_ARGS[@]}" "${_PIP_UPGRADE_ARGS[@]}" pip;
4747
# Install RAPIDS dependency file generator, conda-merge, and toml
48-
/usr/bin/python3 -m pip install "${_PIP_INSTALL_ARGS[@]}" \
48+
python3 -m pip install "${_PIP_INSTALL_ARGS[@]}" \
4949
'rapids-dependency-file-generator<1.14' \
5050
conda-merge \
5151
toml;
@@ -112,11 +112,11 @@ mkdir -p /etc/bash_completion.d/;
112112
yq shell-completion bash | tee /etc/bash_completion.d/yq >/dev/null;
113113

114114
# Activate venv in /etc/bash.bashrc
115-
append_to_etc_bashrc "$(cat .bashrc)";
115+
append_to_etc_bashrc "$(cat .bashrc)\n\nexport ORIG_PYTHON=$(which python3)";
116116
# Activate venv in ~/.bashrc
117-
append_to_all_bashrcs "$(cat .bashrc)";
117+
append_to_all_bashrcs "$(cat .bashrc)\n\nexport ORIG_PYTHON=$(which python3)";
118118
# export envvars in /etc/profile.d
119-
add_etc_profile_d_script rapids-build-utils "$(cat .bashrc)";
119+
add_etc_profile_d_script rapids-build-utils "$(cat .bashrc)\n\nexport ORIG_PYTHON=$(which python3)";
120120

121121
# Clean up
122122
# rm -rf /tmp/*;

‎features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ make_conda_dependencies() {
6363
local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR:-12}.${CUDA_VERSION_MINOR:-0}}";
6464
cuda_version="$(grep -o '^[0-9]*.[0-9]' <<< "${cuda_version}")";
6565

66-
local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}";
66+
local python_version="${PYTHON_VERSION:-$("${ORIG_PYTHON:-python3}" --version 2>&1 | cut -d' ' -f2)}";
6767
python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")";
6868

6969
local -a _matrix_selectors=(

‎features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ make_pip_dependencies() {
6464
local cuda_version="${CUDA_VERSION_MAJOR_MINOR:-}";
6565
local -r cuda_version_major="$(cut -d'.' -f1 <<< "${cuda_version}")";
6666

67-
local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}";
67+
local python_version="${PYTHON_VERSION:-$("${ORIG_PYTHON:-python3}" --version 2>&1 | cut -d' ' -f2)}";
6868
python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")";
6969

7070
# Why default to cuda_suffixed=true?

‎features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-uses-scikit-build-core.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ set -e;
55
test -f "${1}/pyproject.toml";
66

77
# where rapids-build-backend is used, its does a bit of work then forwards on to another build backend... which might be scikit-build-core
8-
[[ "scikit_build_core.build" == "$(/usr/bin/python3 -c "import toml; print(toml.load('${1}/pyproject.toml')['build-system']['build-backend'])" 2>/dev/null)" ]] \
8+
[[ "scikit_build_core.build" == "$("${ORIG_PYTHON:-python3}" -c "import toml; print(toml.load('${1}/pyproject.toml')['build-system']['build-backend'])" 2>/dev/null)" ]] \
99
|| \
10-
[[ "scikit_build_core.build" == "$(/usr/bin/python3 -c "import toml; print(toml.load('${1}/pyproject.toml')['tool']['rapids-build-backend']['build-backend'])" 2>/dev/null)" ]] \
10+
[[ "scikit_build_core.build" == "$("${ORIG_PYTHON:-python3}" -c "import toml; print(toml.load('${1}/pyproject.toml')['tool']['rapids-build-backend']['build-backend'])" 2>/dev/null)" ]] \
1111
;

‎features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-uses-scikit-build.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33
set -e;
44

55
test -f "${1}/pyproject.toml";
6-
test "True" = "$(/usr/bin/python3 -c "import toml; print(any('scikit-build-core' not in x and 'scikit-build' in x for x in toml.load('${1}/pyproject.toml')['build-system']['requires']))" 2>/dev/null)";
6+
test "True" = "$("${ORIG_PYTHON:-python3}" -c "import toml; print(any('scikit-build-core' not in x and 'scikit-build' in x for x in toml.load('${1}/pyproject.toml')['build-system']['requires']))" 2>/dev/null)";

‎features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.clean.tmpl.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ clean_${PY_LIB}_python() {
4040
fi
4141
done
4242

43-
local py_ver="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}";
43+
local py_ver="${PYTHON_VERSION:-$("${ORIG_PYTHON:-python3}" --version 2>&1 | cut -d' ' -f2)}";
4444
py_ver="$(grep -Po '^[0-9]+\.[0-9]+' <<< "${py_ver}")";
4545

4646
if test -d "${PY_SRC}/build"; then

‎features/src/utils/devcontainer-feature.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "devcontainer-utils",
33
"id": "utils",
4-
"version": "25.6.2",
4+
"version": "25.6.3",
55
"description": "A feature to install RAPIDS devcontainer utility scripts",
66
"containerEnv": {
77
"BASH_ENV": "/etc/bash.bash_env"

‎features/src/utils/install.sh

+7-6
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@ PKGS=(
2121
ca-certificates
2222
);
2323

24-
if ! command -v /usr/bin/python3 >/dev/null 2>&1; then
24+
if ! command -v python3 >/dev/null 2>&1; then
2525
PKGS+=(python3 python3-pip);
26-
elif ! /usr/bin/python3 -m pip >/dev/null 2>&1; then
26+
elif ! python3 -m pip >/dev/null 2>&1; then
2727
PKGS+=(python3-pip);
2828
fi
2929

@@ -42,7 +42,7 @@ if [[ "${DISTRIB_RELEASE}" > "22.04" ]]; then
4242
fi
4343
fi
4444

45-
/usr/bin/python3 -m pip install "${_PIP_INSTALL_ARGS[@]}" "${_PIP_UPGRADE_ARGS[@]}" pip;
45+
python3 -m pip install "${_PIP_INSTALL_ARGS[@]}" "${_PIP_UPGRADE_ARGS[@]}" pip;
4646

4747
# Install yq if not installed
4848
if ! command -v yq >/dev/null 2>&1; then
@@ -133,11 +133,12 @@ for_each_user_bashrc 'sed -i -re "s/^#(export GCC_COLORS)/\1/g" "$0"';
133133
for_each_user_bashrc 'sed -i -re "s/^(HIST(FILE)?SIZE=).*$/\1/g" "$0"';
134134

135135
# export envvars in bashrc files
136-
append_to_etc_bashrc "$(cat .bashrc)";
137-
append_to_all_bashrcs "$(cat .bashrc)";
136+
append_to_etc_bashrc "$(cat .bashrc)\n\nexport ORIG_PYTHON=$(which python3)";
137+
138+
append_to_all_bashrcs "$(cat .bashrc)\n\nexport ORIG_PYTHON=$(which python3)";
138139

139140
# export envvars in /etc/profile.d
140-
add_etc_profile_d_script devcontainer-utils "$(cat .bashrc)";
141+
add_etc_profile_d_script devcontainer-utils "$(cat .bashrc)\n\nexport ORIG_PYTHON=$(which python3)";
141142

142143
# Add GitHub's key fingerprints to known_hosts (curl -s https://api.github.com/meta | jq -r '.ssh_keys | map("github.com \(.)") | .[]')
143144
# Add GitLab's key fingerprints to known_hosts (https://docs.gitlab.com/ee/user/gitlab_com/index.html#ssh-known_hosts-entries)

‎matrix.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ x-nvhpc-curr: &nvhpc_curr { name: "nvhpc", version: "25.3" }
3030
x-nvhpc-env: &nvhpc_env { CC: "nvc", CXX: "nvc++", CUDAHOSTCXX: "nvc++" }
3131

3232
x-mambaforge: &conda { name: "mambaforge" }
33-
x-python: &python { name: "ghcr.io/devcontainers/features/python:1.6.2", version: "os-provided", installTools: "false", hide: true }
34-
x-python-rapids: &python_rapids { name: "ghcr.io/devcontainers/features/python:1.6.2", version: "3.12", installTools: "false", hide: true }
33+
x-python: &python { name: "ghcr.io/devcontainers/features/python:1.7.1", version: "os-provided", installTools: false, enableShared: true, optimize: true, hide: true }
34+
x-python-rapids: &python_rapids { name: "ghcr.io/devcontainers/features/python:1.7.1", version: "3.12", installTools: false, enableShared: true, optimize: true, hide: true }
3535
x-ucx-rapids: &ucx_rapids { name: "ucx", version: "1.18.0" }
3636
x-openmpi: &openmpi { name: "openmpi" }
3737

0 commit comments

Comments
 (0)