From 6f767c3620ac09d8db3d09d7d2ba7b57c9cbe909 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 1 Oct 2024 15:44:56 -0700 Subject: [PATCH 1/6] pybind11: 2.13.0+ --- NEWS.rst | 2 +- README.md | 2 +- docs/source/dev/dependencies.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 9cad584bfb..eafc2eec49 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -15,7 +15,7 @@ Note that ADIOS2 does not support compression in BP3 files. CMake 3.22.0 is now the minimally supported version for CMake. pybind11 2.13.0 is now the minimally supported version for Python support. -Python 3.12 & 3.13 are now supported, Python 3.7 is removed. +Python 3.12 & 3.13 are now supported, Python 3.7 & 3.8 are removed. The ``len(...)`` of many classes has been reworked for consistency and returns now the number of entries (iterations, record components, etc.). Previously, this sporadically returned the number of attributes, which is better queried via ``len(.attributes)``. diff --git a/README.md b/README.md index 674ae342b8..f99b7c96f9 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ while those can be built either with or without: Optional language bindings: * Python: - * Python 3.8 - 3.13 + * Python 3.9 - 3.13 * pybind11 2.13.0+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index c0318aa683..284b534e56 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -38,7 +38,7 @@ Optional: language bindings * Python: - * Python 3.8 - 3.13 + * Python 3.9 - 3.13 * pybind11 2.13.0+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) From 09fc3d9a761da7ebd96970674ba07bcc55c6e16f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 1 Oct 2024 15:48:51 -0700 Subject: [PATCH 2/6] Python 3.9-3.13 Python 3.13 is released. Python 3.8 is EOL as of October 2024. --- Dockerfile | 15 --------------- Singularity | 2 +- cmake/dependencies/pybind11.cmake | 4 ++-- conda.yml | 2 +- docs/source/install/install.rst | 2 +- setup.py | 3 +-- 6 files changed, 6 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index eb31c61c51..36fd45f4a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -112,21 +112,6 @@ RUN for whl in /opt/src/dist/*.whl; do \ && du -hs /opt/src/dist/* \ && du -hs /wheelhouse/* -# test in fresh env: Debian:Sid + Python 3.8 -FROM debian:sid -ENV DEBIAN_FRONTEND noninteractive -COPY --from=build-env /wheelhouse/openPMD_api-*-cp38-cp38-manylinux2010_x86_64.whl . -RUN apt-get update \ - && apt-get install -y --no-install-recommends python3.8 python3-distutils ca-certificates curl \ - && rm -rf /var/lib/apt/lists/* -RUN python3.8 --version \ - && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ - && python3.8 get-pip.py \ - && python3.8 -m pip install openPMD_api-*-cp38-cp38-manylinux2010_x86_64.whl -RUN python3.8 -c "import openpmd_api as io; print(io.__version__); print(io.variants)" -RUN python3.8 -m openpmd_api.ls --help -RUN openpmd-ls --help - # test in fresh env: Debian:Bullseye + Python 3.9 FROM debian:bullseye ENV DEBIAN_FRONTEND noninteractive diff --git a/Singularity b/Singularity index f58b39a2d5..5d77c6dd55 100644 --- a/Singularity +++ b/Singularity @@ -42,7 +42,7 @@ Supported frontends are C++11 and Python3. -DopenPMD_USE_PYTHON=ON \ -DopenPMD_BUILD_TESTING=OFF \ -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DCMAKE_INSTALL_PYTHONDIR=lib/python3.6/dist-packages + -DCMAKE_INSTALL_PYTHONDIR=lib/python3.12/dist-packages make # make test make install diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index 742aafa7dc..ae23c66c3d 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -78,9 +78,9 @@ set(openPMD_pybind11_branch "v2.13.6" "Repository branch for openPMD_pybind11_repo if(openPMD_USE_INTERNAL_PYBIND11)") if(openPMD_USE_PYTHON STREQUAL AUTO) - find_package(Python 3.7.0 COMPONENTS Interpreter Development.Module) + find_package(Python 3.9.0 COMPONENTS Interpreter Development.Module) elseif(openPMD_USE_PYTHON) - find_package(Python 3.7.0 COMPONENTS Interpreter Development.Module REQUIRED) + find_package(Python 3.9.0 COMPONENTS Interpreter Development.Module REQUIRED) else() set(openPMD_HAVE_PYTHON FALSE) endif() diff --git a/conda.yml b/conda.yml index e82567b2ae..0ed0278a73 100644 --- a/conda.yml +++ b/conda.yml @@ -36,7 +36,7 @@ dependencies: - pre-commit - pyarrow # for dask # - pybind11 # shipped internally - - python>=3.8 + - python>=3.9 # just a note for later hackery, we could install pip packages inside the env, too: # - pip: diff --git a/docs/source/install/install.rst b/docs/source/install/install.rst index a0268a2a99..08e56dba5b 100644 --- a/docs/source/install/install.rst +++ b/docs/source/install/install.rst @@ -192,7 +192,7 @@ CMake will summarize the install paths for you before the build step. # Note that one some systems, /lib might need to be replaced with /lib64. # change path to your python MAJOR.MINOR version - export PYTHONPATH=$HOME/somepath/lib/python3.8/site-packages:$PYTHONPATH + export PYTHONPATH=$HOME/somepath/lib/python3.12/site-packages:$PYTHONPATH Adding those lines to your ``$HOME/.bashrc`` and re-opening your terminal will set them permanently. diff --git a/setup.py b/setup.py index 1c4e7476d3..4a821d9e91 100644 --- a/setup.py +++ b/setup.py @@ -196,7 +196,7 @@ def build_extension(self, ext): cmdclass=dict(build_ext=CMakeBuild), # scripts=['openpmd-ls'], zip_safe=False, - python_requires='>=3.8', + python_requires='>=3.9', # tests_require=['pytest'], install_requires=install_requires, # see: src/bindings/python/cli @@ -225,7 +225,6 @@ def build_extension(self, ext): 'Topic :: Database :: Front-Ends', 'Programming Language :: C++', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', From bac91a0ee18938a222aaa8eb4ce7877082edfb1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 2 Oct 2024 16:19:57 +0200 Subject: [PATCH 3/6] Try python3.9 for ASAN UBSAN run with venv --- .github/workflows/tooling.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tooling.yml b/.github/workflows/tooling.yml index 03d5070d86..226f695064 100644 --- a/.github/workflows/tooling.yml +++ b/.github/workflows/tooling.yml @@ -47,9 +47,7 @@ jobs: - name: Install run: | sudo apt-get update - sudo apt-get install clang-10 libc++-dev libc++abi-dev python3 gfortran libopenmpi-dev - python3 -m pip install -U pip - python3 -m pip install -U numpy + sudo apt-get install clang-10 libc++-dev libc++abi-dev python3.9 python3.9-dev python3.9-venv gfortran libopenmpi-dev sudo .github/workflows/dependencies/install_spack - name: Build env: {CC: mpicc, CXX: mpic++, OMPI_CC: clang-10, OMPI_CXX: clang++-10, CXXFLAGS: -Werror, OPENPMD_HDF5_CHUNKS: none, OPENPMD_TEST_NFILES_MAX: 100} @@ -58,6 +56,9 @@ jobs: spack install SOURCEPATH="$(pwd)" share/openPMD/download_samples.sh build + python3.9 -m venv env + source env/bin/activate + pip install numpy export LDFLAGS="${LDFLAGS} -fsanitize=address,undefined -shared-libsan" export CXXFLAGS="${CXXFLAGS} -fsanitize=address,undefined -shared-libsan" cmake -S . -B build \ @@ -66,6 +67,7 @@ jobs: -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON \ + -DPython_EXECUTABLE="$(which python)" \ -DCMAKE_VERBOSE_MAKEFILE=ON cmake --build build --parallel 2 export ASAN_OPTIONS=detect_stack_use_after_return=1:detect_leaks=1:check_initialization_order=true:strict_init_order=true:detect_stack_use_after_scope=1:fast_unwind_on_malloc=0 From 4656f000a654c4a9e46bf2bf2706db8e52d0fb93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 7 Oct 2024 14:53:46 +0200 Subject: [PATCH 4/6] Upgrade Linux runs to Python 3.9 --- .github/workflows/linux.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9592a71223..095a35f951 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -213,7 +213,7 @@ jobs: # gcc-4.8.5_py35_nompi_h5 # gcc-7.4.0_py_ompi_h5_ad2_coveralls - gcc7_py36_pd_dd_ompi_h5_ad2: + gcc7_py39_pd_dd_ompi_h5_ad2: runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false steps: @@ -224,19 +224,16 @@ jobs: - name: Install run: | sudo apt-get update - sudo apt-get install g++-7 gfortran libopenmpi-dev python3 python3-setuptools + sudo apt-get install g++-7 gfortran libopenmpi-dev python3.9 python3.9-dev python3.9-venv sudo .github/workflows/dependencies/install_spack - python3 -m pip install -U pip - python3 -m pip install -U numpy - python3 -m pip install -U mpi4py - python3 -m pip install -U pandas - python3 -m pip install -U dask - python3 -m pip install -U pyarrow - name: Build env: {CC: gcc-7, CXX: g++-7, CXXFLAGS: -Werror} run: | eval $(spack env activate --sh .github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/) spack install + python3.9 -m venv env + source env/bin/activate + pip install numpy mpi4py pandas dask pyarrow share/openPMD/download_samples.sh build cmake -S . -B build \ @@ -249,7 +246,7 @@ jobs: cd build ctest --output-on-failure - gcc9_py38_pd_nompi_h5_ad2_libcpp: + gcc9_py39_pd_nompi_h5_ad2_libcpp: runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false steps: @@ -257,11 +254,14 @@ jobs: - name: Install run: | sudo apt-get update - sudo apt-get install g++ libopenmpi-dev libhdf5-openmpi-dev python3 python3-numpy python3-mpi4py python3-pandas + sudo apt-get install g++ libopenmpi-dev libhdf5-openmpi-dev python3.9 python3.9-dev python3.9-venv # TODO ADIOS2 - name: Build env: {CXXFLAGS: -Werror, PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig} run: | + python3.9 -m venv env + source env/bin/activate + pip install numpy mpi4py pandas dask pyarrow share/openPMD/download_samples.sh build cmake -S . -B build \ -DopenPMD_USE_PYTHON=ON \ From 8891edea5cf58a4ddbee94f119b11edd743f07aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 7 Oct 2024 15:09:16 +0200 Subject: [PATCH 5/6] Upgrade to Python 3.9 in Appveyor builds --- .appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 55a5a78f70..c87fa55cee 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,14 +1,14 @@ environment: matrix: - TARGET_ARCH: x64 - CONDA_PY: 3.7 + CONDA_PY: 3.9 CONDA_INSTALL_LOCN: C:\\Miniconda37-x64 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 platform: x64 SHARED: OFF - TARGET_ARCH: x86 - CONDA_PY: 3.7 + CONDA_PY: 3.9 CONDA_INSTALL_LOCN: C:\\Miniconda37 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 platform: x86 From 31258f76c1d168fc0d9a6ad69a2643cb7bfb3b12 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 14 Jan 2025 10:53:16 -0800 Subject: [PATCH 6/6] Do not enforce 3.9 yet --- cmake/dependencies/pybind11.cmake | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index ae23c66c3d..e377d94f22 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -78,9 +78,9 @@ set(openPMD_pybind11_branch "v2.13.6" "Repository branch for openPMD_pybind11_repo if(openPMD_USE_INTERNAL_PYBIND11)") if(openPMD_USE_PYTHON STREQUAL AUTO) - find_package(Python 3.9.0 COMPONENTS Interpreter Development.Module) + find_package(Python 3.8.0 COMPONENTS Interpreter Development.Module) elseif(openPMD_USE_PYTHON) - find_package(Python 3.9.0 COMPONENTS Interpreter Development.Module REQUIRED) + find_package(Python 3.8.0 COMPONENTS Interpreter Development.Module REQUIRED) else() set(openPMD_HAVE_PYTHON FALSE) endif() diff --git a/setup.py b/setup.py index 4a821d9e91..646fded0ae 100644 --- a/setup.py +++ b/setup.py @@ -196,7 +196,7 @@ def build_extension(self, ext): cmdclass=dict(build_ext=CMakeBuild), # scripts=['openpmd-ls'], zip_safe=False, - python_requires='>=3.9', + python_requires='>=3.8', # left for Ubuntu 20.04, truly ">=3.9" # tests_require=['pytest'], install_requires=install_requires, # see: src/bindings/python/cli