From 2ebb9af92647291ad0974facd7edde74f79008d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20David--Cl=C3=A9ris?= Date: Fri, 19 Sep 2025 15:38:47 +0200 Subject: [PATCH 01/15] [Draft] 2nd attempt at registering a external python module as shamrock submodule --- src/pylib/utils/__init__.py | 2 ++ src/shambindings/src/pybindings.cpp | 19 +++++++++++++++++++ src/shambindings/src/start_python.cpp | 1 + 3 files changed, 22 insertions(+) create mode 100644 src/pylib/utils/__init__.py diff --git a/src/pylib/utils/__init__.py b/src/pylib/utils/__init__.py new file mode 100644 index 000000000..7dad5c27f --- /dev/null +++ b/src/pylib/utils/__init__.py @@ -0,0 +1,2 @@ +def hello_world(): + print("Hello, World!") \ No newline at end of file diff --git a/src/shambindings/src/pybindings.cpp b/src/shambindings/src/pybindings.cpp index f838f59a4..130512b7d 100644 --- a/src/shambindings/src/pybindings.cpp +++ b/src/shambindings/src/pybindings.cpp @@ -115,6 +115,23 @@ void register_py_to_sham_print(py::module &m) { }); } +void register_extra_lib(py::module &m) { + + m.def("register_extra_lib", [&]() { + try { + auto sys = py::module::import("sys"); + + sys.attr("path").attr("insert")( + 0, "/Users/davidclt/Documents/shamrock-dev/Shamrock_cursor/src/pylib"); + + m.attr("utils") = py::module_::import("utils"); + + } catch (std::exception &e) { + shambase::throw_with_loc(e.what()); + } + }); +} + namespace shambindings { enum { None = 0, Lib = 1, Embed = 2 } init_state = None; @@ -149,6 +166,8 @@ namespace shambindings { if (hook_stdout) { register_py_to_sham_print(m); m.attr("hook_stdout")(); + register_extra_lib(m); + m.attr("register_extra_lib")(); } } diff --git a/src/shambindings/src/start_python.cpp b/src/shambindings/src/start_python.cpp index 40fac4174..c3994e0ac 100644 --- a/src/shambindings/src/start_python.cpp +++ b/src/shambindings/src/start_python.cpp @@ -14,6 +14,7 @@ * */ +#include "shambase/exception.hpp" #include "shambase/popen.hpp" #include "shambase/print.hpp" #include "shambindings/pybindaliases.hpp" From 5aa801a327da4292ae8486c0b7518c1ddc36e26f Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Sun, 28 Dec 2025 17:51:26 +0000 Subject: [PATCH 02/15] [autofix.ci] fix file authorship & pre-commit --- src/pylib/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pylib/utils/__init__.py b/src/pylib/utils/__init__.py index 7dad5c27f..f3abad5c0 100644 --- a/src/pylib/utils/__init__.py +++ b/src/pylib/utils/__init__.py @@ -1,2 +1,2 @@ def hello_world(): - print("Hello, World!") \ No newline at end of file + print("Hello, World!") From 13377433a9ecbf3b5e6b02c4303f281c538e6625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20David--Cl=C3=A9ris?= Date: Tue, 30 Dec 2025 10:23:00 +0100 Subject: [PATCH 03/15] better --- src/shambindings/CMakeLists.txt | 1 + src/shambindings/src/pybindings.cpp | 19 ------------------- src/shambindings/src/start_python.cpp | 18 ++++++++++++++++++ 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/shambindings/CMakeLists.txt b/src/shambindings/CMakeLists.txt index f72dccc4b..8a46e3f28 100644 --- a/src/shambindings/CMakeLists.txt +++ b/src/shambindings/CMakeLists.txt @@ -58,6 +58,7 @@ file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/configure_time_py_sys_path.cpp " const char* configure_time_py_sys_path() { return \"${__PYTHON_SYS_PATH_OUT}\"; } const char* configure_time_py_executable() { return \"${PYTHON_EXECUTABLE}\"; } + const char* configure_time_pylib_path() { return \"${CMAKE_SOURCE_DIR}/src/pylib\"; } " ) diff --git a/src/shambindings/src/pybindings.cpp b/src/shambindings/src/pybindings.cpp index 130512b7d..f838f59a4 100644 --- a/src/shambindings/src/pybindings.cpp +++ b/src/shambindings/src/pybindings.cpp @@ -115,23 +115,6 @@ void register_py_to_sham_print(py::module &m) { }); } -void register_extra_lib(py::module &m) { - - m.def("register_extra_lib", [&]() { - try { - auto sys = py::module::import("sys"); - - sys.attr("path").attr("insert")( - 0, "/Users/davidclt/Documents/shamrock-dev/Shamrock_cursor/src/pylib"); - - m.attr("utils") = py::module_::import("utils"); - - } catch (std::exception &e) { - shambase::throw_with_loc(e.what()); - } - }); -} - namespace shambindings { enum { None = 0, Lib = 1, Embed = 2 } init_state = None; @@ -166,8 +149,6 @@ namespace shambindings { if (hook_stdout) { register_py_to_sham_print(m); m.attr("hook_stdout")(); - register_extra_lib(m); - m.attr("register_extra_lib")(); } } diff --git a/src/shambindings/src/start_python.cpp b/src/shambindings/src/start_python.cpp index c3994e0ac..5cabbe5ec 100644 --- a/src/shambindings/src/start_python.cpp +++ b/src/shambindings/src/start_python.cpp @@ -35,6 +35,9 @@ extern const char *configure_time_py_sys_path(); /// @brief path of the python executable that was used to configure sys.path extern const char *configure_time_py_executable(); +/// @brief Path to shamrock utils lib a config time +extern const char *configure_time_pylib_path(); + /** * @brief Script to run ipython * @@ -84,6 +87,17 @@ if not cur_path.startswith(sysprefix): namespace shambindings { + std::string get_pylib_path(bool do_print) { + + std::string ret = std::string(configure_time_pylib_path()); + + if (do_print) { + shambase::println("using pylib path : " + ret); + } + + return ret; + } + void setpypath(std::string path) { runtime_set_pypath = path; } void setpypath_from_binary(std::string binary_path) { @@ -108,6 +122,10 @@ namespace shambindings { std::string modify_path = std::string("paths = ") + get_pypath() + "\n"; modify_path += R"(import sys;sys.path = paths)"; py::exec(modify_path); + + std::string pylib_path = get_pylib_path(do_print); + std::string modify_path_lib = std::string("sys.path.insert(0, \"") + pylib_path + "\")\n"; + py::exec(modify_path_lib); } void start_ipython(bool do_print) { From 994565ba15f1460514cb1f5f6bb501d36661275d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 30 Dec 2025 09:29:25 +0000 Subject: [PATCH 04/15] [gh-action] trigger CI with empty commit From 61d927c6d2f5f8f20c9b2c471527047b38d99237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20David--Cl=C3=A9ris?= Date: Tue, 30 Dec 2025 11:33:28 +0100 Subject: [PATCH 05/15] better --- src/CMakeLists.txt | 9 ++++--- src/main.cpp | 2 +- src/main_lib.cpp | 2 +- src/main_test.cpp | 2 +- src/pylib/shamrock/__init__.py | 8 +++++++ src/pylib/{ => shamrock}/utils/__init__.py | 1 + src/shambindings/src/start_python.cpp | 28 ++++++++++++++++++++++ 7 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 src/pylib/shamrock/__init__.py rename src/pylib/{ => shamrock}/utils/__init__.py (97%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 842ad8d03..bc134b6f3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -96,7 +96,7 @@ target_link_lib_and_install(shamrock_exe) target_link_lib_and_install(shamrock_pylib) set_property(TARGET shamrock_exe PROPERTY OUTPUT_NAME shamrock) -set_property(TARGET shamrock_pylib PROPERTY OUTPUT_NAME shamrock) +set_property(TARGET shamrock_pylib PROPERTY OUTPUT_NAME pyshamrock) ######################################################################################## @@ -131,10 +131,9 @@ if(CMAKE_INSTALL_PYTHONDIR) # Set the install dir for the shamrock package set(SHAMROCK_PYTHON_INSTALL_DIR "${CMAKE_INSTALL_PYTHONDIR}/shamrock") - # Create and install __init__.py file for Python package - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/__init__.py" "from .shamrock import *\n") - install(FILES "${CMAKE_CURRENT_BINARY_DIR}/__init__.py" - DESTINATION "${SHAMROCK_PYTHON_INSTALL_DIR}") + # Copy content of pylib/shamrock to the install directory + install(DIRECTORY "${CMAKE_SOURCE_DIR}/src/pylib/shamrock" + DESTINATION "${CMAKE_INSTALL_PYTHONDIR}") # Install the Python module shared library install(TARGETS shamrock_pylib diff --git a/src/main.cpp b/src/main.cpp index 5d7366cc7..5c54a19ef 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -45,7 +45,7 @@ //%Impl status : Should rewrite /// Call bindings init for the shamrock python module -PYBIND11_EMBEDDED_MODULE(shamrock, m) { shambindings::init_embed(m); } +PYBIND11_EMBEDDED_MODULE(pyshamrock, m) { shambindings::init_embed(m); } int main(int argc, char *argv[]) { diff --git a/src/main_lib.cpp b/src/main_lib.cpp index e9d8fb123..f1c8cedfa 100644 --- a/src/main_lib.cpp +++ b/src/main_lib.cpp @@ -18,4 +18,4 @@ #include /// Call bindings init for the shamrock python module -PYBIND11_MODULE(shamrock, m) { shambindings::init_lib(m); } +PYBIND11_MODULE(pyshamrock, m) { shambindings::init_lib(m); } diff --git a/src/main_test.cpp b/src/main_test.cpp index 4e84cca6b..d5551fe4e 100644 --- a/src/main_test.cpp +++ b/src/main_test.cpp @@ -39,7 +39,7 @@ #include /// Call bindings init for the shamrock python module -PYBIND11_EMBEDDED_MODULE(shamrock, m) { shambindings::init_embed(m); } +PYBIND11_EMBEDDED_MODULE(pyshamrock, m) { shambindings::init_embed(m); } int main(int argc, char *argv[]) { diff --git a/src/pylib/shamrock/__init__.py b/src/pylib/shamrock/__init__.py new file mode 100644 index 000000000..585215353 --- /dev/null +++ b/src/pylib/shamrock/__init__.py @@ -0,0 +1,8 @@ +try: + from .pyshamrock import * +except ImportError: + # the one above will fail if pyshamrock is declared by the embed python interpreter + # so we try to import it from the global namespace + from pyshamrock import * + +print(f"pyshamrock imported from {__file__}") \ No newline at end of file diff --git a/src/pylib/utils/__init__.py b/src/pylib/shamrock/utils/__init__.py similarity index 97% rename from src/pylib/utils/__init__.py rename to src/pylib/shamrock/utils/__init__.py index f3abad5c0..8b099a851 100644 --- a/src/pylib/utils/__init__.py +++ b/src/pylib/shamrock/utils/__init__.py @@ -1,2 +1,3 @@ + def hello_world(): print("Hello, World!") diff --git a/src/shambindings/src/start_python.cpp b/src/shambindings/src/start_python.cpp index 5cabbe5ec..78d03f50d 100644 --- a/src/shambindings/src/start_python.cpp +++ b/src/shambindings/src/start_python.cpp @@ -20,8 +20,10 @@ #include "shambindings/pybindaliases.hpp" #include "shambindings/pybindings.hpp" #include "shambindings/start_python.hpp" +#include "shamcmdopt/env.hpp" #include #include +#include #include #include @@ -85,12 +87,38 @@ if not cur_path.startswith(sysprefix): )"; +// env var to set the path to the pylib +std::optional pylib_path_env_var = shamcmdopt::getenv_str("SHAMROCK_PYLIB_PATH"); + namespace shambindings { std::string get_pylib_path(bool do_print) { + // Get the path to the current binary + std::filesystem::path binary_path = std::filesystem::read_symlink("/proc/self/exe"); + std::filesystem::path binary_dir = binary_path.parent_path(); + std::filesystem::path pyshamrock_path = binary_dir / ".." / "pyshamrock"; + + std::vector possible_paths = { + "pyshamrock", + pyshamrock_path.string(), + std::string(configure_time_pylib_path())}; + + if (pylib_path_env_var.has_value()) { + possible_paths.push_back(pylib_path_env_var.value()); + } + std::string ret = std::string(configure_time_pylib_path()); + for (const auto &path : possible_paths) { + if (std::filesystem::is_directory(path.c_str())) { + ret = path; + break; + }else{ + shambase::println("pylib path " + path + " does not exist, skipping"); + } + } + if (do_print) { shambase::println("using pylib path : " + ret); } From 4a76a274fcb574c4734e1706c31739c19696501f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20David--Cl=C3=A9ris?= Date: Tue, 30 Dec 2025 11:33:46 +0100 Subject: [PATCH 06/15] better --- src/pylib/shamrock/__init__.py | 4 ++-- src/pylib/shamrock/utils/__init__.py | 1 - src/shambindings/src/start_python.cpp | 14 ++++++-------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/pylib/shamrock/__init__.py b/src/pylib/shamrock/__init__.py index 585215353..8e4f486cb 100644 --- a/src/pylib/shamrock/__init__.py +++ b/src/pylib/shamrock/__init__.py @@ -4,5 +4,5 @@ # the one above will fail if pyshamrock is declared by the embed python interpreter # so we try to import it from the global namespace from pyshamrock import * - -print(f"pyshamrock imported from {__file__}") \ No newline at end of file + +print(f"pyshamrock imported from {__file__}") diff --git a/src/pylib/shamrock/utils/__init__.py b/src/pylib/shamrock/utils/__init__.py index 8b099a851..f3abad5c0 100644 --- a/src/pylib/shamrock/utils/__init__.py +++ b/src/pylib/shamrock/utils/__init__.py @@ -1,3 +1,2 @@ - def hello_world(): print("Hello, World!") diff --git a/src/shambindings/src/start_python.cpp b/src/shambindings/src/start_python.cpp index 78d03f50d..91dd4a8a8 100644 --- a/src/shambindings/src/start_python.cpp +++ b/src/shambindings/src/start_python.cpp @@ -95,14 +95,12 @@ namespace shambindings { std::string get_pylib_path(bool do_print) { // Get the path to the current binary - std::filesystem::path binary_path = std::filesystem::read_symlink("/proc/self/exe"); - std::filesystem::path binary_dir = binary_path.parent_path(); + std::filesystem::path binary_path = std::filesystem::read_symlink("/proc/self/exe"); + std::filesystem::path binary_dir = binary_path.parent_path(); std::filesystem::path pyshamrock_path = binary_dir / ".." / "pyshamrock"; - - std::vector possible_paths = { - "pyshamrock", - pyshamrock_path.string(), - std::string(configure_time_pylib_path())}; + + std::vector possible_paths + = {"pyshamrock", pyshamrock_path.string(), std::string(configure_time_pylib_path())}; if (pylib_path_env_var.has_value()) { possible_paths.push_back(pylib_path_env_var.value()); @@ -114,7 +112,7 @@ namespace shambindings { if (std::filesystem::is_directory(path.c_str())) { ret = path; break; - }else{ + } else { shambase::println("pylib path " + path + " does not exist, skipping"); } } From 21d48f353a5a9483fd49054396a798c41719abb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20David--Cl=C3=A9ris?= Date: Tue, 30 Dec 2025 11:44:53 +0100 Subject: [PATCH 07/15] better --- src/pylib/shamrock/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pylib/shamrock/__init__.py b/src/pylib/shamrock/__init__.py index 8e4f486cb..06c5e6527 100644 --- a/src/pylib/shamrock/__init__.py +++ b/src/pylib/shamrock/__init__.py @@ -1,8 +1,11 @@ try: - from .pyshamrock import * -except ImportError: - # the one above will fail if pyshamrock is declared by the embed python interpreter - # so we try to import it from the global namespace + # try to import from the global namespace (works if embedded python interpreter is used) from pyshamrock import * - + IMPORT_LOG = "global" +except ImportError: + # then it is a library mode, we import from the local namespace + from .pyshamrock import * + IMPORT_LOG = "local" + print(f"pyshamrock imported from {__file__}") +print(f"import log: {IMPORT_LOG}") \ No newline at end of file From b96e404d277f6e5ac30fcddfc4cc482d5dc62c9d Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 30 Dec 2025 10:45:55 +0000 Subject: [PATCH 08/15] [autofix.ci] fix file authorship & pre-commit --- src/pylib/shamrock/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pylib/shamrock/__init__.py b/src/pylib/shamrock/__init__.py index 06c5e6527..7d5419050 100644 --- a/src/pylib/shamrock/__init__.py +++ b/src/pylib/shamrock/__init__.py @@ -1,11 +1,13 @@ try: # try to import from the global namespace (works if embedded python interpreter is used) from pyshamrock import * + IMPORT_LOG = "global" except ImportError: # then it is a library mode, we import from the local namespace from .pyshamrock import * + IMPORT_LOG = "local" - + print(f"pyshamrock imported from {__file__}") -print(f"import log: {IMPORT_LOG}") \ No newline at end of file +print(f"import log: {IMPORT_LOG}") From a758446cf33b38fafd86567383175e4461c02557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20David--Cl=C3=A9ris?= Date: Wed, 31 Dec 2025 10:47:10 +0100 Subject: [PATCH 09/15] moar tests --- .github/workflows/env_tests.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/env_tests.yml b/.github/workflows/env_tests.yml index da4858bb7..29d15475f 100644 --- a/.github/workflows/env_tests.yml +++ b/.github/workflows/env_tests.yml @@ -231,12 +231,32 @@ jobs: cd build mpirun ${{ matrix.mpiargs }} -n 4 ./shamrock_test --color --smi-full --sycl-cfg 0:0 --loglevel 0 --benchmark-mpi + - name: Create Python test script for next tests + shell: bash # In docker container github action default to sh + if: matrix.runtest + run: | + cat > /tmp/test_rscript.py << 'EOF' + import shamrock + print("Shamrock module location:", shamrock.__file__) + print("Shamrock version:", shamrock.version_string()) + print("Git info:", shamrock.get_git_info()) + shamrock.change_loglevel(125) + shamrock.sys.init('0:0') + EOF + cat /tmp/test_rscript.py + + - name: Try starting Shamrock (build dir) + if: matrix.runtest + run: | + cd build + ./shamrock --smi --sycl-cfg 0:0 --rscript /tmp/test_rscript.py + - name: Try starting Shamrock (installed version) shell: bash # In docker container github action default to sh if: matrix.runtest run: | cd build/install_test/bin - LD_LIBRARY_PATH=../lib:$LD_LIBRARY_PATH ./shamrock --smi --sycl-cfg 0:0 + LD_LIBRARY_PATH=../lib:$LD_LIBRARY_PATH ./shamrock --smi --sycl-cfg 0:0 --rscript /tmp/test_rscript.py - name: Try starting Shamrock (installed python libs) shell: bash # In docker container github action default to sh @@ -248,4 +268,4 @@ jobs: ACPP_DEBUG_LEVEL=3 \ PYTHONPATH=$(pwd)/../pyinstall_test:$PYTHONPATH \ LD_LIBRARY_PATH=$(pwd)/lib:$LD_LIBRARY_PATH \ - python3 -c "import shamrock;print(shamrock.__file__);shamrock.change_loglevel(125);shamrock.sys.init('0:0')" + python3 /tmp/test_rscript.py From 68b8fa6e16921806c1d651da2f4baf6618bd8719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20David--Cl=C3=A9ris?= Date: Wed, 31 Dec 2025 11:02:44 +0100 Subject: [PATCH 10/15] moar tests --- src/shambindings/src/start_python.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/shambindings/src/start_python.cpp b/src/shambindings/src/start_python.cpp index 91dd4a8a8..7418a7204 100644 --- a/src/shambindings/src/start_python.cpp +++ b/src/shambindings/src/start_python.cpp @@ -95,12 +95,12 @@ namespace shambindings { std::string get_pylib_path(bool do_print) { // Get the path to the current binary - std::filesystem::path binary_path = std::filesystem::read_symlink("/proc/self/exe"); - std::filesystem::path binary_dir = binary_path.parent_path(); - std::filesystem::path pyshamrock_path = binary_dir / ".." / "pyshamrock"; + // std::filesystem::path binary_path = std::filesystem::read_symlink("/proc/self/exe"); + // std::filesystem::path binary_dir = binary_path.parent_path(); + // std::filesystem::path pyshamrock_path = binary_dir / ".." / "pyshamrock"; std::vector possible_paths - = {"pyshamrock", pyshamrock_path.string(), std::string(configure_time_pylib_path())}; + = {"pyshamrock", std::string(configure_time_pylib_path())}; if (pylib_path_env_var.has_value()) { possible_paths.push_back(pylib_path_env_var.value()); From 319d106377b7c2929aa506e18047a50bc5a7fa5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20David--Cl=C3=A9ris?= Date: Wed, 31 Dec 2025 11:40:50 +0100 Subject: [PATCH 11/15] moar tests --- .github/workflows/env_tests.yml | 3 +- src/main.cpp | 4 +- .../include/shambindings/start_python.hpp | 11 ++- src/shambindings/src/start_python.cpp | 67 ++++++++++++++++--- src/shamtest/shamtest.cpp | 1 + 5 files changed, 73 insertions(+), 13 deletions(-) diff --git a/.github/workflows/env_tests.yml b/.github/workflows/env_tests.yml index 29d15475f..fe5ed6719 100644 --- a/.github/workflows/env_tests.yml +++ b/.github/workflows/env_tests.yml @@ -241,7 +241,8 @@ jobs: print("Shamrock version:", shamrock.version_string()) print("Git info:", shamrock.get_git_info()) shamrock.change_loglevel(125) - shamrock.sys.init('0:0') + if not shamrock.sys.is_initialized(): + shamrock.sys.init('0:0') EOF cat /tmp/test_rscript.py diff --git a/src/main.cpp b/src/main.cpp index 5c54a19ef..5bffdf2f5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -190,13 +190,13 @@ int main(int argc, char *argv[]) { "cannot run ipython mode with > 1 processes"); } - shambindings::start_ipython(true); + shambindings::start_ipython(true, argc, argv); } else if (opts::has_option("--rscript")) { __shamrock_stack_entry(); std::string fname = std::string(opts::get_option("--rscript")); - shambindings::run_py_file(fname, shamcomm::world_rank() == 0); + shambindings::run_py_file(fname, shamcomm::world_rank() == 0, argc, argv); } else { if (shamcomm::world_rank() == 0) { diff --git a/src/shambindings/include/shambindings/start_python.hpp b/src/shambindings/include/shambindings/start_python.hpp index 341317034..3167c8aff 100644 --- a/src/shambindings/include/shambindings/start_python.hpp +++ b/src/shambindings/include/shambindings/start_python.hpp @@ -27,6 +27,13 @@ namespace shambindings { */ void setpypath(std::string path); + /** + * @brief set the value of sys.argv + * + * This function will throw if bindings were not initialized in embed mode + */ + void set_sys_argv(int argc, char *argv[]); + /** * @brief set the value of sys.path before init from the supplied binary * @@ -42,7 +49,7 @@ namespace shambindings { * @warning This function shall not be called if more than one processes are running * @param do_print print log at python startup */ - void start_ipython(bool do_print); + void start_ipython(bool do_print, int argc, char *argv[]); /** * @brief run python runscript @@ -52,7 +59,7 @@ namespace shambindings { * @param do_print print log at python startup * @param file_path path to the runscript */ - void run_py_file(std::string file_path, bool do_print); + void run_py_file(std::string file_path, bool do_print, int argc, char *argv[]); /** * @brief Modify Python sys.path to point to one detected during cmake invocation diff --git a/src/shambindings/src/start_python.cpp b/src/shambindings/src/start_python.cpp index 7418a7204..26d9a7053 100644 --- a/src/shambindings/src/start_python.cpp +++ b/src/shambindings/src/start_python.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include /** @@ -92,15 +93,46 @@ std::optional pylib_path_env_var = shamcmdopt::getenv_str("SHAMROCK namespace shambindings { - std::string get_pylib_path(bool do_print) { + std::optional get_binary_path() { + + // first try /proc/self/exe + try { + return std::filesystem::read_symlink("/proc/self/exe"); + } catch (const std::filesystem::filesystem_error &e) { + return std::nullopt; + } + + // then try sys.executable from python because why not XD + try { + py::module_ sys = py::module_::import("sys"); + std::string executable = sys.attr("executable").cast(); + return executable; + } catch (const std::exception &e) { + return std::nullopt; + } + } + + std::string locate_pylib_path(bool do_print) { + + auto get_binary_dir = []() -> std::filesystem::path { + auto bpath = get_binary_path(); + if (bpath.has_value()) { + return std::filesystem::path(bpath.value()).parent_path(); + } + return std::filesystem::path("."); + }; // Get the path to the current binary - // std::filesystem::path binary_path = std::filesystem::read_symlink("/proc/self/exe"); - // std::filesystem::path binary_dir = binary_path.parent_path(); - // std::filesystem::path pyshamrock_path = binary_dir / ".." / "pyshamrock"; + std::filesystem::path binary_dir = get_binary_dir(); + + std::filesystem::path pyshamrock_path_relative1 = binary_dir / ".." / "pylib"; + std::filesystem::path pyshamrock_path_relative2 = binary_dir / ".." / "src" / "pylib"; std::vector possible_paths - = {"pyshamrock", std::string(configure_time_pylib_path())}; + = {"pyshamrock", + pyshamrock_path_relative1, + pyshamrock_path_relative2, + std::string(configure_time_pylib_path())}; if (pylib_path_env_var.has_value()) { possible_paths.push_back(pylib_path_env_var.value()); @@ -149,15 +181,33 @@ namespace shambindings { modify_path += R"(import sys;sys.path = paths)"; py::exec(modify_path); - std::string pylib_path = get_pylib_path(do_print); + std::string pylib_path = locate_pylib_path(do_print); std::string modify_path_lib = std::string("sys.path.insert(0, \"") + pylib_path + "\")\n"; py::exec(modify_path_lib); } - void start_ipython(bool do_print) { + void set_sys_argv(int argc, char *argv[]) { + std::vector sys_argv; + for (int i = 0; i < argc; i++) { + sys_argv.push_back(argv[i]); + } + std::stringstream ss; + ss << "["; + for (const auto &arg : sys_argv) { + ss << "\"" << arg << "\", "; + } + ss << "]"; + + std::string cmd = "import sys; sys.argv = " + ss.str(); + + py::exec(cmd); + } + + void start_ipython(bool do_print, int argc, char *argv[]) { py::scoped_interpreter guard{}; modify_py_sys_path(do_print); + set_sys_argv(argc, argv); if (do_print) { shambase::println("--------------------------------------------"); @@ -172,9 +222,10 @@ namespace shambindings { } } - void run_py_file(std::string file_path, bool do_print) { + void run_py_file(std::string file_path, bool do_print, int argc, char *argv[]) { py::scoped_interpreter guard{}; modify_py_sys_path(do_print); + set_sys_argv(argc, argv); if (do_print) { shambase::println("-----------------------------------"); diff --git a/src/shamtest/shamtest.cpp b/src/shamtest/shamtest.cpp index 008262aed..f934af02f 100644 --- a/src/shamtest/shamtest.cpp +++ b/src/shamtest/shamtest.cpp @@ -501,6 +501,7 @@ namespace shamtest { ON_RANK_0(shamcomm::logs::print_faint_row()); shambindings::modify_py_sys_path(shamcomm::world_rank() == 0); + shambindings::set_sys_argv(argc, argv); ON_RANK_0(shamcomm::logs::print_faint_row()); // import shamrock in pybind From bb037a445cde8a592309a58144bad132f4045de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20David--Cl=C3=A9ris?= Date: Wed, 31 Dec 2025 18:54:52 +0100 Subject: [PATCH 12/15] better --- env/helpers/_pysetup.py | 51 ++++++++--------------------------------- 1 file changed, 10 insertions(+), 41 deletions(-) diff --git a/env/helpers/_pysetup.py b/env/helpers/_pysetup.py index 3ba40e3ae..1e99cfeba 100644 --- a/env/helpers/_pysetup.py +++ b/env/helpers/_pysetup.py @@ -50,51 +50,20 @@ def build_extension(self, ext: ShamEnvExtension) -> None: print(f"### {activate_build_dir=}") print(f"### {cwd_is_build=}") - print("-- Activating env") - subprocess.run( - [ - "bash", - "-c", - "source ./activate", - ], - check=True, - ) - - print("-- Configure") - subprocess.run( - [ - "bash", - "-c", - "source ./activate && shamconfigure", - ], - check=True, - ) - - print("-- Compile") - subprocess.run( - [ - "bash", - "-c", - "source ./activate && shammake shamrock shamrock_pylib", - ], - check=True, - ) - print("-- mkdir output dir") print(f" -> mkdir -p {extdir}") subprocess.run(["bash", "-c", f"mkdir -p {extdir}"], check=True) - print("-- Copy lib&exe to output dir") - subprocess.run(["bash", "-c", f"ls {activate_build_dir}"], check=True) - - if not cwd_is_build: - subprocess.run( - ["bash", "-c", f" cp -v {activate_build_dir}/*.so {activate_build_dir}/shamrock ."], - check=True, - ) + install_steps = [ + "source ./activate", + "shamconfigure", + f"cmake . -DCMAKE_INSTALL_PREFIX={extdir} -DCMAKE_INSTALL_PYTHONDIR={extdir}", + "shammake install", + ] - subprocess.run(["bash", "-c", f"ls {activate_build_dir}"], check=True) - subprocess.run(["bash", "-c", f"cp -v {activate_build_dir}/*.so {extdir}"], check=True) + cmd = " && ".join(install_steps) + print(f"-- Run install: {cmd}") + subprocess.run(["bash", "-c", cmd], check=True) # The information here can also be placed in setup.cfg - better separation of @@ -106,7 +75,7 @@ def build_extension(self, ext: ShamEnvExtension) -> None: author_email="tim.shamrock@proton.me", description="SHAMROCK Code for astrophysics", long_description="", - ext_modules=[ShamEnvExtension("shamrock")], + ext_modules=[ShamEnvExtension("shamrock.pyshamrock")], data_files=[("bin", ["shamrock"])], cmdclass={"build_ext": ShamEnvBuild}, zip_safe=False, From ad513d988bd0e11281c89117fddae79a2953f275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20David--Cl=C3=A9ris?= Date: Wed, 31 Dec 2025 19:19:34 +0100 Subject: [PATCH 13/15] better --- env/helpers/_pysetup.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/env/helpers/_pysetup.py b/env/helpers/_pysetup.py index 1e99cfeba..fb2b7f74c 100644 --- a/env/helpers/_pysetup.py +++ b/env/helpers/_pysetup.py @@ -21,7 +21,24 @@ def __init__(self, name: str, sourcedir: str = "") -> None: class ShamEnvBuild(build_ext): + + def init_editable_mode(self) -> bool: + # Detect editable mode + editable_mode = False + + # Method 1: Check self.inplace (most reliable for build_ext) + if hasattr(self, "inplace") and self.inplace: + editable_mode = True + + # Method 2: Check for editable_mode attribute (newer setuptools) + if hasattr(self, "editable_mode") and self.editable_mode: + editable_mode = True + return editable_mode + def build_extension(self, ext: ShamEnvExtension) -> None: + editable_mode = self.init_editable_mode() + print(f"-- Editable mode: {editable_mode}") + # Must be in this form due to bug in .resolve() only fixed in Python 3.10+ ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name) # type: ignore[no-untyped-call] extdir = ext_fullpath.parent.resolve() From deb45b237d9cc0c7d900a194c5ce3fe44b9c01ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20David--Cl=C3=A9ris?= Date: Wed, 31 Dec 2025 20:24:31 +0100 Subject: [PATCH 14/15] better --- .github/workflows/env_tests.yml | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.github/workflows/env_tests.yml b/.github/workflows/env_tests.yml index fe5ed6719..d830ba223 100644 --- a/.github/workflows/env_tests.yml +++ b/.github/workflows/env_tests.yml @@ -270,3 +270,53 @@ jobs: PYTHONPATH=$(pwd)/../pyinstall_test:$PYTHONPATH \ LD_LIBRARY_PATH=$(pwd)/lib:$LD_LIBRARY_PATH \ python3 /tmp/test_rscript.py + + - name: Try pip install Shamrock + if: matrix.runtestpy + run: | + cd build + pip install --verbose . + + - name: Try shamrock startup (pip install) + if: matrix.runtestpy + run: | + cd build + ACPP_VISIBILITY_MASK=omp \ + ACPP_DEBUG_LEVEL=3 \ + python3 /tmp/test_rscript.py + + - name: Try shamrock startup (pip install) + if: matrix.runtestpy + run: | + cd build + ACPP_VISIBILITY_MASK=omp \ + ACPP_DEBUG_LEVEL=3 \ + shamrock --smi --sycl-cfg 0:0 --rscript /tmp/test_rscript.py + + - name: Try pip uninstall Shamrock + if: matrix.runtestpy + run: | + cd build + pip uninstall shamrock + + - name: Try pip install editable Shamrock + if: matrix.runtestpy + run: | + cd build + pip install --verbose -e . + + - name: Try shamrock startup (pip install) + if: matrix.runtestpy + run: | + cd build + ACPP_VISIBILITY_MASK=omp \ + ACPP_DEBUG_LEVEL=3 \ + python3 /tmp/test_rscript.py + + - name: Try shamrock startup (pip install) + if: matrix.runtestpy + run: | + cd build + ACPP_VISIBILITY_MASK=omp \ + ACPP_DEBUG_LEVEL=3 \ + shamrock --smi --sycl-cfg 0:0 --rscript /tmp/test_rscript.py From 4e55c300e6f0f940041b35f3d68b57946f35ae79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20David--Cl=C3=A9ris?= Date: Wed, 31 Dec 2025 21:56:29 +0100 Subject: [PATCH 15/15] better --- .github/workflows/env_tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/env_tests.yml b/.github/workflows/env_tests.yml index d830ba223..71f7a62ea 100644 --- a/.github/workflows/env_tests.yml +++ b/.github/workflows/env_tests.yml @@ -271,6 +271,12 @@ jobs: LD_LIBRARY_PATH=$(pwd)/lib:$LD_LIBRARY_PATH \ python3 /tmp/test_rscript.py + - name: Rm CMake cache + if: matrix.runtestpy + run: | + cd build + rm -rf CMakeCache.txt + - name: Try pip install Shamrock if: matrix.runtestpy run: |