Working on linking all of pytorch as a static lib.

Author: Iainmon

.gitignore

@@ -36,4 +36,6 @@ test/correspondence/gh.out
 !modules/*
 modules/build
 !bridge/*
-bridge/build
+bridge/build
+pytorch/build
+# pytorch/

bridge/CMakeLists.txt

@@ -11,18 +11,12 @@ endif()
 
 set(CMAKE_C_COMPILER "clang")
 set(CMAKE_CXX_COMPILER "clang++")
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 11)
 
 
 set(PROJECT_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(PROJECT_BINARY_DIR "${CMAKE_BINARY_DIR}")
 
-find_package(chpl REQUIRED HINTS ${PROJECT_ROOT_DIR}/cmake/chapel)
-list(APPEND CMAKE_MODULE_PATH "${PROJECT_ROOT_DIR}/cmake")
-list(APPEND CMAKE_MODULE_PATH "${PROJECT_ROOT_DIR}/cmake/chapel")
-
-project(MyProject LANGUAGES CXX C CHPL)
-
 
 # For ExternalProject_Add
 include(FetchContent)
@@ -34,6 +28,10 @@ include(ExternalProject)
 
 # Where to place PyTorch after installation
 set(PYTORCH_INSTALL_DIR "${PROJECT_BINARY_DIR}/pytorch-install")
+set(PYTORCH_BUILD_DIR "${PROJECT_BINARY_DIR}/pytorch-prefix/src/pytorch-build")
+
+set(PYTORCH_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/pytorch-install-prefix")
+file(MAKE_DIRECTORY "${PYTORCH_INSTALL_PREFIX}")
 
 # ExternalProject_Add can fetch from Git, a local path, or a release tarball.
 # Here, for simplicity, we'll fetch from Git. In practice, you might want
@@ -64,17 +62,26 @@ ExternalProject_Add(
       -DUSE_CUDA=OFF                       # Disable CUDA
       -DUSE_CUDNN=OFF                      # Disable cuDNN
       -DUSE_MKLDNN=OFF                     # Disable MKLDNN for simplicity
+      # -DBUILD_BINARY=ON
+      # -DUSE_DISTRIBUTED=ON
+      # -DBUILD_STATIC_RUNTIME_BENCHMARK=ON
+      # -DBUILD_LITE_INTERPRETER=ON
+      # -DUSE_STATIC_MKL=ON
+      # -DSTATIC_DISPATCH_BACKEND=ON
+      # -DCAFFE2_USE_MSVC_STATIC_RUNTIME=ON
+      -DUSE_DISTRIBUTED=ON
       -DCMAKE_BUILD_TYPE=Release
-      -DCMAKE_INSTALL_PREFIX=${PYTORCH_INSTALL_DIR}
+      -DCMAKE_INSTALL_PREFIX=${PYTORCH_INSTALL_PREFIX}
+      # -DCMAKE_POLICY_VERSION_MINIMUM=3.5
 
   INSTALL_DIR         ${PYTORCH_INSTALL_DIR} # Where to install
 
-  LOG_DOWNLOAD ON
-  LOG_UPDATE ON
-  LOG_PATCH ON
-  LOG_CONFIGURE ON
+  # LOG_DOWNLOAD ON
+  # LOG_UPDATE ON
+  # LOG_PATCH ON
+  # LOG_CONFIGURE ON
   # LOG_BUILD ON
-  LOG_INSTALL ON
+  # LOG_INSTALL ON
 
 )
 
@@ -83,6 +90,23 @@ file(GLOB_RECURSE PYTORCH_INCLUDES "${PYTORCH_INSTALL_DIR}/include" "*.h")
 file(GLOB_RECURSE PYTORCH_LIBS "${PYTORCH_INSTALL_DIR}/lib" "*.a")
 
 
+set(PYTORCH_LIBS_LINKER_ARGS "")  # Will hold the list of "-l..." flags.
+foreach(lib_path IN LISTS PYTORCH_LIBS)
+  # Get just the filename without the directory or extension
+  get_filename_component(lib_name "${lib_path}" NAME_WE)
+  # If it starts with "lib", strip that off
+  string(REGEX REPLACE "^lib" "" lib_name "${lib_name}")
+  # Now prepend "-l" to the actual library name
+  list(APPEND MY_LIBS "-l${lib_name}")
+endforeach()
+
+
+find_package(chpl REQUIRED HINTS ${PROJECT_ROOT_DIR}/cmake/chapel)
+list(APPEND CMAKE_MODULE_PATH "${PROJECT_ROOT_DIR}/cmake")
+list(APPEND CMAKE_MODULE_PATH "${PROJECT_ROOT_DIR}/cmake/chapel")
+
+project(MyProject LANGUAGES CXX C CHPL)
+
 # ------------------------------------------------------------------------------
 # 2) Create an INTERFACE library to wrap the installed static libs
 # ------------------------------------------------------------------------------
@@ -99,6 +123,7 @@ add_dependencies(torch_interface pytorch)
 target_include_directories(torch_interface INTERFACE
     "${PYTORCH_INSTALL_DIR}/include"
     "${PYTORCH_INSTALL_DIR}/include/torch/csrc/api/include"
+    # "${PYTORCH_BUILD_DIR}/aten/src/include"
 )
 
 # Link the relevant static libraries.  For a minimal CPU-only build, 
@@ -126,17 +151,22 @@ add_executable(CHPLTest lib/CHPLTest.chpl)
 add_library(torchbridge STATIC "${PROJECT_ROOT_DIR}/lib/bridge.cpp" "${PROJECT_ROOT_DIR}/include/bridge.h")
 add_dependencies(torchbridge torch_interface)
 
-target_include_directories(torchbridge PUBLIC
+target_include_directories(torchbridge PRIVATE
     "${PYTORCH_INSTALL_DIR}/include"
     "${PYTORCH_INSTALL_DIR}/include/torch/csrc/api/include"
     "${PROJECT_ROOT_DIR}/include"
+    # "${PYTORCH_BUILD_DIR}/aten/src/include"
 )
 # target_link_directories(torchbridge PUBLIC
 #     "${PYTORCH_INSTALL_DIR}/lib"
 # )
 target_link_libraries(torchbridge 
-    torch_interface
-    ${PYTORCH_LIBS}
+    PRIVATE 
+      torch_interface
+      ${PYTORCH_LIBS}
+      pthread
+      dl
+      rt
     )
 
 
@@ -145,23 +175,29 @@ target_link_libraries(torchbridge
 
 add_executable(Bridge lib/Bridge.chpl include/bridge.h)
 # add_dependencies(TorchBridge torchbridge)
+add_dependencies(Bridge torchbridge)
 
 target_link_options(Bridge 
   PRIVATE 
     "${PROJECT_ROOT_DIR}/include/bridge.h"
     "-L${PROJECT_BINARY_DIR}"
+    "-ltorchbridge"
     -L. 
     "-ltorchbridge"
-    "-I${PROJECT_BINARY_DIR}"
-    "-L${PYTORCH_INSTALL_DIR}/lib"
-    "-I${PYTORCH_INSTALL_DIR}/include"
-    "-I${PYTORCH_INSTALL_DIR}/include/torch/csrc/api/include"
-    "-ltorch"
-    "-ltorch_cpu"
-    "-lcpuinfo"
-    "-lc10"
-    "-lpthread"
-    "-ldl"
+    # "-I${PROJECT_BINARY_DIR}"
+    # "-L${PYTORCH_INSTALL_DIR}/lib"
+    # "-I${PYTORCH_INSTALL_DIR}/include"
+    # "-I${PYTORCH_INSTALL_DIR}/include/torch/csrc/api/include"
+    # "-ltorch"
+    # "-ltorch_cpu"
+    # "-lcpuinfo"
+    # "-lc10"
+    # "-lsleef"
+    # "-lclog"
+    # "-lprotoc"
+    # ${PYTORCH_LIBS_LINKER_ARGS}
+    # "-lpthread"
+    # "-ldl"
   )
 
 

bridge/include/bridge.h

@@ -4,9 +4,6 @@
 #include <stdio.h>
 
 #ifdef __cplusplus
-#include <torch/torch.h>
-#include <iostream>
-#include <vector>
 extern "C" {
 #endif
 

bridge/lib/Bridge.chpl

@@ -6,7 +6,9 @@ extern proc wrHello(): void;
 extern proc wrHelloTorch(): void;
 extern proc sumArray(arr: [] real(32), sizes: [] int(32), dim: int(32)): real(32);
 
-wrHello();
+
+baz();
+
 wrHello();
 
 

bridge/lib/bridge.cpp

@@ -1,9 +1,24 @@
 #include <bridge.h>
 
+#include <torch/torch.h>
+// #include <torch/script.h>
+// #include <Aten/ATen.h>
+#include <iostream>
+#include <vector>
+
+void secret() {
+    std::cout << "Secret function called!" << std::endl;
+    torch::Tensor tensor = torch::eye(3);
+    // std::cout << "Tensor: " << tensor << std::endl;
+
+    // std::cout << "Secret function called! Tensor: " << tensor << std::endl;
+}
 
 extern "C" int baz(void) {
-    auto x = torch::randn({5, 3});
-    return x.size(0);
+    printf("Hello from baz!\n");
+    secret();
+    // auto x = torch::randn({5, 3});
+    // return x.size(0);
 }
 
 
@@ -14,27 +29,27 @@ extern "C" void wrHello(void) {
 
 extern "C" void wrHelloTorch(void) {
     printf("Hello from wrHelloTorch!\n");
-    auto t = torch::ones({2, 3});
-    std::cout << t << std::endl;
+    // auto t = torch::ones({2, 3});
+    // std::cout << t << std::endl;
 }
 
 extern "C" float sumArray(float* arr, int* sizes, int dim) {
     // Convert sizes to std::vector<int64_t>
 
     printf("sumArray called with arr: %p, sizes: %p, dim: %d\n", arr, sizes, dim);
 
-    std::vector<int64_t> sizes_vec(sizes, sizes + dim);
-    std::cout << sizes_vec << std::endl;
+    // std::vector<int64_t> sizes_vec(sizes, sizes + dim);
+    // std::cout << sizes_vec << std::endl;
 
-    auto shape = at::IntArrayRef(sizes_vec);
-    std::cout << shape << std::endl;
+    // auto shape = at::IntArrayRef(sizes_vec);
+    // std::cout << shape << std::endl;
 
-    auto t = torch::from_blob(arr, shape, torch::kFloat);
-    std::cout << t << std::endl;
+    // auto t = torch::from_blob(arr, shape, torch::kFloat);
+    // std::cout << t << std::endl;
 
-    return t.sum().item<float>();
+    // return t.sum().item<float>();
 
-    // return 0.0f;
+    return 0.0f;
 
     // float sum = 0.0f;
     // for (int i = 0; i < size; ++i) {

pytorch

@@ -0,0 +1 @@
+Subproject commit 58ede0cca3e73a05659bd8de8f131f2e83601b5c

requirements.txt

@@ -0,0 +1,79 @@
+annotated-types==0.7.0
+anyio==4.8.0
+appnope==0.1.4
+asttokens==3.0.0
+astunparse==1.6.3
+attrs==25.3.0
+certifi==2024.12.14
+charset-normalizer==3.4.1
+cmake==4.0.0
+comm==0.2.2
+contourpy==1.3.1
+cycler==0.12.1
+debugpy==1.8.12
+decorator==5.1.1
+executing==2.1.0
+expecttest==0.3.0
+filelock==3.16.1
+fonttools==4.56.0
+fsspec==2024.12.0
+h11==0.14.0
+httpcore==1.0.7
+httpx==0.28.1
+hypothesis==6.130.10
+idna==3.10
+ipykernel==6.29.5
+ipython==8.31.0
+jedi==0.19.2
+Jinja2==3.1.5
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+kagglehub==0.3.10
+kiwisolver==1.4.8
+lintrunner==0.12.7
+MarkupSafe==3.0.2
+matplotlib==3.10.1
+matplotlib-inline==0.1.7
+mpmath==1.3.0
+nest-asyncio==1.6.0
+networkx==3.4.2
+ninja==1.11.1.4
+numpy==2.2.1
+ollama==0.4.7
+opencv-python==4.11.0.86
+optree==0.15.0
+packaging==24.2
+parso==0.8.4
+pexpect==4.9.0
+pillow==11.0.0
+platformdirs==4.3.6
+prompt_toolkit==3.0.48
+psutil==6.1.1
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pydantic==2.10.6
+pydantic_core==2.27.2
+Pygments==2.19.1
+pyparsing==3.2.1
+python-dateutil==2.9.0.post0
+PyYAML==6.0.2
+pyzmq==26.2.0
+requests==2.32.3
+setuptools==75.8.0
+six==1.17.0
+sniffio==1.3.1
+sortedcontainers==2.4.0
+stack-data==0.6.3
+sympy==1.13.3
+tinygrad==0.10.2
+torch==2.5.1
+torchaudio==2.5.1
+torchvision==0.20.1
+tornado==6.4.2
+tqdm==4.67.1
+traitlets==5.14.3
+types-dataclasses==0.6.6
+typing_extensions==4.12.2
+urllib3==2.3.0
+wcwidth==0.2.13
+wheel==0.45.1

to_run.sh

@@ -0,0 +1,9 @@
+git clone --recursive https://github.com/pytorch/pytorch
+cd pytorch
+pip install cmake ninja
+pip install -r requirements.txt
+pip install pkg-config libuv
+MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ python setup.py build --cmake-only
+
+
+python3 setup.py develop