Skip to content

Commit c0d8bce

Browse files
BrandynTucknottBrandynTucknott
authored
Merge branch 'Iainmon:main' into main
2 parents 72c109e + b5dbff2 commit c0d8bce

63 files changed

Lines changed: 1763 additions & 553 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.dockerignore‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
presentation
2+
libtorch_static
3+
libtorch
4+
target
5+
docs
6+
dependencies
7+
build
8+
cmake-build-debug
9+
.git
10+
learning
11+
.cache

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,7 @@ libtorch/
4545
build-release/
4646
libtorch_static/
4747
examples/vgg/**/*.pt
48+
segmentation_models.pytorch/
49+
.idea/
50+
cmake-build-debug/
51+
libtorch_old/

‎CMakeLists.txt‎

Lines changed: 70 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
cmake_minimum_required(VERSION 3.31 FATAL_ERROR)
1+
cmake_minimum_required(VERSION 3.20 FATAL_ERROR)
22
# project(MyProject LANGUAGES CXX)
33

44
include(CMakePrintHelpers)
@@ -15,7 +15,7 @@ set(PROJECT_CACHE_DIR "${PROJECT_ROOT_DIR}/.cache")
1515
set(LIBTORCH_DIR "${PROJECT_ROOT_DIR}/libtorch")
1616
# set(LIBTORCH_STATIC_INSTALL_DIR "${PROJECT_ROOT_DIR}/libtorch_static")
1717
set(BRIDGE_DIR "${PROJECT_ROOT_DIR}/bridge")
18-
18+
set(Torch_DIR "${PROJECT_ROOT_DIR}/libtorch")
1919

2020
find_package(chpl REQUIRED HINTS ${PROJECT_ROOT_DIR}/cmake/chapel)
2121
list(APPEND CMAKE_MODULE_PATH "${PROJECT_ROOT_DIR}/cmake")
@@ -29,7 +29,7 @@ if(APPLE)
2929
set(CMAKE_C_COMPILER "/usr/bin/clang")
3030
set(CMAKE_CXX_COMPILER "/usr/bin/clang++")
3131
endif()
32-
set(CMAKE_CXX_STANDARD 23)
32+
set(CMAKE_CXX_STANDARD 20)
3333

3434

3535
include(LibTorchDL)
@@ -38,63 +38,16 @@ download_libtorch(
3838
DESTINATION ${LIBTORCH_DIR}
3939
)
4040

41+
list(APPEND CMAKE_PREFIX_PATH "${LIBTORCH_DIR}/share/cmake")
42+
find_package(Torch REQUIRED)
43+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++23 -lm -ldl ${TORCH_CXX_FLAGS}")
4144

42-
include(FetchContent)
43-
include(ExternalProject)
44-
45-
46-
# if(NOT EXISTS "${LIBTORCH_STATIC_INSTALL_DIR}/lib/libtorch.a")
47-
# ExternalProject_Add(
48-
# pytorch
49-
# GIT_REPOSITORY https://github.com/pytorch/pytorch.git
50-
# GIT_TAG v2.6.0 # Example: specify a particular release
51-
# UPDATE_COMMAND "" # Don’t auto-run 'git pull'
52-
# PATCH_COMMAND "" # No custom patch step
53-
54-
# # DOWNLOAD_DIR "${CMAKE_BINARY_DIR}/pytorch-download" # Where to download the repo
55-
56-
# # We need all PyTorch submodules. By default, ExternalProject won't do submodule init.
57-
# # So we can do that in a separate step if we want a full build. For a minimal CPU build,
58-
# # you might not need them all, but let's be safe:
59-
# STEP_TARGETS clone
60-
# # After 'clone', run "git submodule update --init --recursive"
61-
# # to fetch all submodules.
62-
# # We can use a little trick with COMMAND.
63-
# # PATCH_COMMAND "git submodule update --init --recursive"
64-
65-
# # CMAKE_ARGS
66-
# # -DBUILD_SHARED_LIBS=OFF # Build static libraries
67-
# # -DBUILD_PYTHON=OFF # Don’t build Python bindings
68-
# # -DBUILD_TEST=OFF # Don’t build tests
69-
# # -DUSE_CUDA=OFF # Disable CUDA
70-
# # -DUSE_CUDNN=OFF # Disable cuDNN
71-
# # -DUSE_MKLDNN=OFF # Disable MKLDNN for simplicity
72-
# # # -DBUILD_BINARY=ON
73-
# # # -DUSE_DISTRIBUTED=ON
74-
# # # -DBUILD_STATIC_RUNTIME_BENCHMARK=ON
75-
# # # -DBUILD_LITE_INTERPRETER=ON
76-
# # # -DUSE_STATIC_MKL=ON
77-
# # # -DSTATIC_DISPATCH_BACKEND=ON
78-
# # # -DCAFFE2_USE_MSVC_STATIC_RUNTIME=ON
79-
# # # -DUSE_DISTRIBUTED=ON
80-
# # # -DCMAKE_BUILD_TYPE=Release
81-
# # -DCMAKE_INSTALL_PREFIX=${PYTORCH_INSTALL_DIR}
82-
# # # -DCMAKE_POLICY_VERSION_MINIMUM=3.5
83-
# CMAKE_ARGS
84-
# -DBUILD_SHARED_LIBS=OFF # Build static libraries
85-
# -DBUILD_PYTHON=OFF # Don’t build Python bindings
86-
# -DBUILD_TEST=OFF # Don’t build tests
87-
# -DUSE_CUDA=OFF # Disable CUDA
88-
# -DUSE_CUDNN=OFF # Disable cuDNN
89-
# -DUSE_MKLDNN=OFF # Disable MKLDNN for simplicity
90-
# -DCMAKE_BUILD_TYPE=Release
91-
# # -DSTATIC_DISPATCH_BACKEND=ON
92-
# -DCMAKE_INSTALL_PREFIX=${LIBTORCH_STATIC_INSTALL_DIR}
93-
# -DCMAKE_POLICY_VERSION_MINIMUM=3.5
94-
95-
# INSTALL_DIR ${LIBTORCH_STATIC_INSTALL_DIR} # Where to install
96-
# )
97-
# endif()
45+
46+
cmake_print_variables(TORCH_LIBRARIES)
47+
cmake_print_variables(TORCH_INCLUDE_DIRS)
48+
cmake_print_variables(TORCH_INSTALL_PREFIX)
49+
cmake_print_variables(TORCH_CXX_FLAGS)
50+
cmake_print_variables(TORCH_LIBRARY)
9851

9952

10053
file(GLOB CHAI_LIB_FILES "${PROJECT_ROOT_DIR}/lib/*.chpl")
@@ -152,25 +105,39 @@ function(watch)
152105
)
153106
endfunction()
154107

108+
109+
155110
# cmake_print_variables(CHAI_LIB_FILES)
156111

157-
add_library(bridge STATIC ${BRIDGE_DIR}/include/bridge.h ${BRIDGE_DIR}/lib/bridge.cpp)
112+
add_library(bridge OBJECT ${BRIDGE_DIR}/include/bridge.h ${BRIDGE_DIR}/lib/bridge.cpp)
113+
114+
target_link_libraries(
115+
bridge
116+
PRIVATE
117+
${TORCH_LIBRARIES}
118+
# ${LIBTORCH_DIR}/lib/libtorch.so
119+
# ${LIBTORCH_DIR}/lib/libtorch_cpu.so
120+
# ${LIBTORCH_DIR}/lib/libc10.so
121+
# ${LIBTORCH_DIR}/lib/libtorch_global_deps.so
122+
)
158123

159124
target_include_directories(
160125
bridge
161126
PRIVATE
162127
${BRIDGE_DIR}/include
163128
${LIBTORCH_DIR}/include
164129
${LIBTORCH_DIR}/include/torch/csrc/api/include
130+
# ${BRIDGE_DIR}/util
165131
)
166132

133+
167134
set(BRIDGE_OBJECT_FILES $<TARGET_OBJECTS:bridge>)
168135

169136

170137

171138

172139
file(GLOB LIBTORCH_ALL_LIB_FILES
173-
"${LIBTORCH_DIR}/lib/*.a"
140+
# "${LIBTORCH_DIR}/lib/*.a"
174141
"${LIBTORCH_DIR}/lib/*.dylib"
175142
"${LIBTORCH_DIR}/lib/*.so")
176143

@@ -185,7 +152,6 @@ set(REQUIRED_LIBS
185152
"libtorch"
186153
"libtorch_cpu"
187154
"libc10"
188-
"libtorch_global_deps"
189155
)
190156

191157
set(DISALLOWED_LIBS
@@ -206,11 +172,20 @@ foreach(lib_name IN LISTS LIBTORCH_ALL_LIBS)
206172
list(APPEND LIBTORCH_LIBS_LINKER_ARGS "-l${lib_name_short}")
207173
endforeach()
208174

209-
# cmake_print_variables(LIBTORCH_LIBS_LINKER_ARGS)
175+
cmake_print_variables(LIBTORCH_LIBS_LINKER_ARGS)
210176
# cmake_print_variables(${BRIDGE_OBJECT_FILES})
211177
# cmake_print_variables(BRIDGE_OBJECT_FILES)
212178

213179

180+
set(LIBTORCH_LIBS_LINKER_ARGS
181+
"-ltorch"
182+
# "-ltorch_cpu"
183+
# "-lc10"
184+
# "-ltorch_global_deps"
185+
)
186+
cmake_print_variables(LIBTORCH_LIBS_LINKER_ARGS)
187+
188+
214189
add_executable(TorchBridge ${BRIDGE_DIR}/lib/Bridge.chpl)
215190
add_dependencies(TorchBridge bridge)
216191
add_dependencies(TorchBridge ChAI)
@@ -219,10 +194,10 @@ target_link_options(TorchBridge
219194
${BRIDGE_DIR}/include/bridge.h
220195
${BRIDGE_OBJECT_FILES}
221196
-L ${LIBTORCH_DIR}/lib
222-
# "-ltorch"
223-
# "-ltorch_cpu"
224-
# "-lc10"
225-
# "-ltorch_global_deps"
197+
"-ltorch"
198+
"-ltorch_cpu"
199+
"-lc10"
200+
"-ltorch_global_deps"
226201
${LIBTORCH_LIBS_LINKER_ARGS}
227202
--ldflags "-Wl,-rpath,${LIBTORCH_DIR}/lib"
228203
)
@@ -292,5 +267,33 @@ add_custom_command(
292267

293268
# ./vgg images/frog.jpg
294269

270+
# add_subdirectory(bridge)
271+
295272
add_subdirectory(examples)
296273
add_subdirectory("test")
274+
add_subdirectory(demos)
275+
276+
277+
278+
add_executable(MPSTest
279+
"${PROJECT_ROOT_DIR}/test/tiny/mps.cpp"
280+
)
281+
282+
target_include_directories(MPSTest
283+
PRIVATE
284+
${LIBTORCH_DIR}/include
285+
${LIBTORCH_DIR}/include/torch/csrc/api/include
286+
)
287+
288+
target_link_directories(MPSTest
289+
PRIVATE
290+
${LIBTORCH_DIR}/lib
291+
)
292+
293+
target_link_libraries(MPSTest
294+
PRIVATE
295+
-ltorch
296+
-ltorch_cpu
297+
-lc10
298+
-ltorch_global_deps
299+
)

‎Dockerfile‎

Lines changed: 64 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,79 @@
1-
FROM ubuntu:22.04
1+
# ────────────────────────────────
2+
# 1. Build stage
3+
# ────────────────────────────────
4+
FROM ubuntu:24.04 AS builder
25

36

4-
# RUN apt update > /dev/null && apt install -y apt-utils && apt upgrade -y > /dev/null
57

8+
ENV DEBIAN_FRONTEND=noninteractive
69

710
COPY deb /
811

9-
RUN export DEBIAN_FRONTEND=noninteractive && \
10-
apt update > /dev/null && \
11-
apt upgrade -y > /dev/null && \
12-
apt install -y apt-utils > /dev/null && \
13-
apt install -y python3-pip > /dev/null && \
14-
(apt install -y ./chapel-2.3.0-1.ubuntu22.arm64.deb > /dev/null) || (apt install -y ./chapel-2.3.0-1.ubuntu22.amd64.deb > /dev/null)
12+
# -------- toolchain & helpers
13+
RUN apt-get update && apt-get install -y --no-install-recommends \
14+
build-essential g++ cmake ninja-build \
15+
git curl wget ca-certificates unzip \
16+
python3 python3-pip libssl-dev \
17+
apt-utils \
18+
&& ((apt-get install -y --no-install-recommends ./chapel-2.4.0-1.ubuntu24.arm64.deb) || (apt-get install -y --no-install-recommends ./chapel-2.4.0-1.ubuntu24.amd64.deb)) \
19+
&& rm -rf /var/lib/apt/lists/*
1520

21+
# && (apt-get install -y --no-install-recommends ./chapel-2.4.0-1.ubuntu24.arm64.deb) || (apt install -y ./chapel-2.4.0-1.ubuntu24.amd64.deb)
1622

23+
# -------- LibTorch (change CUDA tag for GPU builds)
24+
# ARG TORCH_VERSION=2.2.0 # ← keep in sync with your sources
25+
# ARG TORCH_VARIANT=cpu # cpu | cu118 | cu121 …
26+
# RUN mkdir -p /opt \
27+
# && cd /opt \
28+
# && wget -q --show-progress \
29+
# https://download.pytorch.org/libtorch/${TORCH_VARIANT}/libtorch-cxx11-abi-shared-with-deps-${TORCH_VERSION}%2B${TORCH_VARIANT}.zip \
30+
# && unzip libtorch-cxx11-abi-shared-with-deps-${TORCH_VERSION}+${TORCH_VARIANT}.zip \
31+
# && rm libtorch-cxx11-abi-shared-with-deps-${TORCH_VERSION}+${TORCH_VARIANT}.zip
1732

18-
RUN pip3 install numpy > /dev/null
1933

20-
RUN pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu > /dev/null
34+
RUN mkdir -p /app \
35+
&& cd /app \
36+
&& wget -q --show-progress \
37+
https://download.pytorch.org/libtorch/nightly/cpu/libtorch-cxx11-abi-shared-with-deps-latest.zip \
38+
&& unzip libtorch-cxx11-abi-shared-with-deps-latest.zip \
39+
&& mkdir -p /app/.cache \
40+
&& mv /app/libtorch-cxx11-abi-shared-with-deps-latest.zip /app/.cache/libtorch_cache.zip
2141

2242

23-
COPY lib /lib
24-
COPY examples /examples
25-
COPY test /test
43+
ENV Torch_DIR=/app/libtorch
44+
ENV CMAKE_PREFIX_PATH=/app/libtorch/libtorch/share/cmake
45+
# lets CMake find TorchConfig.cmake
2646

27-
# Run tests
28-
RUN cd test/correspondence && python3 correspondence.py
29-
RUN cd test/correspondence && python3 correspondence.py --print-compiler-errors
3047

48+
# -------- build your project
49+
WORKDIR /app
50+
COPY . /app
51+
52+
# (uncomment if you have submodules)
53+
# RUN git submodule update --init --recursive
54+
55+
RUN ls
56+
57+
RUN cd /app \
58+
&& mkdir -p build \
59+
&& cd build \
60+
&& cmake -DCMAKE_BUILD_TYPE=Debug .. \
61+
&& make MyExample
62+
63+
# RUN mkdir build \
64+
# && cd build \
65+
# && cmake -DCMAKE_BUILD_TYPE=Debug .. \
66+
# && make TinyLayerTest -j$(nproc)
67+
68+
#RUN cd build && ./TinyLayerTest
69+
70+
# && cmake -DCMAKE_PREFIX_PATH=/opt/libtorch -DCMAKE_BUILD_TYPE=Release .. \
71+
72+
# configure-and-build
73+
# RUN cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=Release \
74+
# && cmake --build build --target all -j$(nproc)
75+
76+
# RUN cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=Release
77+
78+
# RUN cmake --build build --target all -j$(nproc)
3179

32-
# Build instructions
33-
# force (last resort): docker system prune --all --force
34-
# docker builder prune --all
35-
# docker build --no-cache -t chapel-deb .
36-
# docker build -t chapel-deb .
37-
# docker container run -it chapel-deb bash

‎Dockerfile.testing‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
FROM ubuntu:24.04
2+
3+
4+
# RUN apt update > /dev/null && apt install -y apt-utils && apt upgrade -y > /dev/null
5+
6+
7+
COPY deb /
8+
9+
RUN export DEBIAN_FRONTEND=noninteractive && \
10+
apt update > /dev/null && \
11+
apt upgrade -y > /dev/null && \
12+
apt install -y apt-utils > /dev/null && \
13+
apt install -y python3-pip > /dev/null && \
14+
(apt install -y ./chapel-2.4.0-1.ubuntu25.arm64.deb > /dev/null) || (apt install -y ./chapel-2.4.0-1.ubuntu25.amd64.deb > /dev/null)
15+
16+
17+
18+
RUN pip3 install numpy > /dev/null
19+
20+
RUN pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu > /dev/null
21+
22+
23+
COPY lib /lib
24+
COPY examples /examples
25+
COPY test /test
26+
27+
# Run tests
28+
RUN cd test/correspondence && python3 correspondence.py
29+
RUN cd test/correspondence && python3 correspondence.py --print-compiler-errors
30+
31+
32+
# Build instructions
33+
# force (last resort): docker system prune --all --force
34+
# docker builder prune --all
35+
# docker build --no-cache -t chapel-deb .
36+
# docker build -t chapel-deb .
37+
# docker container run -it chapel-deb bash

0 commit comments

Comments
 (0)