Use CMake's FetchContent to install third_party dependencies (#68)

Use CMake's FetchContent to install third_party dependencies & remove git submodules.
Managing software versions within a build allows for more easily reproducible builds.

Author: jbcoe

.github/workflows/test_and_deploy.yml

@@ -36,8 +36,6 @@ jobs:
               software-properties-common
           fi
       - uses: actions/checkout@v4
-        with:
-          submodules: true
       - name: Install from repo in test mode
         run: pip install -e '.[dev]'
       - name: Run tests
@@ -54,8 +52,6 @@ jobs:
       - name: Ensure latest pip, wheel & setuptools
         run: python -m pip install -q --upgrade pip wheel setuptools
       - uses: actions/checkout@v4
-        with:
-          submodules: true
       - name: Generate sdist
         run: |
           NLE_RELEASE_BUILD=1 python setup.py sdist
@@ -82,8 +78,6 @@ jobs:
       - name: Ensure latest pip, wheel & setuptools
         run: python -m pip install -q --upgrade pip wheel setuptools
       - uses: actions/checkout@v4
-        with:
-          submodules: true
       - name: Generate sdist
         run: |
           NLE_RELEASE_BUILD=1 python setup.py sdist
@@ -113,8 +107,6 @@ jobs:
         os: [ubuntu-latest, ubuntu-24.04-arm]
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: recursive
       - name: Build wheels
         if: github.event_name != 'release'
         uses: pypa/cibuildwheel@v3.1.3 # The main configuration is in pyproject.toml
@@ -147,8 +139,6 @@ jobs:
       - name: Ensure latest pip, wheel & setuptools
         run: python -m pip install -q --upgrade pip wheel tools
       - uses: actions/checkout@v4
-        with:
-          submodules: true
       - name: Get wheels artifacts
         uses: actions/download-artifact@v4
         with:

.gitmodules

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.15)
+cmake_minimum_required(VERSION 3.28)
 file(STRINGS "version.txt" NLE_VERSION)
 # Remove any rcXX suffix from the version number as CMake doesn't like it
 string(REGEX REPLACE "rc[0-9+]$" "" CMAKE_NLE_VERSION ${NLE_VERSION})
@@ -53,6 +53,22 @@ message(STATUS "HACKDIR set to: ${HACKDIR}")
 set(VARDIR ${HACKDIR})
 set(INSTDIR ${HACKDIR})
 
+# pybind11 via FetchContent
+include(FetchContent)
+FetchContent_Declare(
+  pybind11
+  GIT_REPOSITORY https://github.com/pybind/pybind11.git
+  GIT_TAG v3.0.1
+  EXCLUDE_FROM_ALL)
+FetchContent_MakeAvailable(pybind11)
+
+# de-boost-ified version of boost.context via FetchContent
+FetchContent_Declare(
+  deboost_context
+  GIT_REPOSITORY https://github.com/septag/deboost.context.git
+  GIT_HASH "259fc4103bad6bb484d5ff426ace56ac557107a4" EXCLUDE_FROM_ALL)
+FetchContent_MakeAvailable(deboost_context)
+
 add_compile_definitions(
   GCC_WARN
   NOCLIPPING
@@ -78,9 +94,6 @@ set(NLE_DAT_GEN ${nle_BINARY_DIR}/dat)
 set(NLE_UTIL_GEN ${nle_BINARY_DIR}/util)
 
 set(CMAKE_INSTALL_MESSAGE LAZY) # Don't tell us about up-to-date files.
-
-# EXCLUDE_FROM_ALL: Don't install this static library into /usr/local.
-add_subdirectory(third_party/deboost.context EXCLUDE_FROM_ALL)
 add_subdirectory(util)
 add_subdirectory(dat)
 
@@ -96,17 +109,27 @@ file(
   "win/tty/*.c"
   "win/rl/winrl.cc")
 
-# static version of bzip2 library
-add_library(
-  bz2_static STATIC
-  "third_party/bzip2/blocksort.c"
-  "third_party/bzip2/bzlib.c"
-  "third_party/bzip2/compress.c"
-  "third_party/bzip2/crctable.c"
-  "third_party/bzip2/decompress.c"
-  "third_party/bzip2/huffman.c"
-  "third_party/bzip2/randtable.c")
-target_link_libraries(bz2_static)
+# FetchContent for bzip2
+include(FetchContent)
+FetchContent_Declare(
+  bzip2
+  GIT_REPOSITORY git://sourceware.org/git/bzip2.git
+  GIT_HASH fbc4b11da543753b3b803e5546f56e26ec90c2a7 EXCLUDE_FROM_ALL)
+FetchContent_MakeAvailable(bzip2)
+
+# Manually add bzip2 source files from the downloaded directory
+set(BZIP2_SRC
+    ${bzip2_SOURCE_DIR}/blocksort.c
+    ${bzip2_SOURCE_DIR}/bzlib.c
+    ${bzip2_SOURCE_DIR}/compress.c
+    ${bzip2_SOURCE_DIR}/crctable.c
+    ${bzip2_SOURCE_DIR}/decompress.c
+    ${bzip2_SOURCE_DIR}/huffman.c
+    ${bzip2_SOURCE_DIR}/randtable.c)
+
+# EXCLUDE_FROM_ALL: Don't install this static library into /usr/local.
+add_library(bz2_static STATIC EXCLUDE_FROM_ALL ${BZIP2_SRC})
+target_include_directories(bz2_static PUBLIC ${bzip2_SOURCE_DIR})
 
 # terminal emulator library
 add_library(tmt STATIC "third_party/libtmt/tmt.c")
@@ -119,8 +142,7 @@ set_target_properties(nethack PROPERTIES CXX_STANDARD 14 SUFFIX ".so")
 target_include_directories(
   nethack
   PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${NLE_INC_GEN} /usr/local/include
-         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libtmt
-         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/bzip2)
+         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libtmt ${bzip2_SOURCE_DIR})
 # target_link_directories(nethack PUBLIC /usr/local/lib)
 
 # Careful with -DMONITOR_HEAP: Ironically, it fails to fclose FILE* heaplog.
@@ -131,9 +153,8 @@ target_link_libraries(nethack PUBLIC m fcontext bz2_static tmt)
 # dlopen wrapper library
 add_library(nethackdl STATIC "sys/unix/nledl.c")
 target_include_directories(
-  nethackdl
-  PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
-         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/deboost.context/include)
+  nethackdl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
+                   ${deboost_context_SOURCE_DIR}/include)
 target_link_libraries(nethackdl PUBLIC dl)
 
 # rlmain C++ (test) binary
@@ -144,7 +165,6 @@ target_include_directories(rlmain PUBLIC ${NLE_INC_GEN})
 add_dependencies(rlmain util) # For pm.h.
 
 # pybind11 core python library.
-add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/pybind11)
 pybind11_add_module(
   _pynethack
   win/rl/pynethack.cc
@@ -165,8 +185,7 @@ add_library(
 target_include_directories(
   converter
   PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libtmt
-         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/converter
-         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/bzip2)
+         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/converter ${bzip2_SOURCE_DIR})
 target_link_libraries(converter PUBLIC bz2_static tmt)
 if(CMAKE_BUILD_TYPE MATCHES Debug)
   target_compile_options(converter PRIVATE -Wall -Wextra -pedantic -Werror)

CMakeLists.txt

@@ -37,7 +37,7 @@ with other gym (or Gynmasium) / RL environments.
 
 ## Installation
 
-NLE requires `python>=3.10`, `cmake>=3.18` to be installed and available both when building the
+NLE requires `python>=3.10`, `cmake>=3.28` to be installed and available both when building the
 package, and at runtime.
 
 On **MacOS**, one can use `Homebrew` as follows:
@@ -75,7 +75,7 @@ $ pip install nle
 NOTE: If you want to extend / develop NLE, please install the package as follows:
 
 ``` bash
-$ git clone https://github.com/NetHack-LE/nle --recursive
+$ git clone https://github.com/NetHack-LE/nle
 $ pip install -e ".[dev]"
 $ pre-commit install
 ```

README.md

@@ -4,7 +4,7 @@ Getting Started
 Dependencies
 ************
 
-NLE requires `python>=3.10`, `cmake>=3.18` and some NetHack dependencies
+NLE requires `python>=3.10`, `cmake>=3.28` and some NetHack dependencies
 (e.g. `libncurses`) to be installed and available both when building the
 package, and at runtime.
 
@@ -47,7 +47,7 @@ Optionally, one can clone the repository and install the package manually.
 
 .. code-block:: bash
 
-   $ git clone https://github.com/NetHack-LE/nle --recursive
+   $ git clone https://github.com/NetHack-LE/nle
    $ conda activate nledev
    $ pip install .
 

doc/nle/source/getting_started.rst

@@ -15,7 +15,7 @@ To build and run an image (e.g. `Dockerfile-jammy`) for your local
 architecture do:
 
 ```bash
-$ git clone https://github.com/NetHack-LE/nle --recursive
+$ git clone https://github.com/NetHack-LE/nle
 $ cd nle
 $ docker build --file docker/Dockerfile-jammy . --tag nle
 $ docker run -it --gpus all --rm --name nle nle
@@ -34,4 +34,4 @@ $ docker buildx build --platform linux/amd64,linux/arm64 -t nle -f docker/Docker
 ```
 
 The run instructions are as before. Docker will load the correct
-binaries for the architecture you are running the container on.
+binaries for the architecture you are running the container on.