Merge pull request #2 from liamjdavis/mamba-conda-env

Micromamba/Conda environment for Dependencies

Author: liamjdavis

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ on:
 
 defaults:
   run:
-    shell: sh
+    shell: bash -el {0}
 
 jobs:
   build:
@@ -51,38 +51,31 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v5
+      - uses: mamba-org/setup-micromamba@v2
         with:
-          python-version: "3.9"
+          environment-file: environment.yml
+          environment-name: marabou
+          cache-environment: true
+          init-shell: bash
+          post-cleanup: none
 
-      - name: Install system packages
+      - name: Install system build tools
         if: runner.os == 'Linux'
         run: |
           sudo apt-get update -y
-          sudo apt-get install -y \
-            build-essential \
-            ccache \
-            cmake \
-            cxxtest \
-            protobuf-compiler \
-            libprotoc-dev
-
-      - name: Install system packages
+          sudo apt-get install -y build-essential ccache
+
+      - name: Install system build tools
         if: runner.os == 'macOS'
-        run: |
-          brew install \
-            ccache \
-            cmake \
-            cxxtest \
-            protobuf-c
+        run: brew install ccache
 
       - name: Install Python packages
         run: |
-          python -m pip install --user --upgrade pip
-          python -m pip install --user --upgrade setuptools
-          python -m pip install --user wheel
-          python -m pip install --user bumpver
-          python -m pip install --user -r maraboupy/test_requirements.txt --cache-dir $HOME/.pip-cache
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade setuptools
+          python -m pip install wheel
+          python -m pip install bumpver
+          python -m pip install -r maraboupy/test_requirements.txt --cache-dir $HOME/.pip-cache
           echo "/usr/lib/ccache" >> $GITHUB_PATH
 
       - name: Test BumpVer Configuration
@@ -120,13 +113,7 @@ jobs:
           ccache -z
 
       - name: Configure CMake
-        # Use a bash shell so we can use the same syntax for environment variable bb
-        # access regardless of the host operating system
-        shell: bash
         working-directory: build
-        # Note the current convention is to use the -S and -B options here to specify source
-        # and build directories, but this is only available with CMake 3.13 and higher.
-        # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
         run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DBUILD_PYTHON=ON -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
 
       - name: Build

CMakeLists.txt

@@ -46,6 +46,18 @@ execute_process(
 )
 add_definitions("-DGIT_COMMIT_HASH=\"${GIT_COMMIT_HASH}\"")
 
+############################
+## Conda env detection    ##
+############################
+
+if(DEFINED ENV{CONDA_PREFIX} AND NOT DEFINED ENV{CONDA_BUILD})
+  list(INSERT CMAKE_PREFIX_PATH 0 "$ENV{CONDA_PREFIX}")
+  message(STATUS "Conda environment detected: $ENV{CONDA_PREFIX}")
+elseif(DEFINED ENV{CONDA_BUILD})
+  list(INSERT CMAKE_PREFIX_PATH 0 "$ENV{PREFIX}" "$ENV{BUILD_PREFIX}")
+  message(STATUS "Conda build environment detected")
+endif()
+
 #########################
 ## Basic configuration ##
 #########################
@@ -80,56 +92,17 @@ include_directories(SYSTEM ${CVC4_DIR} ${CVC4_DIR}/include)
 ## Boost ##
 ###########
 
-# Avoid using deprecated operations
 add_definitions(-DBOOST_NO_CXX98_FUNCTION_BASE)
-set(BOOST_VERSION 1.84.0)
-set(BOOST_DIR "${TOOLS_DIR}/boost-${BOOST_VERSION}")
-if (MSVC)
-  set(BOOST_ROOT "${BOOST_DIR}/win_installed")
-  set(Boost_NAMESPACE libboost)
-elseif (${CMAKE_SIZEOF_VOID_P} EQUAL 4 AND NOT MSVC)
-  set(BOOST_ROOT "${BOOST_DIR}/installed32")
-else()
-  set(BOOST_ROOT "${BOOST_DIR}/installed")
-endif()
-
-set(Boost_USE_DEBUG_RUNTIME FALSE)
-find_package(Boost ${BOOST_VERSION} COMPONENTS program_options timer chrono thread)
-# Find boost
-if (NOT ${Boost_FOUND})
-  execute_process(COMMAND ${TOOLS_DIR}/download_boost.${SCRIPT_EXTENSION} ${BOOST_VERSION})
-  find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS program_options timer chrono thread regex)
-endif()
+find_package(Boost 1.84.0 REQUIRED COMPONENTS program_options timer chrono thread)
 set(LIBS_INCLUDES ${Boost_INCLUDE_DIRS})
 list(APPEND LIBS ${Boost_LIBRARIES})
 
 ##############
 ## Protobuf ##
 ##############
-# Protobuf is needed to compile ONNX
 
-set(PROTOBUF_VERSION 3.19.2)
-set(PROTOBUF_DEFAULT_DIR "${TOOLS_DIR}/protobuf-${PROTOBUF_VERSION}")
-if (NOT PROTOBUF_DIR)
-    set(PROTOBUF_DIR ${PROTOBUF_DEFAULT_DIR})
-endif()
-
-if(NOT EXISTS "${PROTOBUF_DIR}/installed/lib/libprotobuf.a")
-    message("Can't find protobuf, installing. If protobuf is installed please use the PROTOBUF_DIR parameter to pass the path")
-    if (${PROTOBUF_DIR} STREQUAL ${PROTOBUF_DEFAULT_DIR})
-        message("installing protobuf")
-        execute_process(COMMAND ${TOOLS_DIR}/download_protobuf.sh ${PROTOBUF_VERSION})
-    else()
-        message(FATAL_ERROR "Can't find protobuf in the supplied directory")
-    endif()
-endif()
-
-set(PROTOBUF_LIB protobuf)
-add_library(${PROTOBUF_LIB} SHARED IMPORTED)
-set_property(TARGET ${PROTOBUF_LIB} PROPERTY POSITION_INDEPENDENT_CODE ON)
-set_target_properties(${PROTOBUF_LIB} PROPERTIES IMPORTED_LOCATION ${PROTOBUF_DIR}/installed/lib/libprotobuf.a)
-target_include_directories(${PROTOBUF_LIB} INTERFACE ${PROTOBUF_DIR}/installed/include)
-list(APPEND LIBS ${PROTOBUF_LIB})
+find_package(Protobuf 3.19.2 REQUIRED)
+list(APPEND LIBS protobuf::libprotobuf)
 
 ##########
 ## ONNX ##
@@ -141,7 +114,7 @@ set(ONNX_DIR "${TOOLS_DIR}/onnx-${ONNX_VERSION}")
 
 if(NOT EXISTS "${ONNX_DIR}/onnx.proto3.pb.h")
     message("generating ONNX protobuf file")
-    execute_process(COMMAND ${TOOLS_DIR}/download_onnx.sh ${ONNX_VERSION} ${PROTOBUF_VERSION})
+    execute_process(COMMAND ${TOOLS_DIR}/download_onnx.sh ${ONNX_VERSION} ${Protobuf_PROTOC_EXECUTABLE})
 endif()
 file(GLOB DEPS_ONNX "${ONNX_DIR}/*.cc")
 include_directories(SYSTEM ${ONNX_DIR})
@@ -183,30 +156,15 @@ endif()
 ##############
 
 if (NOT MSVC AND ${ENABLE_OPENBLAS})
-  set(OPENBLAS_VERSION 0.3.19)
-
-  set(OPENBLAS_LIB openblas)
-  set(OPENBLAS_DEFAULT_DIR "${TOOLS_DIR}/OpenBLAS-${OPENBLAS_VERSION}")
-  if (NOT OPENBLAS_DIR)
-      set(OPENBLAS_DIR ${OPENBLAS_DEFAULT_DIR})
-  endif()
-
   message(STATUS "Using OpenBLAS for matrix multiplication")
   add_compile_definitions(ENABLE_OPENBLAS)
-  if(NOT EXISTS "${OPENBLAS_DIR}/installed/lib/libopenblas.a")
-    message("Can't find OpenBLAS, installing. If OpenBLAS is installed please use the OPENBLAS_DIR parameter to pass the path")
-      if (${OPENBLAS_DIR} STREQUAL ${OPENBLAS_DEFAULT_DIR})
-        message("Installing OpenBLAS")
-        execute_process(COMMAND ${TOOLS_DIR}/download_openBLAS.sh ${OPENBLAS_VERSION})
-      else()
-        message(FATAL_ERROR "Can't find OpenBLAS in the supplied directory")
-      endif()
+  find_library(OPENBLAS_LIB openblas REQUIRED)
+  find_path(OPENBLAS_INCLUDE_DIR openblas_config.h PATH_SUFFIXES openblas)
+  if (NOT OPENBLAS_INCLUDE_DIR)
+    message(FATAL_ERROR "Could not find OpenBLAS headers (openblas_config.h)")
   endif()
-
-  add_library(${OPENBLAS_LIB} SHARED IMPORTED)
-  set_target_properties(${OPENBLAS_LIB} PROPERTIES IMPORTED_LOCATION ${OPENBLAS_DIR}/installed/lib/libopenblas.a)
   list(APPEND LIBS ${OPENBLAS_LIB})
-  target_include_directories(${OPENBLAS_LIB} INTERFACE ${OPENBLAS_DIR}/installed/include)
+  list(APPEND LIBS_INCLUDES ${OPENBLAS_INCLUDE_DIR})
 endif()
 
 ###########
@@ -370,9 +328,6 @@ endif()
 
 # Actually build Python
 if (${BUILD_PYTHON})
-   set(PYBIND11_VERSION 2.13.6)
-   set(PYBIND11_DIR "${TOOLS_DIR}/pybind11-${PYBIND11_VERSION}")
-
     # This is suppose to set the PYTHON_EXECUTABLE variable
     # First try to find the default python version
     find_package(PythonInterp ${DEFAULT_PYTHON_VERSION})
@@ -381,11 +336,7 @@ if (${BUILD_PYTHON})
         find_package(PythonInterp REQUIRED)
     endif()
 
-    if (NOT EXISTS ${PYBIND11_DIR})
-        message("didnt find pybind, getting it")
-	execute_process(COMMAND ${TOOLS_DIR}/download_pybind11.${SCRIPT_EXTENSION} ${PYBIND11_VERSION})
-    endif()
-    add_subdirectory(${PYBIND11_DIR})
+    find_package(pybind11 REQUIRED)
 
     set(MARABOU_PY MarabouCore)
     pybind11_add_module(${MARABOU_PY} ${PYTHON_API_DIR}/MarabouCore.cpp)
@@ -479,6 +430,6 @@ add_custom_target(build_input_parsers)
 add_dependencies(build_input_parsers ${MPS_PARSER} ${ACAS_PARSER}
     ${BERKELEY_PARSER})
 
-add_subdirectory(${SRC_DIR})
 add_subdirectory(${TOOLS_DIR})
+add_subdirectory(${SRC_DIR})
 add_subdirectory(${REGRESS_DIR})

README.md

@@ -42,12 +42,24 @@ The Python interface currently supports Python 3.8, 3.9, 3.10 and 3.11.
 
 #### Build dependencies
 
-The build process uses CMake version 3.16 (or later).
-You can get CMake [here](https://cmake.org/download/).
+Marabou uses a conda environment to manage its build dependencies
+(Boost, Protobuf, OpenBLAS, pybind11, CMake, Python, etc.).
+CxxTest is bundled in `tools/` and built automatically.
 
-Marabou depends on the Boost and the OpenBLAS libraries, which are automatically 
-downloaded and built when you run make. Library CXXTEST comes included in the 
-repository.
+Using [conda/anaconda](https://docs.conda.io/en/latest/):
+```bash
+conda env create -f environment.yml
+conda activate marabou
+```
+
+Using [micromamba](https://mamba.readthedocs.io/en/latest/installation/micromamba-installation.html):
+```bash
+micromamba create -f environment.yml
+micromamba activate marabou
+```
+
+CMake automatically detects the active conda environment and finds all
+dependencies from it.
 
 The current version of Marabou can be built for Linux or MacOS machines using CMake:
 ```bash
@@ -62,9 +74,8 @@ To enable multiprocess build change the last command to:
 ```bash
 cmake --build . -j PROC_NUM
 ```
-After building Marabou, the compiled binary is located at `build/Marabou`, and the 
-shared library for the Python API is located in `maraboupy/`. Building the Python 
-interface requires *pybind11* (which is automatically downloaded).
+After building Marabou, the compiled binary is located at `build/Marabou`, and the
+shared library for the Python API is located in `maraboupy/`.
 
 Export `Marabou` folder to Python and Jupyter paths:
 ```

environment.yml

@@ -0,0 +1,13 @@
+name: marabou
+channels:
+  - conda-forge
+dependencies:
+  - boost-cpp=1.84.0
+  - libprotobuf=3.19.2
+  - protobuf=3.19.2
+  - openblas>=0.3.19
+  - pybind11>=2.13.6
+  - cmake>=3.16
+  - ninja
+  - python>=3.9
+  - numpy>=1.21.0,<2

maraboupy/test/test_network.py

@@ -74,7 +74,7 @@ def test_disjunction_constraint():
         network.setLowerBound(var, 0)
         network.setUpperBound(var, 1)
 
-    exitCode1, vals1, stats1 = network.solve()
+    exitCode1, vals1, stats1 = network.solve(options=OPT)
 
     for var in network.inputVars[0][0]:
         assert(abs(vals1[var] - 1) < 0.0000001 or abs(vals1[var]) < 0.0000001)

maraboupy/test/test_nnet_extensions.py

@@ -4,6 +4,7 @@
 from maraboupy import Marabou
 
 # Global settings
+OPT = Marabou.createOptions(verbosity = 0) # Turn off printing
 TOL = 1e-8
 
 NETWORK_FOLDER = "../../resources/nnet/"
@@ -33,7 +34,7 @@ def test_evaluate_network():
         without_marabou_output = nnet_object.evaluate(np.array([inputs]),useMarabou=False)[0].flatten()
         without_marabou_output_rounded = np.array([float(round(y,8)) for y in without_marabou_output])
 
-        with_marabou_output = nnet_object.evaluate(np.array([inputs]),useMarabou=True)[0].flatten()
+        with_marabou_output = nnet_object.evaluate(np.array([inputs]),useMarabou=True, options=OPT)[0].flatten()
         with_marabou_output_rounded = np.array([float(round(y,8)) for y in with_marabou_output])
 
         # Assert that all of the above agree, at least up to 10^-8
@@ -88,7 +89,7 @@ def test_write_read_evaluate(tmpdir):
 
         # Compare evaluation with and without Marabou
         without_marabou_output = nnet_object_a.evaluate(np.array([inputs]), useMarabou=False)[0].flatten()
-        with_marabou_output = nnet_object_a.evaluate(np.array([inputs]), useMarabou=True)[0].flatten()
+        with_marabou_output = nnet_object_a.evaluate(np.array([inputs]), useMarabou=True, options=OPT)[0].flatten()
 
         # Assert that all of the above agree up to TOL
         assert (output2 == without_marabou_output).all()
@@ -123,7 +124,7 @@ def test_normalize_read_flag(tmpdir):
         without_marabou_output = nnet_object_a.evaluate(np.array([inputs]), useMarabou=False)[0].flatten()
         assert (output2 == without_marabou_output).all()
 
-        with_marabou_output = nnet_object_a.evaluate(np.array([inputs]), useMarabou=True)[0].flatten()
+        with_marabou_output = nnet_object_a.evaluate(np.array([inputs]), useMarabou=True, options=OPT)[0].flatten()
         assert (abs(without_marabou_output - with_marabou_output) < TOL).all()
 
 def test_reset_network():
@@ -173,9 +174,9 @@ def test_reset_network():
         assert (abs(output1_norm - output2_norm) < TOL).all()
 
         without_marabou_output1 = nnet_object1.evaluate(np.array([inputs]), useMarabou=False)[0].flatten()
-        with_marabou_output1 = nnet_object1.evaluate(np.array([inputs]), useMarabou=True)[0].flatten()
+        with_marabou_output1 = nnet_object1.evaluate(np.array([inputs]), useMarabou=True, options=OPT)[0].flatten()
         without_marabou_output2 = nnet_object2.evaluate(np.array([inputs]), useMarabou=False)[0].flatten()
-        with_marabou_output2 = nnet_object2.evaluate(np.array([inputs]), useMarabou=True)[0].flatten()
+        with_marabou_output2 = nnet_object2.evaluate(np.array([inputs]), useMarabou=True, options=OPT)[0].flatten()
 
         assert (abs(with_marabou_output1 - with_marabou_output2) < TOL).all()
         assert (abs(without_marabou_output1 - without_marabou_output2) < TOL).all()

pyproject.toml

@@ -61,9 +61,20 @@ test-extras = ["test"]
 test-skip = []
 
 [tool.cibuildwheel.linux]
-before-all = "yum install -y wget"
+manylinux-x86_64-image = "quay.io/pypa/manylinux_2_34_x86_64"
+before-all = [
+    "curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj -C /usr/local bin/micromamba",
+    "micromamba create -f {package}/environment.yml -p /tmp/mamba-env -y",
+]
+environment = { CONDA_PREFIX = "/tmp/mamba-env" }
 
 [tool.cibuildwheel.macos]
+before-all = [
+    "curl -Ls https://micro.mamba.pm/api/micromamba/osx-arm64/latest | tar -xvj -C /usr/local bin/micromamba",
+    "micromamba create -f {package}/environment.yml -p /tmp/mamba-env -y",
+]
+environment = { CONDA_PREFIX = "/tmp/mamba-env" }
+
 test-skip = [
   # 2023-04-15: Skip trying to test arm64 builds on Intel Macs
   "*-macosx_arm64",

src/common/tests/Test_ConstSimpleData.h

@@ -62,7 +62,7 @@ class ConstSimpleDataTestSuite : public CxxTest::TestSuite
 
     void test_as_char()
     {
-        char data[] = { 'a', 'b', 'c' };
+        char data[] = { 'a', 'b', 'c', '\0' };
 
         ConstSimpleData constSimpleData( data, sizeof( data ) );