Download test data from external repo (#113)

* Download test data from external repo
* Add option to use existing tests data
* Convert OBJs to PLYs
* Python download data at test time

Author: zfergus

.gitignore

@@ -243,19 +243,12 @@ core*
 docs/source/doxyoutput
 docs/source/api
 !docs/Makefile
-# Private data
-tests/data/private
-# Git repo of sample CCD queries
-tests/data/ccd-queries
-# Too large to store in git
-tests/data/friction/chain
 logo
 ipctk
 
 # Python bindings
 !python/src/bindings*
 !python/src/**/bindings*
-python/bunny.obj
 
 IPCToolkitOptions.cmake
 
@@ -265,5 +258,7 @@ tests/src/tests/config.hpp
 docs/_doxygen
 
 notebooks/*.[ch]pp
-tests/data/cloth_ball_bf_ccd_candidates.json
 python/update_bindings.py
+
+# External test data
+tests/data

cmake/ipc_toolkit/ipc_toolkit_tests_data.cmake

@@ -0,0 +1,40 @@
+# ipc-toolkit-tests-data (https://github.com/ipc-sim/ipc-toolkit-tests-data)
+# License: MIT
+
+if(TARGET ipc_toolkit_test_data_download)
+    return()
+endif()
+
+include(ExternalProject)
+include(FetchContent)
+
+set(IPC_TOOLKIT_TESTS_DATA_DIR "${PROJECT_SOURCE_DIR}/tests/data/" CACHE PATH "Where should we download the tests data?")
+option(IPC_TOOLKIT_USE_EXISTING_TESTS_DATA_DIR "Use an existing test data directory instead of downloading it" OFF)
+
+if(IPC_TOOLKIT_USE_EXISTING_TESTS_DATA_DIR)
+    ExternalProject_Add(
+        ipc_toolkit_test_data_download
+        PREFIX "${FETCHCONTENT_BASE_DIR}/tests/data"
+        SOURCE_DIR ${IPC_TOOLKIT_TESTS_DATA_DIR}
+
+        # NOTE: No download step
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        LOG_DOWNLOAD ON
+    )
+else()
+    ExternalProject_Add(
+        ipc_toolkit_test_data_download
+        PREFIX "${FETCHCONTENT_BASE_DIR}/tests/data"
+        SOURCE_DIR ${IPC_TOOLKIT_TESTS_DATA_DIR}
+
+        GIT_REPOSITORY https://github.com/ipc-sim/ipc-toolkit-tests-data.git
+        GIT_TAG 333aed25f87c5820bec8a7f82d7959ff87a9a502
+
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        LOG_DOWNLOAD ON
+    )
+endif()

cmake/recipes/ccd_query_io.cmake

@@ -6,8 +6,10 @@ endif()
 
 message(STATUS "Third-party: creating target 'ccd_io::ccd_io'")
 
+include(ipc_toolkit_tests_data)
+
 set(CCD_IO_DOWNLOAD_SAMPLE_QUERIES ON CACHE BOOL "Download sample CCD queries" FORCE)
-set(CCD_IO_SAMPLE_QUERIES_DIR "${PROJECT_SOURCE_DIR}/tests/data/ccd-queries/" CACHE PATH "Where should we download sample queries?")
+set(CCD_IO_SAMPLE_QUERIES_DIR "${IPC_TOOLKIT_TESTS_DATA_DIR}/ccd-queries/" CACHE PATH "Where should we download sample queries?")
 
 include(CPM)
 CPMAddPackage("gh:Continuous-Collision-Detection/CCD-Query-IO#36f6093af81a65acc27d9f05ad32d6b5729e8d15")

docs/source/tutorial/getting_started.rst

@@ -8,7 +8,7 @@ Collision Mesh
 
 First, we need to create a collision mesh. The ``CollisionMesh`` data structure represents the surface geometry used for collision processing.
 
-We will start by creating a collision mesh from a ``bunny.obj`` mesh file (you can find the mesh `here <https://github.com/ipc-sim/ipc-toolkit/blob/main/tests/data/bunny.obj>`_):
+We will start by creating a collision mesh from a ``bunny.ply`` mesh file (you can find the mesh `here <https://github.com/ipc-sim/ipc-toolkit-tests-data/blob/main/bunny.ply>`_):
 
 .. md-tab-set::
 
@@ -20,14 +20,14 @@ We will start by creating a collision mesh from a ``bunny.obj`` mesh file (you c
 
             #include <Eigen/Core>
             #include <Eigen/Sparse>
-            #include <igl/readOBJ.h>
+            #include <igl/read_triangle_mesh.h>
             #include <igl/edges.h>
 
             // ...
 
             Eigen::MatrixXd rest_positions;
             Eigen::MatrixXi edges, faces;
-            igl::readOBJ("bunny.obj", rest_positions, faces);
+            igl::read_triangle_mesh("bunny.ply", rest_positions, faces);
             igl::edges(faces, edges);
 
             ipc::CollisionMesh collision_mesh(rest_positions, edges, faces);
@@ -40,7 +40,7 @@ We will start by creating a collision mesh from a ``bunny.obj`` mesh file (you c
 
             import meshio
 
-            mesh = meshio.read("bunny.obj")
+            mesh = meshio.read("bunny.ply")
             rest_positions = mesh.points
             faces = mesh.cells_dict["triangle"]
             edges = ipctk.edges(faces)

docs/source/tutorial/simulation.rst

@@ -194,7 +194,7 @@ While IPC cannot directly handle nonlinear finite element bases and/or curved me
             Eigen::MatrixXd proxy_rest_positions;
             Eigen::MatrixXi proxy_edges, proxy_faces;
             // Load the proxy mesh from a file
-            igl::readOBJ("proxy.obj", rest_positions, faces);
+            igl::read_triangle_mesh("proxy.ply", rest_positions, faces);
             igl::edges(faces, edges);
             // Or build it from the volumetric mesh
 
@@ -215,7 +215,7 @@ While IPC cannot directly handle nonlinear finite element bases and/or curved me
 
             # Collision proxy mesh
             # Load the proxy mesh from a file
-            proxy_mesh = meshio.read("proxy.obj")
+            proxy_mesh = meshio.read("proxy.ply")
             proxy_rest_positions = proxy_mesh.points
             proxy_faces = proxy_mesh.cells_dict["triangle"]
             proxy_edges = igl.edges(proxy_faces)

python/example.ipynb

@@ -91183,7 +91183,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "in_mesh = meshio.read(\"../tests/data/bunny.obj\")\n",
+    "in_mesh = meshio.read(\"../tests/data/bunny.ply\")\n",
     "V = in_mesh.points\n",
     "F = in_mesh.cells[0].data\n",
     "E = edges(F)"
@@ -91316,7 +91316,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "version": "3.11.9"
   },
   "varInspector": {
    "cols": {

python/tests/test_ccd.py

@@ -8,8 +8,8 @@
 
 
 def test_ccd():
-    V0, E, F = load_mesh("two-cubes-close.obj")
-    V1, E, F = load_mesh("two-cubes-intersecting.obj")
+    V0, E, F = load_mesh("two-cubes-close.ply")
+    V1, E, F = load_mesh("two-cubes-intersecting.ply")
 
     mesh = ipctk.CollisionMesh(V0, E, F)
 

python/tests/test_ipc.py

@@ -41,7 +41,7 @@ def check_ipc_derivatives(broad_phase_method, use_convergent_formulation, mesh_n
 def test_ipc():
     for method in utils.broad_phase_methods():
         for use_convergent_formulation in (True, False):
-            yield check_ipc_derivatives, method, use_convergent_formulation, "cube.obj", np.sqrt(2.0), True
-            yield check_ipc_derivatives, method, use_convergent_formulation, "two-cubes-far.obj", 1e-1, False
-            yield check_ipc_derivatives, method, use_convergent_formulation, "two-cubes-close.obj", 1e-1, False
-            yield check_ipc_derivatives, method, use_convergent_formulation, "bunny.obj", 5e-3, True
+            yield check_ipc_derivatives, method, use_convergent_formulation, "cube.ply", np.sqrt(2.0), True
+            yield check_ipc_derivatives, method, use_convergent_formulation, "two-cubes-far.ply", 1e-1, False
+            yield check_ipc_derivatives, method, use_convergent_formulation, "two-cubes-close.ply", 1e-1, False
+            yield check_ipc_derivatives, method, use_convergent_formulation, "bunny.ply", 5e-3, True

python/tests/utils.py

@@ -5,11 +5,28 @@
 import find_ipctk
 import ipctk
 
-TEST_DATA_DIR = pathlib.Path(__file__).parents[2] / 'tests' / 'data'
+
+def download_test_data_if_needed(directory):
+    if directory.exists():
+        return
+
+    # Clone the test data repository
+    print(f"Downloading test data to {directory}")
+    import subprocess
+    subprocess.run([
+        'git', 'clone', 'https://github.com/ipc-sim/ipc-toolkit-tests-data',
+        str(directory)
+    ])
+
+
+def test_data_dir():
+    _test_data_dir = pathlib.Path(__file__).parents[2] / 'tests' / 'data'
+    download_test_data_if_needed(_test_data_dir)
+    return _test_data_dir
 
 
 def load_mesh(mesh_name):
-    mesh = meshio.read(TEST_DATA_DIR / mesh_name)
+    mesh = meshio.read(test_data_dir() / mesh_name)
     return mesh.points, ipctk.edges(mesh.cells_dict['triangle']), mesh.cells_dict['triangle']
 
 

tests/CMakeLists.txt

@@ -18,9 +18,6 @@ endif()
 set(IPC_TOOLKIT_TESTS_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/tests")
 set(IPC_TOOLKIT_TESTS_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src")
 
-### Data directory
-cmake_path(APPEND IPC_TOOLKIT_TESTS_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}" "data")
-
 ################################################################################
 # IPC Toolkit Tests Executable
 ################################################################################
@@ -29,6 +26,7 @@ cmake_path(APPEND IPC_TOOLKIT_TESTS_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}" "data
 add_executable(ipc_toolkit_tests)
 
 # Fill in configuration options
+include(ipc_toolkit_tests_data)
 configure_file(
   "${IPC_TOOLKIT_TESTS_SOURCE_DIR}/config.hpp.in"
   "${IPC_TOOLKIT_TESTS_SOURCE_DIR}/config.hpp")