Test Mongo against an embedded MongoDB instance, not external

Author: palfrey

.github/workflows/native-cargo.yaml

@@ -49,7 +49,7 @@ jobs:
         run: cargo build --all --profile=smol
 
       - name: Test on ${{ runner.os }}
-        run: cargo test --all --profile=smol
+        run: cargo test --all --profile=smol -- --nocapture
 
       - name: Check schema export
         run: cargo run --bin build-schema --features dev-schema --package nativelink-config

.github/workflows/nix.yaml

@@ -69,7 +69,7 @@ jobs:
       - name: Invoke Cargo build in Nix shell
         run: >
           nix develop --impure --command
-          bash -c "cargo test --all --profile=smol"
+          bash -c "cargo test --all --profile=smol -- --nocapture"
 
   installation:
     strategy:

Cargo.lock

@@ -15,9 +15,11 @@ rust_library(
     deps = [
         "//nativelink-metric",
         "//nativelink-proto",
+        "@crates//:mongodb",
         "@crates//:prost",
         "@crates//:prost-types",
         "@crates//:redis",
+        "@crates//:reqwest",
         "@crates//:rustls-pki-types",
         "@crates//:serde",
         "@crates//:serde_json5",
@@ -26,6 +28,7 @@ rust_library(
         "@crates//:url",
         "@crates//:uuid",
         "@crates//:walkdir",
+        "@crates//:zip",
     ],
 )
 

nativelink-error/BUILD.bazel

@@ -14,9 +14,14 @@ version = "1.0.0-rc4"
 nativelink-metric = { path = "../nativelink-metric" }
 nativelink-proto = { path = "../nativelink-proto" }
 
+mongodb = { version = "3", features = [
+  "compat-3-0-0",
+  "rustls-tls",
+], default-features = false }
 prost = { version = "0.13.5", default-features = false }
 prost-types = { version = "0.13.5", default-features = false }
 redis = { version = "1.0.0", default-features = false }
+reqwest = { version = "0.12", default-features = false }
 rustls-pki-types = { version = "1.13.1", default-features = false }
 serde = { version = "1.0.219", default-features = false }
 serde_json5 = { version = "0.2.1", default-features = false }
@@ -33,3 +38,4 @@ tonic = { version = "0.13.0", features = [
 url = { version = "2.5.7", default-features = false }
 uuid = { version = "1.16.0", default-features = false }
 walkdir = { version = "2.5.0", default-features = false }
+zip = { version = "7.2.0", default-features = false }

nativelink-error/Cargo.toml

@@ -324,6 +324,24 @@ impl From<url::ParseError> for Error {
     }
 }
 
+impl From<mongodb::error::Error> for Error {
+    fn from(value: mongodb::error::Error) -> Self {
+        Self::new(Code::Internal, value.to_string())
+    }
+}
+
+impl From<reqwest::Error> for Error {
+    fn from(value: reqwest::Error) -> Self {
+        Self::new(Code::Internal, value.to_string())
+    }
+}
+
+impl From<zip::result::ZipError> for Error {
+    fn from(value: zip::result::ZipError) -> Self {
+        Self::new(Code::Internal, value.to_string())
+    }
+}
+
 pub trait ResultExt<T> {
     /// # Errors
     ///

nativelink-error/src/lib.rs

@@ -130,6 +130,12 @@ rust_test_suite(
         "tests/size_partitioning_store_test.rs",
         "tests/verify_store_test.rs",
     ],
+    compile_data = [
+        "tests/mongo_runner/mod.rs",
+        "tests/mongo_runner/process.rs",
+        "tests/mongo_runner/extractor.rs",
+        "tests/mongo_runner/downloader.rs",
+    ],
     proc_macro_deps = [
         "//nativelink-macro",
         "@crates//:async-trait",
@@ -149,6 +155,8 @@ rust_test_suite(
         "@crates//:aws-smithy-types",
         "@crates//:bincode",
         "@crates//:bytes",
+        "@crates//:dirs",
+        "@crates//:flate2",
         "@crates//:futures",
         "@crates//:hex",
         "@crates//:http",
@@ -162,16 +170,19 @@ rust_test_suite(
         "@crates//:rand",
         "@crates//:redis",
         "@crates//:redis-test",
+        "@crates//:reqwest",
         "@crates//:serde_json",
         "@crates//:serial_test",
         "@crates//:sha2",
+        "@crates//:tar",
         "@crates//:tempfile",
         "@crates//:tokio",
         "@crates//:tokio-stream",
         "@crates//:tonic",
         "@crates//:tracing",
         "@crates//:tracing-test",
         "@crates//:uuid",
+        "@crates//:zip",
     ],
 )
 

nativelink-store/BUILD.bazel

@@ -126,6 +126,8 @@ aws-smithy-runtime-api = { version = "1.7.4", default-features = false }
 aws-smithy-types = { version = "1.3.0", default-features = false, features = [
   "http-body-1-x",
 ] }
+dirs = { version = "6.0.0", default-features = false }
+flate2 = { version = "1.1.9", default-features = false, features = ["zlib-rs"] }
 futures = { version = "0.3.31", default-features = false, features = [
   "executor",
 ] }
@@ -141,10 +143,14 @@ rand = { version = "0.9.0", default-features = false, features = [
 ] }
 redis-test = { version = "1.0.0", default-features = false, features = ["aio"] }
 serde_json = { version = "1.0.140", default-features = false }
+tar = { version = "0.4.44", default-features = false }
 tempfile = { version = "3.8.1", default-features = false }
 tracing-test = { version = "0.2.5", default-features = false, features = [
   "no-env-filter",
 ] }
+zip = { version = "7.2.0", default-features = false, features = [
+  "deflate-flate2-zlib-rs",
+] }
 
 [package.metadata.cargo-machete]
 # Used by nativelink_test macro

nativelink-store/Cargo.toml

@@ -0,0 +1 @@
+Most of this is based off the defunct https://docs.rs/crate/mongo-embedded/1.0.0/source/ code and should be extracted out again as a separate crate at some point