Test Mongo against an embedded MongoDB instance, not external

Author: palfrey

Cargo.lock

@@ -297,6 +297,26 @@
           (nightlyCraneLibFor p).cargoLlvmCov (coverageArgs
             // {
               cargoArtifacts = nightlyCargoArtifactsFor p;
+              preConfigurePhases = ["tempHome"];
+              tempHome = ''
+                # Default home at this point is /homeless-shelter, which doesn't exist and so breaks things like
+                # the Mongo downloader
+                echo "HOME was ''${HOME}"
+                export HOME=$(mktemp -d fake-homeXXXX --tmpdir)
+                echo "HOME is now ''${HOME}"
+
+                # FIXME: This is a giant pile of hacks, as what we should be doing is downloading mongodb and patching it to work with
+                # Nix. Several hours of patchelf rummaging later, hitting issues like "_Unwind_GetRegionStart: symbol not found" we
+                # instead have this hacky workaround. It adds a symlink to a Nix mongod where the extractor code expects to find it,
+                # which will work. This is the wrong version (7.0.16 v.s. 7.0.11 currently, so not so bad) but TBH our codebase is mostly
+                # fairly version agnostic so far and 7.0.11 is picked out of the air as the one we've used elsewhere.
+
+                mkdir -p $HOME/.cache/mongo/extracted/7.0.11/mongodb-linux-x86_64-ubuntu2204-7.0.11/bin
+                touch $HOME/.cache/mongo/extracted/7.0.11/extracted.marker # As needed by nativelink-store/tests/mongo_runner/mod.rs
+                export MONGOD=$HOME/.cache/mongo/extracted/7.0.11/mongodb-linux-x86_64-ubuntu2204-7.0.11/bin/mongod
+                ln -s ${p.mongodb}/bin/mongod ''${MONGOD}
+                ''${MONGOD} --version
+              '';
               cargoExtraArgs = builtins.concatStringsSep " " [
                 "--all"
                 "--locked"

flake.nix

@@ -16,9 +16,11 @@ rust_library(
     deps = [
         "//nativelink-metric",
         "//nativelink-proto",
+        "@crates//:mongodb",
         "@crates//:prost",
         "@crates//:prost-types",
         "@crates//:redis",
+        "@crates//:reqwest",
         "@crates//:rustls-pki-types",
         "@crates//:serde",
         "@crates//:serde_json5",
@@ -27,6 +29,31 @@ rust_library(
         "@crates//:url",
         "@crates//:uuid",
         "@crates//:walkdir",
+        "@crates//:zip",
+    ],
+)
+
+rust_test_suite(
+    name = "integration",
+    timeout = "short",
+    srcs = [
+        "tests/error_tests.rs",
+    ],
+    deps = [
+        "//nativelink-error",
+        "@crates//:walkdir",
+    ],
+)
+
+rust_test_suite(
+    name = "integration",
+    timeout = "short",
+    srcs = [
+        "tests/error_tests.rs",
+    ],
+    deps = [
+        "//nativelink-error",
+        "@crates//:walkdir",
     ],
 )
 

nativelink-error/BUILD.bazel

@@ -13,11 +13,16 @@ version = "1.0.0-rc4"
 nativelink-metric = { path = "../nativelink-metric" }
 nativelink-proto = { path = "../nativelink-proto" }
 
+mongodb = { version = "3", features = [
+  "compat-3-0-0",
+  "rustls-tls",
+], default-features = false }
 prost = { version = "0.13.5", default-features = false }
 prost-types = { version = "0.13.5", default-features = false, features = [
   "std",
 ] }
 redis = { version = "1.0.0", default-features = false }
+reqwest = { version = "0.12", default-features = false }
 rustls-pki-types = { version = "1.13.1", default-features = false }
 serde = { version = "1.0.219", default-features = false }
 serde_json5 = { version = "0.2.1", default-features = false }
@@ -34,3 +39,4 @@ tonic = { version = "0.13.0", features = [
 url = { version = "2.5.7", default-features = false }
 uuid = { version = "1.16.0", default-features = false, features = ["std"] }
 walkdir = { version = "2.5.0", default-features = false }
+zip = { version = "7.2.0", default-features = false }

nativelink-error/Cargo.toml

@@ -335,6 +335,24 @@ impl From<url::ParseError> for Error {
     }
 }
 
+impl From<mongodb::error::Error> for Error {
+    fn from(err: mongodb::error::Error) -> Self {
+        Self::from_std_err(Code::Internal, &err)
+    }
+}
+
+impl From<reqwest::Error> for Error {
+    fn from(err: reqwest::Error) -> Self {
+        Self::from_std_err(Code::Internal, &err)
+    }
+}
+
+impl From<zip::result::ZipError> for Error {
+    fn from(err: zip::result::ZipError) -> Self {
+        Self::from_std_err(Code::Internal, &err)
+    }
+}
+
 pub trait ResultExt<T> {
     /// # Errors
     ///

nativelink-error/src/lib.rs

@@ -130,6 +130,12 @@ rust_test_suite(
         "tests/size_partitioning_store_test.rs",
         "tests/verify_store_test.rs",
     ],
+    compile_data = [
+        "tests/mongo_runner/mod.rs",
+        "tests/mongo_runner/process.rs",
+        "tests/mongo_runner/extractor.rs",
+        "tests/mongo_runner/downloader.rs",
+    ],
     proc_macro_deps = [
         "//nativelink-macro",
         "@crates//:async-trait",
@@ -149,6 +155,8 @@ rust_test_suite(
         "@crates//:aws-smithy-types",
         "@crates//:bincode",
         "@crates//:bytes",
+        "@crates//:dirs",
+        "@crates//:flate2",
         "@crates//:futures",
         "@crates//:hex",
         "@crates//:http",
@@ -162,16 +170,19 @@ rust_test_suite(
         "@crates//:rand",
         "@crates//:redis",
         "@crates//:redis-test",
+        "@crates//:reqwest",
         "@crates//:serde_json",
         "@crates//:serial_test",
         "@crates//:sha2",
+        "@crates//:tar",
         "@crates//:tempfile",
         "@crates//:tokio",
         "@crates//:tokio-stream",
         "@crates//:tonic",
         "@crates//:tracing",
         "@crates//:tracing-test",
         "@crates//:uuid",
+        "@crates//:zip",
     ],
 )
 

nativelink-store/BUILD.bazel

@@ -126,6 +126,8 @@ aws-smithy-runtime-api = { version = "1.7.4", default-features = false }
 aws-smithy-types = { version = "1.3.0", default-features = false, features = [
   "http-body-1-x",
 ] }
+dirs = { version = "6.0.0", default-features = false }
+flate2 = { version = "1.1.9", default-features = false, features = ["zlib-rs"] }
 futures = { version = "0.3.31", default-features = false, features = [
   "executor",
 ] }
@@ -141,10 +143,14 @@ rand = { version = "0.9.0", default-features = false, features = [
 ] }
 redis-test = { version = "1.0.0", default-features = false, features = ["aio"] }
 serde_json = { version = "1.0.140", default-features = false }
+tar = { version = "0.4.44", default-features = false }
 tempfile = { version = "3.8.1", default-features = false }
 tracing-test = { version = "0.2.5", default-features = false, features = [
   "no-env-filter",
 ] }
+zip = { version = "7.2.0", default-features = false, features = [
+  "deflate-flate2-zlib-rs",
+] }
 
 [package.metadata.cargo-machete]
 # Used by nativelink_test macro

nativelink-store/Cargo.toml

@@ -0,0 +1 @@
+Most of this is based off the defunct https://docs.rs/crate/mongo-embedded/1.0.0/source/ code and should be extracted out again as a separate crate at some point