Content-addressing derivations with shallow traces

maintainers/flake-module.nix

@@ -257,8 +257,6 @@
               ''^src/libstore/pathlocks\.hh$''
               ''^src/libstore/profiles\.cc$''
               ''^src/libstore/profiles\.hh$''
-              ''^src/libstore/realisation\.cc$''
-              ''^src/libstore/realisation\.hh$''
               ''^src/libstore/remote-fs-accessor\.cc$''
               ''^src/libstore/remote-fs-accessor\.hh$''
               ''^src/libstore/remote-store-connection\.hh$''
@@ -280,8 +278,6 @@
               ''^src/libstore/store-dir-config\.hh$''
               ''^src/libstore/build/derivation-goal\.cc$''
               ''^src/libstore/build/derivation-goal\.hh$''
-              ''^src/libstore/build/drv-output-substitution-goal\.cc$''
-              ''^src/libstore/build/drv-output-substitution-goal\.hh$''
               ''^src/libstore/build/entry-points\.cc$''
               ''^src/libstore/build/goal\.cc$''
               ''^src/libstore/build/goal\.hh$''

src/build-remote/build-remote.cc

@@ -345,21 +345,19 @@ static int main_build_remote(int argc, char * * argv)
         }
 
 
-        auto outputHashes = staticOutputHashes(*store, drv);
         std::set<Realisation> missingRealisations;
         StorePathSet missingPaths;
         if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
             for (auto & outputName : wantedOutputs) {
-                auto thisOutputHash = outputHashes.at(outputName);
-                auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
+                auto thisOutputId = DrvOutput{ *drvPath, outputName };
                 if (!store->queryRealisation(thisOutputId)) {
                     debug("missing output %s", outputName);
                     assert(optResult);
                     auto & result = *optResult;
                     auto i = result.builtOutputs.find(outputName);
                     assert(i != result.builtOutputs.end());
                     auto & newRealisation = i->second;
-                    missingRealisations.insert(newRealisation);
+                    missingRealisations.insert({newRealisation, thisOutputId});
                     missingPaths.insert(newRealisation.outPath);
                 }
             }

src/libcmd/built-path.cc

@@ -116,21 +116,16 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
         overloaded{
             [&](const BuiltPath::Opaque & p) { res.insert(p.path); },
             [&](const BuiltPath::Built & p) {
-                auto drvHashes =
-                    staticOutputHashes(store, store.readDerivation(p.drvPath->outPath()));
                 for (auto& [outputName, outputPath] : p.outputs) {
-                    if (experimentalFeatureSettings.isEnabled(
-                                Xp::CaDerivations)) {
-                        auto drvOutput = get(drvHashes, outputName);
-                        if (!drvOutput)
-                            throw Error(
-                                "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
-                                store.printStorePath(p.drvPath->outPath()), outputName);
-                        auto thisRealisation = store.queryRealisation(
-                            DrvOutput{*drvOutput, outputName});
-                        assert(thisRealisation);  // We’ve built it, so we must
-                                                  // have the realisation
-                        res.insert(*thisRealisation);
+                    if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+                        DrvOutput key{
+                            .drvPath = p.drvPath->outPath(),
+                            .outputName = outputName,
+                        };
+                        auto thisRealisation = store.queryRealisation(key);
+                        // We’ve built it, so we must have the realisation.
+                        assert(thisRealisation);
+                        res.insert(Realisation{*thisRealisation, key});
                     } else {
                         res.insert(outputPath);
                     }

src/libexpr/primops.cc

@@ -1522,31 +1522,7 @@ static void derivationStrictInternal(
                 DerivationOutput::Deferred { });
         }
 
-        auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
-        switch (hashModulo.kind) {
-        case DrvHash::Kind::Regular:
-            for (auto & i : outputs) {
-                auto h = get(hashModulo.hashes, i);
-                if (!h)
-                    state.error<AssertionError>(
-                        "derivation produced no hash for output '%s'",
-                        i
-                    ).atPos(v).debugThrow();
-                auto outPath = state.store->makeOutputPath(i, *h, drvName);
-                drv.env[i] = state.store->printStorePath(outPath);
-                drv.outputs.insert_or_assign(
-                    i,
-                    DerivationOutput::InputAddressed {
-                        .path = std::move(outPath),
-                    });
-            }
-            break;
-            ;
-        case DrvHash::Kind::Deferred:
-            for (auto & i : outputs) {
-                drv.outputs.insert_or_assign(i, DerivationOutput::Deferred {});
-            }
-        }
+        resolveInputAddressed(*state.store, drv);
     }
 
     /* Write the resulting term into the Nix store directory. */

src/libstore-test-support/tests/protocol.hh

@@ -64,12 +64,18 @@ public:
     }
 };
 
-#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
+#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
     TEST_F(FIXTURE, NAME ## _read) { \
         readProtoTest(STEM, VERSION, VALUE); \
-    } \
+    }
+
+#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
     TEST_F(FIXTURE, NAME ## _write) { \
         writeProtoTest(STEM, VERSION, VALUE); \
     }
 
+#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)
+
 }

src/libstore-tests/common-protocol.cc

@@ -96,51 +96,6 @@ CHARACTERIZATION_TEST(
         },
     }))
 
-CHARACTERIZATION_TEST(
-    drvOutput,
-    "drv-output",
-    (std::tuple<DrvOutput, DrvOutput> {
-        {
-            .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
-            .outputName = "baz",
-        },
-        DrvOutput {
-            .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
-            .outputName = "quux",
-        },
-    }))
-
-CHARACTERIZATION_TEST(
-    realisation,
-    "realisation",
-    (std::tuple<Realisation, Realisation> {
-        Realisation {
-            .id = DrvOutput {
-                .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
-                .outputName = "baz",
-            },
-            .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
-            .signatures = { "asdf", "qwer" },
-        },
-        Realisation {
-            .id = {
-                .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
-                .outputName = "baz",
-            },
-            .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
-            .signatures = { "asdf", "qwer" },
-            .dependentRealisations = {
-                {
-                    DrvOutput {
-                        .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
-                        .outputName = "quux",
-                    },
-                    StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
-                },
-            },
-        },
-    }))
-
 CHARACTERIZATION_TEST(
     vector,
     "vector",

src/libstore-tests/serve-protocol.cc

@@ -72,16 +72,16 @@ VERSIONED_CHARACTERIZATION_TEST(
 
 VERSIONED_CHARACTERIZATION_TEST(
     ServeProtoTest,
-    drvOutput,
-    "drv-output",
-    defaultVersion,
+    drvOutput_2_8,
+    "drv-output-2.8",
+    2 << 8 | 8,
     (std::tuple<DrvOutput, DrvOutput> {
         {
-            .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
+            .drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
             .outputName = "baz",
         },
         DrvOutput {
-            .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
+            .drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
             .outputName = "quux",
         },
     }))
@@ -90,34 +90,27 @@ VERSIONED_CHARACTERIZATION_TEST(
 
 VERSIONED_CHARACTERIZATION_TEST(
     ServeProtoTest,
-    realisation,
-    "realisation",
-    defaultVersion,
-    (std::tuple<Realisation, Realisation> {
-        Realisation {
-            .id = DrvOutput {
-                .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
-                .outputName = "baz",
-            },
+    unkeyedRealisation_2_8,
+    "unkeyed-realisation-2.8",
+    2 << 8 | 8,
+    (UnkeyedRealisation {
+        .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
+        .signatures = { "asdf", "qwer" },
+    }))
+
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    realisation_2_8,
+    "realisation-2.8",
+    2 << 8 | 8,
+    (Realisation {
+        UnkeyedRealisation {
             .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
             .signatures = { "asdf", "qwer" },
         },
-        Realisation {
-            .id = {
-                .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
-                .outputName = "baz",
-            },
-            .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
-            .signatures = { "asdf", "qwer" },
-            .dependentRealisations = {
-                {
-                    DrvOutput {
-                        .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
-                        .outputName = "quux",
-                    },
-                    StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
-                },
-            },
+        DrvOutput {
+            .drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
+            .outputName = "baz",
         },
     }))
 
@@ -173,7 +166,10 @@ VERSIONED_CHARACTERIZATION_TEST(
         t;
     }))
 
-VERSIONED_CHARACTERIZATION_TEST(
+/* We now do a lossy read which does not allow us to faithfully right
+   back, since we changed the data type. We still however want to test
+   that this read works, and so for that we have a one-way test. */
+VERSIONED_READ_CHARACTERIZATION_TEST(
     ServeProtoTest,
     buildResult_2_6,
     "build-result-2.6",
@@ -200,20 +196,64 @@ VERSIONED_CHARACTERIZATION_TEST(
                     {
                         "foo",
                         {
-                            .id = DrvOutput {
-                                .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
-                                .outputName = "foo",
-                            },
                             .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
                         },
                     },
                     {
                         "bar",
                         {
-                            .id = DrvOutput {
-                                .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
-                                .outputName = "bar",
-                            },
+                            .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" },
+                        },
+                    },
+                },
+                .startTime = 30,
+                .stopTime = 50,
+#if 0
+                // These fields are not yet serialized.
+                // FIXME Include in next version of protocol or document
+                // why they are skipped.
+                .cpuUser = std::chrono::milliseconds(500s),
+                .cpuSystem = std::chrono::milliseconds(604s),
+#endif
+            },
+        };
+        t;
+    }))
+
+
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    buildResult_2_8,
+    "build-result-2.8",
+    2 << 8 | 8,
+    ({
+        using namespace std::literals::chrono_literals;
+        std::tuple<BuildResult, BuildResult, BuildResult> t {
+            BuildResult {
+                .status = BuildResult::OutputRejected,
+                .errorMsg = "no idea why",
+            },
+            BuildResult {
+                .status = BuildResult::NotDeterministic,
+                .errorMsg = "no idea why",
+                .timesBuilt = 3,
+                .isNonDeterministic = true,
+                .startTime = 30,
+                .stopTime = 50,
+            },
+            BuildResult {
+                .status = BuildResult::Built,
+                .timesBuilt = 1,
+                .builtOutputs = {
+                    {
+                        "foo",
+                        {
+                            .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
+                        },
+                    },
+                    {
+                        "bar",
+                        {
                             .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" },
                         },
                     },