feat: Persist the package metadata

Author: akirak

pkgs/emacs/data/default.nix

@@ -25,9 +25,11 @@ in
     inventories,
     flakeLockFile,
     archiveLockFile,
+    metadataJsonFile,
     inputOverrides,
     elispPackagePins,
     defaultMainIsAscii,
+    persistMetadata,
   }: mode: let
     toLockData = {
       nodes,
@@ -48,6 +50,11 @@ in
       then lib.importJSON archiveLockFile
       else {};
 
+    cachedMetadata =
+      if mode == "build" && persistMetadata && pathExists metadataJsonFile
+      then lib.importJSON metadataJsonFile
+      else {};
+
     makeInventory = import ./inventory {
       inherit lib flakeLockData archiveLockData;
     };
@@ -92,8 +99,11 @@ in
           or (throw "Inventory named ${pin} does not exist")
         );
 
+    getPackageData0 =
+      import ./package.nix {inherit lib linkFarm defaultMainIsAscii cachedMetadata;};
+
     getPackageData = revDep: ename:
-      lib.makeExtensible (import ./package.nix {inherit lib linkFarm defaultMainIsAscii;}
+      lib.makeExtensible (getPackageData0
         ename
         # It would be nice if it were possible to set the pin from inside
         # overrideInputs, but it causes infinite recursion unfortunately :(

pkgs/emacs/data/package.nix

@@ -26,6 +26,7 @@ in
     lib,
     linkFarm,
     defaultMainIsAscii,
+    cachedMetadata,
   }: ename: mkAttrs: self: let
     attrs =
       if isAttrs mkAttrs
@@ -58,6 +59,11 @@ in
         )
       else {};
 
+    metadata =
+      if hasAttr ename cachedMetadata && cachedMetadata.${ename}.narHash == (self.src.narHash or null)
+      then cachedMetadata.${ename}
+      else {};
+
     # builtins.readFile fails when the source file contains control characters.
     # pydoc.el is an example. A workaround is to take only the first N bytes of
     # the file using `head` command and read its output. This is IFD, which
@@ -136,26 +142,30 @@ in
 
       # TODO: Check https://github.com/melpa/melpa/issues/2955 on the right versioning scheme
       version =
-        attrs.version
+        metadata.version
+        or attrs.version
         or headers.Version
         or headers.Package-Version
         or packageDesc.version
         # Set a null version if the package lacks a version header.
         or null;
 
-      author = headers.Author or null;
+      author =
+        metadata.author
+        or headers.Author or null;
 
       meta =
-        (lib.optionalAttrs hasPkgFile {
-          description = packageDesc.summary;
-        })
-        // (import ./headers-to-meta.nix {
-          inherit lib;
-          inherit (self) headers;
-        });
+        metadata.meta
+        or ((lib.optionalAttrs hasPkgFile {
+            description = packageDesc.summary;
+          })
+          // (import ./headers-to-meta.nix {
+            inherit lib headers;
+          }));
 
       packageRequires =
-        attrs.packageRequires
+        metadata.packageRequires
+        or attrs.packageRequires
         or packageDesc.packageRequires
         or (
           if headers ? Package-Requires

pkgs/emacs/default.nix

@@ -36,6 +36,9 @@
     if wantExtraOutputs
     then ["info"]
     else [],
+  # Whether to persist package metadata in the lock directory. This is needed to
+  # avoid IFD in certain situations.
+  persistMetadata ? false,
   # Assume the main files of all packages contain only ASCII characters. This is
   # a requirement for avoiding IFD, but some libraries actually contain
   # non-ASCII characters, which cannot be parsed with `builtins.readFile`
@@ -70,6 +73,8 @@ in
 
     archiveLockFile = lockDir + "/archive.lock";
 
+    metadataJsonFile = lockDir + "/metadata.json";
+
     userConfig = lib.pipe self.initFiles [
       (map initReader)
       lib.zipAttrs
@@ -104,9 +109,11 @@ in
         lib
         flakeLockFile
         archiveLockFile
+        metadataJsonFile
         builtinLibraries
         inputOverrides
         defaultMainIsAscii
+        persistMetadata
         ;
       # Just remap the name
       inventories = registries;
@@ -234,6 +241,7 @@ in
           excludeLocalPackages (enumerateConcretePackageSet "update" explicitPackages);
         flakeNix = true;
         archiveLock = true;
+        metadataJson = persistMetadata;
       })
       .writerScript {inherit postCommandOnGeneratingLockDir;};
 
@@ -248,6 +256,7 @@ in
               excludeLocalPackages (enumerateConcretePackageSet "lock" explicitPackages);
             flakeNix = true;
             archiveLock = true;
+            metadataJson = persistMetadata;
             postCommand = "nix flake lock";
           })
         .asAppWritingToRelativeDir
@@ -260,6 +269,7 @@ in
             packageInputs =
               excludeLocalPackages (enumerateConcretePackageSet "update" explicitPackages);
             archiveLock = true;
+            metadataJson = persistMetadata;
           })
         .asAppWritingToRelativeDir
         lockDirName;

pkgs/emacs/lock/default.nix

@@ -13,6 +13,7 @@
   packageInputs,
   flakeNix ? false,
   archiveLock ? false,
+  metadataJson ? false,
   # Command run after writing the directory in asAppWritingToRelativeDir
   postCommand ? null,
 }:
@@ -30,6 +31,20 @@ assert (flakeNix || archiveLock); let
       ]))
   ];
 
+  packageMetadata =
+    mapAttrs (_: {
+      src,
+      version,
+      packageRequires,
+      meta,
+      author,
+      ...
+    }: {
+      inherit (src) narHash;
+      inherit version packageRequires meta author;
+    })
+    packageInputs;
+
   data = {
     flakeNix = lib.toNix {
       description = "THIS IS AN AUTO-GENERATED FILE. PLEASE DON'T EDIT IT MANUALLY.";
@@ -40,11 +55,13 @@ assert (flakeNix || archiveLock); let
       outputs = {...}: {};
     };
     archiveLock = toJSON archiveLockData;
+    metadataJson = toJSON packageMetadata;
   };
 
   passAsFile =
     lib.optional flakeNix "flakeNix"
-    ++ lib.optional archiveLock "archiveLock";
+    ++ lib.optional archiveLock "archiveLock"
+    ++ lib.optional metadataJson "metadataJson";
 
   # HACK: Use sed to convert JSON to Nix
   #
@@ -60,6 +77,10 @@ assert (flakeNix || archiveLock); let
     ${jq}/bin/jq . "$archiveLockPath" > "$out/archive.lock"
   '';
 
+  generateMetadataJson = ''
+    ${jq}/bin/jq . "$metadataJsonPath" > "$out/metadata.json"
+  '';
+
   src =
     runCommandLocal "emacs-twist-lock" ({
         inherit passAsFile;
@@ -70,6 +91,7 @@ assert (flakeNix || archiveLock); let
 
       ${lib.optionalString flakeNix generateFlakeNix}
       ${lib.optionalString archiveLock generateArchiveLock}
+      ${lib.optionalString metadataJson generateMetadataJson}
     '';
 in {
   asAppWritingToRelativeDir = outDir: {