Merge pull request #583 from hercules-ci/add-config-nixSettings

hercules-ci-agent: Add nixSettings config item

Author: roberth

docs/modules/ROOT/pages/agent-config.adoc

@@ -185,6 +185,23 @@ Optional. Control the importance threshold for messages are logged to the system
 
 Defaults to `"InfoS"`. More verbose: `"DebugS"`, less verbose: `"WarningS"`, `"ErrorS"`.
 
+[[nixSettings]]
+== nixSettings
+
+Since hercules-ci-agent 0.10.2
+
+Optional. A key-value map of Nix settings.
+
+Nix interprets keys that start with `extra-` as additions to the existing (e.g. system) value. However, `nixSettings` is unordered, so make sure to use `extra-${x}` or plain `${x}`, but not both.
+
+Example, to ignore other system caches:
+
+```json
+"nixSettings": {
+  "substituters": "https://cache.nixos.org"
+}
+```
+
 [[nixVerbosity]]
 == nixVerbosity
 

hercules-ci-agent/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Added
+
+ - Add `nixSettings` configuration item, to more easily configure Nix settings such as `substituters`, overriding the system Nix settings.
+
 ## [0.10.1] - 2024-02-12
 
 ### Changed

hercules-ci-agent/hercules-ci-agent/Hercules/Agent/Config.hs

@@ -25,7 +25,8 @@ import Data.Aeson.Types qualified as A
 import Data.Profunctor (Star (Star))
 import GHC.Conc (getNumProcessors)
 import Hercules.Agent.Config.Combined
-import Hercules.Agent.Config.Json as Json
+import Hercules.Agent.Config.Json (GCodec (GCodec), (.=.))
+import Hercules.Agent.Config.Json qualified as Json
 import Hercules.Agent.Config.Toml qualified as Toml
 import Hercules.CNix.Verbosity (Verbosity (..))
 import Hercules.Formats.Mountable (Mountable (Mountable))
@@ -72,7 +73,8 @@ data Config purpose = Config
     labels :: Item purpose 'Required (Map Text A.Value),
     allowInsecureBuiltinFetchers :: Item purpose 'Required Bool,
     remotePlatformsWithSameFeatures :: Item purpose 'Optional [Text],
-    effectMountables :: Map Text Mountable
+    effectMountables :: Map Text Mountable,
+    nixSettings :: Map Text Text
   }
   deriving (Generic)
 
@@ -120,6 +122,9 @@ combiCodec =
           (Json.tableMap' (forJson mountableCodec) "effectMountables")
       )
       .=. effectMountables
+    <*> optEmpty
+      (tableMap textAtKey "nixSettings")
+      .=. nixSettings
 
 mountableCodec :: Combi' Mountable
 mountableCodec =
@@ -233,5 +238,6 @@ finalizeConfig loc input = do
         labels = fromMaybe mempty $ labels input,
         allowInsecureBuiltinFetchers = fromMaybe False $ allowInsecureBuiltinFetchers input,
         remotePlatformsWithSameFeatures = remotePlatformsWithSameFeatures input,
-        effectMountables = effectMountables input
+        effectMountables = effectMountables input,
+        nixSettings = nixSettings input
       }

hercules-ci-agent/hercules-ci-agent/Hercules/Agent/Config/Combined.hs

@@ -46,3 +46,6 @@ optEmpty c = dimap Just (fromMaybe mempty) (opt c)
 
 enumBoundedAtKey :: (Bounded a, Enum a, Show a) => Key -> Combi' a
 enumBoundedAtKey k = Combi (Data.Bifunctor.Product.Pair (Toml.enumBounded k) (Json.enumBounded k))
+
+tableMap :: (Key -> Combi' a) -> Key -> Combi' (Map Text a)
+tableMap c k = Combi $ Pair (Toml.tableMap Toml._KeyText (forToml . c) k) (Json.tableMap (forJson . c) k)

hercules-ci-agent/hercules-ci-agent/Hercules/Agent/Config/Json.hs

@@ -39,6 +39,7 @@ import Toml hiding
     dimap,
     first,
     match,
+    tableMap,
     _EnumBounded,
     _Text,
     _TextBy,
@@ -250,12 +251,30 @@ tableMap' ::
   Key ->
   JsonCodec' (Map Text v)
 tableMap' valCodec key =
+  tableMap
+    ( \k ->
+        GCodec
+          { gRead =
+              local
+                (\v -> fromMaybe (panic "tableMap': key disappeared") (v ^? at' k))
+                (gRead valCodec),
+            gWrite = do
+              panic "tableMap': write not implemented"
+          }
+    )
+    key
+
+tableMap ::
+  (Key -> JsonCodec' v) ->
+  Key ->
+  JsonCodec' (Map Text v)
+tableMap valCodec key =
   let c = match (prismWithError "JSON Object expected" (Aeson.Lens._Object . aesonMap)) key
    in GCodec
         { gRead = do
             x <- gRead c
-            fmap M.fromList $ for (M.toList x) $ \(k, v) -> do
-              v' <- local (const v) $ gRead valCodec
+            fmap M.fromList $ for (M.toList x) $ \(k, _v) -> do
+              v' <- gRead (valCodec (key <> (Key $ Piece k :| [])))
               pure (k, v'),
           gWrite = do
             panic "tableMap': write not implemented"

hercules-ci-agent/hercules-ci-agent/Hercules/Agent/Init.hs

@@ -40,7 +40,7 @@ withEnv config logEnv f = do
   concPushes <- newMemo
   concQueries <- newMemo
   withLogging $ Hercules.Agent.Cachix.Init.withEnv config (BC.cachixCaches bcs) \cachix -> liftIO do
-    nix <- Hercules.Agent.Nix.Init.newEnv
+    nix <- Hercules.Agent.Nix.Init.newEnv config
     serviceInfo <- ServiceInfo.newEnv clientEnv
     let env =
           Env

hercules-ci-agent/hercules-ci-agent/Hercules/Agent/Nix/Init.hs

@@ -1,11 +1,19 @@
+{-# LANGUAGE DataKinds #-}
+
 module Hercules.Agent.Nix.Init where
 
+import Data.Map qualified as M
+import Hercules.Agent.Config (Config, Purpose (Final))
+import Hercules.Agent.Config qualified
 import Hercules.Agent.EnvironmentInfo qualified as EnvironmentInfo
 import Hercules.Agent.Nix.Env
+import Hercules.CNix qualified as CNix
 import Protolude
 
-newEnv :: IO Env
-newEnv = do
+newEnv :: Config 'Final -> IO Env
+newEnv config = do
+  for_ (M.toList config.nixSettings) $ \(k, v) -> do
+    CNix.setGlobalOption k v
   nixInfo <- EnvironmentInfo.getNixInfo
   when (EnvironmentInfo.nixNarinfoCacheNegativeTTL nixInfo /= 0) $ do
     putErrText
@@ -33,5 +41,5 @@ newEnv = do
   -- extraOptions here.
   pure
     Env
-      { extraOptions = []
+      { extraOptions = M.toList config.nixSettings
       }

hercules-ci-agent/test/Hercules/Agent/ConfigSpec.hs

@@ -45,6 +45,8 @@ spec =
                   "workDirectory = \"/var/lib/hercules-ci-agent/work\"",
                   "remotePlatformsWithSameFeatures = [\"aarch64-darwin\"]",
                   "nixVerbosity = \"vomit\"",
+                  "[nixSettings]",
+                  "  substituters = \"https://example.com\"",
                   -- This line should not be needed!
                   "[effectMountables]",
                   "[effectMountables.hosts]",
@@ -91,7 +93,8 @@ spec =
                               condition = Hercules.Formats.Secret.Const True
                             }
                         )
-                      ]
+                      ],
+                  nixSettings = M.singleton "substituters" "https://example.com"
                 }
             )
       it "parses empty config" $ do
@@ -119,6 +122,7 @@ spec =
                   "\"secretsJsonPath\": \"/var/lib/hercules-ci-agent/secrets/secrets.json\",",
                   "\"logLevel\": \"DebugS\",",
                   "\"nixVerbosity\": \"Vomit\",",
+                  "\"nixSettings\": { \"substituters\": \"https://example.com\" },",
                   "\"remotePlatformsWithSameFeatures\": [\"aarch64-darwin\"],",
                   "\"labels\": {",
                   "  \"agent\": {\"source\": \"flake\"},",
@@ -169,7 +173,8 @@ spec =
                             condition = Hercules.Formats.Secret.Const True
                           }
                       )
-                    ]
+                    ],
+                nixSettings = M.singleton "substituters" "https://example.com"
               }
 
       it "handles empty config" $ do
@@ -232,5 +237,6 @@ emptyConfig =
       labels = Nothing,
       allowInsecureBuiltinFetchers = Nothing,
       remotePlatformsWithSameFeatures = Nothing,
-      effectMountables = mempty
+      effectMountables = mempty,
+      nixSettings = mempty
     }

internal/nix/settings.nix

@@ -119,6 +119,23 @@ let
         internal = true;
         default = "Talkative";
       };
+      nixSettings = mkOption {
+        description = mdDoc ''
+          Since hercules-ci-agent 0.10.2.
+
+          A key-value map of Nix settings.
+
+          Nix interprets keys that start with `extra-` as additions to the existing (e.g. system) value. However, `nixSettings` is unordered, so make sure to use `extra-''${x}` or plain `''${x}`, but not both.
+        '';
+        type = types.lazyAttrsOf types.str;
+        default = { };
+        example = lib.literalExpression ''
+          {
+            # ignore other system substituters
+            substituters = "https://cache.nixos.org";
+          };
+        '';
+      };
       remotePlatformsWithSameFeatures = mkOption {
         internal = true;
         default = [ ];