implement resolved config versioning (#3349)

[static]

Signed-off-by: Mateusz Błażejewski <mateusz.blazejewski@digitalasset.com>

Author: mblaze-da

.pre-commit-config.yaml

@@ -50,12 +50,14 @@ repos:
         language: system
         entry: "scripts/fix-ts.py"
         types_or: [javascript, jsx, ts, tsx]
-        exclude: "(/[.]prettierrc[.]cjs$|jest.config.ts$)"
-      - id: pulumi_config
-        name: check pulumi configs
+        # TODO(#3334): The current implementation of fix-ts.py makes it impossible to format files
+        #              in the top level pulumi package.
+        exclude: "(/[.]prettierrc[.]cjs$|/pulumi/[^/]+.ts$)"
+      - id: pulumi_tests
+        name: run pulumi tests and check expected files
         language: system
         entry: "make cluster/pulumi/test -j8"
-        files: '^cluster/pulumi/.*[.]ts$'
+        files: '^(cluster/pulumi/.*[.]ts|cluster/.*[.](yaml|yml))$'
         pass_filenames: false
       - id: gha_lint
         name: static check github actions

TESTING.md

@@ -471,6 +471,15 @@ they cover. Please see the [jest "getting started"](https://jestjs.io/docs/getti
 for tips on how to write tests. The type definitions for jest come from the `@jest/globals`
 package usage of which is described [here](https://jestjs.io/docs/getting-started#type-definitions).
 
+### Versioning resolved configuration
+To make sure that changes to cluster configuration are made consciously in light of various
+modularization and reuse mechanisms in the config loader a complete and fully resolved version of
+the cluster configuration file is versioned. Such a file is generated as `config.resolved.yaml`
+for each clusters' main `config.yaml` using `make cluster/deployment/update-resolved-config -j`.
+The `cluster/pulumi/update-expected` make target also includes resolved configuration update.
+Appropriate checks are conducted in pre-commit hooks and later in CI to make sure that these
+resolved configuration files are kept in sync with the original unresolved ones.
+
 ### Pulumi state checks
 To make sure that the impact of changes in the Pulumi resource definitions is well understood we
 rely on checked in `expected` files that need to be updated whenever the expected deployment state changes.

cluster/deployment/local.mk

@@ -0,0 +1,26 @@
+# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+deployment_dir := $(call current_dir)
+
+cluster_dirs := $(foreach config,$(wildcard $(deployment_dir)/*/config.yaml),$(shell dirname $(config)))
+resolved_config_targets := $(foreach cluster_dir,$(cluster_dirs),$(cluster_dir)/config.resolved.yaml)
+
+# We use .PHONY because it is hard to pinpoint exact deps for config resolution as it might depend
+# on many config files and the config loader implementation. At some point, when we get rid of
+# more specific config loading rules, it might make sense to try list the dependencies here.
+define update_resolved_config
+.PHONY: $(1)/config.resolved.yaml
+$(1)/config.resolved.yaml:
+	source $(1)/.envrc.vars && \
+	cd "${SPLICE_ROOT}/cluster/pulumi" && \
+	npm run resolve-config
+endef
+$(foreach cluster_dir,$(cluster_dirs),$(eval $(call update_resolved_config,$(cluster_dir))))
+
+.PHONY: $(deployment_dir)/update-resolved-config
+$(deployment_dir)/update-resolved-config: $(resolved_config_targets)
+
+.PHONY: $(deployment_dir)/check-resolved-config
+$(deployment_dir)/check-resolved-config: $(resolved_config_targets)
+	git diff --exit-code --quiet $(resolved_config_targets)