Support running tests with an older initial package version set

.github/actions/tests/scala_test/action.yml

@@ -83,6 +83,9 @@ inputs:
   failure_notifications_slack_channel:
     description: "Slack channel for failure notifications (for failure notifications)"
     required: true
+  daml_base_version:
+    type: string
+    required: true
 
 runs:
   using: "composite"
@@ -133,6 +136,28 @@ runs:
       with:
         cache_version: 4
 
+    - name: Set Daml package versions
+      uses: ./.github/actions/nix/run_bash_command_in_nix
+      with:
+        cmd: |
+          if [[ "${{ inputs.daml_base_version }}" != "" ]]; then
+            echo "Running with initial package config from ${{ inputs.daml_base_version }}"
+            ./scripts/initial-package-config.py ${{ inputs.daml_base_version }} initial_package_config scala_test_tags
+            echo "Initial package config" $(cat initial_package_config)
+            echo "ScalaTest tags: $(cat scala_test_tags)"
+          else
+            echo "daml_base_version is not set, using the latest versions"
+          fi
+
+    - name: Export Daml package versions output
+      id: daml_package_versions
+      shell: bash
+      run: |
+        # We can't export from within run_bash_command_in_nix so we do it here.
+        if [[ "${{ inputs.daml_base_version }}" != "" ]]; then
+          echo "initial_package_versions='$(cat initial_package_config)'" >> "$GITHUB_OUTPUT"
+        fi
+
     - name: Prepare list of tests to run for sbt
       uses: ./.github/actions/nix/run_bash_command_in_nix
       with:
@@ -147,7 +172,16 @@ runs:
     - name: Set RUN_SPLITTED_TESTS_CMD
       id: list_tests
       shell: bash
-      run: echo "RUN_SPLITTED_TESTS_CMD=\"testOnly $(cat /tmp/tests)\"" >> "$GITHUB_OUTPUT"
+      run: |
+        tags=""
+        if [ -f scala_test_tags ]
+        then
+          tags="$(cat scala_test_tags)"
+          echo "Using scalatest tags: $tags"
+        else
+          echo "No scala test tags configured"
+        fi
+        echo "RUN_SPLITTED_TESTS_CMD=\"testOnly $(cat /tmp/tests) -- $tags\"" >> "$GITHUB_OUTPUT"
 
     # webpack uses inotify to watch for changes, and we need to increase the limit
     - name: increase inotify limit
@@ -191,7 +225,7 @@ runs:
     - name: Run tests
       uses: ./.github/actions/sbt/execute_sbt_command
       with:
-        extra_env_vars: POSTGRES_DB=postgres POSTGRES_HOST=localhost POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres
+        extra_env_vars: "POSTGRES_DB=postgres POSTGRES_HOST=localhost POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres INITIAL_PACKAGE_VERSIONS=${{ steps.daml_package_versions.outputs.initial_package_versions }}"
         artifactory_user: ${{ inputs.artifactory_user }}
         artifactory_password: ${{ inputs.artifactory_password }}
         cmd: ${{ steps.list_tests.outputs.RUN_SPLITTED_TESTS_CMD }}

.github/workflows/build.scala_test.yml

@@ -50,6 +50,9 @@ on:
         type: string
         required: false
         default: ""
+      daml_base_version:
+        type: string
+        required: true
 
 jobs:
 
@@ -130,3 +133,4 @@ jobs:
           failure_notifications_invoker_sa: ${{ secrets.FAILURE_NOTIFICATIONS_INVOKER_SA }}
           failure_notifications_invoker_url: ${{ secrets.FAILURE_NOTIFICATIONS_INVOKER_URL }}
           failure_notifications_slack_channel: ${{ secrets.FAILURE_NOTIFICATIONS_SLACK_CHANNEL }}
+          daml_base_version: ${{ inputs.daml_base_version }}

.github/workflows/build.scala_test_for_compose.yml

@@ -25,6 +25,10 @@ on:
         type: string
         required: false
         default: ""
+      daml_base_version:
+        type: string
+        required: true
+
 
 jobs:
 
@@ -99,3 +103,4 @@ jobs:
           failure_notifications_invoker_url: ${{ secrets.FAILURE_NOTIFICATIONS_INVOKER_URL }}
           failure_notifications_slack_channel: ${{ secrets.FAILURE_NOTIFICATIONS_SLACK_CHANNEL }}
           with_canton: ${{ inputs.with_canton }}
+          daml_base_version: ${{ inputs.daml_base_version }}

.github/workflows/build.scala_test_with_cometbft.yml

@@ -18,6 +18,9 @@ on:
         type: string
         required: false
         default: ""
+      daml_base_version:
+        type: string
+        required: true
 
 jobs:
 
@@ -117,3 +120,4 @@ jobs:
           failure_notifications_invoker_sa: ${{ secrets.FAILURE_NOTIFICATIONS_INVOKER_SA }}
           failure_notifications_invoker_url: ${{ secrets.FAILURE_NOTIFICATIONS_INVOKER_URL }}
           failure_notifications_slack_channel: ${{ secrets.FAILURE_NOTIFICATIONS_SLACK_CHANNEL }}
+          daml_base_version: ${{ inputs.daml_base_version }}

.github/workflows/build.yml

@@ -7,6 +7,21 @@ on:
         required: false
         default: ''
         type: string
+      daml_base_version:
+        type: string
+        required: false
+        default: ""
+  workflow_dispatch:
+    inputs:
+      commit_sha:
+        description: 'The commit SHA to use for the build'
+        required: false
+        default: ''
+        type: string
+      daml_base_version:
+        type: string
+        required: false
+        default: ""
 
 permissions:
   id-token: write # Required for GCP Workload Identity for failure notifications
@@ -66,6 +81,7 @@ jobs:
       parallelism: 4
       test_name: simtime
       commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   scala_test_resource_intensive:
@@ -77,6 +93,7 @@ jobs:
       parallelism: 2
       test_name: resource-intensive
       commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   scala_test_record_time_tolerance:
@@ -88,6 +105,7 @@ jobs:
       parallelism: 1
       test_name: record-time-tolerance
       commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   scala_test_disaster_recovery:
@@ -99,6 +117,7 @@ jobs:
       parallelism: 1
       test_name: disaster-recovery
       commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   scala_test_with_cometbft:
@@ -109,6 +128,7 @@ jobs:
       parallelism: 1
       test_name: cometbft
       commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   scala_test_with_bft:
@@ -120,6 +140,7 @@ jobs:
       parallelism: 1
       test_name: canton bft
       commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
 
@@ -134,6 +155,7 @@ jobs:
       test_name: wall-clock-time
       with_gcp_creds: true
       commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   scala_test_wall_clock_time_bft:
@@ -147,6 +169,7 @@ jobs:
       with_gcp_creds: true
       pre_sbt_cmd: "export SPLICE_USE_BFT_SEQUENCER=1"
       run_if_commit_tag: 'bft'
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   scala_test_frontend_wall_clock_time:
@@ -159,6 +182,7 @@ jobs:
       test_name: frontend-wall-clock-time
       is_frontend_test: true
       commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   scala_test_frontend_simtime:
@@ -171,6 +195,7 @@ jobs:
       test_name: frontend-simtime
       is_frontend_test: true
       commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   scala_test_docker_compose:
@@ -182,6 +207,7 @@ jobs:
       parallelism: 2
       test_name: docker-compose
       commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   scala_test_local_net:
@@ -193,6 +219,7 @@ jobs:
       test_name: docker-compose
       with_canton: false
       start_canton_options: ""
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   scala_test_app_upgrade:
@@ -207,6 +234,7 @@ jobs:
       skip_if_regex: "\\[breaking\\]"
       skip_if_files_changed: "nix/canton-sources.json"
       commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   ui_tests:
@@ -237,6 +265,7 @@ jobs:
       test_name: without-canton
       with_gcp_creds: true
       commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
     secrets: inherit
 
   final_result:

.github/workflows/daml_compat_test.yml

@@ -0,0 +1,36 @@
+name: Integration tests against mainnet Daml version
+on:
+  schedule:
+    - cron: '0 5 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  id-token: write # Required for GCP Workload Identity for failure notifications
+  contents: read
+  pull-requests: read # Required for the static tests
+  issues: read # Required for the static tests
+  actions: write # To cancel itself if not opted in
+
+jobs:
+  get_mainnet_version:
+    runs-on: self-hosted-docker-tiny
+    outputs:
+      version: ${{ steps.get_mainnet_version.outputs.version }}
+    steps:
+      - name: Get MainNet version
+        id: get_mainnet_version
+        run: |
+          set -eou pipefail
+          version="$(curl -sSL --fail-with-body https://docs.global.canton.network.sync.global/info | jq -r '.sv.version')"
+          echo "MainNet version is $version"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+
+  build:
+    needs: get_mainnet_version
+    uses: ./.github/workflows/build.yml
+    with:
+      daml_base_version: ${{ needs.get_mainnet_version.outputs.version }}
+    secrets: inherit

DEVELOPMENT.md

@@ -369,6 +369,26 @@ To edit the files in a particular Daml project, for example, `/apps/wallet/daml`
 *Tip:* if `damlBuild` fails with weird errors, then that might be due to stale `damlBuild` outputs.
 Try forcing a clean rebuild by cleaning via SBT, e.g., `apps-common/clean` and similar for the dependent project.
 
+## Daml Version Guards in Integration Tests
+
+Our backends need to handle the case where we compile against Daml
+version X+1 but the vote to switch to those models is not yet done and
+only Daml version X can be used. This is done by checking the active
+version endpoints in triggers, UIs and similar. However, those checks
+can be easily forgotten and the normal integration tests do not catch
+that as they run against the latest version.
+
+To catch this, we periodically run all integration tests against the
+latest Daml version that was shipped to mainnet. This means tests that
+do actually depend on the newer versions won't work. Default to making
+tests backwards compatible where possible. Where this is not possible
+or does not make sense (e.g. because you're testing functionality that
+did not have an equivalent before), tag the test with a scalatest tag
+corresponding to the version of the Daml package that introduced the
+functionality, e.g.,
+`org.lfdecentralizedtrust.splice.util.scalatesttags.SpliceAmulet_0_1_9`. This
+ensures the test will be excluded if we are running against earlier
+Daml versions.
 
 # Troubleshooting
 

apps/app/src/main/scala/org/lfdecentralizedtrust/splice/config/SpliceConfig.scala

@@ -541,6 +541,10 @@ object SpliceConfig {
     implicit val partyIdConfigReader: ConfigReader[PartyId] = ConfigReader.fromString(str =>
       Codec.decode(Codec.Party)(str).left.map(err => CannotConvert(str, "PartyId", err))
     )
+    implicit val packageVersionConfigReader: ConfigReader[PackageVersion] =
+      ConfigReader.fromString(str =>
+        PackageVersion.fromString(str).left.map(err => CannotConvert(str, "PackageVersion", err))
+      )
     implicit val beneficiaryConfigReader: ConfigReader[BeneficiaryConfig] =
       deriveReader[BeneficiaryConfig]
     implicit val svParticipantClientConfigReader: ConfigReader[SvParticipantClientConfig] =
@@ -890,6 +894,8 @@ object SpliceConfig {
       deriveWriter[PeriodicBackupDumpConfig]
     implicit val partyIdConfigWriter: ConfigWriter[PartyId] =
       implicitly[ConfigWriter[String]].contramap(_.toString)
+    implicit val packageVersionConfigWriter: ConfigWriter[PackageVersion] =
+      implicitly[ConfigWriter[String]].contramap(_.toString)
     implicit val beneficiaryConfigWriter: ConfigWriter[BeneficiaryConfig] =
       deriveWriter[BeneficiaryConfig]
     implicit val svParticipantClientConfigWriter: ConfigWriter[SvParticipantClientConfig] =

apps/app/src/test/resources/include/validator-participant.conf

@@ -37,7 +37,6 @@ canton {
           port = 7575
           address = 0.0.0.0
         }
-        allow-insecure-tokens = true
       }
       parameters {
         initial-protocol-version = 33

apps/app/src/test/resources/simple-topology-canton-simtime.conf

@@ -9,7 +9,6 @@ _sv1Participant_client {
   ledger-api.port = 15101
   http-ledger-api {
     server.port = 16101
-    allow-insecure-tokens = true
   }
 }
 
@@ -18,7 +17,6 @@ _sv2Participant_client {
   ledger-api.port = 15201
   http-ledger-api {
     server.port = 16201
-    allow-insecure-tokens = true
   }
 }
 
@@ -27,7 +25,6 @@ _sv3Participant_client {
   ledger-api.port = 15301
   http-ledger-api {
     server.port = 16301
-    allow-insecure-tokens = true
   }
 }
 
@@ -36,7 +33,6 @@ _sv4Participant_client {
   ledger-api.port = 15401
   http-ledger-api {
     server.port = 16401
-    allow-insecure-tokens = true
   }
 }
 
@@ -46,7 +42,6 @@ _aliceParticipant_client {
   # Used in token standard tests
   http-ledger-api {
     server.port = 16501
-    allow-insecure-tokens = true
   }
 }
 
@@ -55,7 +50,6 @@ _bobParticipant_client {
   ledger-api.port = 15601
   http-ledger-api {
     server.port = 16601
-    allow-insecure-tokens = true
   }
 }
 
@@ -64,7 +58,6 @@ _splitwellParticipant_client {
   ledger-api.port = 15701
   http-ledger-api {
     server.port = 16701
-    allow-insecure-tokens = true
   }
 }