Add 3.3 snapshot to compat tests, (#22183)

samuel-williams-da · samuel-williams-da · commit 09c043bc2e18 · 2025-10-29T10:10:56.000Z
Fixed the "update-versions" script to include the latest 3.3 snapshot
Fixed all of the many many issues with compat tests.
diff --git a/ci/BUILD b/ci/BUILD
diff --git a/ci/bash-lib.yml b/ci/bash-lib.yml
@@ -136,6 +136,9 @@ steps:
       shift
       build=$1
       shift
+
+      az extension add --name azure-devops --allow-preview true || true
+
       az pipelines build queue \
         --definition-name $build \
         --org "https://dev.azure.com/digitalasset" \
diff --git a/ci/blackduck.yml b/ci/blackduck.yml
@@ -209,12 +209,8 @@ jobs:
         sed -i "s|blackduck_script_sha: $current|blackduck_script_sha: $latest|" ../ci/blackduck.yml
         git add ../ci/blackduck.yml
         
-        az extension add --name azure-devops --allow-preview true || true
-        echo ${AZURE_DEVOPS_EXT_PAT} | az devops login --organization="https://dev.azure.com/digitalasset"
-        trap "az devops logout" EXIT
-        
         open_pr "$branch" "bump blackduck script to ${latest:0:8}" "" "" "$(Build.SourceBranchName)"
-        trigger_azure $(System.AccessToken) "PRs" --branch $branch
+        trigger_azure $AZURE_DEVOPS_EXT_PAT "PRs" --branch $branch
       fi
     displayName: 'Bump Blackduck Script if needed'
     env:
diff --git a/ci/cron/daily-compat.yml b/ci/cron/daily-compat.yml
@@ -159,14 +159,8 @@ jobs:
           cd sdk
           eval "$(./dev-env/bin/dade-assist)"
           if [ "$(need_to_build)" == "true" ]; then
-              branch="notices-update-$(Build.BuildId)"
-              az extension add --name azure-devops
-              trap "az devops logout" EXIT
-              echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
-              az pipelines build queue --branch "$branch" \
-                                       --definition-name "PRs" \
-                                       --org "https://dev.azure.com/digitalasset" \
-                                       --project daml
+              BRANCH="notices-update-$(Build.BuildId)"
+              trigger_azure $AZURE_DEVOPS_EXT_PAT PRs --branch $BRANCH
           fi
         env:
           AZURE_DEVOPS_EXT_PAT: $(System.AccessToken)
diff --git a/ci/cron/wednesday.yml b/ci/cron/wednesday.yml
@@ -51,7 +51,7 @@ jobs:
       rotate
       git add .
       open_pr "$BRANCH" "$TITLE" "$BODY" "$PR_NUMBER"
-      trigger_azure $(System.AccessToken) "PRs" --branch $BRANCH
+      trigger_azure $AZURE_DEVOPS_EXT_PAT "PRs" --branch $BRANCH
       request_pr_review $(cat "$PR_NUMBER") "$NEXT_GH"
     env:
       AZURE_DEVOPS_EXT_PAT: $(System.AccessToken)
diff --git a/ci/prs.yml b/ci/prs.yml
@@ -155,9 +155,9 @@ jobs:
       REQUEST=$(git log -n1 --format="%(trailers:key=run-full-compat,valueonly)" $COMMIT)
       if [ $REQUEST == true ]; then
         # Unfortunately recovering the actual branch name is tricky
-        trigger_azure $AZURE_TOKEN digital-asset.daml-daily-compat --commit-id $COMMIT --branch not-main
+        trigger_azure $AZURE_DEVOPS_EXT_PAT digital-asset.daml-daily-compat --commit-id $COMMIT --branch not-main
       fi
     env:
-      AZURE_TOKEN: $(System.AccessToken)
+      AZURE_DEVOPS_EXT_PAT: $(System.AccessToken)
     displayName: "Trigger self-service compatibility test"
 
diff --git a/sdk/bazel_tools/os_info.bzl b/sdk/bazel_tools/os_info.bzl
@@ -13,6 +13,7 @@ is_linux = is_linux_intel or is_linux_arm
 is_windows = cpu_value == "x64_windows"
 os_name = "macos" if is_darwin else "linux" if is_linux_intel or is_linux_arm else "windows"
 is_intel = is_windows or is_darwin or is_linux_intel
+os_arch = "x86_64" if is_windows or is_linux_intel else "arm64" if is_darwin_arm64 else "aarch64"
 """
 
 def _os_info_impl(repository_ctx):
diff --git a/sdk/ci/oracle_image b/sdk/ci/oracle_image
diff --git a/sdk/compatibility/BUILD b/sdk/compatibility/BUILD
@@ -150,9 +150,11 @@ os_name = "linux" if not is_windows else "windows"
 command_targets = {
     "install_from_version": [
         "0.0.0",
+        "3.3.0-snapshot.20250930.0",
     ],
     "build_from_version": [
         "0.0.0",
+        "3.3.0-snapshot.20250930.0",
     ],
     "install_with_custom_version_and_build": [
         "0.0.0",
@@ -200,6 +202,7 @@ command_post_failure_behaviours = {
             ":new_cache",
             ":old_cache",
             ":releases-github-api.json",
+            "@daml-sdk-3.3.0-snapshot.20250926.13852.1.v4f3223e3-tarball//file",
             "@head_sdk//:daml" if not is_windows else "@head_sdk//:daml.exe",
             "@head_sdk//:sdk-release-tarball-ce.tar.gz",
         ],
diff --git a/sdk/compatibility/README.md b/sdk/compatibility/README.md
@@ -18,7 +18,7 @@ tests run on CI.
 We include the following SDK versions in our tests:
 
 1. All stable SDK versions.
-2. All snapshots `x.y.z-snapshot-*` for which there is no stable release `x.y.z`
+2. Latest `3.3.X` snapshot.
 3. HEAD.
 
 Since running all tests can be rather slow, we run them in a daily
diff --git a/sdk/compatibility/WORKSPACE b/sdk/compatibility/WORKSPACE
@@ -85,7 +85,7 @@ load(
     "nixpkgs_package",
     "nixpkgs_python_configure",
 )
-load("//:versions.bzl", "latest_stable_version", "sdk_versions", "version_sha256s")
+load("//:versions.bzl", "internal_sdk_versions", "latest_stable_version", "sdk_versions", "version_sha256s")
 
 common_nix_file_deps = [
     "@daml//nix:bazel.nix",
@@ -389,11 +389,9 @@ load("@scala_version//:index.bzl", "scala_artifacts", "scala_major_version", "sc
 maven_install(
     name = "maven",
     artifacts = [
-        "com.daml:bindings-akka_2.13:{}".format("2.7.6"),
-        "com.daml:daml-lf-archive-reader_2.13:{}".format(latest_stable_version),
-        "com.daml:daml-lf-transaction_2.13:{}".format(latest_stable_version),
-        "com.daml:ledger-api-common_2.13:{}".format(latest_stable_version),
-        "com.daml:lf-value-json_2.13:{}".format(latest_stable_version),
+        "com.daml:daml-lf-archive-reader_2.13:{}".format(internal_sdk_versions[latest_stable_version]),
+        "com.daml:daml-lf-transaction_2.13:{}".format(internal_sdk_versions[latest_stable_version]),
+        "com.daml:lf-value-json_2.13:{}".format(internal_sdk_versions[latest_stable_version]),
         "io.grpc:grpc-api:{}".format(grpc_version),
         "io.grpc:grpc-core:{}".format(grpc_version),
         "io.grpc:grpc-netty:{}".format(grpc_version),
@@ -437,49 +435,13 @@ register_toolchains("@daml//bazel_tools/scala:toolchain")
 # Copies of SDKs at correct versions for daml testing
 
 http_file(
-    name = "daml-sdk-2.7.5-tarball",
-    sha256 = "ea2730b1b50075686c82369a20da5a6b93c899e792d0db756218702427884148",
-    urls = ["https://github.com/digital-asset/daml/releases/download/v2.7.5/daml-sdk-2.7.5-linux.tar.gz"],
+    name = "daml-sdk-3.3.0-snapshot.20250926.13852.1.v4f3223e3-tarball",
+    sha256 = "aac8d2cf931af2122287e0a1a79488fcc2daa2181e5d562cab72d72350a5834e",
+    urls = ["https://github.com/digital-asset/daml/releases/download/v3.3.0-snapshot.20250930.0/daml-sdk-3.3.0-snapshot.20250926.13852.1.v4f3223e3-linux-x86_64.tar.gz"],
 ) if is_linux_intel else None
 
 http_file(
-    name = "daml-sdk-2.7.5-tarball",
-    sha256 = "b8e8bdff694a075431686048311d3a2ad9b62a2c88bf8ef927123c1b5b2addfa",
-    urls = ["https://github.com/digital-asset/daml/releases/download/v2.7.5/daml-sdk-2.7.5-windows.tar.gz"],
-) if is_windows else None
-
-http_file(
-    name = "daml-sdk-2.7.4-tarball",
-    sha256 = "c8625a752006821998b49b48d4c734b41dcedfda5ae72d6d5d6ce3eab443f625",
-    urls = ["https://github.com/digital-asset/daml/releases/download/v2.7.4/daml-sdk-2.7.4-linux.tar.gz"],
-) if is_linux_intel else None
-
-http_file(
-    name = "daml-sdk-2.7.4-tarball",
-    sha256 = "8040d524d487cfe260b0f94bdcfee1556548875b33666200e9d599f22c31e51c",
-    urls = ["https://github.com/digital-asset/daml/releases/download/v2.7.4/daml-sdk-2.7.4-windows.tar.gz"],
-) if is_windows else None
-
-http_file(
-    name = "daml-sdk-2.7.1-tarball",
-    sha256 = "72b6151168b8229e040cf4ed033c9838c1f0d986e63f18186091280dc82f4ea9",
-    urls = ["https://github.com/digital-asset/daml/releases/download/v2.7.1/daml-sdk-2.7.1-linux.tar.gz"],
-) if is_linux_intel else None
-
-http_file(
-    name = "daml-sdk-2.7.1-tarball",
-    sha256 = "200d358103daaf3cd58910966f4a5059bf4bbd3a98964bf65c3414fb3ec6cd92",
-    urls = ["https://github.com/digital-asset/daml/releases/download/v2.7.1/daml-sdk-2.7.1-windows.tar.gz"],
-) if is_windows else None
-
-http_file(
-    name = "daml-sdk-2.8.0-snapshot.20231107.12319.0.v03a51e65-tarball",
-    sha256 = "cf7b706076fa874b2913547bd6b064890155e656523ad94417300ead72ee19f4",
-    urls = ["https://github.com/digital-asset/daml/releases/download/v2.8.0-snapshot.20231109.2/daml-sdk-2.8.0-snapshot.20231107.12319.0.v03a51e65-linux.tar.gz"],
-) if is_linux_intel else None
-
-http_file(
-    name = "daml-sdk-2.8.0-snapshot.20231107.12319.0.v03a51e65-tarball",
-    sha256 = "d2260542d3d63178dc879f96dfd72fcbe703366ce02b3520c28b29ac3817b70f",
-    urls = ["https://github.com/digital-asset/daml/releases/download/v2.8.0-snapshot.20231109.2/daml-sdk-2.8.0-snapshot.20231107.12319.0.v03a51e65-windows.tar.gz"],
+    name = "daml-sdk-3.3.0-snapshot.20250926.13852.1.v4f3223e3-tarball",
+    sha256 = "64c173a473b0d3f02ad48f3666f2869c9bf7ad9f85ae7e3642d3df6b6e30a145",
+    urls = ["https://github.com/digital-asset/daml/releases/download/v3.3.0-snapshot.20250930.0/daml-sdk-3.3.0-snapshot.20250926.13852.1.v4f3223e3-windows-x86_64.tar.gz"],
 ) if is_windows else None
diff --git a/sdk/compatibility/bazel-haskell-deps.bzl b/sdk/compatibility/bazel-haskell-deps.bzl
@@ -63,6 +63,7 @@ def daml_haskell_deps():
             "tasty",
             "tasty-hunit",
             "text",
+            "time",
             "typed-process",
             "unix-compat",
             "unordered-containers",
diff --git a/sdk/compatibility/bazel_tools/daml_ledger/BUILD.bazel b/sdk/compatibility/bazel_tools/daml_ledger/BUILD.bazel
@@ -43,6 +43,7 @@ da_haskell_binary(
         "tasty",
         "tasty-hunit",
         "text",
+        "time",
         "uuid",
     ],
     visibility = ["//visibility:public"],
diff --git a/sdk/compatibility/bazel_tools/daml_ledger/Main.hs b/sdk/compatibility/bazel_tools/daml_ledger/Main.hs
@@ -15,6 +15,7 @@ import Data.Proxy (Proxy (..))
 import Data.SemVer (Version)
 import qualified Data.SemVer as SemVer
 import Data.Tagged (Tagged (..))
+import Data.Time.Clock.POSIX (getPOSIXTime)
 import Sandbox (maxRetries, nullDevice, readCantonPortFile)
 import System.Directory.Extra (withCurrentDirectory)
 import System.Environment (lookupEnv)
@@ -60,7 +61,7 @@ withSandbox getTools mbSecret f =
       in withResource createSandbox destroySandbox (f . fmap fst)
   where
     tweakArg portFile = replace "__PORTFILE__" portFile
-    portArgNames = ["--port=", "--admin-api-port=", "--domain-public-port=", "--domain-admin-port="]
+    portArgNames = ["--port=", "--admin-api-port=", "--sequencer-public-port=", "--sequencer-admin-port=", "--mediator-admin-port=", "--json-api-port="]
 
 newtype DamlOption = DamlOption FilePath
 instance IsOption DamlOption where
@@ -154,8 +155,9 @@ authenticatedUploadTest sdkVersion getTools = do
           withTempDir $ \deployDir -> do
             withCurrentDirectory deployDir $ do
               let tokenFile = deployDir </> "secretToken.jwt"
+              expiration <- JWT.numericDate . (+180) <$> getPOSIXTime
               -- The trailing newline is not required but we want to test that it is supported.
-              writeFileUTF8 tokenFile ("Bearer " <> makeSignedJwt sharedSecret <> "\n")
+              writeFileUTF8 tokenFile ("Bearer " <> makeSignedAdminJwt sharedSecret expiration <> "\n")
               callProcessSilent daml
                 [ "ledger", "list-parties"
                 , "--access-token-file", tokenFile
@@ -168,8 +170,9 @@ authenticatedUploadTest sdkVersion getTools = do
           withTempDir $ \deployDir -> do
             withCurrentDirectory deployDir $ do
               let tokenFile = deployDir </> "secretToken.jwt"
+              expiration <- JWT.numericDate . (+180) <$> getPOSIXTime
               -- The trailing newline is not required but we want to test that it is supported.
-              writeFileUTF8 tokenFile (makeSignedJwt sharedSecret <> "\n")
+              writeFileUTF8 tokenFile (makeSignedAdminJwt sharedSecret expiration <> "\n")
               callProcessSilent daml
                 [ "ledger", "list-parties"
                 , "--access-token-file", tokenFile
@@ -180,13 +183,24 @@ authenticatedUploadTest sdkVersion getTools = do
   where
     sharedSecret = "TheSharedSecret"
 
-makeSignedJwt :: String -> String
-makeSignedJwt sharedSecret = do
-  let urc = JWT.ClaimsMap $ Map.fromList [ ("admin", Aeson.Bool True)]
-  let cs = mempty { JWT.unregisteredClaims = urc }
-  let key = JWT.hmacSecret $ T.pack sharedSecret
-  let text = JWT.encodeSigned key mempty cs
-  T.unpack text
+makeSignedJwt :: String -> String -> Maybe JWT.NumericDate -> String
+makeSignedJwt sharedSecret user expiration = do
+    let urc =
+            JWT.ClaimsMap $
+            Map.fromList
+                [ ("scope", Aeson.String "daml_ledger_api")
+                ]
+    let cs = mempty {
+        JWT.sub = JWT.stringOrURI $ T.pack user
+        , JWT.unregisteredClaims = urc
+        , JWT.exp = expiration
+    }
+    let key = JWT.hmacSecret $ T.pack sharedSecret
+    let text = JWT.encodeSigned key mempty cs
+    T.unpack text
+
+makeSignedAdminJwt :: String -> Maybe JWT.NumericDate -> String
+makeSignedAdminJwt sharedSecret expiration = makeSignedJwt sharedSecret "participant_admin" expiration
 
 unauthenticatedTests :: SdkVersion -> IO Tools -> TestTree
 unauthenticatedTests sdkVersion getTools = do
diff --git a/sdk/compatibility/bazel_tools/daml_ledger/Sandbox.hs b/sdk/compatibility/bazel_tools/daml_ledger/Sandbox.hs
@@ -178,9 +178,9 @@ readPortFileWith parseFn ph n file = do
 
 decodeCantonPort :: String -> String -> Maybe Int
 decodeCantonPort participantName json = do
-    participants :: Map.Map String (Map.Map String Int) <- Aeson.decode (BSL8.pack json)
+    participants :: Map.Map String (Map.Map String (Maybe Int)) <- Aeson.decode (BSL8.pack json)
     ports <- Map.lookup participantName participants
-    Map.lookup "ledgerApi" ports
+    join $ Map.lookup "ledgerApi" ports
 
 decodeCantonSandboxPort :: String -> Maybe Int
 decodeCantonSandboxPort = decodeCantonPort "sandbox"
diff --git a/sdk/compatibility/bazel_tools/daml_script/daml_script.bzl b/sdk/compatibility/bazel_tools/daml_script/daml_script.bzl
@@ -64,7 +64,7 @@ def daml_script_test(
     )
 
     server = daml_runner
-    server_args = ["sandbox", "--canton-port-file", "$$(pwd)/_port_file"]
+    server_args = ["sandbox", "--canton-port-file", "_port_file"]
     server_files = ["$(rootpath {})".format(compiled_dar)]
     server_files_prefix = "--dar="
 
@@ -91,7 +91,7 @@ trap 'status=$$?; kill -TERM $$PID; wait $$PID; exit $$status' INT TERM
 
 if [ {wait_for_port_file} -eq 1 ]; then
     timeout=60
-    while [ ! -e $$(pwd)/_port_file ]; do
+    while [ ! -e _port_file ]; do
         if [ "$$timeout" = 0 ]; then
             echo "Timed out waiting for Canton startup" >&2
             exit 1
diff --git a/sdk/compatibility/bazel_tools/daml_sdk.bzl b/sdk/compatibility/bazel_tools/daml_sdk.bzl
@@ -1,10 +1,11 @@
 # Copyright (c) 2025 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-load("@os_info//:os_info.bzl", "is_windows", "os_name")
+load("@os_info//:os_info.bzl", "is_windows", "os_arch", "os_name")
 load("@daml//bazel_tools/dev_env_tool:dev_env_tool.bzl", "dadew_tool_home", "dadew_where")
 load("@io_bazel_rules_scala//scala:scala_cross_version.bzl", "default_maven_server_urls")
 load("//bazel_tools:versions.bzl", "versions")
+load("//:versions.bzl", "internal_sdk_versions")
 
 runfiles_library = """
 # Copy-pasted from the Bazel Bash runfiles library v2.
@@ -27,6 +28,8 @@ def _daml_sdk_impl(ctx):
     # location and set the symlink ourselves.
     out_dir = ctx.path("sdk").get_child("sdk").get_child(ctx.attr.version)
 
+    internal_sdk_version = internal_sdk_versions.get(ctx.attr.version, default = ctx.attr.version)
+
     if ctx.attr.sdk_tarball:
         ctx.extract(
             ctx.attr.sdk_tarball,
@@ -47,9 +50,9 @@ def _daml_sdk_impl(ctx):
         ctx.download_and_extract(
             output = out_dir,
             url =
-                "https://github.com/digital-asset/daml/releases/download/v{}/daml-sdk-{}-{}.tar.gz".format(ctx.attr.version, ctx.attr.version, ctx.attr.os_name),
+                "https://github.com/digital-asset/daml/releases/download/v{}/daml-sdk-{}-{}-{}.tar.gz".format(ctx.attr.version, internal_sdk_version, ctx.attr.os_name, ctx.attr.arch),
             sha256 = ctx.attr.sdk_sha256[ctx.attr.os_name],
-            stripPrefix = "sdk-{}".format(ctx.attr.version),
+            stripPrefix = "sdk-{}".format(internal_sdk_version),
         )
         sdk_checksum = ctx.attr.sdk_sha256[ctx.attr.os_name]
     else:
@@ -70,7 +73,7 @@ def _daml_sdk_impl(ctx):
         else:
             ctx.download(
                 output = "daml-{}.tgz".format(lib),
-                url = "https://registry.npmjs.org/@daml/{}/-/{}-{}.tgz".format(lib, lib, ctx.attr.version),
+                url = "https://registry.npmjs.org/@daml/{}/-/{}-{}.tgz".format(lib, lib, internal_sdk_version),
                 sha256 = getattr(ctx.attr, "daml_{}_sha256".format(lib)),
             )
 
@@ -133,6 +136,7 @@ _daml_sdk = repository_rule(
     attrs = {
         "version": attr.string(mandatory = True),
         "os_name": attr.string(mandatory = False, default = os_name),
+        "arch": attr.string(mandatory = False, default = os_arch),
         "sdk_sha256": attr.string_dict(mandatory = False),
         "sdk_tarball": attr.label(allow_single_file = True, mandatory = False),
         "daml_types_tarball": attr.label(allow_single_file = True, mandatory = False),
diff --git a/sdk/compatibility/bazel_tools/testing.bzl b/sdk/compatibility/bazel_tools/testing.bzl
@@ -70,8 +70,11 @@ def daml_ledger_test(
 #
 # The proper fix is to use the appropriate version of Daml-LF for every SDK/platform pair.
 
-def daml_lf_compatible(_sdk_version, platform_version):
-    return in_range(platform_version, {"start": "3.0.0-snapshot"})
+def daml_lf_compatible(sdk_version, platform_version):
+    return (
+        # any post 3.4 platform supports any pre 3x SDK
+        in_range(platform_version, {"start": "3.4.0-snapshot"}) and in_range(sdk_version, {"start": "3.0.0-snapshot"})
+    )
 
 def sdk_platform_test(sdk_version, platform_version):
     # SDK components
diff --git a/sdk/compatibility/deps.bzl b/sdk/compatibility/deps.bzl
@@ -52,7 +52,7 @@ load(
     "zlib_sha256",
     "zlib_version",
 )
-load("//:versions.bzl", "latest_stable_version", "version_sha256s")
+load("//:versions.bzl", "version_sha256s")
 load("@os_info//:os_info.bzl", "os_name")
 
 def daml_deps():
diff --git a/sdk/compatibility/maven_install.json b/sdk/compatibility/maven_install.json
diff --git a/sdk/compatibility/test-all-installs.sh b/sdk/compatibility/test-all-installs.sh
diff --git a/sdk/compatibility/update-versions.sh b/sdk/compatibility/update-versions.sh
diff --git a/sdk/compatibility/versions.bzl b/sdk/compatibility/versions.bzl
diff --git a/sdk/compatibility/versions/BUILD b/sdk/compatibility/versions/BUILD
diff --git a/sdk/compatibility/versions/UpdateVersions.hs b/sdk/compatibility/versions/UpdateVersions.hs
