[ci] Merge branch 'main' of github.com:hyperledger-labs/splice into julien/2692-add-scan-connection-metrics

Author: julientinguely-da

.github/actions/scripts/io-utils.sh

@@ -11,7 +11,7 @@
 
 # Copy stdin to stdout, while removing all lines that start with '#' or consist of blanks
 remove_comment_and_blank_lines() {
-  while IFS= read -r # Read a file line by line; IFS= ensures that no separators other than newline are used
+  while IFS= read -r || [[ -n $REPLY ]] # Read a file line by line; IFS= ensures that no separators other than newline are used
   do
     if [[ -n "${REPLY// }" ]] && [[ "$REPLY" != "#"* ]] # Filter out comment lines and blank lines
     then

.github/actions/tests/split_tests/dist/index.js

@@ -22,12 +22,18 @@ function getTestSuiteTimesFromXml(testReportsDir) {
     try {
         fs_1.default.readdirSync(testReportsDir).forEach(file => {
             if (file.endsWith('.xml')) {
-                const path = `${testReportsDir}/${file}`;
-                const XMLdata = fs_1.default.readFileSync(path);
-                const parsed = parser.parse(XMLdata);
-                const testSuiteName = parsed.testsuite['@_name'];
-                const testSuiteTime = parseFloat(parsed.testsuite['@_time']);
-                testTimes[testSuiteName] = testSuiteTime;
+                try {
+                    console.log(`Parsing xml report ${file}`);
+                    const path = `${testReportsDir}/${file}`;
+                    const XMLdata = fs_1.default.readFileSync(path);
+                    const parsed = parser.parse(XMLdata);
+                    const testSuiteName = parsed.testsuite['@_name'];
+                    const testSuiteTime = parseFloat(parsed.testsuite['@_time']);
+                    testTimes[testSuiteName] = testSuiteTime;
+                }
+                catch (e) {
+                    console.error(`Failed to parse xml report ${file}`);
+                }
             }
         });
     }
@@ -69,7 +75,7 @@ function computeBuckets(testReportsDir, testNamesFile, splitTotal) {
     // Build a sorted list of test names, sorted by their estimated test time.
     // We first sort alphabetically, so that tests with the same estimated time
     // are sorted in a deterministic way.
-    const sortedTestNames = testNames.sort().sort((a, b) => estimatedTestTimes[a] - estimatedTestTimes[b]);
+    const sortedTestNames = testNames.sort().sort((a, b) => estimatedTestTimes[a] - estimatedTestTimes[b]).reverse();
     const buckets = splitTests(sortedTestNames, estimatedTestTimes, splitTotal);
     buckets.forEach((bucket, i) => {
         console.log(`bucket ${i}: ${bucket.length} tests, total time: ${bucket.reduce((acc, testName) => acc + estimatedTestTimes[testName], 0)}`);

.github/actions/tests/split_tests/src/split_tests.ts

@@ -1,86 +1,91 @@
-import { getInput, setFailed, setOutput } from '@actions/core';
+import {getInput, setFailed, setOutput} from '@actions/core';
 import xmlParser from 'fast-xml-parser';
 import fs from 'fs';
 
 type TestTimes = { [name: string]: number };
 
 function getTestSuiteTimesFromXml(testReportsDir: string): TestTimes {
 
-  const options = {
-    ignoreAttributes: false
-  };
-  const parser = new xmlParser.XMLParser(options);
-  const testTimes: TestTimes = {};
-  try {
-    fs.readdirSync(testReportsDir).forEach(file => {
-      if (file.endsWith('.xml')) {
-        const path = `${testReportsDir}/${file}`;
-        const XMLdata = fs.readFileSync(path);
-        const parsed = parser.parse(XMLdata);
-        const testSuiteName = parsed.testsuite['@_name'];
-        const testSuiteTime = parseFloat(parsed.testsuite['@_time']);
-        testTimes[testSuiteName] = testSuiteTime;
-      }
-    });
-  } catch (err) {
-    console.error(`Warning: could not read test reports from ${testReportsDir}: ${err}`);
-  }
-
-  return testTimes;
+    const options = {
+        ignoreAttributes: false
+    };
+    const parser = new xmlParser.XMLParser(options);
+    const testTimes: TestTimes = {};
+    try {
+        fs.readdirSync(testReportsDir).forEach(file => {
+            if (file.endsWith('.xml')) {
+                try {
+                    console.log(`Parsing xml report ${file}`)
+                    const path = `${testReportsDir}/${file}`;
+                    const XMLdata = fs.readFileSync(path);
+                    const parsed = parser.parse(XMLdata);
+                    const testSuiteName = parsed.testsuite['@_name'];
+                    const testSuiteTime = parseFloat(parsed.testsuite['@_time']);
+                    testTimes[testSuiteName] = testSuiteTime;
+                } catch (e) {
+                    console.error(`Failed to parse xml report ${file}`)
+                }
+            }
+        });
+    } catch (err) {
+        console.error(`Warning: could not read test reports from ${testReportsDir}: ${err}`);
+    }
+
+    return testTimes;
 }
 
 function estimateTestTimes(testTimes: TestTimes, testNames: string[]): TestTimes {
-  let maxTestTime = Math.max(...Object.values(testTimes));
-  // If maxTestTime is zero, i.e. no runtimes are known, we assign everything an
-  // arbitrary non-zero value of 1.0
-  maxTestTime = Math.max(maxTestTime, 1.0);
-
-  const estimatedTestTimes: TestTimes = {};
-  testNames.forEach(testName => {
-    estimatedTestTimes[testName] = testTimes[testName] || maxTestTime;
-    // Scalatest actually reported occasionally test times with negative numbers,
-    // so we set it to zero in that case.
-    estimatedTestTimes[testName] = Math.max(estimatedTestTimes[testName], 0.0);
-  });
-
-  return estimatedTestTimes
+    let maxTestTime = Math.max(...Object.values(testTimes));
+    // If maxTestTime is zero, i.e. no runtimes are known, we assign everything an
+    // arbitrary non-zero value of 1.0
+    maxTestTime = Math.max(maxTestTime, 1.0);
+
+    const estimatedTestTimes: TestTimes = {};
+    testNames.forEach(testName => {
+        estimatedTestTimes[testName] = testTimes[testName] || maxTestTime;
+        // Scalatest actually reported occasionally test times with negative numbers,
+        // so we set it to zero in that case.
+        estimatedTestTimes[testName] = Math.max(estimatedTestTimes[testName], 0.0);
+    });
+
+    return estimatedTestTimes
 }
 
 function splitTests(sortedTestNames: string[], estimatedTestTimes: TestTimes, splitTotal: number): string[][] {
-  const bucketTimes = Array(splitTotal).fill(0);
-  const buckets = Array.from(Array(splitTotal), () => new Array())
+    const bucketTimes = Array(splitTotal).fill(0);
+    const buckets = Array.from(Array(splitTotal), () => new Array())
 
 
-  sortedTestNames.forEach(testName => {
-    const minBucketIndex = bucketTimes.indexOf(Math.min(...bucketTimes));
-    bucketTimes[minBucketIndex] += estimatedTestTimes[testName];
-    buckets[minBucketIndex].push(testName);
+    sortedTestNames.forEach(testName => {
+        const minBucketIndex = bucketTimes.indexOf(Math.min(...bucketTimes));
+        bucketTimes[minBucketIndex] += estimatedTestTimes[testName];
+        buckets[minBucketIndex].push(testName);
 
-    console.log(`added ${testName} to bucket ${minBucketIndex}, total time: ${bucketTimes[minBucketIndex]}`);
-    console.log(`bucket ${minBucketIndex} has ${buckets[minBucketIndex].length} tests`);
-  });
+        console.log(`added ${testName} to bucket ${minBucketIndex}, total time: ${bucketTimes[minBucketIndex]}`);
+        console.log(`bucket ${minBucketIndex} has ${buckets[minBucketIndex].length} tests`);
+    });
 
-  return buckets;
+    return buckets;
 }
 
 function computeBuckets(testReportsDir: string, testNamesFile: string, splitTotal: number) {
-  const testTimes = getTestSuiteTimesFromXml(testReportsDir);
+    const testTimes = getTestSuiteTimesFromXml(testReportsDir);
 
-  const testNames = fs.readFileSync(testNamesFile).toString().split('\n').filter(name => name.length > 0);
+    const testNames = fs.readFileSync(testNamesFile).toString().split('\n').filter(name => name.length > 0);
 
-  const estimatedTestTimes = estimateTestTimes(testTimes, testNames);
+    const estimatedTestTimes = estimateTestTimes(testTimes, testNames);
 
-  // Build a sorted list of test names, sorted by their estimated test time.
-  // We first sort alphabetically, so that tests with the same estimated time
-  // are sorted in a deterministic way.
-  const sortedTestNames = testNames.sort().sort((a, b) => estimatedTestTimes[a] - estimatedTestTimes[b]);
+    // Build a sorted list of test names, sorted by their estimated test time.
+    // We first sort alphabetically, so that tests with the same estimated time
+    // are sorted in a deterministic way.
+    const sortedTestNames = testNames.sort().sort((a, b) => estimatedTestTimes[a] - estimatedTestTimes[b]).reverse();
 
-  const buckets = splitTests(sortedTestNames, estimatedTestTimes, splitTotal);
+    const buckets = splitTests(sortedTestNames, estimatedTestTimes, splitTotal);
 
-  buckets.forEach((bucket, i) => {
-    console.log(`bucket ${i}: ${bucket.length} tests, total time: ${bucket.reduce((acc, testName) => acc + estimatedTestTimes[testName], 0)}`);
-  });
-  return buckets;
+    buckets.forEach((bucket, i) => {
+        console.log(`bucket ${i}: ${bucket.length} tests, total time: ${bucket.reduce((acc, testName) => acc + estimatedTestTimes[testName], 0)}`);
+    });
+    return buckets;
 }
 
 const buckets = computeBuckets(getInput('test_reports_dir'), getInput('test_names_file'), parseInt(getInput('split_total')));

.github/workflows/build.scala_test.yml

@@ -62,7 +62,10 @@ on:
 jobs:
 
   split_tests:
-    runs-on: ubuntu-24.04
+    # required to have access to the /cache directory to read test-reports
+    runs-on: self-hosted-k8s-small
+    container:
+      image: us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker/splice-test-ci:0.3.12
     name: Split the tests into parallel runs
     outputs:
       indices: ${{ steps.prep.outputs.indices }}

.github/workflows/build.scala_test_for_compose.yml

@@ -33,7 +33,10 @@ on:
 jobs:
 
   split_tests:
-    runs-on: ubuntu-24.04
+    # required to have access to the /cache directory to read test-reports
+    runs-on: self-hosted-k8s-small
+    container:
+      image: us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker/splice-test-ci:0.3.12
     name: Split the tests into parallel runs
     outputs:
       indices: ${{ steps.prep.outputs.indices }}

.github/workflows/build.scala_test_with_cometbft.yml

@@ -25,7 +25,10 @@ on:
 jobs:
 
   split_tests:
-    runs-on: ubuntu-24.04
+    # required to have access to the /cache directory to read test-reports
+    runs-on: self-hosted-k8s-small
+    container:
+      image: us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker/splice-test-ci:0.3.12
     name: Split the tests into parallel runs
     outputs:
       indices: ${{ steps.prep.outputs.indices }}

.github/workflows/bump_gha_runner_version.yml

@@ -0,0 +1,46 @@
+name: Bump GHA runner version
+on:
+  schedule:
+    - cron: '30 6 1,15 * *'  # Monthly on the 1st and 15th at 06:30
+  workflow_dispatch:
+
+jobs:
+  bump_gha_runner_version:
+    name: Bump GHA runner version
+    runs-on: self-hosted-k8s-small
+    container:
+      image: us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker/splice-test-ci:0.3.12
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Setup
+        shell: bash
+        run: ./.github/actions/scripts/common_setup.sh
+
+      - name: Set up Nix (Self hosted)
+        uses: ./.github/actions/nix/setup_nix
+        with:
+          cache_version: 7
+          artifactory_user: dummy
+          artifactory_password: dummy
+      
+      - name: Check for the latest version and create a PR to splice
+        uses: ./.github/actions/nix/run_bash_command_in_nix
+        with:
+          cmd: |
+            git config user.email "splice-maintainers@digitalasset.com"
+            git config user.name "DA Automation"
+            ./scripts/bump-gha-runner-version.sh
+          additional_nix_args: "--keep GH_TOKEN"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Report Failures on Slack & Github
+        if: failure() && (github.event_name == 'push' || github.event_name == 'schedule')
+        uses: ./.github/actions/tests/failure_notifications
+        with:
+          workload_identity_provider: '${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}'
+          service_account: '${{ secrets.FAILURE_NOTIFICATIONS_INVOKER_SA }}'
+          notifications_url: '${{ secrets.FAILURE_NOTIFICATIONS_INVOKER_URL }}'
+          slack_channel: '${{ secrets.FAILURE_NOTIFICATIONS_SLACK_CHANNEL }}'

.gitignore

@@ -1,6 +1,7 @@
 .daml
 *.dar
 !daml/dars/*
+*.iml
 *.log
 *.clog
 .DS_Store

.pre-commit-config.yaml

@@ -50,12 +50,14 @@ repos:
         language: system
         entry: "scripts/fix-ts.py"
         types_or: [javascript, jsx, ts, tsx]
-        exclude: "/[.]prettierrc[.]cjs$"
-      - id: pulumi_config
-        name: check pulumi configs
+        # TODO(#3334): The current implementation of fix-ts.py makes it impossible to format files
+        #              in the top level pulumi package.
+        exclude: "(/[.]prettierrc[.]cjs$|/pulumi/[^/]+.ts$)"
+      - id: pulumi_tests
+        name: run pulumi tests and check expected files
         language: system
         entry: "make cluster/pulumi/test -j8"
-        files: '^cluster/pulumi/.*[.]ts$'
+        files: '^(cluster/pulumi/.*[.]ts|cluster/.*[.](yaml|yml))$'
         pass_filenames: false
       - id: gha_lint
         name: static check github actions

.scalafmt.conf

@@ -12,10 +12,9 @@ newlines.topLevelStatementBlankLines = [
   { blanks { after = 1 }, regex = "^Import" }
 ]
 
-rewrite.scala3.convertToNewSyntax= true
+rewrite.scala3.convertToNewSyntax = true
 
 runner.dialect = "scala213source3"
-runner.dialectOverride.allowQuestionMarkAsTypeWildcard = false
 runner.dialectOverride.allowUnderscoreAsTypePlaceholder = false
 
 fileOverride {