Merge branch 'main' into tcb_checking

microsoft · Feb 28, 2025 · d11b271 · d11b271
2 parents 8d6f848 + da4d63e
commit d11b271
Show file tree

Hide file tree

Showing 39 changed files with 789 additions and 215 deletions.
diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml
@@ -1,24 +1,9 @@
 jobs:
-  - job: generate_ssh_key
-    displayName: "Generate SSH Key"
-    variables:
-      Codeql.SkipTaskAutoInjection: true
-      skipComponentGovernanceDetection: true
-    pool:
-      vmImage: ubuntu-20.04
-    steps:
-      - checkout: none
-
-      - template: generate_ssh_key.yml
-
   - job: deploy_primary_aci
     displayName: "Deploy ACI"
-    dependsOn:
-      - generate_ssh_key
     variables:
       Codeql.SkipTaskAutoInjection: true
       skipComponentGovernanceDetection: true
-      sshKey: $[ dependencies.generate_ssh_key.outputs['generate_ssh_key.sshKey'] ]
     pool:
       name: ado-virtual-ccf-sub # To build CCF quickly
       demands:
@@ -31,9 +16,7 @@ jobs:
         fetchDepth: 0
         fetchTags: true
 
-      - template: install_ssh_key.yml
-        parameters:
-          ssh_key: $(sshKey)
+      - template: generate_ssh_key.yml
 
       - template: azure_cli.yml
         parameters:
@@ -70,7 +53,6 @@ jobs:
             --memory-gb 64 \
             --core-count 16 \
             --aci-setup-timeout 300 \
-            --aci-private-key-b64 $(sshKey) \
             --out ~/aci_ips
           # Create a ~/ipAddresses files which is a list of `<container group name> <IP address>` separated by newlines.
           source ./scripts/azure_deployment/escape_data.sh # Include escape_data to handle newlines.

diff --git a/.azure-pipelines-templates/generate_ssh_key.yml b/.azure-pipelines-templates/generate_ssh_key.yml
@@ -1,7 +1,6 @@
 steps:
   - script: |
       set -ex
+      mkdir -p ~/.ssh
       ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N ""
-      echo "##vso[task.setvariable variable=sshKey;isOutput=true;issecret=true]`base64 -w 0 ~/.ssh/id_rsa`"
-    name: generate_ssh_key
     displayName: "Generate SSH Key"
diff --git a/.azure-pipelines-templates/install_ssh_key.yml b/.azure-pipelines-templates/install_ssh_key.yml
diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
@@ -311,7 +311,7 @@ jobs:
           git config --global --add safe.directory /__w/CCF/CCF
           mkdir build
           cd build
-          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TEST=ON ..
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON ..
           ninja
         shell: bash
 
@@ -326,3 +326,109 @@ jobs:
           # All e2e tests, which are now supported on Mariner.
           ./tests.sh --timeout 1600 --output-on-failure -LE "benchmark"
         shell: bash
+
+  long-asan-azure-linux:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
+    name: "Azure Linux ASAN"
+    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    container:
+      image: mcr.microsoft.com/azurelinux/base/core:3.0
+      options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE
+
+    steps:
+      - name: "Checkout dependencies"
+        shell: bash
+        run: |
+          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
+          tdnf -y update
+          tdnf -y install ca-certificates git
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "Install dependencies"
+        shell: bash
+        run: |
+          set -ex
+          ./scripts/setup-ci.sh
+
+      - name: "Build Debug (Long Test)"
+        run: |
+          set -ex
+          git config --global --add safe.directory /__w/CCF/CCF
+          mkdir build
+          cd build
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON -DSAN=ON ..
+          ninja
+
+      - name: "Test"
+        run: |
+          set +x
+          cd build
+          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark"
+
+      - name: "Upload logs"
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-asan-al
+          path: |
+            build/workspace/*/*.config.json
+            build/workspace/*/out
+            build/workspace/*/err
+            build/workspace/*.ledger/*
+          if-no-files-found: ignore
+
+  long-tsan-azure-linux:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
+    name: "Azure Linux TSAN"
+    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    container:
+      image: mcr.microsoft.com/azurelinux/base/core:3.0
+      options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE
+
+    steps:
+      - name: "Checkout dependencies"
+        shell: bash
+        run: |
+          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
+          tdnf -y update
+          tdnf -y install ca-certificates git
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "Install dependencies"
+        shell: bash
+        run: |
+          set -ex
+          ./scripts/setup-ci.sh
+
+      - name: "Build Debug (Long Test)"
+        run: |
+          set -ex
+          git config --global --add safe.directory /__w/CCF/CCF
+          mkdir build
+          cd build
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON -DTSAN=ON -DWORKER_THREADS=2 ..
+          ninja
+
+      - name: "Test"
+        run: |
+          set +x
+          cd build
+          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark"
+
+      - name: "Upload logs"
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-tsan-al
+          path: |
+            build/workspace/*/*.config.json
+            build/workspace/*/out
+            build/workspace/*/err
+            build/workspace/*.ledger/*
+          if-no-files-found: ignore
diff --git a/.snpcc_canary b/.snpcc_canary
@@ -1,7 +1,7 @@
   ___         ___    ___         \_/
- (. =)     Y (0 0)  (x X)    Y  (___) 
+ (. =)     Y (0 0)  (x X)    Y  (v-v) 
    O     \     o      |    /      |
 /-xXx--//-----x=x--/-xXx--/---x-/--->>>--/
 ...
 /\/\d(-_-)b/\/\
-----vmpl----
+----vmpl----
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Fixed
 
 - `ccf.ledger`/`read_ledger.py` previously enforced too strict a condition on node membership when validating ledger files (#6849).
+- Restore low CPU usage on idle nodes, which had increased in dev20 (#6816).
 
 ## [6.0.0-dev20]
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -716,6 +716,7 @@ if(BUILD_TESTS)
       ${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/unit_strings.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/dl_list.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/nonstd.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/ds/test/work_beacon.cpp
     )
     target_link_libraries(ds_test PRIVATE ${CMAKE_THREAD_LIBS_INIT})
 

diff --git a/doc/audit/builtin_maps.rst b/doc/audit/builtin_maps.rst
@@ -376,9 +376,6 @@ JWT issuers.
    :project: CCF
    :members:
 
-.. doxygenenum:: ccf::JwtIssuerKeyFilter
-   :project: CCF
-
 ``jwt.public_signing_keys``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

diff --git a/doc/build_apps/auth/jwt.rst b/doc/build_apps/auth/jwt.rst
@@ -21,7 +21,6 @@ Before adding public token signing keys to a running CCF network, the IdP has to
           "name": "set_jwt_issuer",
           "args": {
             "issuer": "my_issuer",
-            "key_filter": "all",
             "auto_refresh": false
           }
         }
@@ -95,7 +94,6 @@ Now the issuer can be created with auto-refresh enabled:
           "name": "set_jwt_issuer",
           "args": {
             "issuer": "https://login.microsoftonline.com/common/v2.0",
-            "key_filter": "all",
             "ca_cert_bundle_name": "jwt_ms",
             "auto_refresh": true
           }

diff --git a/doc/host_config_schema/cchost_config.json b/doc/host_config_schema/cchost_config.json
@@ -13,15 +13,15 @@
         },
         "platform": {
           "type": "string",
-          "enum": ["SGX", "SNP", "Virtual"],
-          "default": "SGX",
+          "enum": ["SNP", "Virtual"],
+          "default": "SNP",
           "description": "Trusted Execution Environment platform"
         },
         "type": {
           "type": "string",
           "enum": ["Release", "Debug", "Virtual"],
           "default": "Release",
-          "description": "Type of enclave application (only if platform is SGX). \"Virtual\" is deprecated (use ``platform`` instead)"
+          "description": "Type of enclave application (only if platform is SGX, now deprecated). \"Virtual\" is deprecated (use ``platform`` instead)"
         }
       },
       "description": "This section includes configuration for the enclave application launched by this node",
@@ -522,7 +522,7 @@
           "description": "List of servers used to retrieve attestation report endorsement certificates (SEV-SNP only). The first server in the list is always used and other servers are only specified as fallback. If set, attestation endorsements from ``--snp-security-context-dir-var`` are ignored, but uvm endorsements from that directory are still used."
         }
       },
-      "description": "This section includes configuration for the attestation for AMD SEV-SNP platform (ignored for SGX)",
+      "description": "This section includes configuration for the attestation for AMD SEV-SNP platform.",
       "additionalProperties": false
     },
     "service_data_json_file": {

diff --git a/doc/schemas/gov/2023-06-01-preview/examples/ServiceState_GetJwkInfo.json b/doc/schemas/gov/2023-06-01-preview/examples/ServiceState_GetJwkInfo.json
@@ -9,7 +9,6 @@
       "body": {
         "issuers": {
           "idprovider.myservice.example.com": {
-            "keyFilter": "All",
             "autoRefresh": true,
             "caCertBundleName": "MyIdProviderCa"
           }

diff --git a/doc/schemas/gov/2024-07-01/examples/ServiceState_GetJwkInfo.json b/doc/schemas/gov/2024-07-01/examples/ServiceState_GetJwkInfo.json
@@ -9,7 +9,6 @@
       "body": {
         "issuers": {
           "idprovider.myservice.example.com": {
-            "keyFilter": "All",
             "autoRefresh": true,
             "caCertBundleName": "MyIdProviderCa"
           }

diff --git a/doc/schemas/gov/2024-07-01/gov.json b/doc/schemas/gov/2024-07-01/gov.json
@@ -1914,10 +1914,6 @@
       "type": "object",
       "description": "Description of a JWT issuer or identity provider that the current service will trust tokens from.",
       "properties": {
-        "keyFilter": {
-          "$ref": "#/definitions/ServiceState.JwtIssuerKeyFilter",
-          "description": "Adds restrictions on whether keys should be accepted from this issuer."
-        },
         "keyPolicy": {
           "type": "object",
           "description": "Collection of claims which must be present in SGX attestation to permit updates from this issuer.",
@@ -1935,34 +1931,9 @@
         }
       },
       "required": [
-        "keyFilter",
         "autoRefresh"
       ]
     },
-    "ServiceState.JwtIssuerKeyFilter": {
-      "type": "string",
-      "description": "Possible restrictions on what keys will be accepted from a JWT issuer.",
-      "enum": [
-        "All",
-        "Sgx"
-      ],
-      "x-ms-enum": {
-        "name": "JwtIssuerKeyFilter",
-        "modelAsString": true,
-        "values": [
-          {
-            "name": "All",
-            "value": "All",
-            "description": "Accepts any JWT issuer."
-          },
-          {
-            "name": "Sgx",
-            "value": "Sgx",
-            "description": "Only accepts JWTs issued by a token provider running in SGX, which provides a suitable attestation and additional claims."
-          }
-        ]
-      }
-    },
     "ServiceState.Member": {
       "type": "object",
       "description": "Information on individual members within a consortium.",

diff --git a/doc/schemas/gov_openapi.json b/doc/schemas/gov_openapi.json
@@ -271,22 +271,13 @@
         },
         "type": "object"
       },
-      "JwtIssuerKeyFilter": {
-        "enum": [
-          "all"
-        ],
-        "type": "string"
-      },
       "JwtIssuerMetadata": {
         "properties": {
           "auto_refresh": {
             "$ref": "#/components/schemas/boolean"
           },
           "ca_cert_bundle_name": {
             "$ref": "#/components/schemas/string"
-          },
-          "key_filter": {
-            "$ref": "#/components/schemas/JwtIssuerKeyFilter"
           }
         },
         "type": "object"
@@ -1348,7 +1339,7 @@
   "info": {
     "description": "This API is used to submit and query proposals which affect CCF's public governance tables.",
     "title": "CCF Governance API",
-    "version": "4.6.0"
+    "version": "4.6.1"
   },
   "openapi": "3.0.0",
   "paths": {

diff --git a/include/ccf/service/tables/jwt.h b/include/ccf/service/tables/jwt.h
@@ -21,8 +21,6 @@ namespace ccf
 
   struct JwtIssuerMetadata
   {
-    /// JWT issuer key filter, kept for compatibility with existing ledgers
-    JwtIssuerKeyFilter key_filter = JwtIssuerKeyFilter::All;
     /// Optional CA bundle name used for authentication when auto-refreshing
     std::optional<std::string> ca_cert_bundle_name;
     /// Whether to auto-refresh keys from the issuer
@@ -32,7 +30,7 @@ namespace ccf
   DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(JwtIssuerMetadata);
   DECLARE_JSON_REQUIRED_FIELDS(JwtIssuerMetadata);
   DECLARE_JSON_OPTIONAL_FIELDS(
-    JwtIssuerMetadata, key_filter, ca_cert_bundle_name, auto_refresh);
+    JwtIssuerMetadata, ca_cert_bundle_name, auto_refresh);
 
   using JwtIssuer = std::string;
   using JwtKeyId = std::string;

diff --git a/samples/config/join_config.json b/samples/config/join_config.json
@@ -1,7 +1,7 @@
 {
   "enclave": {
     "file": "libjs_generic.enclave.so.signed",
-    "platform": "SGX",
+    "platform": "SNP",
     "type": "Release"
   },
   "network": {

diff --git a/samples/config/minimal_config.json b/samples/config/minimal_config.json
@@ -1,7 +1,7 @@
 {
   "enclave": {
     "file": "libjs_generic.enclave.so.signed",
-    "platform": "SGX",
+    "platform": "SNP",
     "type": "Release"
   },
   "network": {