Merge branch 'main' into jtschelling/124-azure-maintenance-events

Author: jtschelling

.github/copy-pr-bot.yaml

@@ -20,4 +20,5 @@ additional_trustees:
   - nitz2407
   - XRFXLP
   - mchmarny
-  - tanishagoyal2
+  - tanishagoyal2
+  - ksaur

.github/headers/NOTICE

@@ -6,3 +6,7 @@ This product includes software developed by NVIDIA Corporation (https://www.nvid
 The Initial Developer of the distros/kubernetes/nvsentinel/charts/mongodb-store/charts/mongodb, is Broadcom, Inc. (https://www.broadcom.com/).
 Copyright 2024 Broadcom, Inc. All Rights Reserved.
 Source: https://github.com/bitnami/charts/commit/f001424f6aad534d5fe5a9ad09265b24b1fd6fe7
+
+The Initial Developer of the distros/kubernetes/nvsentinel/charts/mongodb-store/charts/psmdb-operator and distros/kubernetes/nvsentinel/charts/mongodb-store/charts/psmdb-db, is Percona LLC (https://www.percona.com/).
+Copyright 2019 Paul Czarkowski. All Rights Reserved.
+Source: https://github.com/percona/percona-helm-charts

.github/workflows/code-scanning.yml

@@ -40,7 +40,7 @@ jobs:
       pull-requests: write   # Required for posting comments
     steps:
     - name: Checkout repository
-      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+      uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3  # v6.0.0
 
     - name: Setup build environment
       uses: ./.github/actions/setup-ci-env
@@ -90,7 +90,7 @@ jobs:
       packages: read
     steps:
     - name: Checkout repository
-      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+      uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3  # v6.0.0
 
     - name: Setup build environment
       uses: ./.github/actions/setup-ci-env

.github/workflows/container-build-test.yml

@@ -67,7 +67,7 @@ jobs:
           - component: file-server-cleanup
             make_command: 'make -C log-collector docker-build-file-server-cleanup'
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3  # v6.0.0
 
       - name: Setup build environment
         uses: ./.github/actions/setup-ci-env
@@ -122,8 +122,10 @@ jobs:
             path: ./cmd/csp-health-monitor
           - module: health-monitors/csp-health-monitor
             path: ./cmd/maintenance-notifier
+          - module: event-exporter
+            path: .
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3  # v6.0.0
 
       - name: Setup build environment
         uses: ./.github/actions/setup-ci-env

.github/workflows/e2e-test.yml

@@ -15,12 +15,13 @@
 name: E2E Tests
 
 # This workflow runs end-to-end tests on both AMD64 and ARM64 architectures in parallel
-# to ensure compatibility across different hardware platforms.
+# with both MongoDB and PostgreSQL datastores to ensure compatibility across different
+# hardware platforms and database backends.
 #
 # Configuration:
 # - Set RUNNER_ARCH_LARGE_AMD64 variable to override default AMD64 runner
 # - Set RUNNER_ARCH_LARGE_ARM64 variable to override default ARM64 runner
-# - Each architecture gets its own isolated cluster and test artifacts
+# - Each architecture + datastore combination gets its own isolated cluster and test artifacts
 
 on:
   push:
@@ -39,22 +40,37 @@ permissions:
 
 jobs:
   e2e-test:
+    # Run E2E tests on both AMD64 and ARM64 architectures with both MongoDB and PostgreSQL
     strategy:
-      fail-fast: false  # Allow both architectures to complete even if one fails
+      fail-fast: false  # Allow all combinations to complete even if some fail
       matrix:
         include:
           - arch: amd64
             runner: ${{ vars.RUNNER_ARCH_LARGE_AMD64 || 'linux-amd64-cpu32' }}
             arch_name: "AMD64"
+            datastore: "mongodb"
+            datastore_name: "MongoDB"
           - arch: arm64
             runner: ${{ vars.RUNNER_ARCH_LARGE_ARM64 || 'linux-arm64-cpu32' }}
             arch_name: "ARM64"
+            datastore: "mongodb"
+            datastore_name: "MongoDB"
+          - arch: amd64
+            runner: ${{ vars.RUNNER_ARCH_LARGE_AMD64 || 'linux-amd64-cpu32' }}
+            arch_name: "AMD64"
+            datastore: "postgresql"
+            datastore_name: "PostgreSQL"
+          - arch: arm64
+            runner: ${{ vars.RUNNER_ARCH_LARGE_ARM64 || 'linux-arm64-cpu32' }}
+            arch_name: "ARM64"
+            datastore: "postgresql"
+            datastore_name: "PostgreSQL"
 
-    name: "E2E Tests (${{ matrix.arch_name }})"
+    name: "E2E Tests (${{ matrix.arch_name }} + ${{ matrix.datastore_name }})"
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 90
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3  # v6.0.0
 
       - name: Workaround for freeing up more disk space
         run: |
@@ -120,24 +136,32 @@ jobs:
         env:
           CI_COMMIT_REF_NAME: ${{ steps.ref-name.outputs.value }}
           CTLPTL_YAML: .ctlptl.yaml
-          # Make cluster names unique per architecture to avoid conflicts in parallel runs
-          CLUSTER_NAME_SUFFIX: "-${{ matrix.arch }}"
+          # Make cluster names unique per architecture and datastore to avoid conflicts in parallel runs
+          CLUSTER_NAME_SUFFIX: "-${{ matrix.arch }}-${{ matrix.datastore }}"
         run: |
           make cluster-create
 
+      - name: Override MongoDB image for ARM64
+        if: matrix.arch == 'arm64' && matrix.datastore == 'mongodb'
+        run: |
+          sed -i 's/repository: "bitnamilegacy\/mongodb"/repository: "dlavrenuek\/bitnami-mongodb-arm"/' distros/kubernetes/nvsentinel/values-tilt.yaml
+          sed -i 's/tag: "8.0.3-debian-12-r1"/tag: "8.0.4"/' distros/kubernetes/nvsentinel/values-tilt.yaml
+
       - name: Run E2E tests
         env:
           CI_COMMIT_REF_NAME: ${{ steps.ref-name.outputs.value }}
           CTLPTL_YAML: .ctlptl.yaml
           # Use same cluster name suffix for consistency
-          CLUSTER_NAME_SUFFIX: "-${{ matrix.arch }}"
+          CLUSTER_NAME_SUFFIX: "-${{ matrix.arch }}-${{ matrix.datastore }}"
+          # Set USE_POSTGRESQL for PostgreSQL tests (our integrated Tiltfile approach)
+          USE_POSTGRESQL: ${{ matrix.datastore == 'postgresql' && '1' || '0' }}
         run: |
           make e2e-test-ci
 
       - name: Upload test results
         uses: ./.github/actions/upload-test-artifacts
         with:
-          component-name: e2e-test-${{ matrix.arch }}
+          component-name: e2e-test-${{ matrix.arch }}-${{ matrix.datastore }}
           file-paths: |
             tests/results/
             tests/*.log
@@ -152,6 +176,20 @@ jobs:
             kind export logs /tmp/kind-logs --name "$CLUSTER_NAME" || true
           fi
 
+      - name: Pull Kind container logs via crictl
+        if: always()
+        run: |
+          chmod +x ./scripts/pull_kind_logs.sh
+          ./scripts/pull_kind_logs.sh /tmp/kind-container-logs || true
+
+      - name: Upload Kind container logs
+        if: always()
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4  # v5.0.0
+        with:
+          name: e2e-kind-container-logs-${{ matrix.arch }}-${{ matrix.datastore }}-${{ github.run_id }}
+          path: /tmp/kind-container-logs/
+          retention-days: 7
+
       - name: Collect debug artifacts
         if: failure()
         run: |
@@ -162,20 +200,27 @@ jobs:
           kubectl logs --all-namespaces --all-containers=true --tail=500 > /tmp/debug-artifacts/all-logs.txt || true
           docker images > /tmp/debug-artifacts/docker-images.txt || true
           df -h > /tmp/debug-artifacts/disk-usage.txt || true
+          # Comprehensive cluster state dump for debugging
+          kubectl cluster-info dump --all-namespaces --output-directory=/tmp/debug-artifacts/cluster-info-dump || true
+          # Get node details including cordon state
+          kubectl get nodes -o wide > /tmp/debug-artifacts/nodes-wide.txt || true
+          kubectl get nodes -o yaml > /tmp/debug-artifacts/nodes-full.yaml || true
+          # Get node descriptions for annotations and conditions
+          kubectl describe nodes > /tmp/debug-artifacts/nodes-describe.txt || true
 
       - name: Upload Kind logs
         if: always()
         uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4  # v5.0.0
         with:
-          name: e2e-kind-logs-${{ matrix.arch }}-${{ github.run_id }}
+          name: e2e-kind-logs-${{ matrix.arch }}-${{ matrix.datastore }}-${{ github.run_id }}
           path: /tmp/kind-logs/
           retention-days: 7
 
       - name: Upload debug artifacts
         if: failure()
         uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4  # v5.0.0
         with:
-          name: e2e-debug-artifacts-${{ matrix.arch }}-${{ github.run_id }}
+          name: e2e-debug-artifacts-${{ matrix.arch }}-${{ matrix.datastore }}-${{ github.run_id }}
           path: /tmp/debug-artifacts/
           retention-days: 7
 

.github/workflows/integration-aws.yml

@@ -0,0 +1,169 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Integration Tests - AWS
+
+on:
+  workflow_dispatch: {}  # allow manual runs for testing
+  schedule:
+    - cron: '0 8 * * *' # daily at midnight PST, runs on default branch only
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+  actions: read
+  id-token: write
+
+jobs:
+  integration-test-aws:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      CSP: "aws"
+      PREFIX: "nvs"
+      AWS_ACCOUNT_ID: "615299774277"
+      AWS_REGION: "us-east-1"
+      GITHUB_ACTIONS_ROLE_NAME: "github-actions-role"
+      CLUSTER_NAME: "nvs-d${{ github.run_id }}"
+      K8S_VERSION: "1.34"
+      EKSCTL_VERSION: "0.216.0"
+      GPU_AVAILABILITY_ZONE: "e"
+      GPU_NODE_COUNT: "1"
+      CAPACITY_RESERVATION_ID: "cr-0cbe491320188dfa6"
+
+      # Debug
+      SKIP_DELETE: "false"  # skip cluster deletion
+      TEST_TAG: "main-33c1d03"
+      
+    steps:
+      # Checkout
+      - name: Checkout
+        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3  # v6.0.0
+
+      # Auth
+      - name: Configure AWS credentials
+        id: auth
+        uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8  # v5.1.0
+        with:
+          role-to-assume: "arn:aws:iam::${{ env.AWS_ACCOUNT_ID }}:role/${{ env.GITHUB_ACTIONS_ROLE_NAME }}"
+          aws-region: ${{ env.AWS_REGION }}
+          role-session-name: GitHubActions-NVSentinel-Integration
+
+      # Install eksctl
+      - name: Install eksctl
+        run: |
+          set -euox pipefail
+          # Check if eksctl is already installed
+          if command -v eksctl >/dev/null 2>&1; then
+            echo "eksctl is already installed:"
+            eksctl version
+            exit 0
+          fi
+          
+          echo "Installing eksctl..."
+          curl -LO "https://github.com/eksctl-io/eksctl/releases/download/v${EKSCTL_VERSION}/eksctl_linux_amd64.tar.gz"
+          tar -xzf eksctl_linux_amd64.tar.gz
+          chmod +x eksctl
+          sudo mv eksctl /usr/local/bin/
+          rm eksctl_linux_amd64.tar.gz
+          echo "eksctl installed successfully:"
+          eksctl version
+
+      # Cluster
+      - name: Create Cluster
+        id: cluster
+        shell: bash 
+        run: |
+          set -euox pipefail
+          tests/uat/aws/create-eks-cluster.sh
+
+      # Connect
+      - name: Connect to Cluster
+        id: client
+        if: steps.cluster.outcome == 'success'
+        shell: bash
+        run: |
+          set -euox pipefail
+          # Check if kubectl is already installed
+          if command -v kubectl >/dev/null 2>&1; then
+            echo "kubectl is already installed:"
+            kubectl version
+            exit 0
+          fi
+
+          echo "Installing kubectl..."
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          chmod +x kubectl
+          sudo mv kubectl /usr/local/bin/
+          echo "Updating kubeconfig..."
+          aws eks update-kubeconfig --region ${{ env.AWS_REGION }} --name ${{ env.CLUSTER_NAME }}
+          echo "Verifying cluster connection..."
+          kubectl get nodes
+
+      # Image Tag
+      - name: Compute ref name with short SHA
+        id: ref-name
+        if: steps.cluster.outcome == 'success'
+        run: |
+          if [[ "${{ github.ref_type }}" == "tag" ]]; then
+            SAFE_REF="${{ github.ref_name }}"
+          elif [[ "${{ github.ref_name }}" == "main" ]]; then
+            SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
+            SAFE_REF="${{ github.ref_name }}-${SHORT_SHA}"
+          else
+            SAFE_REF="${{ env.TEST_TAG }}"
+          fi
+          # Sanitize ref name: replace slashes with hyphens for Docker tag compatibility
+          SAFE_REF=$(echo "$SAFE_REF" | sed 's/\//-/g')
+          echo "value=$SAFE_REF" >> $GITHUB_OUTPUT
+
+      # Apps
+      - name: Install NVS
+        id: apps
+        if: steps.client.outcome == 'success'
+        shell: bash
+        env:
+          NVSENTINEL_VERSION: "${{ steps.ref-name.outputs.value }}"
+        run: |
+          set -euxo pipefail
+          tests/uat/install-apps.sh
+
+      # Test
+      - name: Run UAT Tests
+        id: tests
+        if: steps.apps.outcome == 'success'
+        shell: bash
+        run: |
+          set -euxo pipefail
+          tests/uat/tests.sh
+
+      # Teardown
+      - name: Destroy Cluster
+        if: always() && steps.cluster.outcome != 'skipped' && env.SKIP_DELETE != 'true'
+        shell: bash
+        run: |
+          set -euxo pipefail
+          tests/uat/aws/delete-eks-cluster.sh
+
+      # Summary
+      - name: Test Summary
+        if: always()
+        run: |
+          echo "## Test Results" >> $GITHUB_STEP_SUMMARY
+          echo "- Cluster: ${{ steps.cluster.outcome }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Connection: ${{ steps.client.outcome }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Apps: ${{ steps.apps.outcome }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Tests: ${{ steps.tests.outcome }}" >> $GITHUB_STEP_SUMMARY