hybrid_system_test.yaml

name: Sequencer - Hybrid Node System Test

on:
  pull_request:
    types:
      - opened
      - reopened
      - edited
      - synchronize

env:
  job_link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
  namespace: sequencer-hybrid-system-test-run-${{ github.run_number }}-attempt-${{ github.run_attempt }}
  layout: hybrid
  overlay: hybrid.testing.node-0
  cluster_name: hybrid-system-test
  crate_triggers: "apollo_node,apollo_deployments,apollo_integration_tests"
  path_triggers: ".github/workflows/hybrid_system_test.yaml,scripts/**,deployments/sequencer/**,deployments/images/**"
  path_triggers_exclude: "scripts/prod/**/*"
  pvc_storage_class_name: "premium-rwo"
  anvil_port: "8545"
  dockerfile_base_path: ./deployments/images/sequencer

permissions:
  contents: read

# On PR events, cancel existing CI runs on this same PR for this workflow.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job }}-pr
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  # https://graphite.com/docs/stacking-and-ci
  optimize_ci:
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    outputs:
      skip: ${{ steps.check_skip.outputs.skip }}
    steps:
      - name: Optimize CI
        id: check_skip
        uses: withgraphite/graphite-ci-action@v0.0.9
        with:
          graphite_token: ${{ secrets.GRAPHITE_CI_OPTIMIZER_TOKEN }}

  check-system-test-trigger:
    runs-on: namespace-profile-small-ubuntu-24-04-amd64
    needs: optimize_ci
    if: needs.optimize_ci.outputs.skip == 'false'
    timeout-minutes: 60
    outputs:
      should_run: ${{ steps.system_check.outputs.should_run }}
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v6
        with:
          python-version: "pypy3.9"

      - name: Set up pip dependency caching
        uses: namespacelabs/nscloud-cache-action@v1
        with:
          cache: python

      - name: Install Python Dependencies
        run: pip install -r scripts/requirements.txt

      - name: Check for system-test-triggering changes
        id: system_check
        run: |
          echo "Checking if any system-test-triggering crates were modified..."
          OUTPUT_FILE=$(mktemp)

          python ./scripts/check_test_trigger.py --output_file $OUTPUT_FILE \
            --commit_id ${{ github.event.pull_request.base.sha }} \
            --crate_triggers ${{ env.crate_triggers }} \
            --path_triggers ${{ env.path_triggers }} \
            --path_triggers_exclude ${{ env.path_triggers_exclude }}

          should_run=$(cat "$OUTPUT_FILE")
          echo "Captured output: $should_run"
          echo "should_run=$should_run" >> $GITHUB_OUTPUT

  build_docker_images:
    runs-on: namespace-profile-small-ubuntu-24-04-amd64
    needs: check-system-test-trigger
    if: success() && needs.check-system-test-trigger.outputs.should_run == 'true'
    timeout-minutes: 60
    permissions:
      id-token: write
      contents: read
    strategy:
      fail-fast: true
      matrix:
        include:
          - svc: sequencer
            dockerfile: Dockerfile
          - svc: dummy_recorder
            dockerfile: dummy_recorder.Dockerfile
          - svc: dummy_exchange_rate_oracle
            dockerfile: dummy_exchange_rate_oracle.Dockerfile
    steps:
      - uses: actions/checkout@v6

      - name: Build ${{ matrix.svc }} image remotely
        uses: docker/build-push-action@v6.18.0
        with:
          context: .
          file: ${{ env.dockerfile_base_path }}/${{ matrix.dockerfile }}
          push: true
          tags: ${{ env.NSC_CONTAINER_REGISTRY }}/${{ matrix.svc }}:${{ github.sha }}

  system_test_hybrid:
    needs:
      - check-system-test-trigger
      - build_docker_images
    runs-on: namespace-profile-large-ubuntu-24-04-amd64
    if: success() && needs.check-system-test-trigger.outputs.should_run == 'true'
    timeout-minutes: 45
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Create registries.yaml for nscr.io
        run: |
          cat << EOF > registries.yaml
          mirrors:
            nscr.io:
              endpoint:
                - https://nscr.io
          configs:
            nscr.io:
              auth:
                username: token
                password: $(cat $NSC_TOKEN_FILE | jq -r .bearer_token)
          EOF

      - uses: ./.github/actions/bootstrap
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}

      - name: Build binaries
        run: cargo build --bin sequencer_node_setup --bin sequencer_simulator

      - name: Create k3d cluster (Local k8s)
        uses: AbsaOSS/k3d-action@v2.4.0
        with:
          # Assumption: only one PR can run per machine at a time.
          cluster-name: ${{ env.cluster_name }}
          args: >-
            --verbose
            --agents 1
            --no-lb
            --wait
            --timeout 120s
            --registry-config registries.yaml

      - name: Install crds
        working-directory: deployments/sequencer
        run: |
          echo "🔧 Installing CRDs..."
          kubectl apply -R -f ./resources/crds
          echo "✅ CRDs installed successfully."

      - name: Install local-path-provisioner (for PVC support)
        run: |
          echo "🔧 Installing local-path-provisioner..."
          kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml

          echo "⏳ Waiting for local-path-provisioner pod to be ready..."
          kubectl wait --for=condition=Ready pod -l app=local-path-provisioner -n local-path-storage --timeout=60s

          echo "✅ local-path-provisioner is ready."

          echo "📦 Setting default StorageClass and volumeBindingMode..."
          # First, set as default StorageClass
          kubectl patch storageclass local-path -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

          # Then, patch the volumeBindingMode in spec
          kubectl patch storageclass local-path --type='merge' -p '{"volumeBindingMode":"WaitForFirstConsumer"}'

          echo "📦 Verifying StorageClass configuration..."
          kubectl get storageclass local-path -o yaml

          echo "🎉 StorageClasses available:"
          kubectl get storageclass

      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"
          cache: pipenv

      - name: Setup pipenv
        run: python3 -m pip install pipenv

      - name: Install dependencies with pipenv
        run: pipenv install kubernetes

      - name: Setup cdk8s-cli
        run: npm install -g cdk8s-cli

      - name: Deploy Dummy Recorder
        env:
          dummy_recorder_namespace: dummy-recorder
        working-directory: deployments/dummy_recorder
        run: |
          echo "Deploying Dummy Recorder..."
          pipenv install
          cdk8s import
          cdk8s synth --app "pipenv run python main.py --namespace ${{ env.dummy_recorder_namespace }} --image ${{ env.NSC_CONTAINER_REGISTRY }}/dummy_recorder:${{ github.sha }}"
          kubectl create namespace ${{ env.dummy_recorder_namespace }}
          kubectl apply -R -f ./dist
          echo "⏳ Waiting for Dummy Recorder to become ready..."

          # Wait for pod to be ready
          kubectl wait --namespace ${{ env.dummy_recorder_namespace }} --for=condition=Ready -l app=dummy-recorder pod --timeout=300s

          echo "🚀 Dummy Recorder deployed successfully."

      - name: Deploy Eth2Strk Oracle
        env:
          dummy_eth_to_strk_namespace: dummy-eth-to-strk
        working-directory: deployments/dummy_eth2strk_oracle
        run: |
          echo "Deploying Dummy Eth2Strk Oracle..."
          pipenv install
          cdk8s import
          cdk8s synth --app "pipenv run python main.py --namespace ${{ env.dummy_eth_to_strk_namespace }} --image ${{ env.NSC_CONTAINER_REGISTRY }}/dummy_exchange_rate_oracle:${{ github.sha }}"
          kubectl create namespace ${{ env.dummy_eth_to_strk_namespace }}
          kubectl apply -R -f ./dist
          echo "⏳ Waiting for Dummy Eth2Strk Oracle to become ready..."
          kubectl wait --namespace ${{ env.dummy_eth_to_strk_namespace }} --for=condition=Ready -l app=dummy-eth2strk-oracle pod --timeout 60s
          echo "🚀 Dummy Eth2Strk Oracle deployed successfully."

      - name: Deploy Anvil
        env:
          namespace: anvil
        working-directory: deployments/anvil
        run: |
          echo "Deploying Anvil..."
          # Delete namespace if it already exists
          if kubectl get namespace "${{ env.namespace }}" &> /dev/null; then
            echo "🔁 Namespace '${{ env.namespace }}' already exists. Deleting it..."
            kubectl delete namespace "${{ env.namespace }}"
            echo "⏳ Waiting for namespace deletion..."
            while kubectl get namespace "${{ env.namespace }}" &> /dev/null; do sleep 2; done
            echo "✅ Namespace '${{ env.namespace }}' deleted."
          fi
          pipenv install
          cdk8s import
          cdk8s synth --app "pipenv run python main.py --namespace ${{ env.namespace }}"
          kubectl create namespace ${{ env.namespace }}
          kubectl apply -R -f ./dist
          echo "⏳ Waiting for Anvil to become ready..."
          kubectl wait --namespace ${{ env.namespace }} --for=condition=Ready -l app=anvil pod --timeout 120s
          echo "🚀 Anvil deployed successfully."

          echo "🔍 Extracting Anvil addresses from logs."

          ANVIL_POD=$(kubectl get pods -n "${{ env.namespace }}" -l app=anvil -o jsonpath="{.items[0].metadata.name}")
          ANVIL_LOGS=$(kubectl logs -n "${{ env.namespace }}" "$ANVIL_POD")

          echo "🔍 Extracting Anvil addresses from logs..."
          ANVIL_POD=$(kubectl get pods -n ${{ env.namespace }} -l app=anvil -o jsonpath="{.items[0].metadata.name}")
          ADDRESSES=$(kubectl logs -n ${{ env.namespace }} "$ANVIL_POD" | grep -oP '0x[a-fA-F0-9]{40}' | head -n 2)

          SENDER_ADDRESS=$(echo "$ADDRESSES" | head -n 1)
          RECEIVER_ADDRESS=$(echo "$ADDRESSES" | tail -n 1)

          echo "💡 SENDER_ADDRESS=$SENDER_ADDRESS"
          echo "💡 RECEIVER_ADDRESS=$RECEIVER_ADDRESS"

          echo "SENDER_ADDRESS=$SENDER_ADDRESS" >> "$GITHUB_ENV"
          echo "RECEIVER_ADDRESS=$RECEIVER_ADDRESS" >> "$GITHUB_ENV"

      - name: Install Anvil
        uses: foundry-rs/foundry-toolchain@v1
        with:
          version: v1.5.1

      - name: Restore executable permissions
        run: chmod +x ./target/debug/sequencer_node_setup ./target/debug/sequencer_simulator

      - name: Create storage files
        run: ./target/debug/sequencer_node_setup --output-base-dir ./output --data-prefix-path /data --n-consolidated 1 --n-hybrid 0 --n-distributed 0

      - name: Generate k8s manifests
        working-directory: deployments/sequencer
        run: |
          pipenv install
          cdk8s import
          echo "Generating Kubernetes manifests using hybrid layout"
          cdk8s synth --app "pipenv run python main.py --namespace ${{ env.namespace }} -l ${{ env.layout }} -o ${{ env.overlay }} --image ${{ env.NSC_CONTAINER_REGISTRY }}/sequencer:${{ github.sha }}"

      - name: Deploy Sequencer
        working-directory: deployments/sequencer
        run: |
          echo "Deploying Sequencer..."
          kubectl create namespace ${{ env.namespace }}
          kubectl apply -R -f ./dist/

      - name: Set default namespace
        run: kubectl config set-context --current --namespace ${{ env.namespace }}

      - name: Run readiness check
        run: |
          pipenv run python ./scripts/system_tests/readiness_check.py \
            --layout ${{ env.layout }} \
            --overlay ${{ env.overlay }} \
            --namespace ${{ env.namespace }} \
            --verbose

      - name: Test sequencer is alive
        env:
          initial_delay_sec: 10
          check_interval_sec: 2
          check_timeout_sec: 10
        run: |
          pipenv run python ./scripts/system_tests/liveness_check.py \
            --layout ${{ env.layout }} \
            --overlay ${{ env.overlay }} \
            --namespace ${{ env.namespace }} \
            --timeout ${{ env.check_timeout_sec }} \
            --interval ${{ env.check_interval_sec }} \
            --initial-delay ${{ env.initial_delay_sec }} \
            --verbose

      - name: Copy state and restart pod
        run: |
          pipenv run python ./scripts/system_tests/copy_state_and_restart.py \
            --layout ${{ env.layout }} \
            --overlay ${{ env.overlay }} \
            --namespace ${{ env.namespace }} \
            --data-dir "./output/data/node_0" \
            --verbose

      - name: Run readiness check
        run: |
          pipenv run python ./scripts/system_tests/readiness_check.py \
            --layout ${{ env.layout }} \
            --overlay ${{ env.overlay }} \
            --namespace ${{ env.namespace }} \
            --verbose

      - name: Port-forward Anvil pod to localhost:${{ env.anvil_port }}
        run: |
          echo "🔌 Setting up port-forward to Anvil..."

          ANVIL_POD=$(kubectl get pods -n anvil -l app=anvil -o jsonpath="{.items[0].metadata.name}")
          echo "🌐 Found Anvil pod: $ANVIL_POD"

          # Start port-forwarding in background and keep it running
          kubectl port-forward -n anvil "$ANVIL_POD" ${{ env.anvil_port }}:${{ env.anvil_port }} &
          echo "⏳ Waiting a few seconds to ensure port-forward is established..."
          sleep 2

      - name: Print Second ConfigMap on the list
        run: |
          CONFIGMAP_NAME=$(kubectl get configmap -n "${{ env.namespace }}" -o jsonpath='{.items[1].metadata.name}')
          echo "ConfigMap: $CONFIGMAP_NAME"
          kubectl get configmap "$CONFIGMAP_NAME" -n "${{ env.namespace }}" -o yaml

      - name: Send transactions test
        env:
          RUST_LOG: debug
          RUST_BACKTRACE: full
        run: pipenv run python ./scripts/system_tests/sequencer_simulator.py --state_sync_monitoring_endpoint_port 8082 --http_server_port 8080 --node_type "hybrid" --sender_address "${{ env.SENDER_ADDRESS }}" --receiver_address "${{ env.RECEIVER_ADDRESS }}"

      - name: List all pods in namespace ${{ env.namespace }}
        if: always()
        run: |
          # ============================================
          # List all pods in the namespace
          # ============================================
          kubectl get pods -n "$namespace"
          echo "---"
          echo ""
          echo "✅ Finished listing pods in namespace ${{ env.namespace }}"
          echo "---------------------------------------------"
          echo ""

      - name: Get container logs
        if: always()
        run: |
          pods=$(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}')
          echo "📥 Getting pod logs and descriptions from namespace: $namespace"
          echo "---------------------------------------------"
          echo ""
          # ============================================
          # Describe each pod
          # ============================================
          echo "## 🔍 Pod Descriptions"
          echo "---------------------------------------------"
          echo ""
          for pod in $pods; do
            echo "### Pod: \`$pod\`"
            echo ""
            echo '```'
            kubectl describe pod -n "$namespace" "$pod" || echo "⚠️ Failed to describe pod $pod"
            echo '```'
            echo ""
            echo "---"
            echo ""
          done
          # ============================================
          # Get logs for each pod
          # ============================================
          echo "## 📜 Pod Logs"
          echo "---------------------------------------------"
          echo ""
          for pod in $pods; do
            echo "### Logs for pod: \`$pod\`"
            echo ""
            echo '```'
            kubectl logs -n "$namespace" "$pod" --tail=1000 || echo "⚠️ Failed to get logs for $pod"
            echo '```'
            echo ""
            echo "---"
            echo ""
          done
          echo "✅ Finished collecting pod information"
          echo "---------------------------------------------"
          echo ""