fix(agent/k8s_test): fix the operator-agent tests and add cli building to agent ci for tests

Author: ayuskauskas

.github/workflows/agent-ci.yaml

@@ -224,8 +224,9 @@ jobs:
       
       - name: Run operator-agent tests
         env:
-          AGENT_IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/agent:${{ needs.compute-metadata.outputs.agent-image-tag }}
+          AGENT_IMAGE: ${{ format('{0}/{1}/agent:{2}', toLower(env.REGISTRY), toLower(env.IMAGE_NAME), needs.compute-metadata.outputs.agent-image-tag) }}
         run: |
           cd operator
           echo "Testing with agent image: ${AGENT_IMAGE}"
+          make build-cli
           make setup-kind-cluster operator-agent-tests

k8s-tests/operator-agent/check_node.sh

@@ -23,12 +23,17 @@ node=$1
 cmd=$2
 check=$3
 timeout=${4:-10}
+invert=${5:-false}
 
 
 # loop until the command returns a non-zero exit code or the timeout is reached
 for i in $(seq 1 ${timeout}); do
     data=$(kubectl exec ${node}-debugger -- chroot /host bash -c "${cmd}")
-    if echo "${data}" | grep -q "${check}"; then
+    check_result=$(echo "${data}" | grep -c "${check}")
+    if [ "$invert" == "true" ]; then
+        check_result=$((! check_result))
+    fi
+    if [ $check_result -gt 0 ]; then
         echo "Check passed"
         exit 0
     else

k8s-tests/operator-agent/dont_write_logs/chainsaw-test.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 #
@@ -14,38 +14,39 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 # yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
   name: dont-write-logs-agent-operator
 spec:
   timeouts:
-    assert: 10s
-    exec: 10s
+    assert: 240s
+    exec: 90s
   steps:
   - try:
     - script:
         content: |
           ## remove annotation from last run
-          ../../../../operator/bin/skyhook reset simple-agent-operator --confirm 2>/dev/null || true
+          ../../../operator/bin/skyhook reset dont-write-logs-agent-operator --confirm 2>/dev/null || true
     - script:
         content: |
           ## reinstall the debug pod in case it was deleted
           ../setup.sh kind-worker setup
     - script:
         content: |
           ## clean up any logs from prior runs
-          ../check_node.sh kind-worker "rm -f /var/log/skyhook/dont-write-logs-agent-operator/shellscript/1.1.1/*.log" ".*" 2
+          ../check_node.sh kind-worker "rm -rf /var/log/skyhook/dont-write-logs-agent-operator || true" ".*" 2
+          ../check_node.sh kind-worker "rm -rf /var/lib/skyhook/dont-write-logs-agent-operator || true" ".*" 2
     - apply:
         file: skyhook.yaml
     - assert:
         file: assert.yaml
     - script:
         content: |
-          ../check_node.sh kind-worker "ls /var/lib/skyhook/dont-write-logs-agent-operator/flags/shellscript/1.1.1/" "shellscript_run.sh.*"
-          ../check_node.sh kind-worker "ls /var/log/skyhook/dont-write-logs-agent-operator/shellscript/1.1.1/*.log | wc -l" "0" 2
+          set -e
+          ## Wont even create the log directory
+          ../check_node.sh kind-worker "ls /var/log/skyhook/" "dont-write-logs-agent-operator" 2 true
   - finally:
     - delete:
         file: skyhook.yaml

k8s-tests/operator-agent/dont_write_logs/skyhook.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 #
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 apiVersion: skyhook.nvidia.com/v1alpha1
 kind: Skyhook
 metadata:
@@ -34,7 +33,7 @@ spec:
         - name: SKYHOOK_AGENT_WRITE_LOGS
           value: "false"
       configMap:
-        config.sh: |
+        apply.sh: |
           #!/bin/bash
           echo "Hello, world!"
 

k8s-tests/operator-agent/interrupt/chainsaw-test.yaml

@@ -28,15 +28,21 @@ spec:
     - script:
         content: |
           ## remove annotation from last run
-          ../../../../operator/bin/skyhook reset simple-agent-operator --confirm 2>/dev/null || true
+          ../../../operator/bin/skyhook reset interrupt-agent-operator --confirm 2>/dev/null || true
     - script:
         content: |
           ## reinstall the debug pod in case it was deleted
           ../setup.sh kind-worker setup
+    - script:
+        content: |
+          ## clean up from prior runs
+          ../check_node.sh kind-worker "rm -rf /var/log/skyhook/interrupt-agent-operator || true" ".*" 2
+          ../check_node.sh kind-worker "rm -rf /var/lib/skyhook/interrupt-agent-operator || true" ".*" 2
     - apply:
         file: skyhook.yaml
     - script:
         content: |
+          set -e
           ../check_node.sh kind-worker "ls /var/lib/skyhook/interrupt-agent-operator/interrupts/flags/interrupt-agent-operator*/" "no_op.complete" 60
     - assert:
         file: assert.yaml

k8s-tests/operator-agent/interrupt/skyhook.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 #
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 apiVersion: skyhook.nvidia.com/v1alpha1
 kind: Skyhook
 metadata:
@@ -33,6 +32,6 @@ spec:
       interrupt:
         type: noop
       configMap:
-        config.sh: |
+        apply.sh: |
           #!/bin/bash
           echo "Hello, world!"

k8s-tests/operator-agent/reap_old_logs/chainsaw-test.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 #
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 # yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
@@ -29,11 +28,16 @@ spec:
     - script:
         content: |
           ## remove annotation from last run
-          ../../../../operator/bin/skyhook reset reap-old-logs-agent-operator --confirm 2>/dev/null || true
+          ../../../operator/bin/skyhook reset reap-old-logs-agent-operator --confirm
     - script:
         content: |
           ## reinstall the debug pod in case it was deleted
           ../setup.sh kind-worker setup
+    - script:
+        content: |
+          ## clean up from prior runs
+          ../check_node.sh kind-worker "rm -rf /var/log/skyhook/reap-old-logs-agent-operator || true" ".*" 2
+          ../check_node.sh kind-worker "rm -rf /var/lib/skyhook/reap-old-logs-agent-operator || true" ".*" 2
     - apply:
         file: skyhook.yaml
     - assert:

k8s-tests/operator-agent/reap_old_logs/skyhook.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 #
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 apiVersion: skyhook.nvidia.com/v1alpha1
 kind: Skyhook
 metadata:
@@ -31,12 +30,13 @@ spec:
       version: "1.1.1"
       image: ghcr.io/nvidia/skyhook-packages/shellscript
       configMap:
-        config.sh: |
+        apply.sh: |-
           #!/bin/bash
-          if [ $(ls /var/lib/skyhook/reap-old-logs-agent-operator/flags/shellscript/1.1.1/*.log | wc -l) -eq 5 ]; then
+          sleep 1
+          if [ $(ls /var/log/skyhook/reap-old-logs-agent-operator/shellscript/1.1.1/*.log | wc -l) -eq 5 ]; then
             echo "5 logs found. After this should still be 5."
             exit 0
-          else:
+          else
             echo "Not enough logs yet. Erroring to produce more."
             exit 1
           fi

k8s-tests/operator-agent/simple/chainsaw-test.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 #
@@ -14,26 +14,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 # yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
   name: simple-agent-operator
 spec:
   timeouts:
-    assert: 10s
-    exec: 10s
+    assert: 240s
+    exec: 90s
   steps:
   - try:
     - script:
         content: |
           ## remove annotation from last run
-          ../../../../operator/bin/skyhook reset simple-agent-operator --confirm 2>/dev/null || true
+          ../../../operator/bin/skyhook reset simple-agent-operator --confirm 2>/dev/null || true
     - script:
         content: |
           ## reinstall the debug pod in case it was deleted
           ../setup.sh kind-worker setup
+    - script:
+        content: |
+          ## clean up from prior runs
+          ../check_node.sh kind-worker "rm -rf /var/log/skyhook/simple-agent-operator || true" ".*" 2
+          ../check_node.sh kind-worker "rm -rf /var/lib/skyhook/simple-agent-operator || true" ".*" 2
     - apply:
         file: skyhook.yaml
     - assert:

k8s-tests/operator-agent/simple/skyhook.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 #
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 apiVersion: skyhook.nvidia.com/v1alpha1
 kind: Skyhook
 metadata:
@@ -31,7 +30,7 @@ spec:
       version: "1.1.1"
       image: ghcr.io/nvidia/skyhook-packages/shellscript
       configMap:
-        config.sh: |
+        apply.sh: |
           #!/bin/bash
           echo "Hello, world!"