e2e: relax topology-aware coldstart test

marquiz · marquiz · commit 130ab684af4c · 2023-08-25T11:51:32.000+03:00
Increase the amount of memory allocated by the test container, in an attempt to reduce test flakyness by reducing the relative effect of "memory usage noise" caused by other system processes. Make the error threshold (acceptable marging of error) of coldstart memory allocation proportional, being 10 percent of the allocated size. Also increase the wait duration of coldstart test to avoid flakyness on slow/heavily loaded systems. (cherry picked from commit 0383c38)
diff --git a/test/e2e/policies.test-suite/topology-aware/c4pmem4/test03-coldstart-deprecated-syntax/bb-coldstart.yaml.in b/test/e2e/policies.test-suite/topology-aware/c4pmem4/test03-coldstart-deprecated-syntax/bb-coldstart.yaml.in
@@ -7,7 +7,7 @@ metadata:
       ${NAME}c0: dram,pmem
     cri-resource-manager.intel.com/cold-start: |
       ${NAME}c0:
-        duration: ${DURATION}
+        duration: ${DURATION_S}s
 spec:
   containers:
     - name: ${NAME}c0
@@ -16,9 +16,9 @@ spec:
       command:
         - sh
         - -c
-        - 'cold_alloc=\$(dd if=/dev/zero bs=${COLD_ALLOC} count=1 | tr \"\\\0\" \"x\");
+        - 'cold_alloc=\$(dd if=/dev/zero bs=${COLD_ALLOC_KB}kB count=1 | tr \"\\\0\" \"x\");
            sh -c \"paused after cold_alloc \\\$(sleep inf)\";
-           warm_alloc=\$(dd if=/dev/zero bs=${WARM_ALLOC} count=1 | tr \"\\\0\" \"x\");
+           warm_alloc=\$(dd if=/dev/zero bs=${WARM_ALLOC_KB}kB count=1 | tr \"\\\0\" \"x\");
            sh -c \"paused after warm_alloc \\\$(sleep inf)\";
            echo ${NAME}c0 \$(sleep inf); # needed for pod resource discovery'
       resources:
diff --git a/test/e2e/policies.test-suite/topology-aware/c4pmem4/test03-coldstart-deprecated-syntax/code.var.sh b/test/e2e/policies.test-suite/topology-aware/c4pmem4/test03-coldstart-deprecated-syntax/code.var.sh
@@ -1,11 +1,11 @@
 # Test that a cold-started pod...
 # 1. is allowed to allocate memory only from PMEM nodes
-#    during cold period (of length $DURATION).
+#    during cold period (of length $DURATION_S).
 # 2. is restricted from the very beginning of pod execution:
 #    immediately allocated memory blob consumes PMEM from expected node.
 # 3. is allowed to allocate memory from both PMEM and DRAM after
 #    the cold period.
-# 4. is no more restricted after $DURATION + 1s has passed in pod:
+# 4. is no more restricted after $DURATION_S + 1s has passed in pod:
 #    warm-allocated memory is not taken from PMEM nodes.
 
 PMEM_NODES='{"node4", "node5", "node6", "node7"}'
@@ -22,9 +22,9 @@ CRI_RESMGR_OUTPUT="cat cri-resmgr.output.txt"
 
 PMEM_USED_BEFORE_POD0="$(pmem-used)"
 
-DURATION=10s
-COLD_ALLOC=$((10 * 1024))kB
-WARM_ALLOC=$((20 * 1024))kB
+DURATION_S=10
+COLD_ALLOC_KB=$((50 * 1024))
+WARM_ALLOC_KB=$((100 * 1024))
 MEM=1G
 create bb-coldstart
 
@@ -40,21 +40,21 @@ vm-run-until "pgrep -f '^sh -c paused after cold_alloc'" >/dev/null ||
     error "cold memory allocation timed out"
 
 echo "Verify PMEM consumption during cold period."
-PMEM_ERROR_MARGIN=1024 # meminfo MemUsed vs dd bytes error margin
+# meminfo MemUsed vs dd bytes error margin, use 10%
+PMEM_ERROR_MARGIN=$((COLD_ALLOC_KB / 10))
 sleep 1
 PMEM_USED_COLD_POD0="$(pmem-used)"
 PMEM_COLD_CONSUMED=$(( $PMEM_USED_COLD_POD0 - $PMEM_USED_BEFORE_POD0 ))
-if (( $PMEM_COLD_CONSUMED + $PMEM_ERROR_MARGIN < ${COLD_ALLOC%kB} )); then
-    error "pod0 did not allocate $COLD_ALLOC from PMEM. MemUsed PMEM delta: $PMEM_COLD_CONSUMED"
+if (( $PMEM_COLD_CONSUMED + $PMEM_ERROR_MARGIN < $COLD_ALLOC_KB )); then
+    error "pod0 did not allocate ${COLD_ALLOC_KB}kB from PMEM. MemUsed PMEM delta: $PMEM_COLD_CONSUMED"
 else
-    echo "### Verified: PMEM memory consumed during cold period: $PMEM_COLD_CONSUMED kB, pod script allocated: ${COLD_ALLOC%kB} kB"
+    echo "### Verified: PMEM memory consumed during cold period: $PMEM_COLD_CONSUMED kB, pod script allocated: $COLD_ALLOC_KB kB"
 fi
 
 coldstarts=$(vm-command-q "$CRI_RESMGR_OUTPUT | grep 'finishing coldstart period for pod0:pod0c0' | wc -l")
-echo "Wait that cri-resmgr finishes coldstart period within 5s + $DURATION."
-sleep 5s
-vm-run-until --timeout ${DURATION%s} "[ \$($CRI_RESMGR_OUTPUT | grep 'finishing coldstart period for pod0:pod0c0' | wc -l) -gt $coldstarts ]" ||
-    error "cri-resmgr did not report finishing coldstart period within $DURATION"
+echo "Wait that cri-resmgr finishes coldstart period within $(($DURATION_S + 10)) seconds."
+vm-run-until --timeout $((DURATION_S + 10)) "[ \$($CRI_RESMGR_OUTPUT | grep 'finishing coldstart period for pod0:pod0c0' | wc -l) -gt $coldstarts ]" ||
+    error "cri-resmgr did not report finishing coldstart period within $DURATION_S seconds"
 
 vm-command "$CRI_RESMGR_OUTPUT | grep 'pinning to memory 1,7'" ||
     error "cri-resmgr did not report pinning to expected memory nodes"
@@ -74,7 +74,7 @@ sleep 1
 PMEM_USED_WARM_POD0="$(pmem-used)"
 PMEM_WARM_CONSUMED=$(( $PMEM_USED_WARM_POD0 - $PMEM_USED_COLD_POD0 ))
 if (( $PMEM_WARM_CONSUMED > 0 )); then
-    echo "### Verify (soft) failed: pod0 allocated $WARM_ALLOC from PMEM. Should have been taken from DRAM."
+    echo "### Verify (soft) failed: pod0 allocated $WARM_ALLOC_KB kB from PMEM. Should have been taken from DRAM."
 else
-    echo "### Verified (soft): PMEM memory consumption delta during warm period: $PMEM_WARM_CONSUMED kB, pod script allocated: ${WARM_ALLOC%kB} kB"
+    echo "### Verified (soft): PMEM memory consumption delta during warm period: $PMEM_WARM_CONSUMED kB, pod script allocated: $WARM_ALLOC_KB kB"
 fi
diff --git a/test/e2e/policies.test-suite/topology-aware/c4pmem4/test03-coldstart/bb-coldstart.yaml.in b/test/e2e/policies.test-suite/topology-aware/c4pmem4/test03-coldstart/bb-coldstart.yaml.in
@@ -14,9 +14,9 @@ spec:
       command:
         - sh
         - -c
-        - 'cold_alloc=\$(dd if=/dev/zero bs=${COLD_ALLOC} count=1 | tr \"\\\0\" \"x\");
+        - 'cold_alloc=\$(dd if=/dev/zero bs=${COLD_ALLOC_KB}kB count=1 | tr \"\\\0\" \"x\");
            sh -c \"paused after cold_alloc \\\$(sleep inf)\";
-           warm_alloc=\$(dd if=/dev/zero bs=${WARM_ALLOC} count=1 | tr \"\\\0\" \"x\");
+           warm_alloc=\$(dd if=/dev/zero bs=${WARM_ALLOC_KB}kB count=1 | tr \"\\\0\" \"x\");
            sh -c \"paused after warm_alloc \\\$(sleep inf)\";
            echo ${NAME}c0 \$(sleep inf); # needed for pod resource discovery'
       resources:
diff --git a/test/e2e/policies.test-suite/topology-aware/c4pmem4/test03-coldstart/code.var.sh b/test/e2e/policies.test-suite/topology-aware/c4pmem4/test03-coldstart/code.var.sh
@@ -23,8 +23,8 @@ CRI_RESMGR_OUTPUT="cat cri-resmgr.output.txt"
 PMEM_USED_BEFORE_POD0="$(pmem-used)"
 
 DURATION=10s
-COLD_ALLOC=$((10 * 1024))kB
-WARM_ALLOC=$((20 * 1024))kB
+COLD_ALLOC_KB=$((50 * 1024))
+WARM_ALLOC_KB=$((100 * 1024))
 MEM=1G
 create bb-coldstart
 
@@ -40,14 +40,15 @@ vm-run-until "pgrep -f '^sh -c paused after cold_alloc'" >/dev/null ||
     error "cold memory allocation timed out"
 
 echo "Verify PMEM consumption during cold period."
-PMEM_ERROR_MARGIN=1024 # meminfo MemUsed vs dd bytes error margin
+# meminfo MemUsed vs dd bytes error margin, use 10%
+PMEM_ERROR_MARGIN=$((COLD_ALLOC_KB / 10))
 sleep 1
 PMEM_USED_COLD_POD0="$(pmem-used)"
 PMEM_COLD_CONSUMED=$(( $PMEM_USED_COLD_POD0 - $PMEM_USED_BEFORE_POD0 ))
-if (( $PMEM_COLD_CONSUMED + $PMEM_ERROR_MARGIN < ${COLD_ALLOC%kB} )); then
-    error "pod0 did not allocate $COLD_ALLOC from PMEM. MemUsed PMEM delta: $PMEM_COLD_CONSUMED"
+if (( $PMEM_COLD_CONSUMED + $PMEM_ERROR_MARGIN < $COLD_ALLOC_KB )); then
+    error "pod0 did not allocate ${COLD_ALLOC_KB}kB from PMEM. MemUsed PMEM delta: $PMEM_COLD_CONSUMED"
 else
-    echo "### Verified: PMEM memory consumed during cold period: $PMEM_COLD_CONSUMED kB, pod script allocated: ${COLD_ALLOC%kB} kB"
+    echo "### Verified: PMEM memory consumed during cold period: $PMEM_COLD_CONSUMED kB, pod script allocated: $COLD_ALLOC_KB kB"
 fi
 
 coldstarts=$(vm-command-q "$CRI_RESMGR_OUTPUT | grep 'finishing coldstart period for pod0:pod0c0' | wc -l")
@@ -74,7 +75,7 @@ sleep 1
 PMEM_USED_WARM_POD0="$(pmem-used)"
 PMEM_WARM_CONSUMED=$(( $PMEM_USED_WARM_POD0 - $PMEM_USED_COLD_POD0 ))
 if (( $PMEM_WARM_CONSUMED > 0 )); then
-    echo "### Verify (soft) failed: pod0 allocated $WARM_ALLOC from PMEM. Should have been taken from DRAM."
+    echo "### Verify (soft) failed: pod0 allocated $WARM_ALLOC_KB kB from PMEM. Should have been taken from DRAM."
 else
-    echo "### Verified (soft): PMEM memory consumption delta during warm period: $PMEM_WARM_CONSUMED kB, pod script allocated: ${WARM_ALLOC%kB} kB"
+    echo "### Verified (soft): PMEM memory consumption delta during warm period: $PMEM_WARM_CONSUMED kB, pod script allocated: $WARM_ALLOC_KB kB"
 fi
