[8.17] (backport #17028) Smoke test os retry event assertions (#17065)

* Smoke test os retry event assertions (#17028)

* smoke: Add retries and wait to data stream event assertions

This will improve the reliability of the smoke tests which intermittently fail due to data not yet being available in ES.

* smoke: decrease wait time after sending events since assertion now performs retries

* smoke: decrease elasticsearch flush_interval to 100ms from the 1s default

This will reduce any latency indexing events to help stabilize smoke test and reduce the amount of retries when asserting events.

* smoke: decreased retries for data stream event assertions

(cherry picked from commit b3381c2)

# Conflicts:
#	testing/smoke/lib.sh

* fix backport merge conflicts

---------

Co-authored-by: Isaac Flores <[email protected]>
Co-authored-by: Isaac Flores <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: mergify[bot]

testing/infra/terraform/modules/standalone_apm_server/apm-server.yml.tftpl

@@ -16,6 +16,7 @@ output:
     hosts: [ ${elasticsearch_url} ]
     username: ${elasticsearch_username}
     password: ${elasticsearch_password}
+    flush_interval: 100ms
 logging.level: debug
 logging.to_files: true
 logging.files:

testing/smoke/lib.sh

@@ -125,7 +125,7 @@ assert_entry() {
     if [[ ${HITS} -ne ${ENTRIES} ]]; then
         echo "Didn't find ${ENTRIES} indexed documents ${MSG}, total hits ${HITS}"
         echo ${RESULT}
-        exit 2
+        return 2
     else
         echo "-> Asserted ${ENTRIES} ${MSG} exists"
     fi
@@ -141,7 +141,7 @@ send_events() {
     curl_fail --data-binary @${INTAKE_DATA} -H "${APM_AUTH_HEADER}" -H "${INTAKE_HEADER}" ${APM_SERVER_INTAKE}
 
     # TODO(marclop). It would be best to query Elasticsearch until at least X documents have been ingested.
-    sleep 10
+    sleep 5
 }
 
 delete_all() {
@@ -169,10 +169,10 @@ data_stream_assert_events() {
     local METRICS_INDEX="metrics-apm.internal-*"
     local VERSION=${1}
     local ENTRIES=${2}
-    assert_document ${ERRORS_INDEX} "error.id" "9876543210abcdeffedcba0123456789" ${VERSION} ${ENTRIES}
-    assert_document ${TRACES_INDEX} "span.id" "1234567890aaaade" ${VERSION} ${ENTRIES}
-    assert_document ${TRACES_INDEX} "transaction.id" "4340a8e0df1906ecbfa9" ${VERSION} ${ENTRIES}
-    assert_document ${METRICS_INDEX} "transaction.type" "request" ${VERSION} ${ENTRIES}
+    retry 6 assert_document ${ERRORS_INDEX} "error.id" "9876543210abcdeffedcba0123456789" ${VERSION} ${ENTRIES}
+    retry 6 assert_document ${TRACES_INDEX} "span.id" "1234567890aaaade" ${VERSION} ${ENTRIES}
+    retry 6 assert_document ${TRACES_INDEX} "transaction.id" "4340a8e0df1906ecbfa9" ${VERSION} ${ENTRIES}
+    retry 6 assert_document ${METRICS_INDEX} "transaction.type" "request" ${VERSION} ${ENTRIES}
 }
 
 healthcheck() {
@@ -418,3 +418,23 @@ is_curl_fail_with_body() {
     fi
     return $HAS_FAIL_WITH_BODY
 }
+
+retry() {
+  local retries=$1
+  shift
+
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** count))
+    count=$((count + 1))
+    if [ $count -lt "$retries" ]; then
+      echo "-> Retry cmd: '$*';  $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      echo "-> Retry cmd: '$*'; $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+  done
+  return 0
+}