Keep ESS stack after CI run for troubleshooting

.buildkite/bk.integration.pipeline.yml

@@ -682,51 +682,51 @@ steps:
           - *google_oidc_plugin
           - *gcp_serverless_secrets_plugin
 
-  - label: ESS stack cleanup
-    depends_on:
-      - integration-tests-ubuntu
-      - integration-tests-win
-      - integration-tests-rhel
-      - integration-tests-kubernetes
-      - extended-integration-tests
-      - integration-tests-debian
-    allow_dependency_failure: true
-    command: |
-      buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess"
-      ls -lah test_infra/ess
-      .buildkite/scripts/steps/ess_down.sh
-    agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
-      useCustomGlobalHooks: true
-    plugins:
-      - *vault_ec_key_prod
-  - label: Aggregate test reports
-    # Warning: The key has a hook in pre-command
-    key: aggregate-reports
-    depends_on:
-      - integration-tests-ech
-      - integration-tests-ubuntu
-      - integration-tests-win
-      - integration-tests-rhel
-      - integration-tests-kubernetes
-      - integration-tests-serverless
-      - integration-tests-debian
-    allow_dependency_failure: true
-    command: |
-      buildkite-agent artifact download "build/*.xml" .
-      buildkite-agent artifact download "build\*.xml" .
-    agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
-      useCustomGlobalHooks: true
-    soft_fail:
-      - exit_status: "*"
-    plugins:
-      - elastic/vault-secrets#v0.1.0:
-          path: "kv/ci-shared/platform-ingest/buildkite_analytics_token"
-          field: "token"
-          env_var: "BUILDKITE_ANALYTICS_TOKEN"
-      - test-collector#v1.11.0:
-          files: "build/*.xml"
-          format: "junit"
-          branches: "main"
-          debug: true
+#  - label: ESS stack cleanup
+#    depends_on:
+#      - integration-tests-ubuntu
+#      - integration-tests-win
+#      - integration-tests-rhel
+#      - integration-tests-kubernetes
+#      - extended-integration-tests
+#      - integration-tests-debian
+#    allow_dependency_failure: true
+#    command: |
+#      buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess"
+#      ls -lah test_infra/ess
+#      .buildkite/scripts/steps/ess_down.sh
+#    agents:
+#      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+#      useCustomGlobalHooks: true
+#    plugins:
+#      - *vault_ec_key_prod
+#  - label: Aggregate test reports
+#    # Warning: The key has a hook in pre-command
+#    key: aggregate-reports
+#    depends_on:
+#      - integration-tests-ech
+#      - integration-tests-ubuntu
+#      - integration-tests-win
+#      - integration-tests-rhel
+#      - integration-tests-kubernetes
+#      - integration-tests-serverless
+#      - integration-tests-debian
+#    allow_dependency_failure: true
+#    command: |
+#      buildkite-agent artifact download "build/*.xml" .
+#      buildkite-agent artifact download "build\*.xml" .
+#    agents:
+#      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+#      useCustomGlobalHooks: true
+#    soft_fail:
+#      - exit_status: "*"
+#    plugins:
+#      - elastic/vault-secrets#v0.1.0:
+#          path: "kv/ci-shared/platform-ingest/buildkite_analytics_token"
+#          field: "token"
+#          env_var: "BUILDKITE_ANALYTICS_TOKEN"
+#      - test-collector#v1.11.0:
+#          files: "build/*.xml"
+#          format: "junit"
+#          branches: "main"
+#          debug: true

.buildkite/scripts/steps/ess.sh

@@ -8,7 +8,7 @@ function ess_up() {
   local STACK_VERSION=$1
   local STACK_BUILD_ID=${2:-""}
   local ESS_REGION=${3:-"gcp-us-west2"}
-    
+
   if [ -z "$STACK_VERSION" ]; then
     echo "Error: Specify stack version: ess_up [stack_version]" >&2
     return 1
@@ -17,8 +17,8 @@ function ess_up() {
   BUILDKITE_BUILD_CREATOR="${BUILDKITE_BUILD_CREATOR:-"$(get_git_user_email)"}"
   BUILDKITE_BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-"0"}"
   BUILDKITE_PIPELINE_SLUG="${BUILDKITE_PIPELINE_SLUG:-"elastic-agent-integration-tests"}"
-  
-  pushd "${TF_DIR}"    
+
+  pushd "${TF_DIR}"
   terraform init
   terraform apply \
     -auto-approve \
@@ -44,7 +44,7 @@ function ess_down() {
   local ESS_REGION=${1:-"gcp-us-west2"}
   local WORKSPACE=$(git rev-parse --show-toplevel)
   local TF_DIR="${WORKSPACE}/test_infra/ess/"
-  
+
   pushd "${TF_DIR}"
   terraform init
   terraform destroy -auto-approve \
@@ -54,17 +54,58 @@ function ess_down() {
 
 function get_git_user_email() {
   if ! git rev-parse --is-inside-work-tree &>/dev/null; then
-    echo "unknown"  
+    echo "unknown"
     return
   fi
 
   local email
   email=$(git config --get user.email)
-  
+
   if [ -z "$email" ]; then
-    echo "unknown"  
+    echo "unknown"
   else
     echo "$email"
   fi
 }
 
+function set_default_number_of_replicas() {
+  if [ -z "$ELASTICSEARCH_HOST" ]; then
+    echo "Error: Elasticsearch hostname not specified via ELASTICSEARCH_HOST environment variable"
+    return 3
+  fi
+
+  if [ -z "$ELASTICSEARCH_USERNAME" ]; then
+    echo "Error: Elasticsearch username not specified via ELASTICSEARCH_USERNAME environment variable"
+    return 4
+  fi
+
+  if [ -z "$ELASTICSEARCH_PASSWORD" ]; then
+    echo "Error: Elasticsearch password not specified via ELASTICSEARCH_PASSWORD environment variable"
+    return 5
+  fi
+
+  resp=$(curl \
+    -s \
+    --fail-with-body \
+    -X PUT \
+    -u "${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+    -H "Content-Type: application/json" \
+    -d '{
+      "index_patterns": ["*"],
+      "template": {
+        "settings": {
+          "number_of_replicas": 0
+        }
+      }
+    }'\
+    "${ELASTICSEARCH_HOST}/_index_template/global_default_replicas")
+
+  echo "$resp"
+
+  # Parse response body for any errors
+  num_errors=$(echo "$resp" | jq '.items[].statusCode | select(.>=400)' | wc -l)
+  if [ "$num_errors" -gt 0 ]; then
+    echo "$resp"
+    return 6
+  fi
+}

.buildkite/scripts/steps/ess_start.sh

@@ -15,6 +15,7 @@ ESS_REGION="${ESS_REGION:-gcp-us-west2}"
 
 ess_up "$STACK_VERSION" "$STACK_BUILD_ID" "$ESS_REGION"
 
+# set_default_number_of_replicas
 preinstall_fleet_packages
 
 echo "ES_HOST: ${ELASTICSEARCH_HOST}"

test_infra/ess/deployment.tf

@@ -113,6 +113,9 @@ resource "ec_deployment" "integration-testing" {
     }
     config = {
       docker_image = local.elasticsearch_docker_image
+      user_settings_json = jsonencode({
+        "cluster.max_shards_per_node" = 2000
+      })
     }
   }
   kibana = {