Merge pull request #3599 from buildkite/pb-968-create-a-testing-framework

Add end-to-end testing infra

Author: DrJosh9000

.buildkite/Dockerfile-e2e

@@ -0,0 +1 @@
+FROM public.ecr.aws/docker/library/golang:1.24.10@sha256:7b13449f08287fdb53114d65bdf20eb3965e4e54997903b5cb9477df0ea37c12

.buildkite/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.5'
-
 services:
   lint:
     build:
@@ -32,6 +30,23 @@ services:
       - GOMODCACHE=/gomodcache
       - BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN
 
+  e2e:
+    build:
+      context: .
+      dockerfile: Dockerfile-e2e
+    volumes:
+      - ../:/work:cached
+    working_dir: /work
+    environment:
+      - BUILDKITE_BUILD_NUMBER
+      - BUILDKITE_JOB_ID
+      - BUILDKITE_BUILD_CREATOR_EMAIL
+      - BUILDKITE_BUILD_CREATOR
+      - CI_E2E_TESTS_BUILDKITE_API_TOKEN
+      - CI_E2E_TESTS_AGENT_TOKEN
+      - CI_E2E_TESTS_TARGET_ORG
+      - CI_E2E_TESTS_TARGET_CLUSTER
+
   ruby:
     image: ruby:2.7.6
     volumes:

.buildkite/pipeline.e2e.yml

@@ -0,0 +1,19 @@
+agents:
+  queue: agent-runners-linux-amd64
+
+env:
+  CI_E2E_TESTS_TARGET_ORG: agent-e2e-testing
+  CI_E2E_TESTS_TARGET_CLUSTER: e1773a06-b43c-4b77-84e5-5e088a737f6a
+
+steps:
+  - label: ":go::sleuth_or_spy: Run agent end-to-end tests"
+    command: .buildkite/steps/e2e-tests.sh
+    secrets:
+      - CI_E2E_TESTS_BUILDKITE_API_TOKEN
+      - CI_E2E_TESTS_AGENT_TOKEN
+    plugins:
+      - docker-compose#v4.14.0:
+          config: .buildkite/docker-compose.yml
+          cli-version: 2
+          mount-buildkite-agent: true
+          run: e2e

.buildkite/steps/e2e-tests.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# TODO: download the agent to test as an artifact
+#buildkite-agent artifact download buildkite-agent . --build "${BUILDKITE_TRIGGERED_FROM_BUILD_ID}"
+#export CI_E2E_TESTS_AGENT_PATH="${PWD}/buildkite-agent"
+
+# For now, e2e test the agent that's currently running
+export CI_E2E_TESTS_AGENT_PATH="$(which buildkite-agent)"
+go test -v -tags e2e ./internal/e2e/... | sed -e 's/^/>  /'

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/aws/smithy-go v1.24.0
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.3.0
+	github.com/buildkite/go-buildkite/v4 v4.11.0
 	github.com/buildkite/go-pipeline v0.16.0
 	github.com/buildkite/interpolate v0.1.5
 	github.com/buildkite/roko v1.4.0
@@ -112,6 +113,7 @@ require (
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
 	github.com/buildkite/test-engine-client v1.6.0 // indirect
+	github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect

go.sum

@@ -132,6 +132,8 @@ github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf/go.mod h1:CeKhh8xSs3WZAc50xABMxu+FlfAAd5PNumo7NfOv7EE=
 github.com/buildkite/bintest/v3 v3.3.0 h1:RTWcSaJRlOT6t/K311ejPf+0J3LE/QEODzVG3vlLnWo=
 github.com/buildkite/bintest/v3 v3.3.0/go.mod h1:btqpTsVODiJcb0NMdkkmtMQ6xoFc2W/nY5yy+3I0zcs=
+github.com/buildkite/go-buildkite/v4 v4.11.0 h1:rEvvUwITrqv433W9JWf6mj+NkkcM45s+ObhNs6C17i4=
+github.com/buildkite/go-buildkite/v4 v4.11.0/go.mod h1:DlebrRJqpZttXDjCW+MJ1QyW9AN++ZWt/UbPtKdbSSk=
 github.com/buildkite/go-pipeline v0.16.0 h1:wEgWUMRAgSg1ZnWOoA3AovtYYdTvN0dLY1zwUWmPP+4=
 github.com/buildkite/go-pipeline v0.16.0/go.mod h1:VE37qY3X5pmAKKUMoDZvPsHOQuyakB9cmXj9Qn6QasA=
 github.com/buildkite/interpolate v0.1.5 h1:v2Ji3voik69UZlbfoqzx+qfcsOKLA61nHdU79VV+tPU=
@@ -144,6 +146,8 @@ github.com/buildkite/test-engine-client v1.6.0 h1:yk/gdkFFU8B1+M16mxPNmxJgVoYffI
 github.com/buildkite/test-engine-client v1.6.0/go.mod h1:J6LrqenaJPfVCffiWW1/QxjICFb+OkqCvdCd7qAI0AE=
 github.com/buildkite/zstash v0.5.0 h1:e70mf8U2EjEB1eixXR78s6bsLgfo6bWLisVlRv58wCI=
 github.com/buildkite/zstash v0.5.0/go.mod h1:h70JfAEa2Ys1GDQQ6CNoKIMfMgJ0LZkNmQnzK710PHQ=
+github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf h1:yxlp0s+Sge9UsKEK0Bsvjiopb9XRk+vxylmZ9eGBfm8=
+github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

internal/e2e/basic_test.go

@@ -0,0 +1,28 @@
+//go:build e2e
+
+package e2e
+
+import "testing"
+
+func TestBasicE2E(t *testing.T) {
+	ctx := t.Context()
+	tc := newTestCase(t, `
+agents:
+  queue: {{.queue}}
+steps:
+  - command: echo hello world
+`)
+
+	tc.startAgent()
+
+	build := tc.triggerBuild()
+	state, err := tc.waitForBuild(ctx, build)
+	if err != nil {
+		t.Fatalf("tc.waitForBuild(build %s) error = %v", build.ID, err)
+	}
+	if got, want := state, "passed"; got != want {
+		t.Errorf("Build state = %q, want %q", got, want)
+	}
+
+	// TODO: add ability to inspect job logs
+}

internal/e2e/doc.go

@@ -0,0 +1,4 @@
+// Package e2e holds the end-to-end tests and test framework.
+// Test files are tagged go:build e2e so they are not run by default
+// (e.g. with plain `go test ./...`).
+package e2e