Stabilize Spanner integration tests and capture failure logs

.changeset/spanner-test-stability.md

@@ -0,0 +1,2 @@
+---
+---

.github/workflows/ci.yml

@@ -90,7 +90,19 @@ jobs:
           cache: true
       # Spanner tests use a Docker testcontainer emulator, not the embedded
       # postgres binary, so no Maven binary cache is needed here.
-      - run: go test -v -tags spanner_integration -timeout=10m ./...
+      - name: Spanner integration tests
+        run: |
+          mkdir -p test-output/go
+          set -o pipefail
+          go test -json -v -tags spanner_integration -timeout=10m ./... | tee test-output/go/spanner-integration.json
+      - name: Upload Spanner test log on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: spanner-integration-test-log
+          path: test-output/go/spanner-integration.json
+          if-no-files-found: ignore
+          retention-days: 7
 
   goreleaser-snapshot:
     runs-on: depot-ubuntu-24.04-8

internal/api/integration_test/flow_definition_test.go

@@ -15,8 +15,6 @@ import (
 )
 
 func TestCreateFlowDefinitionUnauthenticated(t *testing.T) {
-	t.Parallel()
-
 	userSchema := "https://some-tenant.com/schemas/unknown-user-schema.yaml"
 	userSchemaURI, err := url.Parse(userSchema)
 	require.NoError(t, err)
@@ -49,12 +47,11 @@ func TestCreateFlowDefinitionUnauthenticated(t *testing.T) {
 }
 
 func TestCreateFlowDefinition(t *testing.T) {
-	t.Parallel()
 	project, err := harness.EnsureProjectService(t).Create(t.Context(), nil)
 	require.NoError(t, err)
-	harness.CreateUserSchema(t, project.ID, harness.TestData.Schemas.CreateSchemaRequestUserSchema)
 
-	u := "https://raw.githubusercontent.com/zitadel/nextgen/refs/heads/main/api/openapi/endpoints/schemas/examples/user-schema-example.yaml"
+	u := flowDefinitionSchemaURL(t)
+	harness.EnsureUserSchemaWithID(t, project.ID, harness.TestData.Schemas.CreateSchemaRequestUserSchema, u)
 	userSchemaURI, err := url.Parse(u)
 	require.NoError(t, err)
 
@@ -290,7 +287,6 @@ func TestCreateFlowDefinition(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
 			client := harness.EnsureAPIClient(t, project.ID)
 			resp, err := client.CreateFlowDefinition(t.Context(), tt.req)
 			assert.NoError(t, err)
@@ -365,8 +361,12 @@ func validSteps() []api.FlowDefinitionStep {
 	}
 }
 
+func flowDefinitionSchemaURL(t *testing.T) string {
+	t.Helper()
+	return "https://test.example.schemas.com/integration/" + url.PathEscape(t.Name()) + ".json"
+}
+
 func TestGetFlowDefinitionUnauthenticated(t *testing.T) {
-	t.Parallel()
 	client := harness.EnsureAnonymousAPIClient(t)
 	getResp, err := client.GetFlowDefinition(t.Context(), api.GetFlowDefinitionParams{
 		ID:        "flowDef_1234",
@@ -385,11 +385,10 @@ func TestGetFlowDefinitionUnauthenticated(t *testing.T) {
 }
 
 func TestGetFlowDefinition(t *testing.T) {
-	t.Parallel()
 	project, err := harness.EnsureProjectService(t).Create(t.Context(), nil)
 	require.NoError(t, err)
-	harness.CreateUserSchema(t, project.ID, harness.TestData.Schemas.CreateSchemaRequestUserSchema)
-	u := "https://raw.githubusercontent.com/zitadel/nextgen/refs/heads/main/api/openapi/endpoints/schemas/examples/user-schema-example.yaml"
+	u := flowDefinitionSchemaURL(t)
+	harness.EnsureUserSchemaWithID(t, project.ID, harness.TestData.Schemas.CreateSchemaRequestUserSchema, u)
 	userSchemaURI, err := url.Parse(u)
 	require.NoError(t, err)
 
@@ -442,7 +441,6 @@ func TestGetFlowDefinition(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
 			client := harness.EnsureAPIClient(t, project.ID)
 			resp, err := client.GetFlowDefinition(t.Context(), tt.req)
 			assert.NoError(t, err)
@@ -452,8 +450,6 @@ func TestGetFlowDefinition(t *testing.T) {
 }
 
 func TestListFlowDefinitionsUnauthenticated(t *testing.T) {
-	t.Parallel()
-
 	client := harness.EnsureAnonymousAPIClient(t)
 	getResp, err := client.ListFlowDefinitions(t.Context(), api.ListFlowDefinitionsParams{
 		ProjectID: "proj_1234",
@@ -471,19 +467,18 @@ func TestListFlowDefinitionsUnauthenticated(t *testing.T) {
 }
 
 func TestListFlowDefinitions(t *testing.T) {
-	t.Parallel()
 	project1, err := harness.EnsureProjectService(t).Create(t.Context(), nil)
 	require.NoError(t, err)
-	harness.CreateUserSchema(t, project1.ID, harness.TestData.Schemas.CreateSchemaRequestUserSchema)
 
 	project2, err := harness.EnsureProjectService(t).Create(t.Context(), nil)
 	require.NoError(t, err)
-	harness.CreateUserSchema(t, project2.ID, harness.TestData.Schemas.CreateSchemaRequestUserSchema)
 
 	project3, err := harness.EnsureProjectService(t).Create(t.Context(), nil)
 	require.NoError(t, err)
 
-	u := "https://raw.githubusercontent.com/zitadel/nextgen/refs/heads/main/api/openapi/endpoints/schemas/examples/user-schema-example.yaml"
+	u := flowDefinitionSchemaURL(t)
+	harness.EnsureUserSchemaWithID(t, project1.ID, harness.TestData.Schemas.CreateSchemaRequestUserSchema, u)
+	harness.EnsureUserSchemaWithID(t, project2.ID, harness.TestData.Schemas.CreateSchemaRequestUserSchema, u)
 	userSchemaURI, err := url.Parse(u)
 	require.NoError(t, err)
 
@@ -666,7 +661,6 @@ func TestListFlowDefinitions(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
 			client := harness.EnsureAPIClient(t, project1.ID)
 			resp, err := client.ListFlowDefinitions(t.Context(), tt.req)
 			assert.NoError(t, err)
@@ -699,7 +693,6 @@ func TestListFlowDefinitions(t *testing.T) {
 }
 
 func TestDeleteFlowDefinitionUnauthenticated(t *testing.T) {
-	t.Parallel()
 	client := harness.EnsureAnonymousAPIClient(t)
 	resp, err := client.DeleteFlowDefinition(t.Context(), api.DeleteFlowDefinitionParams{
 		ID:        "flowDef_1234",
@@ -717,11 +710,10 @@ func TestDeleteFlowDefinitionUnauthenticated(t *testing.T) {
 }
 
 func TestDeleteFlowDefinition(t *testing.T) {
-	t.Parallel()
 	project, err := harness.EnsureProjectService(t).Create(t.Context(), nil)
 	require.NoError(t, err)
-	harness.CreateUserSchema(t, project.ID, harness.TestData.Schemas.CreateSchemaRequestUserSchema)
-	u := "https://raw.githubusercontent.com/zitadel/nextgen/refs/heads/main/api/openapi/endpoints/schemas/examples/user-schema-example.yaml"
+	u := flowDefinitionSchemaURL(t)
+	harness.EnsureUserSchemaWithID(t, project.ID, harness.TestData.Schemas.CreateSchemaRequestUserSchema, u)
 	userSchemaURI, err := url.Parse(u)
 	require.NoError(t, err)
 
@@ -776,7 +768,6 @@ func TestDeleteFlowDefinition(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
 			client := harness.EnsureAPIClient(t, project.ID)
 			resp, err := client.DeleteFlowDefinition(t.Context(), tt.req)
 			assert.NoError(t, err)

internal/api/integration_test/helpers/auth_attempt.go

@@ -10,27 +10,47 @@ import (
 
 func (h *Harness) EnsureAuthAttemptService(t *testing.T) service.AuthAttemptService {
 	t.Helper()
+	h.mu.Lock()
+	svc := h.AuthAttemptService
+	h.mu.Unlock()
+	if svc != nil {
+		return svc
+	}
+	svc = service.NewAuthAttemptService(
+		h.EnsureDBPool(t),
+		h.EnsureAuthAttemptRepo(t),
+		h.EnsureSessionRepo(t),
+		h.EnsureProjectRepo(t),
+		h.EnsureUserRepo(t),
+		h.EnsureUserPasswordRepo(t),
+		h.EnsureUserPasskeyRepo(t),
+		h.EnsureHashVerifier(t),
+	)
+	h.mu.Lock()
 	if h.AuthAttemptService == nil {
-		h.AuthAttemptService = service.NewAuthAttemptService(
-			h.EnsureDBPool(t),
-			h.EnsureAuthAttemptRepo(t),
-			h.EnsureSessionRepo(t),
-			h.EnsureProjectRepo(t),
-			h.EnsureUserRepo(t),
-			h.EnsureUserPasswordRepo(t),
-			h.EnsureUserPasskeyRepo(t),
-			h.EnsureHashVerifier(t),
-		)
+		h.AuthAttemptService = svc
 	}
-	return h.AuthAttemptService
+	svc = h.AuthAttemptService
+	h.mu.Unlock()
+	return svc
 }
 
 func (h *Harness) EnsureAuthAttemptRepo(t *testing.T) domain.AuthAttemptRepository {
 	t.Helper()
+	h.mu.Lock()
+	repo := h.AuthAttemptRepo
+	h.mu.Unlock()
+	if repo != nil {
+		return repo
+	}
+	repo = repository.NewAuthAttemptRepository(
+		h.EnsureDBPool(t),
+	)
+	h.mu.Lock()
 	if h.AuthAttemptRepo == nil {
-		h.AuthAttemptRepo = repository.NewAuthAttemptRepository(
-			h.EnsureDBPool(t),
-		)
+		h.AuthAttemptRepo = repo
 	}
-	return h.AuthAttemptRepo
+	repo = h.AuthAttemptRepo
+	h.mu.Unlock()
+	return repo
 }

internal/api/integration_test/helpers/client.go

@@ -11,56 +11,101 @@ import (
 
 func (h *Harness) EnsureAPIClient(t *testing.T, projectID string) *api.Client {
 	t.Helper()
-	serv := h.EnsureTestServer(t)
-	if h.apiClients == nil {
-		h.apiClients = make(map[string]*api.Client)
-	}
-	if client, ok := h.apiClients[projectID]; ok {
-		return client
+	h.mu.Lock()
+	if h.apiClients != nil {
+		if client, ok := h.apiClients[projectID]; ok {
+			h.mu.Unlock()
+			return client
+		}
 	}
+	h.mu.Unlock()
+
+	serv := h.EnsureTestServer(t)
 	client, err := api.NewClient(
 		serv.URL,
 		h.EnsureFakeSecuritySource(t, projectID),
 	)
 	require.NoError(t, err)
+	h.mu.Lock()
+	if h.apiClients == nil {
+		h.apiClients = make(map[string]*api.Client)
+	}
+	if existing, ok := h.apiClients[projectID]; ok {
+		h.mu.Unlock()
+		return existing
+	}
 	h.apiClients[projectID] = client
+	h.mu.Unlock()
 	return client
 }
 
 func (h *Harness) EnsureAnonymousAPIClient(t *testing.T) *api.Client {
 	t.Helper()
+	h.mu.Lock()
+	client := h.anonymousClient
+	h.mu.Unlock()
+	if client != nil {
+		return client
+	}
+
 	serv := h.EnsureTestServer(t)
+	client, err := api.NewClient(
+		serv.URL,
+		h.EnsureAnonymousSecuritySource(t),
+	)
+	require.NoError(t, err)
+	h.mu.Lock()
 	if h.anonymousClient == nil {
-		client, err := api.NewClient(
-			serv.URL,
-			h.EnsureAnonymousSecuritySource(t),
-		)
-		require.NoError(t, err)
 		h.anonymousClient = client
 	}
-	return h.anonymousClient
+	client = h.anonymousClient
+	h.mu.Unlock()
+	return client
 }
 
 func (h *Harness) EnsureFakeSecuritySource(t *testing.T, projectID string) *FakeSecuritySource {
 	t.Helper()
+	h.mu.Lock()
+	if h.fakeSecuritySources != nil {
+		if source, ok := h.fakeSecuritySources[projectID]; ok {
+			h.mu.Unlock()
+			return source
+		}
+	}
+	h.mu.Unlock()
+
+	source := &FakeSecuritySource{
+		projectID: projectID,
+	}
+	h.mu.Lock()
 	if h.fakeSecuritySources == nil {
 		h.fakeSecuritySources = make(map[string]*FakeSecuritySource)
 	}
 	if source, ok := h.fakeSecuritySources[projectID]; ok {
+		h.mu.Unlock()
 		return source
 	}
-	h.fakeSecuritySources[projectID] = &FakeSecuritySource{
-		projectID: projectID,
-	}
-	return h.fakeSecuritySources[projectID]
+	h.fakeSecuritySources[projectID] = source
+	h.mu.Unlock()
+	return source
 }
 
 func (h *Harness) EnsureAnonymousSecuritySource(t *testing.T) *FakeSecuritySource {
 	t.Helper()
+	h.mu.Lock()
+	source := h.anonymousSecuritySource
+	h.mu.Unlock()
+	if source != nil {
+		return source
+	}
+	source = &FakeSecuritySource{}
+	h.mu.Lock()
 	if h.anonymousSecuritySource == nil {
-		h.anonymousSecuritySource = &FakeSecuritySource{}
+		h.anonymousSecuritySource = source
 	}
-	return h.anonymousSecuritySource
+	source = h.anonymousSecuritySource
+	h.mu.Unlock()
+	return source
 }
 
 type FakeSecuritySource struct {