@@ -4,92 +4,133 @@ import (
44 "context"
55 "fmt"
66 "net/http"
7+ "strings"
8+ "sync"
79 "testing"
810 "time"
911
1012 "github.com/Khan/genqlient/graphql"
1113 "github.com/buildkite/agent-stack-k8s/v2/internal/integration/api"
14+ "github.com/buildkite/go-buildkite/v4"
1215 "github.com/buildkite/roko"
13- "github.com/stretchr/testify/assert"
14- "github.com/stretchr/testify/require"
1516)
1617
17- func TestCleanupOrphanedPipelines (t * testing.T ) {
18+ func TestCleanupOrphanedResources (t * testing.T ) {
1819 if ! cleanupPipelines {
1920 t .Skip ("not cleaning orphaned pipelines" )
2021 }
2122
2223 ctx := context .Background ()
2324 graphqlClient := api .NewGraphQLClient (cfg .BuildkiteToken , cfg .GraphQLEndpoint )
24- orgSlug := getOrgSlug (t )
25+ tc := testcase {T : t }.Init ()
26+ orgSlug := tc .Org
27+ clusterUUID := tc .ClusterUUID
2528
2629 allPipelines , err := searchAllPipelines (ctx , t , graphqlClient , orgSlug , "test-" )
27- require .NoError (t , err )
30+ if err != nil {
31+ t .Fatalf ("failed to search for pipelines: %v" , err )
32+ }
2833
2934 t .Logf ("found %d total pipelines to delete" , len (allPipelines ))
3035
36+ bkClient , err := buildkite .NewOpts (buildkite .WithTokenAuth (cfg .BuildkiteToken ))
37+ if err != nil {
38+ t .Fatalf ("failed to create buildkite client: %v" , err )
39+ }
40+
41+ // First, go through all the pipelines and cancel any running builds they might have
42+ wg := sync.WaitGroup {}
3143 for _ , pipeline := range allPipelines {
32- t .Run (pipeline .Node .Name , func (t * testing.T ) {
33- tc := testcase {
34- T : t ,
35- GraphQL : graphqlClient ,
36- PipelineName : pipeline .Node .Name ,
37- }.Init ()
38-
39- builds , err := api .GetBuilds (
40- ctx ,
41- graphqlClient ,
42- fmt .Sprintf ("%s/%s" , tc .Org , pipeline .Node .Name ),
43- []api.BuildStates {api .BuildStatesRunning },
44- 100 ,
45- )
46- require .NoError (t , err )
47-
48- for _ , build := range builds .Pipeline .Builds .Edges {
49- _ , err = api .BuildCancel (
50- ctx ,
51- graphqlClient ,
52- api.BuildCancelInput {Id : build .Node .Id },
53- )
54- assert .NoError (t , err )
44+ wg .Go (func () {
45+ cancelAllBuildsForPipeline (ctx , t , graphqlClient , orgSlug , pipeline .Node .Name )
46+
47+ // Then, delete the pipeline
48+ deletePipeline (ctx , t , bkClient , orgSlug , pipeline .Node .Name )
49+ })
50+ }
51+
52+ wg .Wait ()
53+
54+ // Now that all the pipelines have been deleted, go through and delete all the queues that are hanging around
55+ queues , _ , err := bkClient .ClusterQueues .List (ctx , orgSlug , clusterUUID , nil )
56+ if err != nil {
57+ t .Fatalf ("failed to list queues: %v" , err )
58+ }
59+
60+ testQueues := make ([]buildkite.ClusterQueue , 0 , len (queues ))
61+ for _ , queue := range queues {
62+ if strings .HasPrefix (queue .Key , "test-" ) {
63+ testQueues = append (testQueues , queue )
64+ }
65+ }
66+
67+ t .Logf ("found %d test queues to clean up" , len (testQueues ))
68+
69+ for _ , queue := range testQueues {
70+ wg .Go (func () {
71+ _ , err := bkClient .ClusterQueues .Delete (ctx , orgSlug , clusterUUID , queue .ID )
72+ if err != nil {
73+ t .Errorf ("failed to delete queue %q (%s): %v" , queue .Key , queue .ID , err )
74+ return
5575 }
5676
57- tc . deletePipeline ( ctx )
77+ t . Logf ( "deleted queue %q" , queue . Key )
5878 })
5979 }
80+
81+ wg .Wait ()
6082}
6183
62- func ( t testcase ) deletePipeline ( ctx context.Context ) {
84+ func cancelAllBuildsForPipeline ( ctx context.Context , t * testing. T , graphqlClient graphql. Client , orgSlug , pipelineSlug string ) {
6385 t .Helper ()
6486
65- EnsureCleanup (t .T , func () {
66- if err := roko .NewRetrier (
67- roko .WithMaxAttempts (10 ),
68- roko .WithStrategy (roko .Exponential (time .Second , 5 * time .Second )),
69- ).DoWithContext (ctx , func (r * roko.Retrier ) error {
70- resp , err := t .Buildkite .Pipelines .Delete (t .Org , t .PipelineName )
87+ builds , err := api .GetBuilds (ctx , graphqlClient , fmt .Sprintf ("%s/%s" , orgSlug , pipelineSlug ), []api.BuildStates {api .BuildStatesRunning }, 100 )
88+ if err != nil {
89+ t .Errorf ("failed to get builds for pipeline %q while attempting to delete it: %v" , pipelineSlug , err )
90+ return
91+ }
92+
93+ t .Logf ("Found %d builds to cancel for pipeline %q" , len (builds .Pipeline .Builds .Edges ), pipelineSlug )
94+
95+ wg := sync.WaitGroup {}
96+ for _ , build := range builds .Pipeline .Builds .Edges {
97+ wg .Go (func () {
98+ _ , err = api .BuildCancel (ctx , graphqlClient , api.BuildCancelInput {Id : build .Node .Id })
7199 if err != nil {
72- if resp .StatusCode == http .StatusNotFound {
73- return nil
74- }
75- t .Logf ("waiting for build to be canceled on pipeline %s" , t .PipelineName )
76- return err
100+ t .Errorf ("failed to cancel build %q while attempting to delete pipeline %q: %v" , build .Node .Id , pipelineSlug , err )
101+ return
77102 }
78- return nil
79- }); err != nil {
80- t .Errorf ("failed to cleanup pipeline %s: %v" , t .PipelineName , err )
81- return
82- }
83103
84- t .Logf ("deleted pipeline! %s" , t .PipelineName )
85- })
104+ t .Logf ("cancelled build %q" , build .Node .Id )
105+ })
106+ }
107+
108+ wg .Wait ()
86109}
87110
88- func getOrgSlug (t * testing.T ) string {
89- tc := testcase {
90- T : t ,
91- }.Init ()
92- return tc .Org
111+ func deletePipeline (ctx context.Context , t * testing.T , apiClient * buildkite.Client , orgSlug , pipelineSlug string ) {
112+ t .Helper ()
113+
114+ if err := roko .NewRetrier (
115+ roko .WithMaxAttempts (10 ),
116+ roko .WithStrategy (roko .Exponential (time .Second , 5 * time .Second )),
117+ ).DoWithContext (ctx , func (r * roko.Retrier ) error {
118+ resp , err := apiClient .Pipelines .Delete (ctx , orgSlug , pipelineSlug )
119+ if err != nil {
120+ if resp .StatusCode == http .StatusNotFound {
121+ return nil
122+ }
123+
124+ return err
125+ }
126+
127+ return nil
128+ }); err != nil {
129+ t .Errorf ("failed to delete pipeline %q: %v" , pipelineSlug , err )
130+ return
131+ }
132+
133+ t .Logf ("deleted pipeline %q" , pipelineSlug )
93134}
94135
95136// searchAllPipelines fetches all pipelines matching the search prefix, paginating through all results.
0 commit comments