Retry creating and getting queue, submitting job

Signed-off-by: Enrico Minack <github@enrico.minack.dev>

Author: EnricoMi

.github/workflows/integration.yml

@@ -23,10 +23,73 @@ jobs:
         run: make kind-all
 
       - name: Create test queue
-        run: ./bin/app/armadactl create queue test
+        run: |
+          # Create test queue
+          for second in {1..60}
+          do
+            if ./bin/app/armadactl create queue test 2> armadactl.log
+            then
+              if [[ second -eq 1 ]]
+              then
+                echo "Successfully created queue 'test'"
+              else
+                echo "Successfully created queue 'test' after ${second}s"
+                echo "Previously failed due to:"
+                cat armadactl.log
+              fi
+              exit
+            fi
+            sleep 1
+          done
+
+          echo "Failed to create queue 'test' after ${second}s"
+          cat armadactl.log >&2
+          exit 1
 
       - name: Test queue exists
-        run: ./bin/app/armadactl get queue test
+        run: |
+          # Test queue exists
+          for second in {1..60}
+          do
+            if ./bin/app/armadactl get queue test 2> armadactl.log
+            then
+              if [[ second -eq 1 ]]
+              then
+                echo "Successfully inspected queue 'test'"
+              else
+                echo "Successfully inspected queue 'test' after ${second}s"
+                echo "Previously failed due to:"
+                cat armadactl.log
+              fi
+              exit
+            fi
+            sleep 1
+          done
+
+          echo "Failed to inspected queue 'test' after ${second}s"
+          cat armadactl.log >&2
+          exit 1
 
       - name: Submit job
-        run: ./bin/app/armadactl submit dev/quickstart/example-job.yaml
+        run: |
+          # Submit job
+          for second in {1..60}
+          do
+            if ./bin/app/armadactl submit dev/quickstart/example-job.yaml 2> armadactl.log
+            then
+              if [[ attempt -eq 1 ]]
+              then
+                echo "Successfully submitted job"
+              else
+                echo "Successfully submitted job after ${second}s"
+                echo "Previously failed due to:"
+                cat armadactl.log
+              fi
+              exit
+            fi
+            sleep 1
+          done
+
+          echo "Failed to submitted job after ${second}s"
+          cat armadactl.log >&2
+          exit 1