|
| 1 | +package function |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "errors" |
| 6 | + "fmt" |
| 7 | + "net/http" |
| 8 | + "net/http/httptest" |
| 9 | + "strings" |
| 10 | + "testing" |
| 11 | + |
| 12 | + computepb "cloud.google.com/go/compute/apiv1/computepb" |
| 13 | + "google.golang.org/api/option" |
| 14 | +) |
| 15 | + |
| 16 | +// engine is what Compute Engine answers: zones from discovery (nil when it |
| 17 | +// fails), each zone's catalogue (absent when unreadable), each insert's error. |
| 18 | +type engine struct { |
| 19 | + zones []string |
| 20 | + catalogue map[string][]*MachineTypeInfo |
| 21 | + insert map[string]error |
| 22 | +} |
| 23 | + |
| 24 | +var fourVCPUs = []*MachineTypeInfo{{Name: "c3d-highcpu-4", Family: "c3d", VCPUs: 4, MemoryMB: 8192}} |
| 25 | + |
| 26 | +// queuedJob delivers the queued task for job "manifest" of run 7 asking for |
| 27 | +// preset "deploy", with the repository's config file and Compute as given, |
| 28 | +// and reports the status handed back. |
| 29 | +func queuedJob(t *testing.T, gh *github, config string, gce engine) int { |
| 30 | + t.Helper() |
| 31 | + t.Setenv("GCRUNNER_ZONES", "") |
| 32 | + t.Setenv("GCE_REGION", "europe-west3") |
| 33 | + t.Setenv("GCP_PROJECT", "p") |
| 34 | + useGitHub(t, gh) |
| 35 | + originalFetch, originalZones, originalOptions := fetchRepositoryFile, listZones, computeOptions |
| 36 | + t.Cleanup(func() { |
| 37 | + fetchRepositoryFile, listZones, computeOptions = originalFetch, originalZones, originalOptions |
| 38 | + configCache.configs = map[string]configCacheEntry{} |
| 39 | + zoneCache.zones = map[string]zoneCacheEntry{} |
| 40 | + machineTypeCache.types = map[string]machineTypeCacheEntry{} |
| 41 | + }) |
| 42 | + configCache.configs = map[string]configCacheEntry{} |
| 43 | + zoneCache.zones = map[string]zoneCacheEntry{} |
| 44 | + machineTypeCache.types = map[string]machineTypeCacheEntry{} |
| 45 | + zoneOffset.Store(0) |
| 46 | + empty := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, `{"items": {}}`) })) |
| 47 | + t.Cleanup(empty.Close) |
| 48 | + computeOptions = []option.ClientOption{option.WithEndpoint(empty.URL), option.WithoutAuthentication()} |
| 49 | + fetchRepositoryFile = func(context.Context, string, string, string, string) ([]byte, error) { return []byte(config), nil } |
| 50 | + listZones = func(context.Context, string, string) ([]string, error) { |
| 51 | + if gce.zones == nil { |
| 52 | + return nil, errors.New("compute unavailable") |
| 53 | + } |
| 54 | + return gce.zones, nil |
| 55 | + } |
| 56 | + withMachineTypes(t, func(_ context.Context, _, zone string) ([]*MachineTypeInfo, error) { |
| 57 | + if types, ok := gce.catalogue[zone]; ok { |
| 58 | + return types, nil |
| 59 | + } |
| 60 | + return nil, errors.New("compute unavailable in " + zone) |
| 61 | + }) |
| 62 | + withInsert(t, func(_ context.Context, zone string, _ *computepb.Instance) error { return gce.insert[zone] }) |
| 63 | + return deliverTask(t, "/task/queued", WorkflowJobEvent{ |
| 64 | + Action: "queued", |
| 65 | + WorkflowJob: WorkflowJob{ID: 42, Name: "manifest", RunID: 7, HeadSHA: "6d8b388", Labels: []string{"gcrunner=7/runner=deploy"}}, |
| 66 | + Repository: cloudRepository, |
| 67 | + }) |
| 68 | +} |
| 69 | + |
| 70 | +const ( |
| 71 | + twoVCPUsConfig = "runners:\n deploy:\n family: c3d\n cpu: 2\n ram: 8\n" |
| 72 | + exactNameConfig = "runners:\n deploy:\n machine: n2d-standard-999\n" |
| 73 | +) |
| 74 | + |
| 75 | +func TestAJobThatCanNeverStartFailsItsRunOnce(t *testing.T) { |
| 76 | + missing := errors.New(missingMachineTypeError) |
| 77 | + two := engine{zones: []string{"europe-west3-a", "europe-west3-b"}, catalogue: map[string][]*MachineTypeInfo{"europe-west3-a": fourVCPUs, "europe-west3-b": fourVCPUs}} |
| 78 | + for name, c := range map[string]struct { |
| 79 | + config string |
| 80 | + gce engine |
| 81 | + reason string |
| 82 | + }{ |
| 83 | + "preset not defined": {"runners:\n build:\n cpu: 4\n", two, `runner "deploy" is not defined`}, |
| 84 | + "no zone offers the size": {twoVCPUsConfig, two, "no machine type matching"}, |
| 85 | + "no zone offers the exact name": {exactNameConfig, engine{zones: two.zones, insert: map[string]error{"europe-west3-a": missing, "europe-west3-b": missing}}, "does not exist"}, |
| 86 | + } { |
| 87 | + gh := &github{checkStatus: http.StatusCreated} |
| 88 | + if code := queuedJob(t, gh, c.config, c.gce); code != http.StatusOK { |
| 89 | + t.Errorf("%s: status %d, want 200 so Cloud Tasks does not retry", name, code) |
| 90 | + } |
| 91 | + if len(gh.checks) != 1 || gh.cancels != 1 { |
| 92 | + t.Fatalf("%s: %d check runs and %d cancels, want one of each", name, len(gh.checks), gh.cancels) |
| 93 | + } |
| 94 | + check := gh.checks[0] |
| 95 | + output, _ := check["output"].(map[string]any) |
| 96 | + summary, _ := output["summary"].(string) |
| 97 | + if check["head_sha"] != "6d8b388" || check["conclusion"] != "failure" || output["title"] != "manifest cannot start" || !strings.Contains(summary, c.reason) { |
| 98 | + t.Errorf("%s: check run %v does not carry the reason %q", name, check, c.reason) |
| 99 | + } |
| 100 | + } |
| 101 | +} |
| 102 | + |
| 103 | +// Only Compute answering in every zone proves the workflow wrong. A zone |
| 104 | +// that cannot be read, a zone out of quota, guessed zones after discovery |
| 105 | +// failed, and GitHub itself being down all leave the task to be retried. |
| 106 | +func TestATransientFailureIsRetriedWithoutFailingTheRun(t *testing.T) { |
| 107 | + missing := errors.New(missingMachineTypeError) |
| 108 | + zones := []string{"europe-west3-a", "europe-west3-b"} |
| 109 | + for name, c := range map[string]struct { |
| 110 | + config string |
| 111 | + gce engine |
| 112 | + check int |
| 113 | + }{ |
| 114 | + "one catalogue unreadable": {twoVCPUsConfig, engine{zones: zones, catalogue: map[string][]*MachineTypeInfo{"europe-west3-a": fourVCPUs}}, http.StatusCreated}, |
| 115 | + "one zone out of quota": {exactNameConfig, engine{zones: zones, insert: map[string]error{"europe-west3-a": missing, "europe-west3-b": errors.New("QUOTA_EXCEEDED")}}, http.StatusCreated}, |
| 116 | + "zones guessed": {twoVCPUsConfig, engine{catalogue: map[string][]*MachineTypeInfo{"europe-west3-a": fourVCPUs, "europe-west3-b": fourVCPUs, "europe-west3-c": fourVCPUs}}, http.StatusCreated}, |
| 117 | + "GitHub down for the check": {twoVCPUsConfig, engine{zones: zones, catalogue: map[string][]*MachineTypeInfo{"europe-west3-a": fourVCPUs, "europe-west3-b": fourVCPUs}}, http.StatusBadGateway}, |
| 118 | + } { |
| 119 | + gh := &github{checkStatus: c.check} |
| 120 | + if code := queuedJob(t, gh, c.config, c.gce); code != http.StatusInternalServerError { |
| 121 | + t.Errorf("%s: status %d, want 500 so Cloud Tasks retries", name, code) |
| 122 | + } |
| 123 | + if gh.cancels != 0 { |
| 124 | + t.Errorf("%s: cancelled the run", name) |
| 125 | + } |
| 126 | + } |
| 127 | +} |
| 128 | + |
| 129 | +// Without checks: write the report cannot be made, but retrying would not |
| 130 | +// grant it, so the task is still acknowledged. |
| 131 | +func TestARefusedCheckRunIsNotRetried(t *testing.T) { |
| 132 | + gh := &github{checkStatus: http.StatusForbidden} |
| 133 | + if code := queuedJob(t, gh, "runners: {}\n", engine{}); code != http.StatusOK { |
| 134 | + t.Errorf("status %d, want 200", code) |
| 135 | + } |
| 136 | + if gh.cancels != 0 { |
| 137 | + t.Errorf("cancelled the run without having explained why") |
| 138 | + } |
| 139 | +} |
| 140 | + |
| 141 | +func withMachineTypes(t *testing.T, fn func(ctx context.Context, project, zone string) ([]*MachineTypeInfo, error)) { |
| 142 | + t.Helper() |
| 143 | + original := listMachineTypes |
| 144 | + listMachineTypes = fn |
| 145 | + t.Cleanup(func() { listMachineTypes = original }) |
| 146 | +} |
0 commit comments