INT-7: Add support for retry attempts in test results reporting (#1072)

* Add support for retry attempts in test results reporting

- Updated GitHub summary to include attempts in the report header.
- Enhanced JUnit reporter to add properties for retry attempts.
- Added tests for rendering reports with retry attempts in both JSON and GitHub formats.
- Modified TestResult struct to include attempts in the JSON output.

* updated time import position in code

* show retries in summary only when there was retry attempt made.

* fix whitespace linter issues

* refactor: replace custom string search function with strings.Contains

* remove white space

Author: vtvipul-saucelabs

internal/report/github/summary.go

@@ -44,25 +44,35 @@ func hasDevice(results []report.TestResult) bool {
 	return false
 }
 
+func hasRetries(results []report.TestResult) bool {
+	for _, t := range results {
+		if len(t.Attempts) > 1 {
+			return true
+		}
+	}
+	return false
+}
+
 func (r *Reporter) Render() {
 	if !r.isActive() {
 		return
 	}
 
 	endTime := time.Now()
 	hasDevices := hasDevice(r.results)
+	showRetries := hasRetries(r.results)
 	errors := 0
 	inProgress := 0
 
-	content := renderHeader(hasDevices)
+	content := renderHeader(hasDevices, showRetries)
 	for _, result := range r.results {
 		if result.Status == job.StateInProgress || result.Status == job.StateNew {
 			inProgress++
 		}
 		if result.Status == job.StateFailed || result.Status == job.StateError {
 			errors++
 		}
-		content += renderTestResult(result, hasDevices)
+		content += renderTestResult(result, hasDevices, showRetries)
 	}
 	content += renderFooter(errors, inProgress, len(r.results), endTime.Sub(r.startTime))
 
@@ -82,15 +92,21 @@ func (r *Reporter) ArtifactRequirements() []report.ArtifactType {
 	return []report.ArtifactType{}
 }
 
-func renderHeader(hasDevices bool) string {
+func renderHeader(hasDevices bool, showRetries bool) string {
 	deviceTitle := ""
 	deviceSeparator := ""
 	if hasDevices {
 		deviceTitle = " Device |"
 		deviceSeparator = " --- |"
 	}
-	content := fmt.Sprintf("| | Name | Duration | Status | Browser | Platform |%s\n", deviceTitle)
-	content += fmt.Sprintf("| --- | --- | --- | --- | --- | --- |%s\n", deviceSeparator)
+	retriesTitle := ""
+	retriesSeparator := ""
+	if showRetries {
+		retriesTitle = " Attempts |"
+		retriesSeparator = " --- |"
+	}
+	content := fmt.Sprintf("| | Name | Duration | Status | Browser | Platform |%s%s\n", deviceTitle, retriesTitle)
+	content += fmt.Sprintf("| --- | --- | --- | --- | --- | --- |%s%s\n", deviceSeparator, retriesSeparator)
 	return content
 }
 
@@ -109,17 +125,21 @@ func statusToEmoji(status string) string {
 	}
 }
 
-func renderTestResult(t report.TestResult, hasDevices bool) string {
+func renderTestResult(t report.TestResult, hasDevices bool, showRetries bool) string {
 	content := ""
 
 	mark := statusToEmoji(t.Status)
 	deviceValue := ""
 	if hasDevices {
 		deviceValue = fmt.Sprintf(" %s |", t.DeviceName)
 	}
+	retriesValue := ""
+	if showRetries {
+		retriesValue = fmt.Sprintf(" %d |", len(t.Attempts))
+	}
 
-	content += fmt.Sprintf("| %s | [%s](%s) | %.0fs | %s | %s | %s |%s\n",
-		mark, t.Name, t.URL, t.Duration.Seconds(), t.Status, t.Browser, t.Platform, deviceValue)
+	content += fmt.Sprintf("| %s | [%s](%s) | %.0fs | %s | %s | %s |%s%s\n",
+		mark, t.Name, t.URL, t.Duration.Seconds(), t.Status, t.Browser, t.Platform, deviceValue, retriesValue)
 	return content
 }
 

internal/report/github/summary_test.go

@@ -0,0 +1,72 @@
+package github
+
+import (
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/saucelabs/saucectl/internal/job"
+	"github.com/saucelabs/saucectl/internal/report"
+)
+
+func TestReporter_RenderWithAttempts(t *testing.T) {
+	f, err := os.CreateTemp("", "github-step-summary")
+	if err != nil {
+		t.Fatalf("Failed to create temp file: %s", err)
+	}
+	defer os.Remove(f.Name())
+	f.Close()
+
+	r := &Reporter{
+		startTime:       time.Now(),
+		stepSummaryFile: f.Name(),
+	}
+
+	r.Add(report.TestResult{
+		Name:     "Login Tests",
+		Duration: 90 * time.Second,
+		Status:   job.StatePassed,
+		Browser:  "Chrome 120",
+		Platform: "Windows 11",
+		URL:      "https://app.saucelabs.com/tests/job-3",
+		Attempts: []report.Attempt{
+			{ID: "job-1", Status: job.StateFailed},
+			{ID: "job-2", Status: job.StateFailed},
+			{ID: "job-3", Status: job.StatePassed},
+		},
+	})
+	r.Add(report.TestResult{
+		Name:     "Checkout Tests",
+		Duration: 45 * time.Second,
+		Status:   job.StatePassed,
+		Browser:  "Firefox 121",
+		Platform: "macOS 13",
+		URL:      "https://app.saucelabs.com/tests/job-4",
+		Attempts: []report.Attempt{
+			{ID: "job-4", Status: job.StatePassed},
+		},
+	})
+
+	r.Render()
+
+	data, err := os.ReadFile(f.Name())
+	if err != nil {
+		t.Fatalf("Failed to read output file: %s", err)
+	}
+
+	content := string(data)
+
+	// Verify header has Attempts column
+	if !strings.Contains(content, "Attempts |") {
+		t.Errorf("Expected 'Attempts' column in header, got:\n%s", content)
+	}
+	// Verify Login Tests row shows 3 attempts
+	if !strings.Contains(content, "| 3 |") {
+		t.Errorf("Expected '| 3 |' for Login Tests attempts, got:\n%s", content)
+	}
+	// Verify Checkout Tests row shows 1 attempt
+	if !strings.Contains(content, "| 1 |") {
+		t.Errorf("Expected '| 1 |' for Checkout Tests attempts, got:\n%s", content)
+	}
+}

internal/report/json/json_test.go

@@ -0,0 +1,99 @@
+package json
+
+import (
+	"encoding/json"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/saucelabs/saucectl/internal/job"
+	"github.com/saucelabs/saucectl/internal/report"
+)
+
+func TestReporter_RenderWithAttempts(t *testing.T) {
+	f, err := os.CreateTemp("", "saucectl-report.json")
+	if err != nil {
+		t.Fatalf("Failed to create temp file: %s", err)
+	}
+	defer os.Remove(f.Name())
+	f.Close()
+
+	r := &Reporter{
+		Filename: f.Name(),
+		Results: []report.TestResult{
+			{
+				Name:     "Login Tests",
+				Duration: 90 * time.Second,
+				Status:   job.StatePassed,
+				Browser:  "Chrome 120",
+				Platform: "Windows 11",
+				Attempts: []report.Attempt{
+					{ID: "job-1", Status: job.StateFailed, Duration: 30 * time.Second},
+					{ID: "job-2", Status: job.StateFailed, Duration: 28 * time.Second},
+					{ID: "job-3", Status: job.StatePassed, Duration: 25 * time.Second},
+				},
+			},
+		},
+	}
+	r.Render()
+
+	data, err := os.ReadFile(f.Name())
+	if err != nil {
+		t.Fatalf("Failed to read output file: %s", err)
+	}
+
+	// Parse back to verify structure
+	var results []report.TestResult
+	if err := json.Unmarshal(data, &results); err != nil {
+		t.Fatalf("Failed to parse JSON output: %s", err)
+	}
+
+	if len(results) != 1 {
+		t.Fatalf("Expected 1 result, got %d", len(results))
+	}
+	if len(results[0].Attempts) != 3 {
+		t.Fatalf("Expected 3 attempts, got %d", len(results[0].Attempts))
+	}
+	if results[0].Attempts[0].Status != job.StateFailed {
+		t.Errorf("Expected attempt 0 status %q, got %q", job.StateFailed, results[0].Attempts[0].Status)
+	}
+	if results[0].Attempts[0].ID != "job-1" {
+		t.Errorf("Expected attempt 0 ID %q, got %q", "job-1", results[0].Attempts[0].ID)
+	}
+	if results[0].Attempts[2].Status != job.StatePassed {
+		t.Errorf("Expected attempt 2 status %q, got %q", job.StatePassed, results[0].Attempts[2].Status)
+	}
+}
+
+func TestReporter_RenderNoAttempts(t *testing.T) {
+	f, err := os.CreateTemp("", "saucectl-report.json")
+	if err != nil {
+		t.Fatalf("Failed to create temp file: %s", err)
+	}
+	defer os.Remove(f.Name())
+	f.Close()
+
+	r := &Reporter{
+		Filename: f.Name(),
+		Results: []report.TestResult{
+			{
+				Name:     "Simple Test",
+				Duration: 10 * time.Second,
+				Status:   job.StatePassed,
+			},
+		},
+	}
+	r.Render()
+
+	data, err := os.ReadFile(f.Name())
+	if err != nil {
+		t.Fatalf("Failed to read output file: %s", err)
+	}
+
+	// With omitempty, the "attempts" key should not appear when slice is empty
+	raw := string(data)
+	if strings.Contains(raw, `"attempts"`) {
+		t.Errorf("Expected no 'attempts' key in JSON when Attempts is empty, got:\n%s", raw)
+	}
+}

internal/report/junit/junit.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"strconv"
 	"sync"
+	"time"
 
 	"github.com/rs/zerolog/log"
 	"github.com/saucelabs/saucectl/internal/junit"
@@ -94,6 +95,22 @@ func extractProperties(r report.TestResult) []junit.Property {
 		},
 	}
 
+	// Add retry attempt properties when more than one attempt was made.
+	if len(r.Attempts) > 1 {
+		props = append(props,
+			junit.Property{Name: "retried", Value: "true"},
+			junit.Property{Name: "retries", Value: strconv.Itoa(len(r.Attempts))},
+		)
+		for i, a := range r.Attempts {
+			prefix := fmt.Sprintf("attempt.%d.", i)
+			props = append(props,
+				junit.Property{Name: prefix + "id", Value: a.ID},
+				junit.Property{Name: prefix + "status", Value: a.Status},
+				junit.Property{Name: prefix + "duration", Value: fmt.Sprintf("%.0f", a.Duration.Truncate(time.Second).Seconds())},
+			)
+		}
+	}
+
 	var filtered []junit.Property
 	for _, p := range props {
 		// we don't want to display properties with empty values

internal/report/junit/junit_test.go

@@ -92,6 +92,47 @@ func TestReporter_Render(t *testing.T) {
     </properties>
   </testsuite>
 </testsuites>
+`,
+		},
+		{
+			name: "with retries",
+			fields: fields{
+				TestResults: []report.TestResult{
+					{
+						Name:     "Chrome",
+						Duration: 90 * time.Second,
+						Status:   job.StatePassed,
+						Browser:  "Chrome",
+						Platform: "Windows 11",
+						URL:      "https://app.saucelabs.com/tests/job-3",
+						Attempts: []report.Attempt{
+							{ID: "job-1", Status: job.StateFailed, Duration: 30 * time.Second},
+							{ID: "job-2", Status: job.StateFailed, Duration: 28 * time.Second},
+							{ID: "job-3", Status: job.StatePassed, Duration: 25 * time.Second},
+						},
+					},
+				},
+			},
+			want: `<testsuites>
+  <testsuite name="Chrome" tests="0" time="90">
+    <properties>
+      <property name="url" value="https://app.saucelabs.com/tests/job-3"></property>
+      <property name="browser" value="Chrome"></property>
+      <property name="platform" value="Windows 11"></property>
+      <property name="retried" value="true"></property>
+      <property name="retries" value="3"></property>
+      <property name="attempt.0.id" value="job-1"></property>
+      <property name="attempt.0.status" value="failed"></property>
+      <property name="attempt.0.duration" value="30"></property>
+      <property name="attempt.1.id" value="job-2"></property>
+      <property name="attempt.1.status" value="failed"></property>
+      <property name="attempt.1.duration" value="28"></property>
+      <property name="attempt.2.id" value="job-3"></property>
+      <property name="attempt.2.status" value="passed"></property>
+      <property name="attempt.2.duration" value="25"></property>
+    </properties>
+  </testsuite>
+</testsuites>
 `,
 		},
 	}

internal/report/report.go

@@ -39,7 +39,7 @@ type TestResult struct {
 	RDC           bool          `json:"-"`
 	TimedOut      bool          `json:"-"`
 	PassThreshold bool          `json:"-"`
-	Attempts      []Attempt     `json:"-"`
+	Attempts      []Attempt     `json:"attempts,omitempty"`
 }
 
 // ArtifactType represents the type of assets (e.g. a junit report). Semantically similar to Content-Type.