playwright.go

package runner

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"slices"

	"github.com/buildkite/test-engine-client/internal/debug"
	"github.com/buildkite/test-engine-client/internal/plan"
	"github.com/kballard/go-shellquote"
)

type Playwright struct {
	RunnerConfig
}

func (p Playwright) Name() string {
	return "Playwright"
}

func NewPlaywright(p RunnerConfig) Playwright {
	if p.TestCommand == "" {
		p.TestCommand = "npx playwright test"
	}

	if p.TestFilePattern == "" {
		p.TestFilePattern = "**/{*.spec,*.test}.{ts,js}"
	}

	return Playwright{
		RunnerConfig: p,
	}
}

func (p Playwright) SupportedFeatures() SupportedFeatures {
	return SupportedFeatures{
		SplitByFile:     true,
		SplitByExample:  false,
		FilterTestFiles: true,
		AutoRetry:       true,
		Mute:            true,
		Skip:            false,
	}
}

func (p Playwright) Run(result *RunResult, testCases []plan.TestCase, retry bool) error {
	testPaths := make([]string, len(testCases))
	for i, tc := range testCases {
		testPaths[i] = tc.Path
	}

	cmdName, cmdArgs, err := p.commandNameAndArgs(p.TestCommand, testPaths)
	if err != nil {
		return fmt.Errorf("failed to build command: %w", err)
	}

	cmd := exec.Command(cmdName, cmdArgs...)

	err = runAndForwardSignal(cmd)

	if ProcessSignaledError := new(ProcessSignaledError); errors.As(err, &ProcessSignaledError) {
		return err
	}

	report, parseErr := p.parseReport(p.ResultPath)
	if parseErr != nil {
		fmt.Printf("Buildkite Test Engine Client: Failed to read Playwright output, tests will not be retried: %v", parseErr)
		return err
	}

	for _, suite := range report.Suites {
		testResults := p.getTestResultsFromSuite(suite, suite.Title)
		for _, testResult := range testResults {
			result.RecordTestResult(testResult.TestCase, testResult.Status)
		}
	}

	if len(report.Errors) > 0 {
		result.error = fmt.Errorf("Playwright failed with errors")
	}

	return nil

}

// getTestCasesFromSuite recursively traverses the Playwright report suite and returns all test cases.
// Playwright's report format is a tree structure, where each suite can contain multiple specs and sub-suites.
// The function traverses the tree and collects failed test cases from the leaf nodes.
func (p Playwright) getTestResultsFromSuite(suite PlaywrightReportSuite, suiteName string) []TestResult {
	var testResults []TestResult

	for _, spec := range suite.Specs {
		projectName := spec.Tests[0].ProjectName
		var status TestStatus
		if !spec.Ok {
			status = TestStatusFailed
		} else if spec.Tests[0].Status == "skipped" {
			status = TestStatusSkipped
		} else {
			status = TestStatusPassed
		}

		testResults = append(testResults, TestResult{
			TestCase: plan.TestCase{
				Name: spec.Title,
				Path: fmt.Sprintf("%s:%d", spec.File, spec.Line),
				// The scope has to match with the scope generated by Buildkite test collector.
				// In Buildkite test collector, the scope is generated using Playwright built-in reporter function, titlePath().
				// titlePath function returns an array of suite's title from the root suite down to the current test,
				// which is then joined with a space separator to form the scope.
				// For more details, see:
				// [Buildkite Test Collector - Playwright implementation](https://github.com/buildkite/test-collector-javascript/blob/42b803a618a15a07edf0169038ef4b5eba88f98d/playwright/reporter.js#L47)
				// [Playwright titlePath implementation](https://github.com/microsoft/playwright/blob/523e50088a7f982dd96aacdb260dfbd1189159b1/packages/playwright/src/common/test.ts#L126)
				// [Playwright suite structure](https://playwright.dev/docs/api/class-suite)
				Scope: fmt.Sprintf(" %s %s %s", projectName, suiteName, spec.Title),
			},
			Status: status,
		})
	}

	for _, subSuite := range suite.Suites {
		testResults = append(testResults, p.getTestResultsFromSuite(subSuite, fmt.Sprintf("%s %s", suiteName, subSuite.Title))...)
	}

	return testResults
}

func (p Playwright) commandNameAndArgs(cmd string, testCases []string) (string, []string, error) {
	words, err := shellquote.Split(cmd)
	if err != nil {
		return "", []string{}, err
	}
	idx := slices.Index(words, "{{testExamples}}")
	if idx < 0 {
		words = append(words, testCases...)
	} else {
		words = slices.Replace(words, idx, idx+1, testCases...)
	}

	return words[0], words[1:], nil
}

func (p Playwright) parseReport(path string) (PlaywrightReport, error) {
	var report PlaywrightReport
	data, err := os.ReadFile(path)
	if err != nil {
		return PlaywrightReport{}, fmt.Errorf("failed to read playwright output: %v", err)
	}

	if err := json.Unmarshal(data, &report); err != nil {
		return PlaywrightReport{}, fmt.Errorf("failed to parse playwright output: %s", err)
	}

	return report, nil
}

func (p Playwright) GetFiles() ([]string, error) {
	debug.Println("Discovering test files with include pattern:", p.TestFilePattern, "exclude pattern:", p.TestFileExcludePattern)
	files, err := discoverTestFiles(p.TestFilePattern, p.TestFileExcludePattern)
	debug.Println("Discovered", len(files), "files")

	if err != nil {
		return nil, err
	}

	if len(files) == 0 {
		return nil, fmt.Errorf("no files found with pattern %q and exclude pattern %q", p.TestFilePattern, p.TestFileExcludePattern)
	}

	return files, nil
}

func (p Playwright) GetExamples(files []string) ([]plan.TestCase, error) {
	return nil, fmt.Errorf("not supported in Playwright")
}

type PlaywrightTest struct {
	ProjectName string
	Status      string
}

type PlaywrightSpec struct {
	File   string
	Line   int
	Column int
	Id     string
	Title  string
	Ok     bool
	Tests  []PlaywrightTest
}

type PlaywrightReportSuite struct {
	Title  string
	Specs  []PlaywrightSpec
	Suites []PlaywrightReportSuite
}

type PlaywrightReport struct {
	Suites []PlaywrightReportSuite
	Stats  struct {
		Expected   int
		Unexpected int
	}
	Errors []struct {
		Message string
	}
}