cli_test.go

package main

import (
	"context"
	"reflect"
	"testing"
	"time"

	"github.com/buildkite/test-engine-client/v2/internal/config"
	"github.com/urfave/cli/v3"
)

func TestPreviewSelectionEnabled(t *testing.T) {
	truthyValues := []string{"1", "true", "TRUE", "yes", "on", "t", "y"}
	for _, value := range truthyValues {
		t.Run(value, func(t *testing.T) {
			t.Setenv(previewSelectionEnvVar, value)
			if !previewSelectionEnabled() {
				t.Fatalf("previewSelectionEnabled() = false, want true for %q", value)
			}
		})
	}

	falsyValues := []string{"", "0", "false", "off", "no", "random"}
	for _, value := range falsyValues {
		t.Run(value, func(t *testing.T) {
			t.Setenv(previewSelectionEnvVar, value)
			if previewSelectionEnabled() {
				t.Fatalf("previewSelectionEnabled() = true, want false for %q", value)
			}
		})
	}
}

func TestSelectionFlagsAreGatedByPreviewEnv(t *testing.T) {
	t.Setenv(previewSelectionEnvVar, "")
	if hasSelectionFlag(runCommandFlags()) {
		t.Fatalf("runCommandFlags() unexpectedly includes selection flags when preview is disabled")
	}
	if hasSelectionFlag(planCommandFlags()) {
		t.Fatalf("planCommandFlags() unexpectedly includes selection flags when preview is disabled")
	}

	t.Setenv(previewSelectionEnvVar, "true")
	if !hasSelectionFlag(runCommandFlags()) {
		t.Fatalf("runCommandFlags() missing selection flags when preview is enabled")
	}
	if !hasSelectionFlag(planCommandFlags()) {
		t.Fatalf("planCommandFlags() missing selection flags when preview is enabled")
	}
}

func TestCollectGitMetadataFlagIsGatedByPreviewEnv(t *testing.T) {
	t.Setenv(previewSelectionEnvVar, "")
	if hasFlag(planCommandFlags(), "collect-git-metadata") {
		t.Fatalf("planCommandFlags() unexpectedly includes --collect-git-metadata when preview is disabled")
	}

	t.Setenv(previewSelectionEnvVar, "true")
	if !hasFlag(planCommandFlags(), "collect-git-metadata") {
		t.Fatalf("planCommandFlags() missing --collect-git-metadata when preview is enabled")
	}
}

func TestPlanCommandIncludesParallelismFlag(t *testing.T) {
	if !hasFlag(planCommandFlags(), "parallelism") {
		t.Fatalf("planCommandFlags() missing --parallelism flag; BUILDKITE_PARALLEL_JOB_COUNT will not be bound to cfg.Parallelism for `bktec plan`, breaking split-by-example slow-file detection")
	}
}

func TestPlanCommandIncludesPlanIdentifierFlag(t *testing.T) {
	if !hasFlag(planCommandFlags(), "plan-identifier") {
		t.Fatalf("planCommandFlags() missing --plan-identifier flag; an off-agent `bktec plan` cannot set the plan cache key or skip the BUILDKITE_BUILD_ID/BUILDKITE_STEP_ID guards")
	}
}

// --plan-out is registered in the plan command's mutually-exclusive PLAN
// OUTPUT group, so it is a valid output mode and cannot be combined with --json
// or --pipeline-upload.
func TestPlanCommandIncludesPlanOutOutputFlag(t *testing.T) {
	var planCmd *cli.Command
	for _, c := range cliCommand.Commands {
		if c.Name == "plan" {
			planCmd = c
		}
	}
	if planCmd == nil {
		t.Fatal("cliCommand missing the plan subcommand")
	}

	found := false
	for _, group := range planCmd.MutuallyExclusiveFlags {
		if group.Category != "PLAN OUTPUT" {
			continue
		}
		for _, flags := range group.Flags {
			if hasFlag(flags, "plan-out") {
				found = true
			}
		}
	}

	if !found {
		t.Fatal("plan command's PLAN OUTPUT group missing --plan-out")
	}
}

func TestApplyPlanRequestContext_ClearsCollectGitMetadataWhenPreviewDisabled(t *testing.T) {
	t.Setenv(previewSelectionEnvVar, "")

	cfg.CollectGitMetadata = true
	cfg.SelectionStrategy = "percent"
	cfg.Metadata = map[string]string{"key": "val"}

	// Create a minimal command to satisfy the function signature.
	cmd := &cli.Command{}

	if err := applyPlanRequestContext(cmd); err != nil {
		t.Fatalf("applyPlanRequestContext() error = %v", err)
	}

	if cfg.CollectGitMetadata {
		t.Errorf("cfg.CollectGitMetadata = true, want false when preview is disabled")
	}
	if cfg.SelectionStrategy != "" {
		t.Errorf("cfg.SelectionStrategy = %q, want empty when preview is disabled", cfg.SelectionStrategy)
	}
	if cfg.Metadata != nil {
		t.Errorf("cfg.Metadata = %v, want nil when preview is disabled", cfg.Metadata)
	}
}

func hasSelectionFlag(flags []cli.Flag) bool {
	for _, flag := range flags {
		for _, name := range flag.Names() {
			if name == "selection-strategy" || name == "selection-param" || name == "metadata" {
				return true
			}
		}
	}

	return false
}

func hasFlag(flags []cli.Flag, name string) bool {
	for _, flag := range flags {
		for _, n := range flag.Names() {
			if n == name {
				return true
			}
		}
	}
	return false
}

// TestRunCommandEnvVarsBindToConfig verifies that every env var wired to a run
// command flag actually lands in the cfg struct. This guards against accidental
// removal of the Destination field from a flag definition.
func TestRunCommandEnvVarsBindToConfig(t *testing.T) {
	cfg = config.New()
	t.Cleanup(func() { cfg = config.New() })

	t.Setenv("BUILDKITE_ORGANIZATION_SLUG", "my-org")
	t.Setenv("BUILDKITE_BUILD_ID", "build-1")
	t.Setenv("BUILDKITE_JOB_ID", "job-2")
	t.Setenv("BUILDKITE_STEP_ID", "step-3")
	t.Setenv("BUILDKITE_BRANCH", "main")
	t.Setenv("BUILDKITE_RETRY_COUNT", "2")
	t.Setenv("BUILDKITE_PARALLEL_JOB", "1")
	t.Setenv("BUILDKITE_PARALLEL_JOB_COUNT", "4")
	t.Setenv("BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN", "access-token")
	t.Setenv("BUILDKITE_ANALYTICS_TOKEN", "upload-token")
	t.Setenv("BUILDKITE_TEST_ENGINE_SUITE_SLUG", "my-suite")
	t.Setenv("BUILDKITE_TEST_ENGINE_BASE_URL", "https://example.com")
	t.Setenv("BUILDKITE_TEST_ENGINE_TAG_FILTERS", "fast")
	t.Setenv("BUILDKITE_TEST_ENGINE_TEST_CMD", "go test ./...")
	t.Setenv("BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN", "**/*_test.go")
	t.Setenv("BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN", "vendor/**")
	t.Setenv("BUILDKITE_TEST_ENGINE_TEST_RUNNER", "gotest")
	t.Setenv("BUILDKITE_TEST_ENGINE_RESULT_PATH", "/tmp/results.json")
	t.Setenv("BUILDKITE_TEST_ENGINE_SPLIT_BY_EXAMPLE", "true")
	t.Setenv("BUILDKITE_TEST_ENGINE_SELECTOR_SPLITTING", "true")
	t.Setenv("BUILDKITE_TEST_ENGINE_SELECTOR_FILE", "selectors.txt")
	t.Setenv("BUILDKITE_TEST_ENGINE_FAIL_ON_NO_TESTS", "true")
	t.Setenv("BUILDKITE_TEST_ENGINE_LOCATION_PREFIX", "app/")
	t.Setenv("BUILDKITE_TEST_ENGINE_RETRY_COUNT", "3")
	t.Setenv("BUILDKITE_TEST_ENGINE_DISABLE_RETRY_FOR_MUTED_TEST", "true")
	t.Setenv("BUILDKITE_TEST_ENGINE_RETRY_CMD", "go test -run .")
	t.Setenv("BUILDKITE_TEST_ENGINE_PLAN_IDENTIFIER", "my-plan")
	t.Setenv("BUILDKITE_TEST_ENGINE_DEBUG_ENABLED", "true")
	t.Setenv("BUILDKITE_TEST_ENGINE_OIDC", "false")
	t.Setenv("BUILDKITE_TEST_ENGINE_OIDC_LIFETIME", "1h")
	t.Setenv("BUILDKITE_TEST_ENGINE_TAGS", "env=production,region=us-east-1")

	cmd := &cli.Command{
		Name:  "bktec",
		Flags: freshFlags([]cli.Flag{debugFlag}),
		Commands: []*cli.Command{
			{
				Name:                      "run",
				DisableSliceFlagSeparator: true,
				Action:                    func(ctx context.Context, cmd *cli.Command) error { return nil },
				Flags:                     runCommandFlags(),
			},
		},
	}

	if err := cmd.Run(context.Background(), []string{"bktec", "run"}); err != nil {
		t.Fatalf("unexpected error: %v", err)
	}

	checks := []struct {
		name string
		got  any
		want any
	}{
		{"OrganizationSlug", cfg.OrganizationSlug, "my-org"},
		{"BuildID", cfg.BuildID, "build-1"},
		{"JobID", cfg.JobID, "job-2"},
		{"StepID", cfg.StepID, "step-3"},
		{"Branch", cfg.Branch, "main"},
		{"JobRetryCount", cfg.JobRetryCount, 2},
		{"NodeIndex", cfg.NodeIndex, 1},
		{"Parallelism", cfg.Parallelism, 4},
		{"AccessToken", cfg.AccessToken, "access-token"},
		{"UploadToken", cfg.UploadToken, "upload-token"},
		{"SuiteSlug", cfg.SuiteSlug, "my-suite"},
		{"ServerBaseURL", cfg.ServerBaseURL, "https://example.com"},
		{"TagFilters", cfg.TagFilters, "fast"},
		{"TestCommand", cfg.TestCommand, "go test ./..."},
		{"TestFilePattern", cfg.TestFilePattern, "**/*_test.go"},
		{"TestFileExcludePattern", cfg.TestFileExcludePattern, "vendor/**"},
		{"TestRunner", cfg.TestRunner, "gotest"},
		{"ResultPath", cfg.ResultPath, "/tmp/results.json"},
		{"SplitByExample", cfg.SplitByExample, true},
		{"SelectorSplitting", cfg.SelectorSplitting, true},
		{"SelectorListPath", cfg.SelectorListPath, "selectors.txt"},
		{"FailOnNoTests", cfg.FailOnNoTests, true},
		{"LocationPrefix", cfg.LocationPrefix, "app/"},
		{"MaxRetries", cfg.MaxRetries, 3},
		// DISABLE_RETRY_FOR_MUTED_TEST=true means RetryForMutedTest should be false (flag Action inverts the bool)
		{"RetryForMutedTest", cfg.RetryForMutedTest, false},
		{"RetryCommand", cfg.RetryCommand, "go test -run ."},
		{"Identifier", cfg.Identifier, "my-plan"},
		{"DebugEnabled", cfg.DebugEnabled, true},
		{"OIDC", cfg.OIDC, false},
		{"OIDCLifetime", cfg.OIDCLifetime, time.Hour},
	}

	for _, c := range checks {
		if c.got != c.want {
			t.Errorf("cfg.%s = %v, want %v", c.name, c.got, c.want)
		}
	}

	wantUploadTags := map[string]string{"env": "production", "region": "us-east-1"}
	if !reflect.DeepEqual(cfg.UploadTags, wantUploadTags) {
		t.Errorf("cfg.UploadTags = %v, want %v", cfg.UploadTags, wantUploadTags)
	}
}

func TestSelectorSplittingFlagBindsToConfig(t *testing.T) {
	cfg = config.New()
	t.Cleanup(func() { cfg = config.New() })

	cmd := &cli.Command{
		Name: "bktec",
		Commands: []*cli.Command{
			{
				Name:   "run",
				Action: func(ctx context.Context, cmd *cli.Command) error { return nil },
				Flags:  runCommandFlags(),
			},
		},
	}

	if err := cmd.Run(context.Background(), []string{"bktec", "run", "--selector-splitting"}); err != nil {
		t.Fatalf("unexpected error: %v", err)
	}

	if !cfg.SelectorSplitting {
		t.Fatalf("cfg.SelectorSplitting = false, want true")
	}
}

func TestSelectorSplittingFlagBindsToPlanConfig(t *testing.T) {
	cfg = config.New()
	t.Cleanup(func() { cfg = config.New() })

	cmd := &cli.Command{
		Name: "bktec",
		Commands: []*cli.Command{
			{
				Name:   "plan",
				Action: func(ctx context.Context, cmd *cli.Command) error { return nil },
				Flags:  planCommandFlags(),
			},
		},
	}

	if err := cmd.Run(context.Background(), []string{"bktec", "plan", "--selector-splitting"}); err != nil {
		t.Fatalf("unexpected error: %v", err)
	}

	if !cfg.SelectorSplitting {
		t.Fatalf("cfg.SelectorSplitting = false, want true")
	}
}

// TestRunCommandFlagsDoNotShareParseState guards against the order-dependent
// failure from TE-6257: runCommandFlags() must hand out fresh flag instances so
// that explicitly setting a flag on one command does not leave urfave/cli's
// hasBeenSet state on a shared flag object, which would make a later command
// ignore the flag's env var source.
func TestRunCommandFlagsDoNotShareParseState(t *testing.T) {
	cfg = config.New()
	t.Cleanup(func() { cfg = config.New() })

	// First command parses --selector-splitting on the CLI.
	first := &cli.Command{
		Name: "bktec",
		Commands: []*cli.Command{
			{
				Name:   "run",
				Action: func(ctx context.Context, cmd *cli.Command) error { return nil },
				Flags:  runCommandFlags(),
			},
		},
	}
	if err := first.Run(context.Background(), []string{"bktec", "run", "--selector-splitting"}); err != nil {
		t.Fatalf("unexpected error: %v", err)
	}

	// Second command, built independently, relies on the env var source. If the
	// two commands shared the same flag instance, the flag's hasBeenSet state
	// from the first parse would suppress the env var here.
	cfg = config.New()
	t.Setenv("BUILDKITE_TEST_ENGINE_SELECTOR_SPLITTING", "true")
	second := &cli.Command{
		Name: "bktec",
		Commands: []*cli.Command{
			{
				Name:   "run",
				Action: func(ctx context.Context, cmd *cli.Command) error { return nil },
				Flags:  runCommandFlags(),
			},
		},
	}
	if err := second.Run(context.Background(), []string{"bktec", "run"}); err != nil {
		t.Fatalf("unexpected error: %v", err)
	}

	if !cfg.SelectorSplitting {
		t.Fatalf("cfg.SelectorSplitting = false, want true; flag parse state leaked between commands")
	}
}

// TestCommandFlagsAllReturnFreshInstances asserts every flag handed out by the
// run and plan helpers is a distinct instance, i.e. freshFlag matched a concrete
// case rather than falling through to its default (which returns the shared
// global unchanged). This catches a new flag type being added without a
// corresponding freshFlag case, which would silently reintroduce the TE-6257
// parse-state leak for that flag.
func TestCommandFlagsAllReturnFreshInstances(t *testing.T) {
	// Enable preview so the selection flags are included in the sweep.
	t.Setenv(previewSelectionEnvVar, "true")

	for _, tc := range []struct {
		name  string
		flags []cli.Flag
	}{
		{"run", runCommandFlags()},
		{"plan", planCommandFlags()},
	} {
		for _, f := range tc.flags {
			if freshFlag(f) == f {
				t.Errorf("%s: freshFlag(%v) returned the shared global (type %T); add a case to freshFlag so its parse state is isolated", tc.name, f.Names(), f)
			}
		}
	}
}