[draft] DO NOT MERGE: rayapp test prototype

build_rayapp.sh

@@ -6,6 +6,6 @@ echo "--- Building rayapp"
 mkdir -p ci/bin
 go build -o ci/bin/rayapp ./rayapp/rayapp
 
-echo "--- Building template releases"
+echo "--- Running rayapp commands"
 rm -rf _build
 exec ci/bin/rayapp "$@"

rayapp/anyscale_cli.go

@@ -0,0 +1,175 @@
+package rayapp
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+type WorkspaceState int
+
+const (
+    StateTerminated WorkspaceState = iota
+    StateStarting
+    StateRunning
+)
+
+var WorkspaceStateName = map[WorkspaceState]string{
+    StateTerminated: "TERMINATED",
+    StateStarting: "STARTING",
+    StateRunning:  "RUNNING",
+}
+
+func (ws WorkspaceState) String() string {
+    return WorkspaceStateName[ws]
+}
+
+type AnyscaleCLI struct {
+	token string
+}
+
+var errAnyscaleNotInstalled = errors.New("anyscale is not installed")
+
+func NewAnyscaleCLI(token string) *AnyscaleCLI {
+	return &AnyscaleCLI{token: token}
+}
+
+func isAnyscaleInstalled() bool {
+	_, err := exec.LookPath("anyscale")
+	return err == nil
+}
+
+func (ac *AnyscaleCLI) Authenticate() error {
+	cmd := exec.Command("anyscale", "login")
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("anyscale auth login failed, please set ANYSCALE_CLI_TOKEN & ANYSCALE_HOST env variables: %w", err)
+	}
+	return nil
+}
+
+// RunAnyscaleCLI runs the anyscale CLI with the given arguments.
+// Returns the combined output and any error that occurred.
+// Output is displayed to the terminal with colors preserved.
+func (ac *AnyscaleCLI) runAnyscaleCLI(args []string) (string, error) {
+	if !isAnyscaleInstalled() {
+		return "", errAnyscaleNotInstalled
+	}
+
+	fmt.Println("anyscale cli args: ", args)
+	cmd := exec.Command("anyscale", args...)
+
+	// Capture output while also displaying to terminal with colors
+	var outputBuf bytes.Buffer
+	cmd.Stdout = io.MultiWriter(os.Stdout, &outputBuf)
+	cmd.Stderr = io.MultiWriter(os.Stderr, &outputBuf)
+
+	if err := cmd.Run(); err != nil {
+		return outputBuf.String(), fmt.Errorf("anyscale error: %w", err)
+	}
+
+	return outputBuf.String(), nil
+}
+
+func (ac *AnyscaleCLI) createEmptyWorkspace(config *WorkspaceTestConfig) error {
+	args := []string{"workspace_v2", "create"}
+	// get image URI and ray version from build ID
+	imageURI, rayVersion, err := convertBuildIdToImageURI(config.template.ClusterEnv.BuildID)
+	if err != nil {
+		return fmt.Errorf("convert build ID to image URI failed: %w", err)
+	}
+	args = append(args, "--name", config.workspaceName)
+	args = append(args, "--image-uri", imageURI)
+	args = append(args, "--ray-version", rayVersion)
+	if config.computeConfig != "" {
+		args = append(args, "--compute-config", "tmpl-test-basic-serverless-aws:1")
+	}
+	output, err := ac.runAnyscaleCLI(args)
+	if err != nil {
+		return fmt.Errorf("create empty workspace failed: %w", err)
+	}
+	fmt.Println("create empty workspace output:\n", output)
+	return nil
+}
+
+func (ac *AnyscaleCLI) terminateWorkspace(workspaceName string) error {
+	output, err := ac.runAnyscaleCLI([]string{"workspace_v2", "terminate", "--name", workspaceName})
+	if err != nil {
+		return fmt.Errorf("delete workspace failed: %w", err)
+	}
+	fmt.Println("terminate workspace output:\n", output)
+	return nil
+}
+
+func (ac *AnyscaleCLI) copyTemplateToWorkspace(config *WorkspaceTestConfig) error {
+	output, err := ac.runAnyscaleCLI([]string{"workspace_v2", "push", "--name", config.workspaceName, "--local-dir", config.template.Dir})
+	if err != nil {
+		return fmt.Errorf("copy template to workspace failed: %w", err)
+	}
+	fmt.Println("copy template to workspace output:\n", output)
+	return nil
+}
+
+func (ac *AnyscaleCLI) runCmdInWorkspace(config *WorkspaceTestConfig, cmd string) error {
+	output, err := ac.runAnyscaleCLI([]string{"workspace_v2", "run_command", "--name", config.workspaceName, cmd})
+	if err != nil {
+		return fmt.Errorf("run command in workspace failed: %w", err)
+	}
+	fmt.Println("run command in workspace output:\n", output)
+	return nil
+}
+
+func (ac *AnyscaleCLI) startWorkspace(config *WorkspaceTestConfig) error {
+	output, err := ac.runAnyscaleCLI([]string{"workspace_v2", "start", "--name", config.workspaceName})
+	if err != nil {
+		return fmt.Errorf("start workspace failed: %w", err)
+	}
+	fmt.Println("start workspace output:\n", output)
+	return nil
+}
+
+func (ac *AnyscaleCLI) getWorkspaceStatus(workspaceName string) (string, error) {
+	output, err := ac.runAnyscaleCLI([]string{"workspace_v2", "status", "--name", workspaceName})
+	if err != nil {
+		return "", fmt.Errorf("get workspace state failed: %w", err)
+	}
+	return output, nil
+}
+
+func convertBuildIdToImageURI(buildId string) (string, string, error) {
+	// Convert build ID like "anyscaleray2441-py312-cu128" to "anyscale/ray:2.44.1-py312-cu128"
+	const prefix = "anyscaleray"
+	if !strings.HasPrefix(buildId, prefix) {
+		return "", "", fmt.Errorf("build ID must start with %q: %s", prefix, buildId)
+	}
+
+	// Remove the prefix to get "2441-py312-cu128"
+	remainder := strings.TrimPrefix(buildId, prefix)
+
+	// Find the first hyphen to separate version from suffix
+	hyphenIdx := strings.Index(remainder, "-")
+	var versionStr, suffix string
+	if hyphenIdx == -1 {
+		versionStr = remainder
+		suffix = ""
+	} else {
+		versionStr = remainder[:hyphenIdx]
+		suffix = remainder[hyphenIdx:] // includes the hyphen
+	}
+
+	// Parse version: "2441" -> "2.44.1"
+	// Format: first digit = major, next two = minor, rest = patch
+	if len(versionStr) < 4 {
+		return "", "", fmt.Errorf("version string too short: %s", versionStr)
+	}
+
+	major := versionStr[0:1]
+	minor := versionStr[1:3]
+	patch := versionStr[3:]
+
+	return fmt.Sprintf("anyscale/ray:%s.%s.%s%s", major, minor, patch, suffix), fmt.Sprintf("%s.%s.%s", major, minor, patch), nil
+}

rayapp/anyscale_cli_test.go

@@ -0,0 +1,112 @@
+package rayapp
+
+import (
+	"errors"
+	"os"
+	"strings"
+	"testing"
+)
+
+var anyscaleCLI *AnyscaleCLI
+
+// setupMockAnyscale creates a mock anyscale script and returns a cleanup function.
+func setupMockAnyscale(t *testing.T, script string) {
+	t.Helper()
+	tmp := t.TempDir()
+
+	if script != "" {
+		mockScript := tmp + "/anyscale"
+		if err := os.WriteFile(mockScript, []byte(script), 0755); err != nil {
+			t.Fatalf("failed to create mock script: %v", err)
+		}
+	}
+
+	origPath := os.Getenv("PATH")
+	t.Cleanup(func() { os.Setenv("PATH", origPath) })
+	os.Setenv("PATH", tmp)
+}
+
+func TestRunAnyscaleCLI(t *testing.T) {
+	tests := []struct {
+		name       string
+		script     string
+		args       []string
+		wantErr    error
+		wantSubstr string
+	}{
+		{
+			name:    "anyscale not installed",
+			script:  "", // empty PATH, no script
+			args:    []string{"--version"},
+			wantErr: errAnyscaleNotInstalled,
+		},
+		{
+			name:       "success",
+			script:     "#!/bin/sh\nif [ \"$1\" = \"login\" ]; then exit 0; fi\necho \"output: $@\"",
+			args:       []string{"service", "deploy"},
+			wantSubstr: "output: service deploy",
+		},
+		{
+			name:       "empty args",
+			script:     "#!/bin/sh\nif [ \"$1\" = \"login\" ]; then exit 0; fi\necho \"help\"",
+			args:       []string{},
+			wantSubstr: "help",
+		},
+		{
+			name:       "command fails with stderr",
+			script:     "#!/bin/sh\nif [ \"$1\" = \"login\" ]; then exit 0; fi\necho \"error msg\" >&2; exit 1",
+			args:       []string{"deploy"},
+			wantSubstr: "error msg",
+			wantErr:    errors.New("anyscale error"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			setupMockAnyscale(t, tt.script)
+
+			output, err := anyscaleCLI.RunAnyscaleCLI(tt.args)
+
+			if tt.wantErr != nil {
+				if err == nil {
+					t.Fatal("expected error, got nil")
+				}
+				if errors.Is(tt.wantErr, errAnyscaleNotInstalled) {
+					if !errors.Is(err, errAnyscaleNotInstalled) {
+						t.Errorf("expected errAnyscaleNotInstalled, got: %v", err)
+					}
+				} else if !strings.Contains(err.Error(), tt.wantErr.Error()) {
+					t.Errorf("error %q should contain %q", err.Error(), tt.wantErr.Error())
+				}
+				// For error cases, also check wantSubstr against error message
+				if tt.wantSubstr != "" && !strings.Contains(err.Error(), tt.wantSubstr) {
+					t.Errorf("error %q should contain %q", err.Error(), tt.wantSubstr)
+				}
+				return
+			}
+
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if tt.wantSubstr != "" && !strings.Contains(output, tt.wantSubstr) {
+				t.Errorf("output %q should contain %q", output, tt.wantSubstr)
+			}
+		})
+	}
+}
+
+func TestIsAnyscaleInstalled(t *testing.T) {
+	t.Run("not installed", func(t *testing.T) {
+		setupMockAnyscale(t, "")
+		if isAnyscaleInstalled() {
+			t.Error("should return false when not in PATH")
+		}
+	})
+
+	t.Run("installed", func(t *testing.T) {
+		setupMockAnyscale(t, "#!/bin/sh\necho mock")
+		if !isAnyscaleInstalled() {
+			t.Error("should return true when in PATH")
+		}
+	})
+}

rayapp/rayapp/main.go

@@ -3,30 +3,78 @@ package main
 import (
 	"flag"
 	"fmt"
-	"github.com/ray-project/rayci/rayapp"
 	"log"
+	"os"
+
+	"github.com/ray-project/rayci/rayapp"
 )
 
 func main() {
-	base := flag.String("base", ".", "base directory")
-	output := flag.String("output", "_build", "output directory")
-	buildFile := flag.String("build", "BUILD.yaml", "build file")
+	if len(os.Args) < 2 {
+		printUsage()
+		os.Exit(1)
+	}
+
+	// Build command flags (shared by build and build-all)
+	buildFlags := flag.NewFlagSet("build", flag.ExitOnError)
+	base := buildFlags.String("base", ".", "base directory")
+	output := buildFlags.String("output", "_build", "output directory")
+	buildFile := buildFlags.String("build", "BUILD.yaml", "build file")
 
-	flag.Parse()
+	// Test command flags
+	testFlags := flag.NewFlagSet("test", flag.ExitOnError)
+	testBuildFile := testFlags.String("build", "BUILD.yaml", "build file")
+	// workspaceName := testFlags.String("workspace-name", "", "workspace name (required)")
+	// templateDir := testFlags.String("template-dir", "", "template directory (required)")
+	// config := testFlags.String("config", "config.yml", "config file path (required)")
 
-	args := flag.Args()
-	switch args[0] {
+	switch os.Args[1] {
 	case "build-all":
+		buildFlags.Parse(os.Args[2:])
 		if err := rayapp.BuildAll(*buildFile, *base, *output); err != nil {
 			log.Fatal(err)
 		}
 	case "build":
-		if err := rayapp.Build(*buildFile, args[1], *base, *output); err != nil {
+		buildFlags.Parse(os.Args[2:])
+		args := buildFlags.Args()
+		if len(args) < 1 {
+			log.Fatal("build requires a template name")
+		}
+		if err := rayapp.Build(*buildFile, args[0], *base, *output); err != nil {
+			log.Fatal(err)
+		}
+	case "test":
+		testFlags.Parse(os.Args[2:])
+		args := testFlags.Args()
+		if len(args) < 1 {
+			log.Fatal("test requires <template-name> flag")
+		}
+		if err := rayapp.Test(args[0], *testBuildFile); err != nil {
 			log.Fatal(err)
 		}
 	case "help":
-		fmt.Println("Usage: rayapp build-all | build <template-name> | help")
+		printUsage()
 	default:
-		log.Fatal("unknown command")
+		log.Fatalf("unknown command: %s", os.Args[1])
 	}
 }
+
+func printUsage() {
+	fmt.Println("Usage: rayapp <command> [flags]")
+	fmt.Println()
+	fmt.Println("Commands:")
+	fmt.Println("  build-all              Build all templates")
+	fmt.Println("  build <template-name>  Build a specific template")
+	fmt.Println("  test                   Test templates")
+	fmt.Println("  help                   Show this help message")
+	fmt.Println()
+	fmt.Println("Build flags (build, build-all):")
+	fmt.Println("  --base string      Base directory (default \".\")")
+	fmt.Println("  --output string    Output directory (default \"_build\")")
+	fmt.Println("  --build string     Build file (default \"BUILD.yaml\")")
+	fmt.Println()
+	fmt.Println("Test flags:")
+	fmt.Println("  --workspace string     Workspace name (required)")
+	fmt.Println("  --template-dir string  Template directory (required)")
+	fmt.Println("  --config string        Config file path (required)")
+}