feat(raycicmd): set RAYCI_ARTIFACTS_DIR for wanda steps

Set RAYCI_ARTIFACTS_DIR environment variable for wanda steps based on the target platform. Uses /c/tmp/artifacts for Windows (detected via instance_type containing "windows") and /tmp/artifacts for other platforms.

This enables wanda to extract build artifacts to the correct location for Buildkite artifact upload.

Topic: raycicmd-artifacts
Relative: wanda-artifact-cli
Signed-off-by: andrew <andrew@anyscale.com>

Author: andrew-anyscale

.buildkite/artifact-test.Dockerfile

@@ -0,0 +1,13 @@
+FROM alpine:latest
+
+RUN <<EOF
+#!/bin/bash
+
+set -euo pipefail
+
+mkdir -p /build/dist /app/bin
+echo "wheel-content-1.0" > /build/dist/mypackage-1.0.0.whl
+echo "wheel-content-1.1" > /build/dist/mypackage-1.1.0.whl
+echo "binary-content" > /app/bin/myapp
+
+EOF

.buildkite/artifact-test.wanda.yaml

@@ -0,0 +1,12 @@
+name: artifact-test
+froms:
+  - alpine:latest
+dockerfile: .buildkite/artifact-test.Dockerfile
+artifacts:
+  - src: /app/bin/myapp
+    dst: bin/myapp
+  - src: /build/dist/*.whl
+    dst: wheels/
+  - src: /nonexistent/optional-file.txt
+    dst: optional.txt
+    optional: true

.buildkite/forge.rayci.yaml

@@ -34,3 +34,9 @@ steps:
   - name: forge-arm64
     wanda: .buildkite/forge-arm64.wanda.yaml
     instance_type: builder-arm64
+
+  - name: artifact-test
+    tags:
+      - wanda
+      - raycicmd
+    wanda: .buildkite/artifact-test.wanda.yaml

raycicmd/bk_pipeline.go

@@ -220,4 +220,7 @@ var (
 
 	defaultArtifactPaths = []string{"/tmp/artifacts/**/*"}
 	windowsArtifactPaths = []string{`C:\tmp\artifacts\**\*`}
+
+	defaultArtifactsMountDir = "/tmp/artifacts"
+	windowsArtifactsMountDir = "/c/tmp/artifacts"
 )

raycicmd/converter_test.go

@@ -139,6 +139,11 @@ func TestConvertPipelineStep(t *testing.T) {
 			"broken":  skipQueue,
 		},
 
+		BuilderQueues: map[string]string{
+			"builder":         "fakebuilder",
+			"builder-windows": "fakewinbuilder",
+		},
+
 		Env: map[string]string{
 			"BUILDKITE_BAZEL_CACHE_URL": "https://bazel-build-cache",
 		},
@@ -354,15 +359,18 @@ func TestConvertPipelineStep(t *testing.T) {
 			"matrix":     []any{"py36", "py37"},
 		},
 		out: map[string]any{
-			"label":    "my forge",
-			"key":      "forge",
-			"commands": wandaCommands("beta"),
+			"label":          "my forge",
+			"key":            "forge",
+			"commands":       wandaCommands("beta"),
+			"agents":         newBkAgents("fakebuilder"),
+			"artifact_paths": defaultArtifactPaths,
 			"env": map[string]string{
 				"RAYCI_BUILD_ID":            buildID,
 				"RAYCI_TEMP":                "s3://ci-temp/abc123/",
 				"BUILDKITE_BAZEL_CACHE_URL": "https://bazel-build-cache",
 				"RAYCI_WORK_REPO":           "fakeecr",
 				"RAYCI_BRANCH":              "beta",
+				"RAYCI_ARTIFACTS_DIR":       "/tmp/artifacts",
 
 				"RAYCI_WANDA_FILE": "ci/forge.wanda.yaml",
 				"RAYCI_WANDA_NAME": "forge",
@@ -378,6 +386,37 @@ func TestConvertPipelineStep(t *testing.T) {
 			},
 			"timeout_in_minutes": 300,
 		},
+	}, {
+		// Windows wanda step
+		in: map[string]any{
+			"name":          "windows-forge",
+			"wanda":         "ci/windows.wanda.yaml",
+			"instance_type": "builder-windows",
+		},
+		out: map[string]any{
+			"label":          "wanda: windows-forge",
+			"key":            "windows-forge",
+			"commands":       wandaCommands("beta"),
+			"artifact_paths": windowsArtifactPaths,
+			"env": map[string]string{
+				"RAYCI_BUILD_ID":            buildID,
+				"RAYCI_TEMP":                "s3://ci-temp/abc123/",
+				"BUILDKITE_BAZEL_CACHE_URL": "https://bazel-build-cache",
+				"RAYCI_WORK_REPO":           "fakeecr",
+				"RAYCI_BRANCH":              "beta",
+				"RAYCI_ARTIFACTS_DIR":       "/c/tmp/artifacts",
+
+				"RAYCI_WANDA_FILE": "ci/windows.wanda.yaml",
+				"RAYCI_WANDA_NAME": "windows-forge",
+
+				"BUILDKITE_ARTIFACT_UPLOAD_DESTINATION": artifactDest,
+			},
+			"retry": map[string]any{
+				"automatic": map[string]any{"limit": 1},
+			},
+			"timeout_in_minutes": 300,
+			"agents":             newBkAgents("fakewinbuilder"),
+		},
 	}, {
 		in: map[string]any{
 			"label":         "windows job",

raycicmd/wanda.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"path"
 	"sort"
+	"strings"
 )
 
 const rawGitHubURL = "https://raw.githubusercontent.com/"
@@ -72,6 +73,12 @@ func (s *wandaStep) buildkiteStep() map[string]any {
 		"timeout_in_minutes": defaultTimeoutInMinutes,
 	}
 
+	if strings.Contains(instanceType, "windows") {
+		bkStep["artifact_paths"] = windowsArtifactPaths
+	} else {
+		bkStep["artifact_paths"] = defaultArtifactPaths
+	}
+
 	if s.dependsOn != nil {
 		bkStep["depends_on"] = s.dependsOn
 	}
@@ -192,6 +199,11 @@ func (c *wandaConverter) convert(id string, step map[string]any) (
 			}
 		}
 	}
+	if strings.Contains(instanceType, "windows") {
+		envs["RAYCI_ARTIFACTS_DIR"] = windowsArtifactsMountDir
+	} else {
+		envs["RAYCI_ARTIFACTS_DIR"] = defaultArtifactsMountDir
+	}
 
 	s := &wandaStep{
 		name:           name,