feat(raycicmd): add artifacts_dir to wanda steps

Configured dependent on GOOS. This way we can upload Wheels etc. as part of the Wanda step itself.

Labels: draft
Topic: raycicmd-artifacts
Relative: wanda-artifact-cli
Signed-off-by: andrew <andrew@anyscale.com>

Author: andrew-anyscale

.buildkite/artifact-test.Dockerfile

@@ -0,0 +1,7 @@
+FROM alpine:latest
+
+RUN mkdir -p /build/dist /app/bin \
+    && echo "wheel-content-1.0" > /build/dist/mypackage-1.0.0.whl \
+    && echo "wheel-content-1.1" > /build/dist/mypackage-1.1.0.whl \
+    && echo "binary-content" > /app/bin/myapp \
+    && chmod +x /app/bin/myapp

.buildkite/artifact-test.wanda.yaml

@@ -0,0 +1,12 @@
+name: artifact-test
+froms:
+  - alpine:latest
+dockerfile: .buildkite/artifact-test.Dockerfile
+artifacts:
+  - src: /app/bin/myapp
+    dst: bin/myapp
+  - src: /build/dist/*.whl
+    dst: wheels/
+  - src: /nonexistent/optional-file.txt
+    dst: optional.txt
+    optional: true

.buildkite/forge.rayci.yaml

@@ -34,3 +34,7 @@ steps:
   - name: forge-arm64
     wanda: .buildkite/forge-arm64.wanda.yaml
     instance_type: builder-arm64
+
+  - name: artifact-test
+    tags: wanda
+    wanda: .buildkite/artifact-test.wanda.yaml

raycicmd/converter.go

@@ -2,6 +2,7 @@ package raycicmd
 
 import (
 	"fmt"
+	"runtime"
 )
 
 type converter struct {
@@ -31,6 +32,12 @@ func newConverter(config *config, info *buildInfo) *converter {
 		envMap["RAYCI_FORGE_PREFIX"] = config.ForgePrefix
 	}
 
+	if runtime.GOOS == "windows" {
+		envMap["RAYCI_ARTIFACTS_DIR"] = "/c/artifact-mount"
+	} else {
+		envMap["RAYCI_ARTIFACTS_DIR"] = "/artifact-mount"
+	}
+
 	if c.config.ArtifactsBucket != "" && info.gitCommit != "" {
 		dest := fmt.Sprintf(
 			"s3://%s/%s/%s",

raycicmd/converter_test.go

@@ -174,6 +174,7 @@ func TestConvertPipelineStep(t *testing.T) {
 				"RAYCI_WORK_REPO":           "fakeecr",
 				"RAYCI_BRANCH":              "beta",
 				"RAYCI_STEP_ID":             fakeStepID,
+				"RAYCI_ARTIFACTS_DIR":       "/artifact-mount",
 
 				"BUILDKITE_ARTIFACT_UPLOAD_DESTINATION": artifactDest,
 			},
@@ -198,6 +199,7 @@ func TestConvertPipelineStep(t *testing.T) {
 				"RAYCI_WORK_REPO":           "fakeecr",
 				"RAYCI_BRANCH":              "beta",
 				"RAYCI_STEP_ID":             fakeStepID,
+				"RAYCI_ARTIFACTS_DIR":       "/artifact-mount",
 
 				"BUILDKITE_ARTIFACT_UPLOAD_DESTINATION": artifactDest,
 			},
@@ -223,6 +225,7 @@ func TestConvertPipelineStep(t *testing.T) {
 				"RAYCI_WORK_REPO":           "fakeecr",
 				"RAYCI_BRANCH":              "beta",
 				"RAYCI_STEP_ID":             fakeStepID,
+				"RAYCI_ARTIFACTS_DIR":       "/artifact-mount",
 
 				"BUILDKITE_ARTIFACT_UPLOAD_DESTINATION": artifactDest,
 			},
@@ -252,6 +255,7 @@ func TestConvertPipelineStep(t *testing.T) {
 				"RAYCI_WORK_REPO":           "fakeecr",
 				"RAYCI_BRANCH":              "beta",
 				"RAYCI_STEP_ID":             fakeStepID,
+				"RAYCI_ARTIFACTS_DIR":       "/artifact-mount",
 
 				"BUILDKITE_ARTIFACT_UPLOAD_DESTINATION": artifactDest,
 			},
@@ -275,6 +279,7 @@ func TestConvertPipelineStep(t *testing.T) {
 				"RAYCI_WORK_REPO":           "fakeecr",
 				"RAYCI_BRANCH":              "beta",
 				"RAYCI_STEP_ID":             fakeStepID,
+				"RAYCI_ARTIFACTS_DIR":       "/artifact-mount",
 
 				"BUILDKITE_ARTIFACT_UPLOAD_DESTINATION": artifactDest,
 			},
@@ -307,6 +312,7 @@ func TestConvertPipelineStep(t *testing.T) {
 				"RAYCI_WORK_REPO":           "fakeecr",
 				"RAYCI_BRANCH":              "beta",
 				"RAYCI_STEP_ID":             fakeStepID,
+				"RAYCI_ARTIFACTS_DIR":       "/artifact-mount",
 
 				"BUILDKITE_ARTIFACT_UPLOAD_DESTINATION": artifactDest,
 			},
@@ -340,6 +346,7 @@ func TestConvertPipelineStep(t *testing.T) {
 				"RAYCI_WORK_REPO":           "fakeecr",
 				"RAYCI_BRANCH":              "beta",
 				"RAYCI_STEP_ID":             fakeStepID,
+				"RAYCI_ARTIFACTS_DIR":       "/artifact-mount",
 
 				"BUILDKITE_ARTIFACT_UPLOAD_DESTINATION": artifactDest,
 			},
@@ -363,6 +370,7 @@ func TestConvertPipelineStep(t *testing.T) {
 				"BUILDKITE_BAZEL_CACHE_URL": "https://bazel-build-cache",
 				"RAYCI_WORK_REPO":           "fakeecr",
 				"RAYCI_BRANCH":              "beta",
+				"RAYCI_ARTIFACTS_DIR":       "/artifact-mount",
 
 				"RAYCI_WANDA_FILE": "ci/forge.wanda.yaml",
 				"RAYCI_WANDA_NAME": "forge",
@@ -402,6 +410,7 @@ func TestConvertPipelineStep(t *testing.T) {
 				"RAYCI_WORK_REPO":           "fakeecr",
 				"RAYCI_BRANCH":              "beta",
 				"RAYCI_STEP_ID":             fakeStepID,
+				"RAYCI_ARTIFACTS_DIR":       "/artifact-mount",
 
 				"BUILDKITE_ARTIFACT_UPLOAD_DESTINATION": artifactDest,
 			},
@@ -430,6 +439,7 @@ func TestConvertPipelineStep(t *testing.T) {
 				"RAYCI_WORK_REPO":           "fakeecr",
 				"RAYCI_BRANCH":              "beta",
 				"RAYCI_STEP_ID":             fakeStepID,
+				"RAYCI_ARTIFACTS_DIR":       "/artifact-mount",
 
 				"BUILDKITE_ARTIFACT_UPLOAD_DESTINATION": artifactDest,
 			},
@@ -460,6 +470,7 @@ func TestConvertPipelineStep(t *testing.T) {
 				"RAYCI_TEMP":                "s3://ci-temp/abc123/",
 				"RAYCI_WORK_REPO":           "fakeecr",
 				"RAYCI_STEP_ID":             fakeStepID,
+				"RAYCI_ARTIFACTS_DIR":       "/artifact-mount",
 			},
 			"parallelism": 4,
 		},
@@ -505,6 +516,7 @@ func TestConvertPipelineStep(t *testing.T) {
 				"RAYCI_STEP_ID":                         "fakeid",
 				"RAYCI_TEMP":                            "s3://ci-temp/abc123/",
 				"RAYCI_WORK_REPO":                       "fakeecr",
+				"RAYCI_ARTIFACTS_DIR":                   "/artifact-mount",
 			},
 			"label":       "say hello [fakeid]",
 			"command":     "echo hello",