diff --git a/.buildkite/artifact-test.Dockerfile b/.buildkite/artifact-test.Dockerfile new file mode 100644 index 00000000..c0994000 --- /dev/null +++ b/.buildkite/artifact-test.Dockerfile @@ -0,0 +1,15 @@ +# syntax=docker/dockerfile:1.3-labs + +FROM alpine:latest + +RUN < /build/dist/mypackage-1.0.0.whl +echo "wheel-content-1.1" > /build/dist/mypackage-1.1.0.whl +echo "binary-content" > /app/bin/myapp +chmod +x /app/bin/myapp + +EOF diff --git a/.buildkite/artifact-test.wanda.yaml b/.buildkite/artifact-test.wanda.yaml new file mode 100644 index 00000000..f6c07c47 --- /dev/null +++ b/.buildkite/artifact-test.wanda.yaml @@ -0,0 +1,12 @@ +name: artifact-test +froms: + - alpine:latest +dockerfile: .buildkite/artifact-test.Dockerfile +artifacts: + - src: /app/bin/myapp + dst: bin/myapp + - src: /build/dist/*.whl + dst: wheels/ + - src: /nonexistent/optional-file.txt + dst: optional.txt + optional: true diff --git a/.buildkite/forge.rayci.yaml b/.buildkite/forge.rayci.yaml index 9316c80c..d3a32363 100644 --- a/.buildkite/forge.rayci.yaml +++ b/.buildkite/forge.rayci.yaml @@ -34,3 +34,9 @@ steps: - name: forge-arm64 wanda: .buildkite/forge-arm64.wanda.yaml instance_type: builder-arm64 + + - name: artifact-test + tags: + - wanda + - raycicmd + wanda: .buildkite/artifact-test.wanda.yaml diff --git a/raycicmd/bk_pipeline.go b/raycicmd/bk_pipeline.go index eaad0db8..28b63b2b 100644 --- a/raycicmd/bk_pipeline.go +++ b/raycicmd/bk_pipeline.go @@ -220,4 +220,7 @@ var ( defaultArtifactPaths = []string{"/tmp/artifacts/**/*"} windowsArtifactPaths = []string{`C:\tmp\artifacts\**\*`} + + defaultArtifactsMountDir = "/tmp/artifacts" + windowsArtifactsMountDir = "/c/tmp/artifacts" ) diff --git a/raycicmd/converter_test.go b/raycicmd/converter_test.go index 69d7c5f4..4c4a90dc 100644 --- a/raycicmd/converter_test.go +++ b/raycicmd/converter_test.go @@ -139,6 +139,11 @@ func TestConvertPipelineStep(t *testing.T) { "broken": skipQueue, }, + BuilderQueues: map[string]string{ + "builder": "fakebuilder", + "builder-windows": "fakewinbuilder", + }, + Env: map[string]string{ "BUILDKITE_BAZEL_CACHE_URL": "https://bazel-build-cache", }, @@ -354,15 +359,18 @@ func TestConvertPipelineStep(t *testing.T) { "matrix": []any{"py36", "py37"}, }, out: map[string]any{ - "label": "my forge", - "key": "forge", - "commands": wandaCommands("beta"), + "label": "my forge", + "key": "forge", + "commands": wandaCommands("beta"), + "agents": newBkAgents("fakebuilder"), + "artifact_paths": defaultArtifactPaths, "env": map[string]string{ "RAYCI_BUILD_ID": buildID, "RAYCI_TEMP": "s3://ci-temp/abc123/", "BUILDKITE_BAZEL_CACHE_URL": "https://bazel-build-cache", "RAYCI_WORK_REPO": "fakeecr", "RAYCI_BRANCH": "beta", + "RAYCI_ARTIFACTS_DIR": "/tmp/artifacts", "RAYCI_WANDA_FILE": "ci/forge.wanda.yaml", "RAYCI_WANDA_NAME": "forge", @@ -378,6 +386,37 @@ func TestConvertPipelineStep(t *testing.T) { }, "timeout_in_minutes": 300, }, + }, { + // Windows wanda step + in: map[string]any{ + "name": "windows-forge", + "wanda": "ci/windows.wanda.yaml", + "instance_type": "builder-windows", + }, + out: map[string]any{ + "label": "wanda: windows-forge", + "key": "windows-forge", + "commands": wandaCommands("beta"), + "artifact_paths": windowsArtifactPaths, + "env": map[string]string{ + "RAYCI_BUILD_ID": buildID, + "RAYCI_TEMP": "s3://ci-temp/abc123/", + "BUILDKITE_BAZEL_CACHE_URL": "https://bazel-build-cache", + "RAYCI_WORK_REPO": "fakeecr", + "RAYCI_BRANCH": "beta", + "RAYCI_ARTIFACTS_DIR": "/c/tmp/artifacts", + + "RAYCI_WANDA_FILE": "ci/windows.wanda.yaml", + "RAYCI_WANDA_NAME": "windows-forge", + + "BUILDKITE_ARTIFACT_UPLOAD_DESTINATION": artifactDest, + }, + "retry": map[string]any{ + "automatic": map[string]any{"limit": 1}, + }, + "timeout_in_minutes": 300, + "agents": newBkAgents("fakewinbuilder"), + }, }, { in: map[string]any{ "label": "windows job", diff --git a/raycicmd/wanda.go b/raycicmd/wanda.go index 46a13415..358c93ba 100644 --- a/raycicmd/wanda.go +++ b/raycicmd/wanda.go @@ -5,6 +5,7 @@ import ( "log" "path" "sort" + "strings" ) const rawGitHubURL = "https://raw.githubusercontent.com/" @@ -72,6 +73,12 @@ func (s *wandaStep) buildkiteStep() map[string]any { "timeout_in_minutes": defaultTimeoutInMinutes, } + if strings.Contains(instanceType, "windows") { + bkStep["artifact_paths"] = windowsArtifactPaths + } else { + bkStep["artifact_paths"] = defaultArtifactPaths + } + if s.dependsOn != nil { bkStep["depends_on"] = s.dependsOn } @@ -192,6 +199,11 @@ func (c *wandaConverter) convert(id string, step map[string]any) ( } } } + if strings.Contains(instanceType, "windows") { + envs["RAYCI_ARTIFACTS_DIR"] = windowsArtifactsMountDir + } else { + envs["RAYCI_ARTIFACTS_DIR"] = defaultArtifactsMountDir + } s := &wandaStep{ name: name,