From 5f3b9be60a17f9022135f440aeb34aea4ec7da54 Mon Sep 17 00:00:00 2001 From: Adam Singer Date: Wed, 7 Aug 2024 20:34:00 +0000 Subject: [PATCH] Add no_prelude remote execution to github actions Enabling testing of remote execution on github actions for `no_prelude` * Create github action that generates .buckconfig.local using `NATIVELINK_HEADER_RW_KEY` from gha secrets. * Set `container-image` used for remote execution specific to setup of buck2. * Update .gitignore to include .buckconfig.local. * Include a `platforms` configuration for the `no_prelude` example. * Update `go_toolchain.bzl` to be remote enabled friendly, no need to download a toolchain when remote worker has one. Open question: * Was the downloading of go toolchain to be remote execution compatible prior to this change? --- .../action.yml | 27 +++++++++++++++ .github/workflows/build-and-test.yml | 3 ++ .gitignore | 1 + examples/no_prelude/go/rules.bzl | 2 +- examples/no_prelude/platforms/BUCK | 3 ++ examples/no_prelude/platforms/defs.bzl | 33 +++++++++++++++++++ .../no_prelude/toolchains/go_toolchain.bzl | 33 ++++++++++++------- 7 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 .github/actions/build_example_nativelink_no_prelude/action.yml create mode 100644 examples/no_prelude/platforms/BUCK create mode 100644 examples/no_prelude/platforms/defs.bzl diff --git a/.github/actions/build_example_nativelink_no_prelude/action.yml b/.github/actions/build_example_nativelink_no_prelude/action.yml new file mode 100644 index 0000000000000..f6f4409351b63 --- /dev/null +++ b/.github/actions/build_example_nativelink_no_prelude/action.yml @@ -0,0 +1,27 @@ +name: build_example_nativelink_no_prelude +inputs: + NATIVELINK_HEADER_RW_KEY_SECRET: + description: '' + required: true +runs: + using: composite + steps: + - name: Build example/no_prelude directory using remote execution + run: |- + { + echo "[buck2_re_client] + engine_address = grpc://scheduler-buck2.build-faster.nativelink.net:443 + action_cache_address = grpc://cas-buck2.build-faster.nativelink.net:443 + cas_address = grpc://cas-buck2.build-faster.nativelink.net:443 + http_headers = x-nativelink-api-key:$NATIVELINK_HEADER_RW_KEY + tls = true + instance_name = main + enabled = true + [build] + execution_platforms = root//platforms:platforms" + } > examples/no_prelude/.buckconfig.local + cd examples/no_prelude + $RUNNER_TEMP/artifacts/buck2 build //... + env: + NATIVELINK_HEADER_RW_KEY: ${{ inputs.NATIVELINK_HEADER_RW_KEY_SECRET }} + shell: bash diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 568e8535542ae..e26185866a1ba 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -48,6 +48,9 @@ jobs: $RUNNER_TEMP/artifacts/buck2 build //... -v 2 $RUNNER_TEMP/artifacts/buck2 test //... -v 2 - uses: ./.github/actions/build_example_no_prelude + - uses: ./.github/actions/build_example_nativelink_no_prelude + with: + NATIVELINK_HEADER_RW_KEY_SECRET: ${{ secrets.NATIVELINK_HEADER_RW_KEY_SECRET }} - uses: ./.github/actions/setup_reindeer - uses: ./.github/actions/build_bootstrap linux-build-examples: diff --git a/.gitignore b/.gitignore index 5ca9410a5c866..bfeb8157ca3f1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ Cargo.lock buck-out /.direnv +.buckconfig.local # symlinks /examples/with_prelude/prelude diff --git a/examples/no_prelude/go/rules.bzl b/examples/no_prelude/go/rules.bzl index 0a0b1aeed8843..ec96767127393 100644 --- a/examples/no_prelude/go/rules.bzl +++ b/examples/no_prelude/go/rules.bzl @@ -14,7 +14,7 @@ def _go_binary_impl(ctx: AnalysisContext) -> list[Provider]: cmd = cmd_args([ctx.attrs.toolchain[GoCompilerInfo].compiler_path, "build", "-o", out.as_output()] + sources) - ctx.actions.run(cmd, category = "compile") + ctx.actions.run(cmd, env = {"GOCACHE":ctx.attrs.toolchain[GoCompilerInfo].GOCACHE}, category = "compile") return [ DefaultInfo(default_output = out), diff --git a/examples/no_prelude/platforms/BUCK b/examples/no_prelude/platforms/BUCK new file mode 100644 index 0000000000000..63f852afecbda --- /dev/null +++ b/examples/no_prelude/platforms/BUCK @@ -0,0 +1,3 @@ +load(":defs.bzl", "platforms") + +platforms(name = "platforms") diff --git a/examples/no_prelude/platforms/defs.bzl b/examples/no_prelude/platforms/defs.bzl new file mode 100644 index 0000000000000..5d065f7a8855f --- /dev/null +++ b/examples/no_prelude/platforms/defs.bzl @@ -0,0 +1,33 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +def _platforms(ctx): + configuration = ConfigurationInfo( + constraints = {}, + values = {}, + ) + + platform = ExecutionPlatformInfo( + label = ctx.label.raw_target(), + configuration = configuration, + executor_config = CommandExecutorConfig( + local_enabled = True, + remote_enabled = True, + use_limited_hybrid = True, + # Set those up based on what workers you've registered with NativeLink. + remote_execution_properties = { + "OSFamily": "linux", + "container-image": "docker://buck2-github:latest", + }, + remote_execution_use_case = "buck2-default", + remote_output_paths = "output_paths", + ), + ) + + return [DefaultInfo(), ExecutionPlatformRegistrationInfo(platforms = [platform])] + +platforms = rule(attrs = {}, impl = _platforms) diff --git a/examples/no_prelude/toolchains/go_toolchain.bzl b/examples/no_prelude/toolchains/go_toolchain.bzl index 6d921aed9d9f9..b80c50eff2ee9 100644 --- a/examples/no_prelude/toolchains/go_toolchain.bzl +++ b/examples/no_prelude/toolchains/go_toolchain.bzl @@ -7,24 +7,33 @@ GoCompilerInfo = provider( doc = "Information about how to invoke the go compiler.", - fields = ["compiler_path", "GOROOT"], + fields = ["compiler_path", "GOROOT", "GOCACHE"], ) +def is_remote_enabled() -> bool: + re_enabled = read_root_config("buck2_re_client", "enabled", "false") + return re_enabled == "true" + def _go_toolchain_impl(ctx): - download = _download_toolchain(ctx) + is_re_enabled = is_remote_enabled() + gocache="/tmp/gocache" + if is_re_enabled: + return [DefaultInfo(), GoCompilerInfo(compiler_path = "go", GOROOT = "", GOCACHE=gocache)] + else: + download = _download_toolchain(ctx) - compiler_dst = ctx.actions.declare_output("compiler.exe" if host_info().os.is_windows else "compiler") + compiler_dst = ctx.actions.declare_output("compiler.exe" if host_info().os.is_windows else "compiler") - cmd = cmd_args() - if host_info().os.is_windows: - compiler_src = cmd_args(download, format = "{}\\go\\bin\\go.exe", relative_to = (compiler_dst, 1)) - cmd.add([ctx.attrs._symlink_bat, compiler_dst.as_output(), compiler_src]) - else: - compiler_src = cmd_args(download, format = "{}/go/bin/go", relative_to = (compiler_dst, 1)) - cmd.add(["ln", "-sf", compiler_src, compiler_dst.as_output()]) + cmd = cmd_args() + if host_info().os.is_windows: + compiler_src = cmd_args(download, format = "{}\\go\\bin\\go.exe", relative_to = (compiler_dst, 1)) + cmd.add([ctx.attrs._symlink_bat, compiler_dst.as_output(), compiler_src]) + else: + compiler_src = cmd_args(download, format = "{}/go/bin/go", relative_to = (compiler_dst, 1)) + cmd.add(["ln", "-sf", compiler_src, compiler_dst.as_output()]) - ctx.actions.run(cmd, category = "cp_compiler") - return [DefaultInfo(default_output = download), GoCompilerInfo(compiler_path = compiler_dst, GOROOT = "")] + ctx.actions.run(cmd, category = "cp_compiler") + return [DefaultInfo(default_output = download), GoCompilerInfo(compiler_path = compiler_dst, GOROOT = "", GOCACHE=gocache)] go_toolchain = rule( impl = _go_toolchain_impl,