From 6550bdc91678a1d3cccf6aaaabb1af1e2ef2a451 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 25 Sep 2024 16:43:13 +0200 Subject: [PATCH 01/36] Add example for remote persistent workers --- examples/persistent_worker/.buckconfig | 17 ++ .../persistent_worker/.buckconfig.buildbuddy | 15 ++ .../.buckconfig.buildbuddy-persistent-workers | 15 ++ .../.buckconfig.local-persistent-workers | 2 + .../persistent_worker/.buckconfig.no-workers | 2 + examples/persistent_worker/.buckroot | 0 examples/persistent_worker/.gitignore | 6 + examples/persistent_worker/BUCK | 21 +++ examples/persistent_worker/README.md | 101 +++++++++++ examples/persistent_worker/defs.bzl | 43 +++++ examples/persistent_worker/flake.lock | 61 +++++++ examples/persistent_worker/flake.nix | 62 +++++++ .../persistent_worker/persistent_worker.py | 171 ++++++++++++++++++ examples/persistent_worker/platforms/BUCK | 33 ++++ .../platforms/buildbuddy.bzl | 52 ++++++ .../persistent_worker/platforms/local.bzl | 29 +++ examples/persistent_worker/proto/BUCK | 5 + examples/persistent_worker/proto/bazel/BUCK | 24 +++ .../proto/bazel/worker_protocol.proto | 98 ++++++++++ examples/persistent_worker/proto/buck2/BUCK | 24 +++ .../proto/buck2/worker.proto | 47 +++++ .../persistent_worker/toolchains/.buckconfig | 1 + examples/persistent_worker/toolchains/BUCK | 25 +++ 23 files changed, 854 insertions(+) create mode 100644 examples/persistent_worker/.buckconfig create mode 100644 examples/persistent_worker/.buckconfig.buildbuddy create mode 100644 examples/persistent_worker/.buckconfig.buildbuddy-persistent-workers create mode 100644 examples/persistent_worker/.buckconfig.local-persistent-workers create mode 100644 examples/persistent_worker/.buckconfig.no-workers create mode 100644 examples/persistent_worker/.buckroot create mode 100644 examples/persistent_worker/.gitignore create mode 100644 examples/persistent_worker/BUCK create mode 100644 examples/persistent_worker/README.md create mode 100644 examples/persistent_worker/defs.bzl create mode 100644 examples/persistent_worker/flake.lock create mode 100644 examples/persistent_worker/flake.nix create mode 100644 examples/persistent_worker/persistent_worker.py create mode 100644 examples/persistent_worker/platforms/BUCK create mode 100644 examples/persistent_worker/platforms/buildbuddy.bzl create mode 100644 examples/persistent_worker/platforms/local.bzl create mode 100644 examples/persistent_worker/proto/BUCK create mode 100644 examples/persistent_worker/proto/bazel/BUCK create mode 100644 examples/persistent_worker/proto/bazel/worker_protocol.proto create mode 100644 examples/persistent_worker/proto/buck2/BUCK create mode 100644 examples/persistent_worker/proto/buck2/worker.proto create mode 100644 examples/persistent_worker/toolchains/.buckconfig create mode 100644 examples/persistent_worker/toolchains/BUCK diff --git a/examples/persistent_worker/.buckconfig b/examples/persistent_worker/.buckconfig new file mode 100644 index 000000000000..49063dec2535 --- /dev/null +++ b/examples/persistent_worker/.buckconfig @@ -0,0 +1,17 @@ +[cells] +root = . +prelude = prelude +toolchains = toolchains +none = none + +[cell_aliases] +config = prelude +fbcode = none +fbsource = none +buck = none + +[external_cells] + prelude = bundled + +[parser] +target_platform_detector_spec = target:root//...->prelude//platforms:default diff --git a/examples/persistent_worker/.buckconfig.buildbuddy b/examples/persistent_worker/.buckconfig.buildbuddy new file mode 100644 index 000000000000..e13c0b2fab19 --- /dev/null +++ b/examples/persistent_worker/.buckconfig.buildbuddy @@ -0,0 +1,15 @@ +[buck2] +digest_algorithms = SHA256 + +[buck2_re_client] +engine_address = grpc://remote.buildbuddy.io +action_cache_address = grpc://remote.buildbuddy.io +cas_address = grpc://remote.buildbuddy.io +tls = true +http_headers = \ + x-buildbuddy-api-key:$BUILDBUDDY_API_KEY, \ + x-buildbuddy-platform.container-registry-username:$BUILDBUDDY_CONTAINER_USER, \ + x-buildbuddy-platform.container-registry-password:$BUILDBUDDY_CONTAINER_PASSWORD + +[build] +execution_platforms = root//platforms:buildbuddy diff --git a/examples/persistent_worker/.buckconfig.buildbuddy-persistent-workers b/examples/persistent_worker/.buckconfig.buildbuddy-persistent-workers new file mode 100644 index 000000000000..231de6969edf --- /dev/null +++ b/examples/persistent_worker/.buckconfig.buildbuddy-persistent-workers @@ -0,0 +1,15 @@ +[buck2] +digest_algorithms = SHA256 + +[buck2_re_client] +engine_address = grpc://remote.buildbuddy.io +action_cache_address = grpc://remote.buildbuddy.io +cas_address = grpc://remote.buildbuddy.io +tls = true +http_headers = \ + x-buildbuddy-api-key:$BUILDBUDDY_API_KEY, \ + x-buildbuddy-platform.container-registry-username:$BUILDBUDDY_CONTAINER_USER, \ + x-buildbuddy-platform.container-registry-password:$BUILDBUDDY_CONTAINER_PASSWORD + +[build] +execution_platforms = root//platforms:buildbuddy-persistent-workers diff --git a/examples/persistent_worker/.buckconfig.local-persistent-workers b/examples/persistent_worker/.buckconfig.local-persistent-workers new file mode 100644 index 000000000000..fef3def8a1f3 --- /dev/null +++ b/examples/persistent_worker/.buckconfig.local-persistent-workers @@ -0,0 +1,2 @@ +[build] +execution_platforms = root//platforms:local-persistent-workers diff --git a/examples/persistent_worker/.buckconfig.no-workers b/examples/persistent_worker/.buckconfig.no-workers new file mode 100644 index 000000000000..c731b17c7756 --- /dev/null +++ b/examples/persistent_worker/.buckconfig.no-workers @@ -0,0 +1,2 @@ +[build] +execution_platforms = root//platforms:local diff --git a/examples/persistent_worker/.buckroot b/examples/persistent_worker/.buckroot new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/examples/persistent_worker/.gitignore b/examples/persistent_worker/.gitignore new file mode 100644 index 000000000000..55014d7e91cb --- /dev/null +++ b/examples/persistent_worker/.gitignore @@ -0,0 +1,6 @@ +.buckconfig.local +.direnv +.envrc.private +image.digest +prelude +toolchains/.buckconfig.nix diff --git a/examples/persistent_worker/BUCK b/examples/persistent_worker/BUCK new file mode 100644 index 000000000000..1384003d20d4 --- /dev/null +++ b/examples/persistent_worker/BUCK @@ -0,0 +1,21 @@ +load("defs.bzl", "demo", "worker") + +python_binary( + name = "worker_py", + main = "persistent_worker.py", + deps = [ + "//proto/bazel:worker_protocol_pb2", + "//proto/buck2:worker_pb2", + ], +) + +worker( + name = "worker", + worker = ":worker_py", + visibility = ["PUBLIC"], +) + +[ + demo(name = "demo-" + str(i)) + for i in range(10) +] diff --git a/examples/persistent_worker/README.md b/examples/persistent_worker/README.md new file mode 100644 index 000000000000..44626ebe9b7b --- /dev/null +++ b/examples/persistent_worker/README.md @@ -0,0 +1,101 @@ +# Persistent Worker Demo + +At the time of writing (2024-09-25) Buck2 supports persistent workers +for local builds through a dedicated Buck2 persistent worker gRPC +protocol. However, Buck2 does not support persistent workers for builds +that use remote execution. This demo is part of a patch-set that adds +support for remote persistent workers to Buck2, see [#776]. + +[#776]: https://github.com/facebook/buck2/issues/776 + +## Requirements + +The demo uses Nix to provide required system dependencies, such as +Python and grpc-io. As well as direnv to set up the environment. + +- [Nix](https://nixos.org/) +- [direnv](https://direnv.net/) + +Credentials for [BuildBuddy](https://www.buildbuddy.io/) and +[GHCR](https://ghcr.io) stored in `.envrc.private`: +``` +export BUILDBUDDY_API_KEY=... +export BUILDBUDDY_CONTAINER_USER=... # GitHub user name +export BUILDBUDDY_CONTAINER_PASSWORD=... # GitHub access token +export GITHUB_USER=... # GitHub user name +``` + +Upload the Nix Docker image for remote execution as described in +`platforms/buildbuddy.bzl`. + +## Local Build + +Configure a local build without persistent workers: +``` +$ cd examples/persistent_worker +$ echo '' > .buckconfig.local +``` + +Run a clean build: +``` +$ buck2 clean; buck2 build : -vstderr +... +stderr for root//:demo-7 (demo): +... +ONE-SHOT START +... +``` + +## Local Persistent Worker + +Configure a local build with persistent workers: +``` +$ cd examples/persistent_worker +$ echo '' > .buckconfig.local +``` + +Run a clean build: +``` +$ buck2 clean; buck2 build : -vstderr +... +stderr for root//:demo-7 (demo): +... +Buck2 persistent worker ... +... +``` + +## Remote Execution + +Configure a remote build without persistent workers: +``` +$ cd examples/persistent_worker +$ echo '' > .buckconfig.local +``` + +Run a clean build: +``` +$ buck2 clean; buck2 build : -vstderr +... +stderr for root//:demo-7 (demo): +... +ONE-SHOT START +... +``` + +## Remote Persistent Worker + +Configure a remote build with persistent workers: +``` +$ cd examples/persistent_worker +$ echo '' > .buckconfig.local +``` + +Run a clean build: +``` +$ buck2 clean; buck2 build : -vstderr +... +stderr for root//:demo-7 (demo): +... +Bazel persistent worker ... +... +``` diff --git a/examples/persistent_worker/defs.bzl b/examples/persistent_worker/defs.bzl new file mode 100644 index 000000000000..fa5d596e1ff3 --- /dev/null +++ b/examples/persistent_worker/defs.bzl @@ -0,0 +1,43 @@ +load("@prelude//utils:argfile.bzl", "at_argfile") + +def _worker_impl(ctx: AnalysisContext) -> list[Provider]: + return [ + DefaultInfo(), + WorkerInfo( + exe = ctx.attrs.worker[RunInfo].args, + concurrency = None, + remote = True, + ), + ] + +worker = rule( + impl = _worker_impl, + attrs = { + "worker": attrs.dep(providers = [RunInfo]), + }, +) + +def _demo_impl(ctx: AnalysisContext) -> list[Provider]: + output = ctx.actions.declare_output(ctx.label.name) + worker_exe = ctx.attrs._worker[WorkerInfo].exe + argfile = at_argfile( + actions = ctx.actions, + name = "demo." + ctx.label.name + ".args", + args = cmd_args(output.as_output()), + ) + ctx.actions.run( + cmd_args(worker_exe, argfile), + category = "demo", + exe = WorkerRunInfo(worker = ctx.attrs._worker[WorkerInfo]), + ) + return [DefaultInfo(default_output = output)] + +demo = rule( + impl = _demo_impl, + attrs = { + "_worker": attrs.exec_dep( + default = "//:worker", + providers = [WorkerInfo], + ), + }, +) diff --git a/examples/persistent_worker/flake.lock b/examples/persistent_worker/flake.lock new file mode 100644 index 000000000000..6e2aab74f5d0 --- /dev/null +++ b/examples/persistent_worker/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1726560853, + "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1727140925, + "narHash": "sha256-ZHSasdLwEEjSOD/WTW1o7dr3/EjuYsdwYB4NSgICZ2I=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "189e5f171b163feb7791a9118afa778d9a1db81f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/examples/persistent_worker/flake.nix b/examples/persistent_worker/flake.nix new file mode 100644 index 000000000000..3cc82f061f41 --- /dev/null +++ b/examples/persistent_worker/flake.nix @@ -0,0 +1,62 @@ +{ + description = "Buck2 Remote Persistent Worker Example"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + outputs = { + self, + nixpkgs, + flake-utils, + ... + }: + flake-utils.lib.eachDefaultSystem (system: let + pkgs = import nixpkgs { + inherit system; + overlays = []; + config = {}; + }; + python = (pkgs.python3.override { + packageOverrides = self: super: { + protobuf = self.protobuf5; + }; + }).withPackages (ps: with ps; [ + ipython + grpcio + grpcio-tools + ]); + in + { + devShells = { + default = pkgs.mkShellNoCC { + packages = [ python ]; + NIX_PYTHON = "${python}/bin/python"; + shellHook = '' + cat >toolchains/.buckconfig.nix < Response: + try: + print("WORKER", socket.gethostname(), os.getpid(), os.getcwd(), file=sys.stderr) + print("REQUEST", request.argv, file=sys.stderr) + args = self.parser.parse_args(request.argv) + print("ARGS", args, file=sys.stderr) + output = args.outfile + name = os.path.basename(output.name) + print("WRITE", name, file=sys.stderr) + output.write(name + "\n") + print("SLEEP", name, file=sys.stderr) + time.sleep(1) + print("COMPLETED", name, file=sys.stderr) + output.close() + return Response( + exit_code = 0, + stderr = f"wrote to {output.name}") + except ArgumentParserError as e: + return Response(exit_code = 2, stderr = str(e)) + + +class Buck2Servicer(buck2_pb2_grpc.WorkerServicer): + """Buck2 remote persistent worker implementation.""" + def __init__(self): + self.impl = Implementation() + + def Execute(self, request, context): + _ = context + print("BUCK2", request, file=sys.stderr) + # Decode arguments as UTF-8 strings. + # Drop the first argument, which is the worker binary. + # The first argument is needed for Bazel remote execution persistent workers. + argv = [arg.decode("utf-8") for arg in request.argv[1:]] + response = self.impl.execute(Request(argv = argv)) + host = socket.gethostname() + pid = os.getpid() + cwd = os.getcwd() + return buck2_pb2.ExecuteResponse( + exit_code = response.exit_code, + stderr = f"Buck2 persistent worker {host} {pid} {cwd}\n" + response.stderr) + + +class BazelServicer: + def __init__(self): + self.impl = Implementation() + + def Execute(self, request: bazel_pb2.WorkRequest) -> bazel_pb2.WorkResponse: + print("BAZEL", request, file=sys.stderr) + response = self.impl.execute(Request(argv = request.arguments)) + host = socket.gethostname() + pid = os.getpid() + cwd = os.getcwd() + return bazel_pb2.WorkResponse( + exit_code = response.exit_code, + output = f"Bazel persistent worker {host} {pid} {cwd} {request.request_id}\n" + response.stderr, + request_id = request.request_id) + + +def main(): + print("MAIN", socket.gethostname(), os.getpid(), os.getcwd(), file=sys.stderr) + parser = argparse.ArgumentParser( + fromfile_prefix_chars='@', + prog="worker", + description="Buck2/Bazel Local/Remote Persistent Worker") + parser.add_argument( + "--persistent_worker", + action="store_true", + help="Enable persistent worker (Bazel protocol).") + + (args, rest) = parser.parse_known_args() + + if socket_path := os.getenv("WORKER_SOCKET"): + # Buck2 persistent worker mode + print("BUCK2 WORKER START", file=sys.stderr) + if rest: + rest_joined = " ".join(map(shlex.quote, rest)) + print(f"Unexpected arguments: {rest_joined}\n", file=sys.stderr) + parser.print_usage() + sys.exit(2) + + server = grpc.server(futures.ThreadPoolExecutor(max_workers=os.cpu_count() or 1)) + buck2_pb2_grpc.add_WorkerServicer_to_server(Buck2Servicer(), server) + server.add_insecure_port(f"unix://{socket_path}") + server.start() + server.wait_for_termination() + elif args.persistent_worker: + # Bazel persistent worker mode + print("BAZEL WORKER START", file=sys.stderr) + if rest: + rest_joined = " ".join(map(shlex.quote, rest)) + print(f"Unexpected arguments: {rest_joined}\n", file=sys.stderr) + parser.print_usage() + sys.exit(2) + + servicer = BazelServicer() + # uses length prefixed serialization features added in proto version 5.28.0. + # https://github.com/protocolbuffers/protobuf/pull/16965 + while request := proto.parse_length_prefixed(bazel_pb2.WorkRequest, sys.stdin.buffer): + response = servicer.Execute(request) + proto.serialize_length_prefixed(response, sys.stdout.buffer) + sys.stdout.flush() + else: + # One-shot execution mode + print("ONE-SHOT START", file=sys.stderr) + servicer = Implementation() + request = Request(argv = rest) + response = servicer.execute(request) + print(response.stderr, file=sys.stderr) + sys.exit(response.exit_code) + + +if __name__ == "__main__": + main() diff --git a/examples/persistent_worker/platforms/BUCK b/examples/persistent_worker/platforms/BUCK new file mode 100644 index 000000000000..f36ef05b8d88 --- /dev/null +++ b/examples/persistent_worker/platforms/BUCK @@ -0,0 +1,33 @@ +load(":buildbuddy.bzl", "buildbuddy") +load(":local.bzl", "local") + +host_cpu = "prelude//cpu:" + ("arm64" if host_info().arch.is_aarch64 else "x86_64") +host_os = "prelude//os:" + ("macos" if host_info().os.is_macos else "linux") + +buildbuddy( + name = "buildbuddy", + cpu_configuration = host_cpu, + os_configuration = host_os, + use_persistent_workers = False, +) + +buildbuddy( + name = "buildbuddy-persistent-workers", + cpu_configuration = host_cpu, + os_configuration = host_os, + use_persistent_workers = True, +) + +local( + name = "local", + cpu_configuration = host_cpu, + os_configuration = host_os, + use_persistent_workers = False, +) + +local( + name = "local-persistent-workers", + cpu_configuration = host_cpu, + os_configuration = host_os, + use_persistent_workers = True, +) diff --git a/examples/persistent_worker/platforms/buildbuddy.bzl b/examples/persistent_worker/platforms/buildbuddy.bzl new file mode 100644 index 000000000000..a65e07453aef --- /dev/null +++ b/examples/persistent_worker/platforms/buildbuddy.bzl @@ -0,0 +1,52 @@ +# Update the Docker image for remote execution whenever you make a change to the Nix package set. +# +# nix run path:.#dockerBuild \ +# | gzip --fast --no-name \ +# | nix run nixpkgs#skopeo -- copy \ +# --insecure-policy \ +# --digestfile image.digest \ +# --preserve-digests docker-archive:/dev/stdin \ +# docker://ghcr.io/$GITHUB_USER/buck2-remote-persistent-worker:latest +# +# NOTE, Change to a container registry that you have access to. +image = "docker://ghcr.io/aherrmann/buck2-remote-persistent-worker@sha256:23832b49492ef77601111218661b8fcc6eafbcc49cb9abffd08d79988dda540e" + +def _platforms(ctx): + constraints = dict() + constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints) + constraints.update(ctx.attrs.os_configuration[ConfigurationInfo].constraints) + configuration = ConfigurationInfo(constraints = constraints, values = {}) + + platform = ExecutionPlatformInfo( + label = ctx.label.raw_target(), + configuration = configuration, + executor_config = CommandExecutorConfig( + local_enabled = True, + remote_enabled = True, + remote_cache_enabled = True, + allow_cache_uploads = True, + use_limited_hybrid = True, + use_persistent_workers = ctx.attrs.use_persistent_workers, + use_remote_persistent_workers = ctx.attrs.use_persistent_workers, + remote_execution_properties = { + "OSFamily": "Linux", + "container-image": image, + "workload-isolation-type": "podman", + "recycle-runner": True, # required for remote persistent workers + "nonroot-workspace": True, + }, + remote_execution_use_case = "buck2-default", + remote_output_paths = "output_paths", + ), + ) + + return [DefaultInfo(), ExecutionPlatformRegistrationInfo(platforms = [platform])] + +buildbuddy = rule( + attrs = { + "cpu_configuration": attrs.dep(providers = [ConfigurationInfo]), + "os_configuration": attrs.dep(providers = [ConfigurationInfo]), + "use_persistent_workers": attrs.bool(default = False), + }, + impl = _platforms +) diff --git a/examples/persistent_worker/platforms/local.bzl b/examples/persistent_worker/platforms/local.bzl new file mode 100644 index 000000000000..2c9e7d78ac32 --- /dev/null +++ b/examples/persistent_worker/platforms/local.bzl @@ -0,0 +1,29 @@ +def _platforms(ctx): + constraints = dict() + constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints) + constraints.update(ctx.attrs.os_configuration[ConfigurationInfo].constraints) + configuration = ConfigurationInfo(constraints = constraints, values = {}) + + platform = ExecutionPlatformInfo( + label = ctx.label.raw_target(), + configuration = configuration, + executor_config = CommandExecutorConfig( + local_enabled = True, + remote_enabled = False, + remote_cache_enabled = False, + allow_cache_uploads = False, + use_persistent_workers = ctx.attrs.use_persistent_workers, + use_remote_persistent_workers = False, + ), + ) + + return [DefaultInfo(), ExecutionPlatformRegistrationInfo(platforms = [platform])] + +local = rule( + attrs = { + "cpu_configuration": attrs.dep(providers = [ConfigurationInfo]), + "os_configuration": attrs.dep(providers = [ConfigurationInfo]), + "use_persistent_workers": attrs.bool(default = False), + }, + impl = _platforms +) diff --git a/examples/persistent_worker/proto/BUCK b/examples/persistent_worker/proto/BUCK new file mode 100644 index 000000000000..90f869b10335 --- /dev/null +++ b/examples/persistent_worker/proto/BUCK @@ -0,0 +1,5 @@ +python_binary( + name = "protoc", + main_module = "grpc_tools.protoc", + visibility = ["PUBLIC"], +) diff --git a/examples/persistent_worker/proto/bazel/BUCK b/examples/persistent_worker/proto/bazel/BUCK new file mode 100644 index 000000000000..ec6eb230d0af --- /dev/null +++ b/examples/persistent_worker/proto/bazel/BUCK @@ -0,0 +1,24 @@ +genrule( + name = "build-pb2", + cmd = r""" + $(exe //proto:protoc) \ + -Iproto/bazel=${SRCDIR} \ + --python_out=${OUT} \ + --pyi_out=${OUT} \ + --grpc_python_out=${OUT} \ + ${SRCS} + mv ${OUT}/proto/bazel/* ${OUT}/ + """, + srcs = ["worker_protocol.proto"], + outs = {"pb2": [ + "worker_protocol_pb2.py", + "worker_protocol_pb2.pyi", + "worker_protocol_pb2_grpc.py", + ]}, +) + +python_library( + name = "worker_protocol_pb2", + srcs = [":build-pb2[pb2]"], + visibility = ["PUBLIC"], +) diff --git a/examples/persistent_worker/proto/bazel/worker_protocol.proto b/examples/persistent_worker/proto/bazel/worker_protocol.proto new file mode 100644 index 000000000000..4bca44874b5e --- /dev/null +++ b/examples/persistent_worker/proto/bazel/worker_protocol.proto @@ -0,0 +1,98 @@ +// Copyright 2015 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package blaze.worker; + +option java_package = "com.google.devtools.build.lib.worker"; + +// An input file. +message Input { + // The path in the file system where to read this input artifact from. This is + // either a path relative to the execution root (the worker process is + // launched with the working directory set to the execution root), or an + // absolute path. + string path = 1; + + // A hash-value of the contents. The format of the contents is unspecified and + // the digest should be treated as an opaque token. This can be empty in some + // cases. + bytes digest = 2; +} + +// This represents a single work unit that Blaze sends to the worker. +message WorkRequest { + repeated string arguments = 1; + + // The inputs that the worker is allowed to read during execution of this + // request. + repeated Input inputs = 2; + + // Each WorkRequest must have either a unique + // request_id or request_id = 0. If request_id is 0, this WorkRequest must be + // processed alone (singleplex), otherwise the worker may process multiple + // WorkRequests in parallel (multiplexing). As an exception to the above, if + // the cancel field is true, the request_id must be the same as a previously + // sent WorkRequest. The request_id must be attached unchanged to the + // corresponding WorkResponse. Only one singleplex request may be sent to a + // worker at a time. + int32 request_id = 3; + + // EXPERIMENTAL: When true, this is a cancel request, indicating that a + // previously sent WorkRequest with the same request_id should be cancelled. + // The arguments and inputs fields must be empty and should be ignored. + bool cancel = 4; + + // Values greater than 0 indicate that the worker may output extra debug + // information to stderr (which will go into the worker log). Setting the + // --worker_verbose flag for Bazel makes this flag default to 10. + int32 verbosity = 5; + + // The relative directory inside the workers working directory where the + // inputs and outputs are placed, for sandboxing purposes. For singleplex + // workers, this is unset, as they can use their working directory as sandbox. + // For multiplex workers, this will be set when the + // --experimental_worker_multiplex_sandbox flag is set _and_ the execution + // requirements for the worker includes 'supports-multiplex-sandbox'. + // The paths in `inputs` will not contain this prefix, but the actual files + // will be placed/must be written relative to this directory. The worker + // implementation is responsible for resolving the file paths. + string sandbox_dir = 6; +} + +// The worker sends this message to Blaze when it finished its work on the +// WorkRequest message. +message WorkResponse { + int32 exit_code = 1; + + // This is printed to the user after the WorkResponse has been received and is + // supposed to contain compiler warnings / errors etc. - thus we'll use a + // string type here, which gives us UTF-8 encoding. + string output = 2; + + // This field must be set to the same request_id as the WorkRequest it is a + // response to. Since worker processes which support multiplex worker will + // handle multiple WorkRequests in parallel, this ID will be used to + // determined which WorkerProxy does this WorkResponse belong to. + int32 request_id = 3; + + // EXPERIMENTAL When true, indicates that this response was sent due to + // receiving a cancel request. The exit_code and output fields should be empty + // and will be ignored. Exactly one WorkResponse must be sent for each + // non-cancelling WorkRequest received by the worker, but if the worker + // received a cancel request, it doesn't matter if it replies with a regular + // WorkResponse or with one where was_cancelled = true. + bool was_cancelled = 4; +} diff --git a/examples/persistent_worker/proto/buck2/BUCK b/examples/persistent_worker/proto/buck2/BUCK new file mode 100644 index 000000000000..bf1df8cbff24 --- /dev/null +++ b/examples/persistent_worker/proto/buck2/BUCK @@ -0,0 +1,24 @@ +genrule( + name = "build-pb2", + cmd = r""" + $(exe //proto:protoc) \ + -Iproto/buck2=${SRCDIR} \ + --python_out=${OUT} \ + --pyi_out=${OUT} \ + --grpc_python_out=${OUT} \ + ${SRCS} + mv ${OUT}/proto/buck2/* ${OUT}/ + """, + srcs = ["worker.proto"], + outs = {"pb2": [ + "worker_pb2.py", + "worker_pb2.pyi", + "worker_pb2_grpc.py", + ]}, +) + +python_library( + name = "worker_pb2", + srcs = [":build-pb2[pb2]"], + visibility = ["PUBLIC"], +) diff --git a/examples/persistent_worker/proto/buck2/worker.proto b/examples/persistent_worker/proto/buck2/worker.proto new file mode 100644 index 000000000000..0fb4ced5062b --- /dev/null +++ b/examples/persistent_worker/proto/buck2/worker.proto @@ -0,0 +1,47 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "com.facebook.buck.worker.model"; +option java_outer_classname = "WorkerProto"; + +package worker; + +message ExecuteCommand { + message EnvironmentEntry { + bytes key = 1; + bytes value = 2; + } + + repeated bytes argv = 1; + repeated EnvironmentEntry env = 2; +} + +message ExecuteResponse { + int32 exit_code = 1; + string stderr = 2; +} + +message ExecuteCancel {} + +message ExecuteEvent { + oneof data { + ExecuteCommand command = 1; + ExecuteCancel cancel = 2; + } +} + +service Worker { + // TODO(ctolliday) delete once workers switch to Exec + rpc Execute(ExecuteCommand) returns (ExecuteResponse) {}; + + rpc Exec(stream ExecuteEvent) returns (ExecuteResponse) {}; +} diff --git a/examples/persistent_worker/toolchains/.buckconfig b/examples/persistent_worker/toolchains/.buckconfig new file mode 100644 index 000000000000..03964db941d5 --- /dev/null +++ b/examples/persistent_worker/toolchains/.buckconfig @@ -0,0 +1 @@ + diff --git a/examples/persistent_worker/toolchains/BUCK b/examples/persistent_worker/toolchains/BUCK new file mode 100644 index 000000000000..90e817c05d0f --- /dev/null +++ b/examples/persistent_worker/toolchains/BUCK @@ -0,0 +1,25 @@ +load("@prelude//toolchains:cxx.bzl", "system_cxx_toolchain") +load("@prelude//toolchains:genrule.bzl", "system_genrule_toolchain") +load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain", "system_python_toolchain") + +system_cxx_toolchain( + name = "cxx", + visibility = ["PUBLIC"], +) + +system_genrule_toolchain( + name = "genrule", + visibility = ["PUBLIC"], +) + +system_python_bootstrap_toolchain( + name = "python_bootstrap", + interpreter = read_config("nix", "python"), + visibility = ["PUBLIC"], +) + +system_python_toolchain( + name = "python", + interpreter = read_config("nix", "python"), + visibility = ["PUBLIC"], +) From 17ac18abc3f0b2688624a2518b930ea859028aac Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 25 Sep 2024 16:43:30 +0200 Subject: [PATCH 02/36] Implement support for remote persistent workers --- .../src/actions/impls/run.rs | 24 +++++++++++++++++++ .../rule_defs/command_executor_config.rs | 3 +++ .../rule_defs/provider/builtin/worker_info.rs | 11 +++++++++ .../src/execution_types/executor_config.rs | 2 ++ .../src/execute/command_executor.rs | 9 ++++++- app/buck2_execute/src/execute/request.rs | 3 ++- app/buck2_execute/src/lib.rs | 1 + app/buck2_server/src/daemon/common.rs | 1 + app/buck2_test/src/orchestrator.rs | 2 ++ 9 files changed, 54 insertions(+), 2 deletions(-) diff --git a/app/buck2_action_impl/src/actions/impls/run.rs b/app/buck2_action_impl/src/actions/impls/run.rs index c1307b4aa113..c1a3c4bf9e45 100644 --- a/app/buck2_action_impl/src/actions/impls/run.rs +++ b/app/buck2_action_impl/src/actions/impls/run.rs @@ -40,6 +40,7 @@ use buck2_core::execution_types::executor_config::RemoteExecutorDependency; use buck2_core::fs::artifact_path_resolver::ArtifactFs; use buck2_core::fs::buck_out_path::BuildArtifactPath; use buck2_core::fs::paths::forward_rel_path::ForwardRelativePathBuf; +use buck2_error::buck2_error; use buck2_error::BuckErrorContext; use buck2_events::dispatch::span_async_simple; use buck2_execute::artifact::fs::ExecutorFs; @@ -263,6 +264,7 @@ struct UnpackedWorkerValues<'v> { id: WorkerId, concurrency: Option, streaming: bool, + remote: bool, } struct UnpackedRunActionValues<'v> { @@ -320,6 +322,7 @@ impl RunAction { id: WorkerId(worker.id), concurrency: worker.concurrency(), streaming: worker.streaming(), + remote: worker.remote(), }); Ok(UnpackedRunActionValues { @@ -352,11 +355,32 @@ impl RunAction { .exe .add_to_command_line(&mut worker_rendered, &mut cli_ctx)?; worker.exe.visit_artifacts(artifact_visitor)?; + let worker_key = if worker.remote { + let mut worker_visitor = SimpleCommandLineArtifactVisitor::new(); + worker.exe.visit_artifacts(&mut worker_visitor)?; + if !worker_visitor.outputs.is_empty() { + // TODO[AH] create appropriate error enum value. + return Err(buck2_error!( + [], + "remote persistent worker command should not produce an output" + )); + } + let worker_inputs: Vec<&ArtifactGroupValues> = worker_visitor + .inputs() + .map(|group| action_execution_ctx.artifact_values(group)) + .collect(); + let (_, worker_digest) = + metadata_content(fs.fs(), &worker_inputs, action_execution_ctx.digest_config())?; + Some(worker_digest) + } else { + None + }; Some(WorkerSpec { exe: worker_rendered, id: worker.id, concurrency: worker.concurrency, streaming: worker.streaming, + remote_key: worker_key, }) } else { None diff --git a/app/buck2_build_api/src/interpreter/rule_defs/command_executor_config.rs b/app/buck2_build_api/src/interpreter/rule_defs/command_executor_config.rs index ba65c66f2bf4..25c93604fcd8 100644 --- a/app/buck2_build_api/src/interpreter/rule_defs/command_executor_config.rs +++ b/app/buck2_build_api/src/interpreter/rule_defs/command_executor_config.rs @@ -86,6 +86,7 @@ pub fn register_command_executor_config(builder: &mut GlobalsBuilder) { /// * `allow_hybrid_fallbacks_on_failure`: Whether to allow fallbacks when the result is failure (i.e. the command failed on the primary, but the infra worked) /// * `use_windows_path_separators`: Whether to use Windows path separators in command line arguments /// * `use_persistent workers`: Whether to use persistent workers for local execution if they are available + /// * `use_remote_persistent_workers`: Whether to use persistent workers for remote execution if they are available /// * `allow_cache_uploads`: Whether to upload local actions to the RE cache /// * `max_cache_upload_mebibytes`: Maximum size to upload in cache uploads /// * `experimental_low_pass_filter`: Whether to use the experimental low pass filter @@ -111,6 +112,7 @@ pub fn register_command_executor_config(builder: &mut GlobalsBuilder) { #[starlark(default = false, require = named)] allow_hybrid_fallbacks_on_failure: bool, #[starlark(default = false, require = named)] use_windows_path_separators: bool, #[starlark(default = false, require = named)] use_persistent_workers: bool, + #[starlark(default = false, require = named)] use_remote_persistent_workers: bool, #[starlark(default = false, require = named)] allow_cache_uploads: bool, #[starlark(default = NoneOr::None, require = named)] max_cache_upload_mebibytes: NoneOr< i32, @@ -323,6 +325,7 @@ pub fn register_command_executor_config(builder: &mut GlobalsBuilder) { PathSeparatorKind::Unix }, output_paths_behavior, + use_remote_persistent_workers, }, } }; diff --git a/app/buck2_build_api/src/interpreter/rule_defs/provider/builtin/worker_info.rs b/app/buck2_build_api/src/interpreter/rule_defs/provider/builtin/worker_info.rs index 51753d1be41a..742aab65d9aa 100644 --- a/app/buck2_build_api/src/interpreter/rule_defs/provider/builtin/worker_info.rs +++ b/app/buck2_build_api/src/interpreter/rule_defs/provider/builtin/worker_info.rs @@ -50,6 +50,8 @@ pub struct WorkerInfoGen { pub concurrency: ValueOfUncheckedGeneric>, // Whether to always run actions using this worker via the streaming API pub streaming: ValueOfUncheckedGeneric, + // Remote execution capable worker + pub remote: ValueOfUncheckedGeneric, pub id: u64, } @@ -68,6 +70,7 @@ fn worker_info_creator(globals: &mut GlobalsBuilder) { ValueOf<'v, usize>, >, #[starlark(require = named, default = NoneType)] streaming: Value<'v>, + #[starlark(require = named, default = false)] remote: bool, eval: &mut Evaluator<'v, '_, '_>, ) -> starlark::Result> { let heap = eval.heap(); @@ -79,6 +82,7 @@ fn worker_info_creator(globals: &mut GlobalsBuilder) { id, concurrency: heap.alloc_typed_unchecked(concurrency).cast(), streaming: ValueOfUnchecked::new(streaming), + remote: heap.alloc_typed_unchecked(remote).cast(), }) } } @@ -105,6 +109,13 @@ impl<'v, V: ValueLike<'v>> WorkerInfoGen { .into_option() .unwrap_or(false) } + + pub fn remote(&self) -> bool { + self.remote + .to_value() + .unpack() + .expect("validated at construction") + } } fn validate_worker_info<'v, V>(info: &WorkerInfoGen) -> buck2_error::Result<()> diff --git a/app/buck2_core/src/execution_types/executor_config.rs b/app/buck2_core/src/execution_types/executor_config.rs index 113bc92acece..03689701c901 100644 --- a/app/buck2_core/src/execution_types/executor_config.rs +++ b/app/buck2_core/src/execution_types/executor_config.rs @@ -283,6 +283,7 @@ impl Default for CacheUploadBehavior { pub struct CommandGenerationOptions { pub path_separator: PathSeparatorKind, pub output_paths_behavior: OutputPathsBehavior, + pub use_remote_persistent_workers: bool, } #[derive(Debug, Eq, PartialEq, Hash, Allocative, Clone)] @@ -314,6 +315,7 @@ impl CommandExecutorConfig { options: CommandGenerationOptions { path_separator: PathSeparatorKind::system_default(), output_paths_behavior: Default::default(), + use_remote_persistent_workers: false, }, }) } diff --git a/app/buck2_execute/src/execute/command_executor.rs b/app/buck2_execute/src/execute/command_executor.rs index 1859e6fb0021..a5cb57c3a759 100644 --- a/app/buck2_execute/src/execute/command_executor.rs +++ b/app/buck2_execute/src/execute/command_executor.rs @@ -187,6 +187,13 @@ impl CommandExecutor { } CommandExecutionInput::ScratchPath(_) => None, }); + let mut platform = self.0.re_platform.clone(); + if self.0.options.use_remote_persistent_workers && let Some(worker) = request.worker() && let Some(key) = worker.remote_key.as_ref() { + platform.properties.push(RE::Property { + name: "persistentWorkerKey".to_owned(), + value: key.to_string(), + }); + } let action = re_create_action( request.all_args_vec(), request.paths().output_paths(), @@ -195,7 +202,7 @@ impl CommandExecutor { input_digest, action_metadata_blobs, request.timeout(), - self.0.re_platform.clone(), + platform, false, digest_config, self.0.options.output_paths_behavior, diff --git a/app/buck2_execute/src/execute/request.rs b/app/buck2_execute/src/execute/request.rs index 7c17d4c946af..a205892c4fac 100644 --- a/app/buck2_execute/src/execute/request.rs +++ b/app/buck2_execute/src/execute/request.rs @@ -278,12 +278,13 @@ impl CommandExecutionPaths { #[derive(Copy, Clone, Dupe, Debug, Display, Allocative, Hash, PartialEq, Eq)] pub struct WorkerId(pub u64); -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct WorkerSpec { pub id: WorkerId, pub exe: Vec, pub concurrency: Option, pub streaming: bool, + pub remote_key: Option, } /// The data contains the information about the command to be executed. diff --git a/app/buck2_execute/src/lib.rs b/app/buck2_execute/src/lib.rs index aebb7770d038..59bc110e4ae0 100644 --- a/app/buck2_execute/src/lib.rs +++ b/app/buck2_execute/src/lib.rs @@ -14,6 +14,7 @@ #![feature(try_trait_v2)] #![feature(used_with_arg)] #![feature(trait_upcasting)] +#![feature(let_chains)] pub mod artifact; pub mod artifact_utils; diff --git a/app/buck2_server/src/daemon/common.rs b/app/buck2_server/src/daemon/common.rs index 2bfa6204f993..61202c8725ce 100644 --- a/app/buck2_server/src/daemon/common.rs +++ b/app/buck2_server/src/daemon/common.rs @@ -500,6 +500,7 @@ pub fn get_default_executor_config(host_platform: HostPlatformOverride) -> Comma options: CommandGenerationOptions { path_separator: get_default_path_separator(host_platform), output_paths_behavior: Default::default(), + use_remote_persistent_workers: false, }, } } diff --git a/app/buck2_test/src/orchestrator.rs b/app/buck2_test/src/orchestrator.rs index b66144f30795..990e7a87c2fc 100644 --- a/app/buck2_test/src/orchestrator.rs +++ b/app/buck2_test/src/orchestrator.rs @@ -1191,6 +1191,7 @@ impl<'b> BuckTestOrchestrator<'b> { options: CommandGenerationOptions { path_separator: PathSeparatorKind::system_default(), output_paths_behavior: Default::default(), + use_remote_persistent_workers: false, }, }; let CommandExecutorResponse { @@ -1733,6 +1734,7 @@ impl<'a> Execute2RequestExpander<'a> { id: WorkerId(worker.id), concurrency: worker.concurrency(), streaming: worker.streaming(), + remote_key: None, }) } _ => None, From 9730b29a781d12ce843c595a5eca265b321cfd69 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 25 Sep 2024 17:52:11 +0200 Subject: [PATCH 03/36] Fix clippy error --- app/buck2_build_api/src/actions/execute/action_executor.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/app/buck2_build_api/src/actions/execute/action_executor.rs b/app/buck2_build_api/src/actions/execute/action_executor.rs index 8297c9ca2856..e0fd9256fb9f 100644 --- a/app/buck2_build_api/src/actions/execute/action_executor.rs +++ b/app/buck2_build_api/src/actions/execute/action_executor.rs @@ -761,6 +761,7 @@ mod tests { CommandGenerationOptions { path_separator: PathSeparatorKind::Unix, output_paths_behavior: Default::default(), + use_remote_persistent_workers: false, }, Default::default(), ), From 9e0b232dd18a88fed63142096e2b4245c040970d Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 25 Sep 2024 20:59:40 +0200 Subject: [PATCH 04/36] fix run_display test --- .../src/interpreter/rule_defs/provider/builtin/worker_info.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/buck2_build_api_tests/src/interpreter/rule_defs/provider/builtin/worker_info.rs b/app/buck2_build_api_tests/src/interpreter/rule_defs/provider/builtin/worker_info.rs index 1124c9e1629b..3fd51e7dae70 100644 --- a/app/buck2_build_api_tests/src/interpreter/rule_defs/provider/builtin/worker_info.rs +++ b/app/buck2_build_api_tests/src/interpreter/rule_defs/provider/builtin/worker_info.rs @@ -24,7 +24,7 @@ fn run_display() { .run_starlark_bzl_test( r#" def test(): - assert_eq('WorkerInfo(exe=cmd_args("x"), concurrency=None, streaming=None)', str(WorkerInfo(exe="x"))) + assert_eq('WorkerInfo(exe=cmd_args("x"), concurrency=None, streaming=None, remote=False)', str(WorkerInfo(exe="x"))) "#, ) .unwrap(); From 58d4a22a04f0594c9ab6ca2df8831ccf3a3777da Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Fri, 4 Oct 2024 17:18:59 +0200 Subject: [PATCH 05/36] Use WorkerRunInfo.exe for non-worker mode `WorkerRunInfo` has two fields `worker` and `exe` for the persistent worker command or the non-worker mode command respectively. This commit changes the remote persistent worker example to use these appropriately to distinguish between worker and non-worker mode execution. --- examples/persistent_worker/BUCK | 5 ++++ examples/persistent_worker/defs.bzl | 12 ++++++-- examples/persistent_worker/one_shot.py | 29 +++++++++++++++++++ .../persistent_worker/persistent_worker.py | 14 ++------- 4 files changed, 46 insertions(+), 14 deletions(-) create mode 100644 examples/persistent_worker/one_shot.py diff --git a/examples/persistent_worker/BUCK b/examples/persistent_worker/BUCK index 1384003d20d4..f16a384c7517 100644 --- a/examples/persistent_worker/BUCK +++ b/examples/persistent_worker/BUCK @@ -1,5 +1,10 @@ load("defs.bzl", "demo", "worker") +python_binary( + name = "one_shot", + main = "one_shot.py", +) + python_binary( name = "worker_py", main = "persistent_worker.py", diff --git a/examples/persistent_worker/defs.bzl b/examples/persistent_worker/defs.bzl index fa5d596e1ff3..38520cb1b9f1 100644 --- a/examples/persistent_worker/defs.bzl +++ b/examples/persistent_worker/defs.bzl @@ -19,22 +19,28 @@ worker = rule( def _demo_impl(ctx: AnalysisContext) -> list[Provider]: output = ctx.actions.declare_output(ctx.label.name) - worker_exe = ctx.attrs._worker[WorkerInfo].exe argfile = at_argfile( actions = ctx.actions, name = "demo." + ctx.label.name + ".args", args = cmd_args(output.as_output()), ) ctx.actions.run( - cmd_args(worker_exe, argfile), + cmd_args(argfile), category = "demo", - exe = WorkerRunInfo(worker = ctx.attrs._worker[WorkerInfo]), + exe = WorkerRunInfo( + worker = ctx.attrs._worker[WorkerInfo], + exe = ctx.attrs._one_shot[RunInfo].args, + ), ) return [DefaultInfo(default_output = output)] demo = rule( impl = _demo_impl, attrs = { + "_one_shot": attrs.exec_dep( + default = "//:one_shot", + providers = [RunInfo], + ), "_worker": attrs.exec_dep( default = "//:worker", providers = [WorkerInfo], diff --git a/examples/persistent_worker/one_shot.py b/examples/persistent_worker/one_shot.py new file mode 100644 index 000000000000..121313b659f0 --- /dev/null +++ b/examples/persistent_worker/one_shot.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import argparse +import os +import sys + + +def main(): + parser = argparse.ArgumentParser( + fromfile_prefix_chars='@', + prog="one_shot", + description="One-shot command") + parser.add_argument( + "outfile", + type=argparse.FileType("w"), + help="Output file.") + + args = parser.parse_args() + + print("one-shot.py", file=sys.stderr) + print("ONE-SHOT START", file=sys.stderr) + name = os.path.basename(args.outfile.name) + args.outfile.write(name + "\n") + args.outfile.close() + print("ONE-SHOT END", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/examples/persistent_worker/persistent_worker.py b/examples/persistent_worker/persistent_worker.py index 7887035af80c..38c0cd24291b 100644 --- a/examples/persistent_worker/persistent_worker.py +++ b/examples/persistent_worker/persistent_worker.py @@ -2,7 +2,6 @@ """Buck2 local and remote persistent worker and action runner. This script can: -- Execute build requests in one-shot mode for builds without persistent workers enabled. - Execute build requests as a Buck2 local persistent worker. - Execute build requests as a remote persistent worker through Bazel protocol. """ @@ -86,9 +85,7 @@ def Execute(self, request, context): _ = context print("BUCK2", request, file=sys.stderr) # Decode arguments as UTF-8 strings. - # Drop the first argument, which is the worker binary. - # The first argument is needed for Bazel remote execution persistent workers. - argv = [arg.decode("utf-8") for arg in request.argv[1:]] + argv = [arg.decode("utf-8") for arg in request.argv] response = self.impl.execute(Request(argv = argv)) host = socket.gethostname() pid = os.getpid() @@ -158,13 +155,8 @@ def main(): proto.serialize_length_prefixed(response, sys.stdout.buffer) sys.stdout.flush() else: - # One-shot execution mode - print("ONE-SHOT START", file=sys.stderr) - servicer = Implementation() - request = Request(argv = rest) - response = servicer.execute(request) - print(response.stderr, file=sys.stderr) - sys.exit(response.exit_code) + print("Expected either 'WORKER_SOCKET' environment variable or '--persistent_worker' argument.", file=sys.stderr) + sys.exit(2) if __name__ == "__main__": From 1bd5a94852118f34ee50ad3083d9a3e07c4e5b48 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Fri, 4 Oct 2024 17:21:05 +0200 Subject: [PATCH 06/36] Use WorkerRunInfo.worker for remote persistent worker The Buck2 Rust implementation needs to use the worker command instead of the non-worker command for remote persistent worker execution mode. --- .../src/execute/command_executor.rs | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/app/buck2_execute/src/execute/command_executor.rs b/app/buck2_execute/src/execute/command_executor.rs index a5cb57c3a759..3c2e50bf7668 100644 --- a/app/buck2_execute/src/execute/command_executor.rs +++ b/app/buck2_execute/src/execute/command_executor.rs @@ -20,7 +20,6 @@ use buck2_core::fs::artifact_path_resolver::ArtifactFs; use buck2_core::fs::project_rel_path::ProjectRelativePath; use buck2_core::fs::project_rel_path::ProjectRelativePathBuf; use buck2_directory::directory::fingerprinted_directory::FingerprintedDirectory; -#[cfg(fbcode_build)] use buck2_error::buck2_error; use buck2_error::BuckErrorContext; use buck2_futures::cancellation::CancellationContext; @@ -188,14 +187,37 @@ impl CommandExecutor { CommandExecutionInput::ScratchPath(_) => None, }); let mut platform = self.0.re_platform.clone(); - if self.0.options.use_remote_persistent_workers && let Some(worker) = request.worker() && let Some(key) = worker.remote_key.as_ref() { + let args = if self.0.options.use_remote_persistent_workers + && let Some(worker) = request.worker() + && let Some(key) = worker.remote_key.as_ref() + { platform.properties.push(RE::Property { name: "persistentWorkerKey".to_owned(), value: key.to_string(), }); - } + // TODO[AH] Ideally, Buck2 could generate an argfile on the fly. + for arg in request.args() { + if !(arg.starts_with("@") + || arg.starts_with("-flagfile") + || arg.starts_with("--flagfile")) + { + return Err(buck2_error!( + [], + "Remote persistent worker arguments must be passed as `@argfile`, `-flagfile=argfile`, or `--flagfile=argfile`." + )); + } + } + worker + .exe + .iter() + .chain(request.args().iter()) + .cloned() + .collect() + } else { + request.all_args_vec() + }; let action = re_create_action( - request.all_args_vec(), + args, request.paths().output_paths(), request.working_directory(), request.env(), From 2c032c97b024272d8bbfbcea187d7c2c9f2805ff Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 7 Oct 2024 11:01:55 +0200 Subject: [PATCH 07/36] Document the worker protocol in the example --- examples/persistent_worker/README.md | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/examples/persistent_worker/README.md b/examples/persistent_worker/README.md index 44626ebe9b7b..0f88180c3cd3 100644 --- a/examples/persistent_worker/README.md +++ b/examples/persistent_worker/README.md @@ -99,3 +99,41 @@ stderr for root//:demo-7 (demo): Bazel persistent worker ... ... ``` + +## Protocol + +### Starlark + +A Buck2 persistent worker is created by a rule that emits the +`WorkerInfo` provider. Setting `remote = True` on this provider +indicates that this worker is remote execution capable. + +Buck2 actions indicate that they can utilize a persistent worker by +setting the `exe` parameter to `ctx.actions.run` to +`WorkerRunInfo(worker, exe)`, where `worker` is a `WorkerInfo` provider, +and `exe` defines the fallback executable for non persistent-worker +execution. + +Buck2 actions that want to utilize a remote persistent worker must pass +command-line arguments in an argument file specified as `@argfile`, +`-flagfile=argfile`, or `--flagfile=argfile` on the command-line. + +### Local Persistent Worker + +A locally executed Buck2 persistent worker falls under the [Buck2 +persistent worker protocol](./proto/buck2/worker.proto): It is started +and managed by Buck2 and passed a file path in the `WORKER_SOCKET` +environment variable where it should create a gRPC Unix domain socket to +serve worker requests over. Multiple requests may be sent in parallel +and expected to be served at the same time depending on the +`concurrency` attribute of the `WorkerInfo` provider. + +### Remote Persistent Worker + +A remotely executed Buck2 persistent worker falls under the [Bazel +persistent worker protocol](./proto/bazel/worker_protocol.proto): It is +started and managed by the remote execution system. Work requests are +sent as length prefixed protobuf objects to the standard input of the +worker process. Work responses are expected as length prefixed protobuf +objects on the standard output of the worker process. The worker process +may not use standard output for anything else. From 4a3e31e88cf09bfba15f9bf446f3baa8d1c0c651 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 8 Oct 2024 11:44:11 +0200 Subject: [PATCH 08/36] Create proto_python_library rule We want to use a Buck2 managed hermetic Python toolchain. However, Python binaries generated by such a toolchain are sensitive to `PWD` and don't work in other working directories than the repository root. Unfortunately, `genrule` changes directory and is hence incompatible with such Python binaries. This commit defines a dedicated rule to invoke protoc without changing the working directory. --- examples/persistent_worker/proto/buck2/BUCK | 8 ++++ examples/persistent_worker/proto/defs.bzl | 46 +++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 examples/persistent_worker/proto/defs.bzl diff --git a/examples/persistent_worker/proto/buck2/BUCK b/examples/persistent_worker/proto/buck2/BUCK index bf1df8cbff24..acdedc62d267 100644 --- a/examples/persistent_worker/proto/buck2/BUCK +++ b/examples/persistent_worker/proto/buck2/BUCK @@ -1,3 +1,11 @@ +load("//proto:defs.bzl", "proto_python_library") + +proto_python_library( + name = "worker", + src = "worker.proto", + visibility = ["PUBLIC"], +) + genrule( name = "build-pb2", cmd = r""" diff --git a/examples/persistent_worker/proto/defs.bzl b/examples/persistent_worker/proto/defs.bzl new file mode 100644 index 000000000000..a834e4a4fbc7 --- /dev/null +++ b/examples/persistent_worker/proto/defs.bzl @@ -0,0 +1,46 @@ +def _proto_python_library_impl(ctx: AnalysisContext) -> list[Provider]: + prefix = ctx.label.package + depth = len(prefix.split("/")) + libname = ctx.attrs.src.basename.removesuffix(".proto") + "_pb2" + python_out = ctx.actions.declare_output(prefix, "{}.py".format(libname)) + pyi_out = ctx.actions.declare_output(prefix, "{}.pyi".format(libname)) + grpc_python_out = ctx.actions.declare_output(prefix, "{}_grpc.py".format(libname)) + ctx.actions.run( + cmd_args( + ctx.attrs._protoc[RunInfo], + cmd_args(ctx.attrs.src, format = "-I{}={}", parent = 1), + cmd_args(python_out.as_output(), format = "--python_out={}", parent = depth + 1), + cmd_args(pyi_out.as_output(), format = "--pyi_out={}", parent = depth + 1), + cmd_args(grpc_python_out.as_output(), format = "--grpc_python_out={}", parent = depth + 1), + ctx.attrs.src, + ), + category = "protoc", + ) + # protoc does not let us control the import prefix and path prefix separately. + # So, we need to copy the generated files into the correct location after the fact. + python_out_copied = ctx.actions.declare_output("{}.py".format(libname)) + pyi_out_copied = ctx.actions.declare_output("{}.pyi".format(libname)) + grpc_python_out_copied = ctx.actions.declare_output("{}_grpc.py".format(libname)) + ctx.actions.copy_file(python_out_copied, python_out) + ctx.actions.copy_file(pyi_out_copied, pyi_out) + ctx.actions.copy_file(grpc_python_out_copied, grpc_python_out) + return [DefaultInfo(default_outputs = [python_out_copied, pyi_out_copied, grpc_python_out_copied])] + +_proto_python_library = rule( + impl = _proto_python_library_impl, + attrs = { + "src": attrs.source(), + "_protoc": attrs.exec_dep(default = "//proto:protoc", providers = [RunInfo]), + }, +) + +def proto_python_library(*, name, src, **kwargs): + _proto_python_library( + name = "{}-gen".format(name), + src = src, + ) + native.python_library( + name = name, + srcs = [":{}-gen".format(name)], + **kwargs + ) From f659e4352f0ee4742b813faa24ec7028d8ac8fea Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 8 Oct 2024 11:04:24 +0200 Subject: [PATCH 09/36] Hermetic Python toolchain and Buck2 installed packages The previous setup used Nix to provide a Python toolchain and packages in a reproducible fashion. However, this requires dedicated remote worker images with the Nix store paths pre-populated which complicates the setup for testing on Buck2 CI. Using a hermetic Python toolchain avoids these issues and works on the standard remote execution image. --- examples/persistent_worker/python/BUCK | 22 ++++++++ examples/persistent_worker/python/defs.bzl | 14 +++++ examples/persistent_worker/toolchains/BUCK | 19 +++++-- .../persistent_worker/toolchains/defs.bzl | 52 +++++++++++++++++++ 4 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 examples/persistent_worker/python/BUCK create mode 100644 examples/persistent_worker/python/defs.bzl create mode 100644 examples/persistent_worker/toolchains/defs.bzl diff --git a/examples/persistent_worker/python/BUCK b/examples/persistent_worker/python/BUCK new file mode 100644 index 000000000000..df744aacb867 --- /dev/null +++ b/examples/persistent_worker/python/BUCK @@ -0,0 +1,22 @@ +load(":defs.bzl", "fetch_python_libraries") + +fetch_python_libraries(pkgs = { + "grpcio-tools": { + "url": "https://files.pythonhosted.org/packages/1d/0f/273d7ac9c7d99b56abb5841d8aff7ffd148fe01b48c2913c8da3de9438e7/grpcio_tools-1.66.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "sha256": "c68642829368f4f83929e0df571dbbc99f1f1553555d8f98d0582da9f6743d9e", + "deps": ["grpcio", "protobuf", "setuptools"], + }, + "grpcio": { + "url": "https://files.pythonhosted.org/packages/2f/86/a86742f3deaa22385c3bff984c5947fc62d47d3fab26c508730037d027e5/grpcio-1.66.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "sha256": "f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46", + "deps": ["protobuf"], + }, + "protobuf": { + "url": "https://files.pythonhosted.org/packages/9b/55/f24e3b801d2e108c48aa2b1b59bb791b5cffba89465cbbf66fc98de89270/protobuf-5.28.2-py3-none-any.whl", + "sha256": "52235802093bd8a2811abbe8bf0ab9c5f54cca0a751fdd3f6ac2a21438bffece", + }, + "setuptools": { + "url": "https://files.pythonhosted.org/packages/ff/ae/f19306b5a221f6a436d8f2238d5b80925004093fa3edea59835b514d9057/setuptools-75.1.0-py3-none-any.whl", + "sha256": "35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2", + }, +}) diff --git a/examples/persistent_worker/python/defs.bzl b/examples/persistent_worker/python/defs.bzl new file mode 100644 index 000000000000..0584ae1aba90 --- /dev/null +++ b/examples/persistent_worker/python/defs.bzl @@ -0,0 +1,14 @@ +def fetch_python_libraries(pkgs): + for name, pkg in pkgs.items(): + native.remote_file( + name = "{}-download".format(name), + url = pkg["url"], + sha256 = pkg["sha256"], + out = "{}.whl".format(name), + ) + native.prebuilt_python_library( + name = name, + binary_src = ":{}-download".format(name), + deps = [":{}".format(dep) for dep in pkg.get("deps", [])], + visibility = ["PUBLIC"], + ) diff --git a/examples/persistent_worker/toolchains/BUCK b/examples/persistent_worker/toolchains/BUCK index 90e817c05d0f..b26965ad7542 100644 --- a/examples/persistent_worker/toolchains/BUCK +++ b/examples/persistent_worker/toolchains/BUCK @@ -1,6 +1,7 @@ load("@prelude//toolchains:cxx.bzl", "system_cxx_toolchain") load("@prelude//toolchains:genrule.bzl", "system_genrule_toolchain") load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain", "system_python_toolchain") +load(":defs.bzl", "python_toolchain") system_cxx_toolchain( name = "cxx", @@ -14,12 +15,24 @@ system_genrule_toolchain( system_python_bootstrap_toolchain( name = "python_bootstrap", - interpreter = read_config("nix", "python"), + interpreter = "python3", visibility = ["PUBLIC"], ) -system_python_toolchain( +remote_file( + name = "python-download", + url = "https://github.com/indygreg/python-build-standalone/releases/download/20241002/cpython-3.13.0rc3+20241002-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + sha256 = "445156c61e1cc167f7b8777ad08cc36e5598e12cd27e07453f6e6dc0f62e421e", + out = "cpython.tar.gz", +) + +extract_archive( + name = "python-extract", + contents_archive = ":python-download", +) + +python_toolchain( name = "python", - interpreter = read_config("nix", "python"), + distribution = ":python-extract", visibility = ["PUBLIC"], ) diff --git a/examples/persistent_worker/toolchains/defs.bzl b/examples/persistent_worker/toolchains/defs.bzl new file mode 100644 index 000000000000..de5aff4d2f36 --- /dev/null +++ b/examples/persistent_worker/toolchains/defs.bzl @@ -0,0 +1,52 @@ +load( + "@prelude//:artifacts.bzl", + "ArtifactGroupInfo", +) +load( + "@prelude//python:toolchain.bzl", + "PythonPlatformInfo", + "PythonToolchainInfo", +) + +def _python_toolchain_impl(ctx): + distribution = ctx.attrs.distribution[DefaultInfo].default_outputs[0] + interpreter = cmd_args(distribution, absolute_suffix = "/python/bin/python") + return [ + DefaultInfo(), + PythonToolchainInfo( + binary_linker_flags = ctx.attrs.binary_linker_flags, + linker_flags = ctx.attrs.linker_flags, + fail_with_message = ctx.attrs.fail_with_message[RunInfo], + generate_static_extension_info = ctx.attrs.generate_static_extension_info, + make_source_db = ctx.attrs.make_source_db[RunInfo], + make_source_db_no_deps = ctx.attrs.make_source_db_no_deps[RunInfo], + host_interpreter = RunInfo(args = ["python3"]), + interpreter = RunInfo(args = [interpreter]), + make_py_package_modules = ctx.attrs.make_py_package_modules[RunInfo], + make_py_package_inplace = ctx.attrs.make_py_package_inplace[RunInfo], + compile = RunInfo(args = ["echo", "COMPILEINFO"]), + package_style = "inplace", + pex_extension = ctx.attrs.pex_extension, + native_link_strategy = "separate", + runtime_library = ctx.attrs.runtime_library, + ), + PythonPlatformInfo(name = "x86_64"), + ] + +python_toolchain = rule( + impl = _python_toolchain_impl, + attrs = { + "distribution": attrs.exec_dep(), + "binary_linker_flags": attrs.default_only(attrs.list(attrs.arg(), default = [])), + "fail_with_message": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//python/tools:fail_with_message")), + "generate_static_extension_info": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//python/tools:generate_static_extension_info")), + "linker_flags": attrs.default_only(attrs.list(attrs.arg(), default = [])), + "make_py_package_inplace": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//python/tools:make_py_package_inplace")), + "make_py_package_modules": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//python/tools:make_py_package_modules")), + "make_source_db": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//python/tools:make_source_db")), + "make_source_db_no_deps": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//python/tools:make_source_db_no_deps")), + "pex_extension": attrs.string(default = ".pex"), + "runtime_library": attrs.default_only(attrs.dep(providers = [ArtifactGroupInfo], default = "prelude//python/runtime:bootstrap_files")), + }, + is_toolchain_rule = True, +) From bd8b6d7ebd888dbce5a95029fb225be147364eba Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 8 Oct 2024 15:30:05 +0200 Subject: [PATCH 10/36] Use proto_python_library rule Remove the old genrule targets to generate the Python gRPC/protobuf bindings. --- examples/persistent_worker/proto/BUCK | 1 + examples/persistent_worker/proto/bazel/BUCK | 23 +++--------------- examples/persistent_worker/proto/buck2/BUCK | 27 +-------------------- examples/persistent_worker/proto/defs.bzl | 1 + 4 files changed, 6 insertions(+), 46 deletions(-) diff --git a/examples/persistent_worker/proto/BUCK b/examples/persistent_worker/proto/BUCK index 90f869b10335..316ba09efa17 100644 --- a/examples/persistent_worker/proto/BUCK +++ b/examples/persistent_worker/proto/BUCK @@ -1,5 +1,6 @@ python_binary( name = "protoc", main_module = "grpc_tools.protoc", + deps = ["//python:grpcio-tools"], visibility = ["PUBLIC"], ) diff --git a/examples/persistent_worker/proto/bazel/BUCK b/examples/persistent_worker/proto/bazel/BUCK index ec6eb230d0af..5728ce76db4a 100644 --- a/examples/persistent_worker/proto/bazel/BUCK +++ b/examples/persistent_worker/proto/bazel/BUCK @@ -1,24 +1,7 @@ -genrule( - name = "build-pb2", - cmd = r""" - $(exe //proto:protoc) \ - -Iproto/bazel=${SRCDIR} \ - --python_out=${OUT} \ - --pyi_out=${OUT} \ - --grpc_python_out=${OUT} \ - ${SRCS} - mv ${OUT}/proto/bazel/* ${OUT}/ - """, - srcs = ["worker_protocol.proto"], - outs = {"pb2": [ - "worker_protocol_pb2.py", - "worker_protocol_pb2.pyi", - "worker_protocol_pb2_grpc.py", - ]}, -) +load("//proto:defs.bzl", "proto_python_library") -python_library( +proto_python_library( name = "worker_protocol_pb2", - srcs = [":build-pb2[pb2]"], + src = "worker_protocol.proto", visibility = ["PUBLIC"], ) diff --git a/examples/persistent_worker/proto/buck2/BUCK b/examples/persistent_worker/proto/buck2/BUCK index acdedc62d267..87b120176bbd 100644 --- a/examples/persistent_worker/proto/buck2/BUCK +++ b/examples/persistent_worker/proto/buck2/BUCK @@ -1,32 +1,7 @@ load("//proto:defs.bzl", "proto_python_library") proto_python_library( - name = "worker", - src = "worker.proto", - visibility = ["PUBLIC"], -) - -genrule( - name = "build-pb2", - cmd = r""" - $(exe //proto:protoc) \ - -Iproto/buck2=${SRCDIR} \ - --python_out=${OUT} \ - --pyi_out=${OUT} \ - --grpc_python_out=${OUT} \ - ${SRCS} - mv ${OUT}/proto/buck2/* ${OUT}/ - """, - srcs = ["worker.proto"], - outs = {"pb2": [ - "worker_pb2.py", - "worker_pb2.pyi", - "worker_pb2_grpc.py", - ]}, -) - -python_library( name = "worker_pb2", - srcs = [":build-pb2[pb2]"], + src = "worker.proto", visibility = ["PUBLIC"], ) diff --git a/examples/persistent_worker/proto/defs.bzl b/examples/persistent_worker/proto/defs.bzl index a834e4a4fbc7..133d0cbcfd2c 100644 --- a/examples/persistent_worker/proto/defs.bzl +++ b/examples/persistent_worker/proto/defs.bzl @@ -42,5 +42,6 @@ def proto_python_library(*, name, src, **kwargs): native.python_library( name = name, srcs = [":{}-gen".format(name)], + deps = ["//python:grpcio"], **kwargs ) From 6f0ce5e0854bf8bfa5953e6382b041ee12dfeaf6 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 8 Oct 2024 17:44:19 +0200 Subject: [PATCH 11/36] Standard BuildBuddy worker image Without Nix it is no longer required to use a custom remote worker image. --- examples/persistent_worker/.buckconfig.buildbuddy | 4 +--- .../.buckconfig.buildbuddy-persistent-workers | 4 +--- .../persistent_worker/platforms/buildbuddy.bzl | 15 --------------- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/examples/persistent_worker/.buckconfig.buildbuddy b/examples/persistent_worker/.buckconfig.buildbuddy index e13c0b2fab19..66227e97615e 100644 --- a/examples/persistent_worker/.buckconfig.buildbuddy +++ b/examples/persistent_worker/.buckconfig.buildbuddy @@ -7,9 +7,7 @@ action_cache_address = grpc://remote.buildbuddy.io cas_address = grpc://remote.buildbuddy.io tls = true http_headers = \ - x-buildbuddy-api-key:$BUILDBUDDY_API_KEY, \ - x-buildbuddy-platform.container-registry-username:$BUILDBUDDY_CONTAINER_USER, \ - x-buildbuddy-platform.container-registry-password:$BUILDBUDDY_CONTAINER_PASSWORD + x-buildbuddy-api-key:$BUILDBUDDY_API_KEY [build] execution_platforms = root//platforms:buildbuddy diff --git a/examples/persistent_worker/.buckconfig.buildbuddy-persistent-workers b/examples/persistent_worker/.buckconfig.buildbuddy-persistent-workers index 231de6969edf..436af6c9a504 100644 --- a/examples/persistent_worker/.buckconfig.buildbuddy-persistent-workers +++ b/examples/persistent_worker/.buckconfig.buildbuddy-persistent-workers @@ -7,9 +7,7 @@ action_cache_address = grpc://remote.buildbuddy.io cas_address = grpc://remote.buildbuddy.io tls = true http_headers = \ - x-buildbuddy-api-key:$BUILDBUDDY_API_KEY, \ - x-buildbuddy-platform.container-registry-username:$BUILDBUDDY_CONTAINER_USER, \ - x-buildbuddy-platform.container-registry-password:$BUILDBUDDY_CONTAINER_PASSWORD + x-buildbuddy-api-key:$BUILDBUDDY_API_KEY [build] execution_platforms = root//platforms:buildbuddy-persistent-workers diff --git a/examples/persistent_worker/platforms/buildbuddy.bzl b/examples/persistent_worker/platforms/buildbuddy.bzl index a65e07453aef..7fc9787d1377 100644 --- a/examples/persistent_worker/platforms/buildbuddy.bzl +++ b/examples/persistent_worker/platforms/buildbuddy.bzl @@ -1,16 +1,3 @@ -# Update the Docker image for remote execution whenever you make a change to the Nix package set. -# -# nix run path:.#dockerBuild \ -# | gzip --fast --no-name \ -# | nix run nixpkgs#skopeo -- copy \ -# --insecure-policy \ -# --digestfile image.digest \ -# --preserve-digests docker-archive:/dev/stdin \ -# docker://ghcr.io/$GITHUB_USER/buck2-remote-persistent-worker:latest -# -# NOTE, Change to a container registry that you have access to. -image = "docker://ghcr.io/aherrmann/buck2-remote-persistent-worker@sha256:23832b49492ef77601111218661b8fcc6eafbcc49cb9abffd08d79988dda540e" - def _platforms(ctx): constraints = dict() constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints) @@ -30,8 +17,6 @@ def _platforms(ctx): use_remote_persistent_workers = ctx.attrs.use_persistent_workers, remote_execution_properties = { "OSFamily": "Linux", - "container-image": image, - "workload-isolation-type": "podman", "recycle-runner": True, # required for remote persistent workers "nonroot-workspace": True, }, From a8ad96f496e98a7466d7d4321b511092d946b0be Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 29 Oct 2024 15:27:33 +0100 Subject: [PATCH 12/36] Remove the Nix flake --- examples/persistent_worker/flake.lock | 61 -------------------------- examples/persistent_worker/flake.nix | 62 --------------------------- 2 files changed, 123 deletions(-) delete mode 100644 examples/persistent_worker/flake.lock delete mode 100644 examples/persistent_worker/flake.nix diff --git a/examples/persistent_worker/flake.lock b/examples/persistent_worker/flake.lock deleted file mode 100644 index 6e2aab74f5d0..000000000000 --- a/examples/persistent_worker/flake.lock +++ /dev/null @@ -1,61 +0,0 @@ -{ - "nodes": { - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1726560853, - "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1727140925, - "narHash": "sha256-ZHSasdLwEEjSOD/WTW1o7dr3/EjuYsdwYB4NSgICZ2I=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "189e5f171b163feb7791a9118afa778d9a1db81f", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/examples/persistent_worker/flake.nix b/examples/persistent_worker/flake.nix deleted file mode 100644 index 3cc82f061f41..000000000000 --- a/examples/persistent_worker/flake.nix +++ /dev/null @@ -1,62 +0,0 @@ -{ - description = "Buck2 Remote Persistent Worker Example"; - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - flake-utils.url = "github:numtide/flake-utils"; - }; - outputs = { - self, - nixpkgs, - flake-utils, - ... - }: - flake-utils.lib.eachDefaultSystem (system: let - pkgs = import nixpkgs { - inherit system; - overlays = []; - config = {}; - }; - python = (pkgs.python3.override { - packageOverrides = self: super: { - protobuf = self.protobuf5; - }; - }).withPackages (ps: with ps; [ - ipython - grpcio - grpcio-tools - ]); - in - { - devShells = { - default = pkgs.mkShellNoCC { - packages = [ python ]; - NIX_PYTHON = "${python}/bin/python"; - shellHook = '' - cat >toolchains/.buckconfig.nix < Date: Tue, 29 Oct 2024 15:32:43 +0100 Subject: [PATCH 13/36] Update the README and direnv configuration --- examples/persistent_worker/.envrc | 3 +++ examples/persistent_worker/README.md | 17 +++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 examples/persistent_worker/.envrc diff --git a/examples/persistent_worker/.envrc b/examples/persistent_worker/.envrc new file mode 100644 index 000000000000..1ab1354014c7 --- /dev/null +++ b/examples/persistent_worker/.envrc @@ -0,0 +1,3 @@ +# specify the following: +# - BUILDBUDDY_API_KEY +source_env_if_exists .envrc.private diff --git a/examples/persistent_worker/README.md b/examples/persistent_worker/README.md index 0f88180c3cd3..8ab4f4c46db1 100644 --- a/examples/persistent_worker/README.md +++ b/examples/persistent_worker/README.md @@ -10,24 +10,21 @@ support for remote persistent workers to Buck2, see [#776]. ## Requirements -The demo uses Nix to provide required system dependencies, such as -Python and grpc-io. As well as direnv to set up the environment. +This demo uses BuildBuddy remote execution to demonstrate remote +persistent workers. You will need an API token for at least a free open +source account. You can use [direnv] to set up the environment: -- [Nix](https://nixos.org/) -- [direnv](https://direnv.net/) - -Credentials for [BuildBuddy](https://www.buildbuddy.io/) and -[GHCR](https://ghcr.io) stored in `.envrc.private`: +Credentials for [BuildBuddy] stored in `.envrc.private`: ``` export BUILDBUDDY_API_KEY=... -export BUILDBUDDY_CONTAINER_USER=... # GitHub user name -export BUILDBUDDY_CONTAINER_PASSWORD=... # GitHub access token -export GITHUB_USER=... # GitHub user name ``` Upload the Nix Docker image for remote execution as described in `platforms/buildbuddy.bzl`. +[direnv]: https://direnv.net/ +[BuildBuddy]: https://www.buildbuddy.io/ + ## Local Build Configure a local build without persistent workers: From 1eebd7dea83e693e9ed1a46c777a5f05d3b24c59 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 29 Oct 2024 15:59:28 +0100 Subject: [PATCH 14/36] Shrink number of test targets It's useful to test multiple builds in parallel to catch potential issues related parallel worker requests. However, we also don't need to be excessive in the number of tests to not unnecessarily waste resources. --- examples/persistent_worker/BUCK | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/persistent_worker/BUCK b/examples/persistent_worker/BUCK index f16a384c7517..63a74c5c29ee 100644 --- a/examples/persistent_worker/BUCK +++ b/examples/persistent_worker/BUCK @@ -22,5 +22,5 @@ worker( [ demo(name = "demo-" + str(i)) - for i in range(10) + for i in range(4) ] From 16b31c43e63205187c3919ea108739da531b5356 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 29 Oct 2024 16:41:30 +0100 Subject: [PATCH 15/36] fix README instructions --- examples/persistent_worker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/persistent_worker/README.md b/examples/persistent_worker/README.md index 8ab4f4c46db1..e0cc236fa6f3 100644 --- a/examples/persistent_worker/README.md +++ b/examples/persistent_worker/README.md @@ -84,7 +84,7 @@ ONE-SHOT START Configure a remote build with persistent workers: ``` $ cd examples/persistent_worker -$ echo '' > .buckconfig.local +$ echo '' > .buckconfig.local ``` Run a clean build: From 8e900f2499d70ece5e116e1f3f594f1c0e2b6658 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 29 Oct 2024 16:47:42 +0100 Subject: [PATCH 16/36] Add an automated test script --- examples/persistent_worker/test.sh | 94 ++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100755 examples/persistent_worker/test.sh diff --git a/examples/persistent_worker/test.sh b/examples/persistent_worker/test.sh new file mode 100755 index 000000000000..38e6d10a2ad8 --- /dev/null +++ b/examples/persistent_worker/test.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "::group::Local build without persistent worker" >&2 +echo '' > .buckconfig.local +buck2 clean; buck2 build : -vstderr +echo "# Verifying Buck2 log" >&2 +buck2 log what-ran --show-std-err --format json \ + | jq -s ' + [ + .[] + | select(.identity | startswith("root//:demo-")) + ] + | if length == 4 then + . + else + error("expected 4 demo targets, got " + (length | tostring)) + end + | .[] + | if .reproducer.executor == "Local" and (.std_err | startswith("one-shot.py")) then + true + else + error("expected local without persistent worker, got " + ([.reproducer.executor, .std_err] | tostring)) + end + ' + +echo "::group::Local build with persistent worker" >&2 +echo '' > .buckconfig.local +buck2 clean; buck2 build : -vstderr +echo "# Verifying Buck2 log" >&2 +buck2 log what-ran --show-std-err --format json \ + | jq -s ' + [ + .[] + | select(.identity | startswith("root//:demo-")) + ] + | if length == 5 then + . + else + error("expected 5 demo targets, got " + (length | tostring)) + end + | .[] + | if (.reproducer.executor == "Worker" or .reproducer.executor == "WorkerInit") and (.std_err | startswith("Buck2 persistent worker")) then + true + else + error("expected local without persistent worker, got " + ([.reproducer.executor, .std_err] | tostring)) + end + ' + +echo "::group::Remote build without persistent worker" >&2 +echo '' > .buckconfig.local +buck2 clean; buck2 build : -vstderr +echo "# Verifying Buck2 log" >&2 +buck2 log what-ran --show-std-err --format json \ + | jq -s ' + [ + .[] + | select(.identity | startswith("root//:demo-")) + ] + | if length == 4 then + . + else + error("expected 4 demo targets, got " + (length | tostring)) + end + | .[] + | if .reproducer.executor == "Re" and (.std_err | startswith("one-shot.py")) then + true + else + error("expected local without persistent worker, got " + ([.reproducer.executor, .std_err] | tostring)) + end + ' + +echo "::group::Remote build with persistent worker" >&2 +echo '' > .buckconfig.local +buck2 clean; buck2 build : -vstderr +echo "# Verifying Buck2 log" >&2 +buck2 log what-ran --show-std-err --format json \ + | jq -s ' + [ + .[] + | select(.identity | startswith("root//:demo-")) + ] + | if length == 4 then + . + else + error("expected 4 demo targets, got " + (length | tostring)) + end + | .[] + | if .reproducer.executor == "Re" and (.std_err | startswith("Bazel persistent worker")) then + true + else + error("expected remote persistent worker, got " + ([.reproducer.executor, .std_err] | tostring)) + end + ' From 1977ac2966fea9ef554b6daca1999206169ed459 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 29 Oct 2024 17:00:54 +0100 Subject: [PATCH 17/36] Test persistent worker example on CI Requires a repository secret to be set up for the BuildBuddy API key named `BUILDBUDDY_API_KEY`. --- .../build_example_persitent_worker/action.yml | 16 ++++++++++++++++ .github/workflows/build-and-test.yml | 3 +++ 2 files changed, 19 insertions(+) create mode 100644 .github/actions/build_example_persitent_worker/action.yml diff --git a/.github/actions/build_example_persitent_worker/action.yml b/.github/actions/build_example_persitent_worker/action.yml new file mode 100644 index 000000000000..2943b0a84ca4 --- /dev/null +++ b/.github/actions/build_example_persitent_worker/action.yml @@ -0,0 +1,16 @@ +name: build_example_persistent_worker +inputs: + buildbuddyApiKey: + description: "The API key for BuildBuddy remote cache and execution." + required: true +runs: + using: composite + steps: + - name: Build examples/persistent_worker directory + env: + BUILDBUDDY_API_KEY: ${{ inputs.buildbuddyApiKey }} + run: |- + cd examples/persistent_worker + export PATH="$RUNNER_TEMP/artifacts:$PATH" + ./test.sh + shell: bash diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 089de65260bc..f1466a0474aa 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -69,6 +69,9 @@ jobs: $RUNNER_TEMP/artifacts/buck2 test //... -v 2 - uses: ./.github/actions/build_example_conan - uses: ./.github/actions/build_example_no_prelude + - uses: ./.github/actions/build_example_persistent_worker + with: + buildbuddyApiKey: ${{ secrets.BUILDBUDDY_API_KEY }} - uses: ./.github/actions/setup_reindeer - uses: ./.github/actions/build_bootstrap windows-build-examples: From f1fd2312a073bb85e67274193d077920c5368be7 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 29 Oct 2024 17:48:44 +0100 Subject: [PATCH 18/36] fix typo --- .../action.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/actions/{build_example_persitent_worker => build_example_persistent_worker}/action.yml (100%) diff --git a/.github/actions/build_example_persitent_worker/action.yml b/.github/actions/build_example_persistent_worker/action.yml similarity index 100% rename from .github/actions/build_example_persitent_worker/action.yml rename to .github/actions/build_example_persistent_worker/action.yml From 34f7ade441673fbb8da2cf0bf3879070cec823a8 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 30 Oct 2024 11:13:52 +0100 Subject: [PATCH 19/36] Remove old Nix toolchain configuration file --- examples/persistent_worker/.gitignore | 2 -- examples/persistent_worker/toolchains/.buckconfig | 1 - 2 files changed, 3 deletions(-) delete mode 100644 examples/persistent_worker/toolchains/.buckconfig diff --git a/examples/persistent_worker/.gitignore b/examples/persistent_worker/.gitignore index 55014d7e91cb..a82178e703cc 100644 --- a/examples/persistent_worker/.gitignore +++ b/examples/persistent_worker/.gitignore @@ -1,6 +1,4 @@ .buckconfig.local .direnv .envrc.private -image.digest prelude -toolchains/.buckconfig.nix diff --git a/examples/persistent_worker/toolchains/.buckconfig b/examples/persistent_worker/toolchains/.buckconfig deleted file mode 100644 index 03964db941d5..000000000000 --- a/examples/persistent_worker/toolchains/.buckconfig +++ /dev/null @@ -1 +0,0 @@ - From f7fc5f70146b90767aa620ba9470fe971be4a4cf Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 30 Oct 2024 14:05:20 +0100 Subject: [PATCH 20/36] close GH actions output groups --- examples/persistent_worker/test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/persistent_worker/test.sh b/examples/persistent_worker/test.sh index 38e6d10a2ad8..996d0cb930ea 100755 --- a/examples/persistent_worker/test.sh +++ b/examples/persistent_worker/test.sh @@ -23,6 +23,7 @@ buck2 log what-ran --show-std-err --format json \ error("expected local without persistent worker, got " + ([.reproducer.executor, .std_err] | tostring)) end ' +echo "::endgroup::" >&2 echo "::group::Local build with persistent worker" >&2 echo '' > .buckconfig.local @@ -46,6 +47,7 @@ buck2 log what-ran --show-std-err --format json \ error("expected local without persistent worker, got " + ([.reproducer.executor, .std_err] | tostring)) end ' +echo "::endgroup::" >&2 echo "::group::Remote build without persistent worker" >&2 echo '' > .buckconfig.local @@ -69,6 +71,7 @@ buck2 log what-ran --show-std-err --format json \ error("expected local without persistent worker, got " + ([.reproducer.executor, .std_err] | tostring)) end ' +echo "::endgroup::" >&2 echo "::group::Remote build with persistent worker" >&2 echo '' > .buckconfig.local @@ -92,3 +95,4 @@ buck2 log what-ran --show-std-err --format json \ error("expected remote persistent worker, got " + ([.reproducer.executor, .std_err] | tostring)) end ' +echo "::endgroup::" >&2 From e259d89269f5e8710a3df8fb70136e46d8de2ca0 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 30 Oct 2024 14:05:34 +0100 Subject: [PATCH 21/36] Generate GH actions annotations on missing token --- examples/persistent_worker/test.sh | 92 ++++++++++++++++-------------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/examples/persistent_worker/test.sh b/examples/persistent_worker/test.sh index 996d0cb930ea..4611ca8f7853 100755 --- a/examples/persistent_worker/test.sh +++ b/examples/persistent_worker/test.sh @@ -50,49 +50,57 @@ buck2 log what-ran --show-std-err --format json \ echo "::endgroup::" >&2 echo "::group::Remote build without persistent worker" >&2 -echo '' > .buckconfig.local -buck2 clean; buck2 build : -vstderr -echo "# Verifying Buck2 log" >&2 -buck2 log what-ran --show-std-err --format json \ - | jq -s ' - [ - .[] - | select(.identity | startswith("root//:demo-")) - ] - | if length == 4 then - . - else - error("expected 4 demo targets, got " + (length | tostring)) - end - | .[] - | if .reproducer.executor == "Re" and (.std_err | startswith("one-shot.py")) then - true - else - error("expected local without persistent worker, got " + ([.reproducer.executor, .std_err] | tostring)) - end - ' +if [[ -z ${BUILDBUDDY_API_KEY:+x} ]]; then + echo "::notice file=${BASH_SOURCE[0]},line=${LINENO}::SKIPPED Missing BuildBuddy token. See examples/persistent_worker/README.md" >&2 +else + echo '' > .buckconfig.local + buck2 clean; buck2 build : -vstderr + echo "# Verifying Buck2 log" >&2 + buck2 log what-ran --show-std-err --format json \ + | jq -s ' + [ + .[] + | select(.identity | startswith("root//:demo-")) + ] + | if length == 4 then + . + else + error("expected 4 demo targets, got " + (length | tostring)) + end + | .[] + | if .reproducer.executor == "Re" and (.std_err | startswith("one-shot.py")) then + true + else + error("expected local without persistent worker, got " + ([.reproducer.executor, .std_err] | tostring)) + end + ' +fi echo "::endgroup::" >&2 echo "::group::Remote build with persistent worker" >&2 -echo '' > .buckconfig.local -buck2 clean; buck2 build : -vstderr -echo "# Verifying Buck2 log" >&2 -buck2 log what-ran --show-std-err --format json \ - | jq -s ' - [ - .[] - | select(.identity | startswith("root//:demo-")) - ] - | if length == 4 then - . - else - error("expected 4 demo targets, got " + (length | tostring)) - end - | .[] - | if .reproducer.executor == "Re" and (.std_err | startswith("Bazel persistent worker")) then - true - else - error("expected remote persistent worker, got " + ([.reproducer.executor, .std_err] | tostring)) - end - ' +if [[ -z ${BUILDBUDDY_API_KEY:+x} ]]; then + echo "::notice file=${BASH_SOURCE[0]},line=${LINENO}::SKIPPED Missing BuildBuddy token. See examples/persistent_worker/README.md" >&2 +else + echo '' > .buckconfig.local + buck2 clean; buck2 build : -vstderr + echo "# Verifying Buck2 log" >&2 + buck2 log what-ran --show-std-err --format json \ + | jq -s ' + [ + .[] + | select(.identity | startswith("root//:demo-")) + ] + | if length == 4 then + . + else + error("expected 4 demo targets, got " + (length | tostring)) + end + | .[] + | if .reproducer.executor == "Re" and (.std_err | startswith("Bazel persistent worker")) then + true + else + error("expected remote persistent worker, got " + ([.reproducer.executor, .std_err] | tostring)) + end + ' +fi echo "::endgroup::" >&2 From e496262123708fe1caef4ef55983316930144eb3 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 30 Oct 2024 14:08:49 +0100 Subject: [PATCH 22/36] Document BuildBuddy token availability --- examples/persistent_worker/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/persistent_worker/README.md b/examples/persistent_worker/README.md index e0cc236fa6f3..5bc396b71dcd 100644 --- a/examples/persistent_worker/README.md +++ b/examples/persistent_worker/README.md @@ -19,8 +19,10 @@ Credentials for [BuildBuddy] stored in `.envrc.private`: export BUILDBUDDY_API_KEY=... ``` -Upload the Nix Docker image for remote execution as described in -`platforms/buildbuddy.bzl`. +On CI the API key is not available for pipelines initiated from forks of +the main Buck2 repository. The corresponding tests will be skipped in +that case. A Meta engineer can manually initiate a pipeline run with the +token set. [direnv]: https://direnv.net/ [BuildBuddy]: https://www.buildbuddy.io/ From bf8580e06298db3f5fef61f017f8ba7f5c38ac88 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 30 Oct 2024 14:09:01 +0100 Subject: [PATCH 23/36] Enable manual pipeline runs --- .github/workflows/build-and-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f1466a0474aa..885d43332813 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -2,6 +2,7 @@ name: Build and test on: push: pull_request: + workflow_dispatch: # allows manual triggering jobs: linux-build-and-test: runs-on: 4-core-ubuntu From a4e55d7d14005e6077db0b0daa04125f7e92b21f Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 30 Oct 2024 16:12:40 +0100 Subject: [PATCH 24/36] Fix annotations file path --- examples/persistent_worker/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/persistent_worker/test.sh b/examples/persistent_worker/test.sh index 4611ca8f7853..9af967a17556 100755 --- a/examples/persistent_worker/test.sh +++ b/examples/persistent_worker/test.sh @@ -51,7 +51,7 @@ echo "::endgroup::" >&2 echo "::group::Remote build without persistent worker" >&2 if [[ -z ${BUILDBUDDY_API_KEY:+x} ]]; then - echo "::notice file=${BASH_SOURCE[0]},line=${LINENO}::SKIPPED Missing BuildBuddy token. See examples/persistent_worker/README.md" >&2 + echo "::notice file=$(realpath --relative-to=../.. ${BASH_SOURCE[0]}),line=${LINENO}::SKIPPED Missing BuildBuddy token. See examples/persistent_worker/README.md" >&2 else echo '' > .buckconfig.local buck2 clean; buck2 build : -vstderr @@ -79,7 +79,7 @@ echo "::endgroup::" >&2 echo "::group::Remote build with persistent worker" >&2 if [[ -z ${BUILDBUDDY_API_KEY:+x} ]]; then - echo "::notice file=${BASH_SOURCE[0]},line=${LINENO}::SKIPPED Missing BuildBuddy token. See examples/persistent_worker/README.md" >&2 + echo "::notice file=$(realpath --relative-to=../.. ${BASH_SOURCE[0]}),line=${LINENO}::SKIPPED Missing BuildBuddy token. See examples/persistent_worker/README.md" >&2 else echo '' > .buckconfig.local buck2 clean; buck2 build : -vstderr From 3b33fb69d6ed08fbbda88a13e0a038c5d896a4eb Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 18 Dec 2024 10:26:06 +0100 Subject: [PATCH 25/36] align formatting --- examples/persistent_worker/BUCK | 2 +- examples/persistent_worker/README.md | 75 +++++++++-------- examples/persistent_worker/defs.bzl | 7 ++ examples/persistent_worker/one_shot.py | 16 ++-- .../persistent_worker/persistent_worker.py | 84 ++++++++++++------- examples/persistent_worker/platforms/BUCK | 1 + .../platforms/buildbuddy.bzl | 11 ++- .../persistent_worker/platforms/local.bzl | 9 +- examples/persistent_worker/proto/BUCK | 2 +- examples/persistent_worker/proto/defs.bzl | 8 ++ examples/persistent_worker/python/BUCK | 22 +++-- examples/persistent_worker/python/defs.bzl | 7 ++ examples/persistent_worker/test.sh | 7 ++ examples/persistent_worker/toolchains/BUCK | 6 +- .../persistent_worker/toolchains/defs.bzl | 9 +- 15 files changed, 179 insertions(+), 87 deletions(-) diff --git a/examples/persistent_worker/BUCK b/examples/persistent_worker/BUCK index 63a74c5c29ee..df7c7bba3241 100644 --- a/examples/persistent_worker/BUCK +++ b/examples/persistent_worker/BUCK @@ -16,8 +16,8 @@ python_binary( worker( name = "worker", - worker = ":worker_py", visibility = ["PUBLIC"], + worker = ":worker_py", ) [ diff --git a/examples/persistent_worker/README.md b/examples/persistent_worker/README.md index 5bc396b71dcd..f381050cbcc3 100644 --- a/examples/persistent_worker/README.md +++ b/examples/persistent_worker/README.md @@ -1,28 +1,28 @@ # Persistent Worker Demo -At the time of writing (2024-09-25) Buck2 supports persistent workers -for local builds through a dedicated Buck2 persistent worker gRPC -protocol. However, Buck2 does not support persistent workers for builds -that use remote execution. This demo is part of a patch-set that adds -support for remote persistent workers to Buck2, see [#776]. +At the time of writing (2024-09-25) Buck2 supports persistent workers for local +builds through a dedicated Buck2 persistent worker gRPC protocol. However, Buck2 +does not support persistent workers for builds that use remote execution. This +demo is part of a patch-set that adds support for remote persistent workers to +Buck2, see [#776]. [#776]: https://github.com/facebook/buck2/issues/776 ## Requirements -This demo uses BuildBuddy remote execution to demonstrate remote -persistent workers. You will need an API token for at least a free open -source account. You can use [direnv] to set up the environment: +This demo uses BuildBuddy remote execution to demonstrate remote persistent +workers. You will need an API token for at least a free open source account. You +can use [direnv] to set up the environment: Credentials for [BuildBuddy] stored in `.envrc.private`: + ``` export BUILDBUDDY_API_KEY=... ``` -On CI the API key is not available for pipelines initiated from forks of -the main Buck2 repository. The corresponding tests will be skipped in -that case. A Meta engineer can manually initiate a pipeline run with the -token set. +On CI the API key is not available for pipelines initiated from forks of the +main Buck2 repository. The corresponding tests will be skipped in that case. A +Meta engineer can manually initiate a pipeline run with the token set. [direnv]: https://direnv.net/ [BuildBuddy]: https://www.buildbuddy.io/ @@ -30,12 +30,14 @@ token set. ## Local Build Configure a local build without persistent workers: + ``` $ cd examples/persistent_worker $ echo '' > .buckconfig.local ``` Run a clean build: + ``` $ buck2 clean; buck2 build : -vstderr ... @@ -48,12 +50,14 @@ ONE-SHOT START ## Local Persistent Worker Configure a local build with persistent workers: + ``` $ cd examples/persistent_worker $ echo '' > .buckconfig.local ``` Run a clean build: + ``` $ buck2 clean; buck2 build : -vstderr ... @@ -66,12 +70,14 @@ Buck2 persistent worker ... ## Remote Execution Configure a remote build without persistent workers: + ``` $ cd examples/persistent_worker $ echo '' > .buckconfig.local ``` Run a clean build: + ``` $ buck2 clean; buck2 build : -vstderr ... @@ -84,12 +90,14 @@ ONE-SHOT START ## Remote Persistent Worker Configure a remote build with persistent workers: + ``` $ cd examples/persistent_worker $ echo '' > .buckconfig.local ``` Run a clean build: + ``` $ buck2 clean; buck2 build : -vstderr ... @@ -103,15 +111,14 @@ Bazel persistent worker ... ### Starlark -A Buck2 persistent worker is created by a rule that emits the -`WorkerInfo` provider. Setting `remote = True` on this provider -indicates that this worker is remote execution capable. +A Buck2 persistent worker is created by a rule that emits the `WorkerInfo` +provider. Setting `remote = True` on this provider indicates that this worker is +remote execution capable. -Buck2 actions indicate that they can utilize a persistent worker by -setting the `exe` parameter to `ctx.actions.run` to -`WorkerRunInfo(worker, exe)`, where `worker` is a `WorkerInfo` provider, -and `exe` defines the fallback executable for non persistent-worker -execution. +Buck2 actions indicate that they can utilize a persistent worker by setting the +`exe` parameter to `ctx.actions.run` to `WorkerRunInfo(worker, exe)`, where +`worker` is a `WorkerInfo` provider, and `exe` defines the fallback executable +for non persistent-worker execution. Buck2 actions that want to utilize a remote persistent worker must pass command-line arguments in an argument file specified as `@argfile`, @@ -119,20 +126,20 @@ command-line arguments in an argument file specified as `@argfile`, ### Local Persistent Worker -A locally executed Buck2 persistent worker falls under the [Buck2 -persistent worker protocol](./proto/buck2/worker.proto): It is started -and managed by Buck2 and passed a file path in the `WORKER_SOCKET` -environment variable where it should create a gRPC Unix domain socket to -serve worker requests over. Multiple requests may be sent in parallel -and expected to be served at the same time depending on the -`concurrency` attribute of the `WorkerInfo` provider. +A locally executed Buck2 persistent worker falls under the +[Buck2 persistent worker protocol](./proto/buck2/worker.proto): It is started +and managed by Buck2 and passed a file path in the `WORKER_SOCKET` environment +variable where it should create a gRPC Unix domain socket to serve worker +requests over. Multiple requests may be sent in parallel and expected to be +served at the same time depending on the `concurrency` attribute of the +`WorkerInfo` provider. ### Remote Persistent Worker -A remotely executed Buck2 persistent worker falls under the [Bazel -persistent worker protocol](./proto/bazel/worker_protocol.proto): It is -started and managed by the remote execution system. Work requests are -sent as length prefixed protobuf objects to the standard input of the -worker process. Work responses are expected as length prefixed protobuf -objects on the standard output of the worker process. The worker process -may not use standard output for anything else. +A remotely executed Buck2 persistent worker falls under the +[Bazel persistent worker protocol](./proto/bazel/worker_protocol.proto): It is +started and managed by the remote execution system. Work requests are sent as +length prefixed protobuf objects to the standard input of the worker process. +Work responses are expected as length prefixed protobuf objects on the standard +output of the worker process. The worker process may not use standard output for +anything else. diff --git a/examples/persistent_worker/defs.bzl b/examples/persistent_worker/defs.bzl index 38520cb1b9f1..d1b347233aac 100644 --- a/examples/persistent_worker/defs.bzl +++ b/examples/persistent_worker/defs.bzl @@ -1,3 +1,10 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + load("@prelude//utils:argfile.bzl", "at_argfile") def _worker_impl(ctx: AnalysisContext) -> list[Provider]: diff --git a/examples/persistent_worker/one_shot.py b/examples/persistent_worker/one_shot.py index 121313b659f0..6efa97162908 100644 --- a/examples/persistent_worker/one_shot.py +++ b/examples/persistent_worker/one_shot.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. import argparse import os @@ -7,13 +13,9 @@ def main(): parser = argparse.ArgumentParser( - fromfile_prefix_chars='@', - prog="one_shot", - description="One-shot command") - parser.add_argument( - "outfile", - type=argparse.FileType("w"), - help="Output file.") + fromfile_prefix_chars="@", prog="one_shot", description="One-shot command" + ) + parser.add_argument("outfile", type=argparse.FileType("w"), help="Output file.") args = parser.parse_args() diff --git a/examples/persistent_worker/persistent_worker.py b/examples/persistent_worker/persistent_worker.py index 38c0cd24291b..931971280ba2 100644 --- a/examples/persistent_worker/persistent_worker.py +++ b/examples/persistent_worker/persistent_worker.py @@ -1,4 +1,11 @@ #!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + """Buck2 local and remote persistent worker and action runner. This script can: @@ -7,29 +14,32 @@ """ import argparse +import os +import shlex +import socket +import sys +import time from concurrent import futures from dataclasses import dataclass + import google.protobuf.proto as proto import grpc -import os -import sys -import time import proto.bazel.worker_protocol_pb2 as bazel_pb2 import proto.buck2.worker_pb2 as buck2_pb2 import proto.buck2.worker_pb2_grpc as buck2_pb2_grpc -import shlex -import socket @dataclass class Request: """Universal worker request, independent of Buck2 or Bazel protocol.""" + argv: list[str] @dataclass class Response: """Universal worker response, independent of Buck2 or Bazel protocol.""" + exit_code: int stderr: str @@ -45,19 +55,26 @@ def error(self, message): class Implementation: """Universal worker implementation, independent of Buck2 or Bazel protocol.""" + def __init__(self): self.parser = RecoverableArgumentParser( - fromfile_prefix_chars='@', + fromfile_prefix_chars="@", prog="worker_py_handler", - description="Persistent Worker Request Handler") + description="Persistent Worker Request Handler", + ) self.parser.add_argument( - "outfile", - type=argparse.FileType("w"), - help="Output file.") + "outfile", type=argparse.FileType("w"), help="Output file." + ) def execute(self, request: Request) -> Response: try: - print("WORKER", socket.gethostname(), os.getpid(), os.getcwd(), file=sys.stderr) + print( + "WORKER", + socket.gethostname(), + os.getpid(), + os.getcwd(), + file=sys.stderr, + ) print("REQUEST", request.argv, file=sys.stderr) args = self.parser.parse_args(request.argv) print("ARGS", args, file=sys.stderr) @@ -69,15 +86,14 @@ def execute(self, request: Request) -> Response: time.sleep(1) print("COMPLETED", name, file=sys.stderr) output.close() - return Response( - exit_code = 0, - stderr = f"wrote to {output.name}") + return Response(exit_code=0, stderr=f"wrote to {output.name}") except ArgumentParserError as e: - return Response(exit_code = 2, stderr = str(e)) + return Response(exit_code=2, stderr=str(e)) class Buck2Servicer(buck2_pb2_grpc.WorkerServicer): """Buck2 remote persistent worker implementation.""" + def __init__(self): self.impl = Implementation() @@ -86,13 +102,14 @@ def Execute(self, request, context): print("BUCK2", request, file=sys.stderr) # Decode arguments as UTF-8 strings. argv = [arg.decode("utf-8") for arg in request.argv] - response = self.impl.execute(Request(argv = argv)) + response = self.impl.execute(Request(argv=argv)) host = socket.gethostname() pid = os.getpid() cwd = os.getcwd() return buck2_pb2.ExecuteResponse( - exit_code = response.exit_code, - stderr = f"Buck2 persistent worker {host} {pid} {cwd}\n" + response.stderr) + exit_code=response.exit_code, + stderr=f"Buck2 persistent worker {host} {pid} {cwd}\n" + response.stderr, + ) class BazelServicer: @@ -101,26 +118,30 @@ def __init__(self): def Execute(self, request: bazel_pb2.WorkRequest) -> bazel_pb2.WorkResponse: print("BAZEL", request, file=sys.stderr) - response = self.impl.execute(Request(argv = request.arguments)) + response = self.impl.execute(Request(argv=request.arguments)) host = socket.gethostname() pid = os.getpid() cwd = os.getcwd() return bazel_pb2.WorkResponse( - exit_code = response.exit_code, - output = f"Bazel persistent worker {host} {pid} {cwd} {request.request_id}\n" + response.stderr, - request_id = request.request_id) + exit_code=response.exit_code, + output=f"Bazel persistent worker {host} {pid} {cwd} {request.request_id}\n" + + response.stderr, + request_id=request.request_id, + ) def main(): print("MAIN", socket.gethostname(), os.getpid(), os.getcwd(), file=sys.stderr) parser = argparse.ArgumentParser( - fromfile_prefix_chars='@', + fromfile_prefix_chars="@", prog="worker", - description="Buck2/Bazel Local/Remote Persistent Worker") + description="Buck2/Bazel Local/Remote Persistent Worker", + ) parser.add_argument( "--persistent_worker", action="store_true", - help="Enable persistent worker (Bazel protocol).") + help="Enable persistent worker (Bazel protocol).", + ) (args, rest) = parser.parse_known_args() @@ -133,7 +154,9 @@ def main(): parser.print_usage() sys.exit(2) - server = grpc.server(futures.ThreadPoolExecutor(max_workers=os.cpu_count() or 1)) + server = grpc.server( + futures.ThreadPoolExecutor(max_workers=os.cpu_count() or 1) + ) buck2_pb2_grpc.add_WorkerServicer_to_server(Buck2Servicer(), server) server.add_insecure_port(f"unix://{socket_path}") server.start() @@ -150,12 +173,17 @@ def main(): servicer = BazelServicer() # uses length prefixed serialization features added in proto version 5.28.0. # https://github.com/protocolbuffers/protobuf/pull/16965 - while request := proto.parse_length_prefixed(bazel_pb2.WorkRequest, sys.stdin.buffer): + while request := proto.parse_length_prefixed( + bazel_pb2.WorkRequest, sys.stdin.buffer + ): response = servicer.Execute(request) proto.serialize_length_prefixed(response, sys.stdout.buffer) sys.stdout.flush() else: - print("Expected either 'WORKER_SOCKET' environment variable or '--persistent_worker' argument.", file=sys.stderr) + print( + "Expected either 'WORKER_SOCKET' environment variable or '--persistent_worker' argument.", + file=sys.stderr, + ) sys.exit(2) diff --git a/examples/persistent_worker/platforms/BUCK b/examples/persistent_worker/platforms/BUCK index f36ef05b8d88..f1d81b78c4b2 100644 --- a/examples/persistent_worker/platforms/BUCK +++ b/examples/persistent_worker/platforms/BUCK @@ -2,6 +2,7 @@ load(":buildbuddy.bzl", "buildbuddy") load(":local.bzl", "local") host_cpu = "prelude//cpu:" + ("arm64" if host_info().arch.is_aarch64 else "x86_64") + host_os = "prelude//os:" + ("macos" if host_info().os.is_macos else "linux") buildbuddy( diff --git a/examples/persistent_worker/platforms/buildbuddy.bzl b/examples/persistent_worker/platforms/buildbuddy.bzl index 7fc9787d1377..2d6db748473c 100644 --- a/examples/persistent_worker/platforms/buildbuddy.bzl +++ b/examples/persistent_worker/platforms/buildbuddy.bzl @@ -1,3 +1,10 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + def _platforms(ctx): constraints = dict() constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints) @@ -17,8 +24,8 @@ def _platforms(ctx): use_remote_persistent_workers = ctx.attrs.use_persistent_workers, remote_execution_properties = { "OSFamily": "Linux", - "recycle-runner": True, # required for remote persistent workers "nonroot-workspace": True, + "recycle-runner": True, # required for remote persistent workers }, remote_execution_use_case = "buck2-default", remote_output_paths = "output_paths", @@ -33,5 +40,5 @@ buildbuddy = rule( "os_configuration": attrs.dep(providers = [ConfigurationInfo]), "use_persistent_workers": attrs.bool(default = False), }, - impl = _platforms + impl = _platforms, ) diff --git a/examples/persistent_worker/platforms/local.bzl b/examples/persistent_worker/platforms/local.bzl index 2c9e7d78ac32..f09c0f61bc58 100644 --- a/examples/persistent_worker/platforms/local.bzl +++ b/examples/persistent_worker/platforms/local.bzl @@ -1,3 +1,10 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + def _platforms(ctx): constraints = dict() constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints) @@ -25,5 +32,5 @@ local = rule( "os_configuration": attrs.dep(providers = [ConfigurationInfo]), "use_persistent_workers": attrs.bool(default = False), }, - impl = _platforms + impl = _platforms, ) diff --git a/examples/persistent_worker/proto/BUCK b/examples/persistent_worker/proto/BUCK index 316ba09efa17..fef4f5e67561 100644 --- a/examples/persistent_worker/proto/BUCK +++ b/examples/persistent_worker/proto/BUCK @@ -1,6 +1,6 @@ python_binary( name = "protoc", main_module = "grpc_tools.protoc", - deps = ["//python:grpcio-tools"], visibility = ["PUBLIC"], + deps = ["//python:grpcio-tools"], ) diff --git a/examples/persistent_worker/proto/defs.bzl b/examples/persistent_worker/proto/defs.bzl index 133d0cbcfd2c..cf7e3a4f39f2 100644 --- a/examples/persistent_worker/proto/defs.bzl +++ b/examples/persistent_worker/proto/defs.bzl @@ -1,3 +1,10 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + def _proto_python_library_impl(ctx: AnalysisContext) -> list[Provider]: prefix = ctx.label.package depth = len(prefix.split("/")) @@ -16,6 +23,7 @@ def _proto_python_library_impl(ctx: AnalysisContext) -> list[Provider]: ), category = "protoc", ) + # protoc does not let us control the import prefix and path prefix separately. # So, we need to copy the generated files into the correct location after the fact. python_out_copied = ctx.actions.declare_output("{}.py".format(libname)) diff --git a/examples/persistent_worker/python/BUCK b/examples/persistent_worker/python/BUCK index df744aacb867..a0b16bbc95e0 100644 --- a/examples/persistent_worker/python/BUCK +++ b/examples/persistent_worker/python/BUCK @@ -1,22 +1,26 @@ load(":defs.bzl", "fetch_python_libraries") fetch_python_libraries(pkgs = { - "grpcio-tools": { - "url": "https://files.pythonhosted.org/packages/1d/0f/273d7ac9c7d99b56abb5841d8aff7ffd148fe01b48c2913c8da3de9438e7/grpcio_tools-1.66.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "sha256": "c68642829368f4f83929e0df571dbbc99f1f1553555d8f98d0582da9f6743d9e", - "deps": ["grpcio", "protobuf", "setuptools"], - }, "grpcio": { - "url": "https://files.pythonhosted.org/packages/2f/86/a86742f3deaa22385c3bff984c5947fc62d47d3fab26c508730037d027e5/grpcio-1.66.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "sha256": "f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46", "deps": ["protobuf"], + "sha256": "f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46", + "url": "https://files.pythonhosted.org/packages/2f/86/a86742f3deaa22385c3bff984c5947fc62d47d3fab26c508730037d027e5/grpcio-1.66.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + }, + "grpcio-tools": { + "deps": [ + "grpcio", + "protobuf", + "setuptools", + ], + "sha256": "c68642829368f4f83929e0df571dbbc99f1f1553555d8f98d0582da9f6743d9e", + "url": "https://files.pythonhosted.org/packages/1d/0f/273d7ac9c7d99b56abb5841d8aff7ffd148fe01b48c2913c8da3de9438e7/grpcio_tools-1.66.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", }, "protobuf": { - "url": "https://files.pythonhosted.org/packages/9b/55/f24e3b801d2e108c48aa2b1b59bb791b5cffba89465cbbf66fc98de89270/protobuf-5.28.2-py3-none-any.whl", "sha256": "52235802093bd8a2811abbe8bf0ab9c5f54cca0a751fdd3f6ac2a21438bffece", + "url": "https://files.pythonhosted.org/packages/9b/55/f24e3b801d2e108c48aa2b1b59bb791b5cffba89465cbbf66fc98de89270/protobuf-5.28.2-py3-none-any.whl", }, "setuptools": { - "url": "https://files.pythonhosted.org/packages/ff/ae/f19306b5a221f6a436d8f2238d5b80925004093fa3edea59835b514d9057/setuptools-75.1.0-py3-none-any.whl", "sha256": "35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2", + "url": "https://files.pythonhosted.org/packages/ff/ae/f19306b5a221f6a436d8f2238d5b80925004093fa3edea59835b514d9057/setuptools-75.1.0-py3-none-any.whl", }, }) diff --git a/examples/persistent_worker/python/defs.bzl b/examples/persistent_worker/python/defs.bzl index 0584ae1aba90..66f1a719f520 100644 --- a/examples/persistent_worker/python/defs.bzl +++ b/examples/persistent_worker/python/defs.bzl @@ -1,3 +1,10 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + def fetch_python_libraries(pkgs): for name, pkg in pkgs.items(): native.remote_file( diff --git a/examples/persistent_worker/test.sh b/examples/persistent_worker/test.sh index 9af967a17556..127a5bfd4787 100755 --- a/examples/persistent_worker/test.sh +++ b/examples/persistent_worker/test.sh @@ -1,4 +1,11 @@ #!/usr/bin/env bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + set -euo pipefail echo "::group::Local build without persistent worker" >&2 diff --git a/examples/persistent_worker/toolchains/BUCK b/examples/persistent_worker/toolchains/BUCK index b26965ad7542..f8e1c79724f6 100644 --- a/examples/persistent_worker/toolchains/BUCK +++ b/examples/persistent_worker/toolchains/BUCK @@ -1,6 +1,6 @@ load("@prelude//toolchains:cxx.bzl", "system_cxx_toolchain") load("@prelude//toolchains:genrule.bzl", "system_genrule_toolchain") -load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain", "system_python_toolchain") +load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain") load(":defs.bzl", "python_toolchain") system_cxx_toolchain( @@ -21,9 +21,9 @@ system_python_bootstrap_toolchain( remote_file( name = "python-download", - url = "https://github.com/indygreg/python-build-standalone/releases/download/20241002/cpython-3.13.0rc3+20241002-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", - sha256 = "445156c61e1cc167f7b8777ad08cc36e5598e12cd27e07453f6e6dc0f62e421e", out = "cpython.tar.gz", + sha256 = "445156c61e1cc167f7b8777ad08cc36e5598e12cd27e07453f6e6dc0f62e421e", + url = "https://github.com/indygreg/python-build-standalone/releases/download/20241002/cpython-3.13.0rc3+20241002-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", ) extract_archive( diff --git a/examples/persistent_worker/toolchains/defs.bzl b/examples/persistent_worker/toolchains/defs.bzl index de5aff4d2f36..c6594923a09e 100644 --- a/examples/persistent_worker/toolchains/defs.bzl +++ b/examples/persistent_worker/toolchains/defs.bzl @@ -1,3 +1,10 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + load( "@prelude//:artifacts.bzl", "ArtifactGroupInfo", @@ -36,8 +43,8 @@ def _python_toolchain_impl(ctx): python_toolchain = rule( impl = _python_toolchain_impl, attrs = { - "distribution": attrs.exec_dep(), "binary_linker_flags": attrs.default_only(attrs.list(attrs.arg(), default = [])), + "distribution": attrs.exec_dep(), "fail_with_message": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//python/tools:fail_with_message")), "generate_static_extension_info": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//python/tools:generate_static_extension_info")), "linker_flags": attrs.default_only(attrs.list(attrs.arg(), default = [])), From d1a8e37767bad4bf3836d3cc984111249e22234e Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 18 Dec 2024 11:28:12 +0100 Subject: [PATCH 26/36] Avoid remote caching of remote persistent worker demo The test wants to make sure that the actions are executed correct using either the remote persistent worker or running as individual actions on the remote execution system. Caching interferes with this test. This injects a cache-silo-key that changes each time to force a re-run of the action. --- examples/persistent_worker/defs.bzl | 7 +++++++ examples/persistent_worker/test.sh | 22 ++++++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/examples/persistent_worker/defs.bzl b/examples/persistent_worker/defs.bzl index d1b347233aac..32f976b7154e 100644 --- a/examples/persistent_worker/defs.bzl +++ b/examples/persistent_worker/defs.bzl @@ -34,6 +34,13 @@ def _demo_impl(ctx: AnalysisContext) -> list[Provider]: ctx.actions.run( cmd_args(argfile), category = "demo", + env = { + # modify this value to force an action rerun even if caching is enabled. + # `--no-remote-cache` does not have the desired effect, because it also causes + # the action to be omitted from `buck2 log what-ran`, which interferes with the + # test setup. + "CACHE_SILO_KEY": read_root_config("build", "cache_silo_key", "0"), + }, exe = WorkerRunInfo( worker = ctx.attrs._worker[WorkerInfo], exe = ctx.attrs._one_shot[RunInfo].args, diff --git a/examples/persistent_worker/test.sh b/examples/persistent_worker/test.sh index 127a5bfd4787..06d77fbd814e 100755 --- a/examples/persistent_worker/test.sh +++ b/examples/persistent_worker/test.sh @@ -9,7 +9,9 @@ set -euo pipefail echo "::group::Local build without persistent worker" >&2 -echo '' > .buckconfig.local +cat >.buckconfig.local < +EOF buck2 clean; buck2 build : -vstderr echo "# Verifying Buck2 log" >&2 buck2 log what-ran --show-std-err --format json \ @@ -33,7 +35,9 @@ buck2 log what-ran --show-std-err --format json \ echo "::endgroup::" >&2 echo "::group::Local build with persistent worker" >&2 -echo '' > .buckconfig.local +cat >.buckconfig.local < +EOF buck2 clean; buck2 build : -vstderr echo "# Verifying Buck2 log" >&2 buck2 log what-ran --show-std-err --format json \ @@ -60,7 +64,12 @@ echo "::group::Remote build without persistent worker" >&2 if [[ -z ${BUILDBUDDY_API_KEY:+x} ]]; then echo "::notice file=$(realpath --relative-to=../.. ${BASH_SOURCE[0]}),line=${LINENO}::SKIPPED Missing BuildBuddy token. See examples/persistent_worker/README.md" >&2 else - echo '' > .buckconfig.local + cat >.buckconfig.local < + +[build] +cache_silo_key=$(date +%s.%N).${GITHUB_RUN_ID-0} +EOF buck2 clean; buck2 build : -vstderr echo "# Verifying Buck2 log" >&2 buck2 log what-ran --show-std-err --format json \ @@ -88,7 +97,12 @@ echo "::group::Remote build with persistent worker" >&2 if [[ -z ${BUILDBUDDY_API_KEY:+x} ]]; then echo "::notice file=$(realpath --relative-to=../.. ${BASH_SOURCE[0]}),line=${LINENO}::SKIPPED Missing BuildBuddy token. See examples/persistent_worker/README.md" >&2 else - echo '' > .buckconfig.local + cat >.buckconfig.local < + +[build] +cache_silo_key=$(date +%s.%N).${GITHUB_RUN_ID-0} +EOF buck2 clean; buck2 build : -vstderr echo "# Verifying Buck2 log" >&2 buck2 log what-ran --show-std-err --format json \ From 72e35ade0a3d2eb3c357bc896099da609d19be93 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 18 Dec 2024 16:26:44 +0100 Subject: [PATCH 27/36] Fix extra space --- examples/persistent_worker/.buckconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/persistent_worker/.buckconfig b/examples/persistent_worker/.buckconfig index 49063dec2535..34a9d2e17746 100644 --- a/examples/persistent_worker/.buckconfig +++ b/examples/persistent_worker/.buckconfig @@ -11,7 +11,7 @@ fbsource = none buck = none [external_cells] - prelude = bundled +prelude = bundled [parser] target_platform_detector_spec = target:root//...->prelude//platforms:default From d2499329e09c5415de2f45d854a2a36534cb4523 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 13 Jan 2025 17:12:23 +0100 Subject: [PATCH 28/36] Update persistent worker example extract_archive See 779fead92ae469b12ace4eefc38eb82e512a9045. --- examples/persistent_worker/toolchains/BUCK | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/persistent_worker/toolchains/BUCK b/examples/persistent_worker/toolchains/BUCK index f8e1c79724f6..1b7ca8f52ebe 100644 --- a/examples/persistent_worker/toolchains/BUCK +++ b/examples/persistent_worker/toolchains/BUCK @@ -28,7 +28,7 @@ remote_file( extract_archive( name = "python-extract", - contents_archive = ":python-download", + src = ":python-download", ) python_toolchain( From 1693bf12e1ecb5dee64c79ca8543e50b02e936c5 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 13 Jan 2025 17:29:38 +0100 Subject: [PATCH 29/36] Pin linux-build-examples Ubuntu version I noticed a discrepancy on external PR GitHub Actions runs vs. upstream main branch GitHub Actions runs: The main branch CI runs on Ubuntu 22.04, while external PRs run on Ubuntu 24.04. This causes CI failures due to version mismatches in the distribution package repository. External PR CI run setup: https://github.com/aherrmann/buck2/actions/runs/12751410326/job/35538421552#step:1:4 Main branch CI run setup: https://github.com/facebook/buck2/actions/runs/12749831677/job/35533176968#step:1:4 External PR CI failure: https://github.com/aherrmann/buck2/actions/runs/12751410326/job/35538421552#step:3:491 Main branch CI success: https://github.com/facebook/buck2/actions/runs/12749831677/job/35533176968#step:3:461 --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 885d43332813..4cc27f787700 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -52,7 +52,7 @@ jobs: - uses: ./.github/actions/setup_reindeer - uses: ./.github/actions/build_bootstrap linux-build-examples: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4.1.0 - uses: ./.github/actions/setup_linux_env From de7f3c05fc308eeb017632fd68e7b31b58c51b2c Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 20 Jan 2025 10:35:22 +0100 Subject: [PATCH 30/36] Revert "Pin linux-build-examples Ubuntu version" This reverts commit 1693bf12e1ecb5dee64c79ca8543e50b02e936c5. No longer needed as of https://github.com/facebook/buck2/pull/845 --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 4cc27f787700..885d43332813 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -52,7 +52,7 @@ jobs: - uses: ./.github/actions/setup_reindeer - uses: ./.github/actions/build_bootstrap linux-build-examples: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4.1.0 - uses: ./.github/actions/setup_linux_env From d9a7303b03a40c2734d986170dac00466d0715d0 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 20 Jan 2025 10:50:00 +0100 Subject: [PATCH 31/36] buck2_error! signature changed see 15d70a31da --- app/buck2_action_impl/src/actions/impls/run.rs | 2 +- app/buck2_execute/src/execute/command_executor.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/buck2_action_impl/src/actions/impls/run.rs b/app/buck2_action_impl/src/actions/impls/run.rs index c1a3c4bf9e45..4d84ee3d9a2c 100644 --- a/app/buck2_action_impl/src/actions/impls/run.rs +++ b/app/buck2_action_impl/src/actions/impls/run.rs @@ -361,7 +361,7 @@ impl RunAction { if !worker_visitor.outputs.is_empty() { // TODO[AH] create appropriate error enum value. return Err(buck2_error!( - [], + buck2_error::ErrorTag::Input, "remote persistent worker command should not produce an output" )); } diff --git a/app/buck2_execute/src/execute/command_executor.rs b/app/buck2_execute/src/execute/command_executor.rs index 3c2e50bf7668..657383d7d8d8 100644 --- a/app/buck2_execute/src/execute/command_executor.rs +++ b/app/buck2_execute/src/execute/command_executor.rs @@ -202,7 +202,7 @@ impl CommandExecutor { || arg.starts_with("--flagfile")) { return Err(buck2_error!( - [], + buck2_error::ErrorTag::Input, "Remote persistent worker arguments must be passed as `@argfile`, `-flagfile=argfile`, or `--flagfile=argfile`." )); } From 19e9583e7c8d57cfa5ef1ca5bf9026ccfffdc57f Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 20 Jan 2025 14:43:45 +0100 Subject: [PATCH 32/36] Explicit Bazel remote persistent worker support Adresses https://github.com/facebook/buck2/pull/787#discussion_r1922141136 --- app/buck2_action_impl/src/actions/impls/run.rs | 6 +++--- .../rule_defs/provider/builtin/worker_info.rs | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/buck2_action_impl/src/actions/impls/run.rs b/app/buck2_action_impl/src/actions/impls/run.rs index 4d84ee3d9a2c..7627fbb1a506 100644 --- a/app/buck2_action_impl/src/actions/impls/run.rs +++ b/app/buck2_action_impl/src/actions/impls/run.rs @@ -264,7 +264,7 @@ struct UnpackedWorkerValues<'v> { id: WorkerId, concurrency: Option, streaming: bool, - remote: bool, + supports_bazel_remote_persistent_worker_protocol: bool, } struct UnpackedRunActionValues<'v> { @@ -322,7 +322,7 @@ impl RunAction { id: WorkerId(worker.id), concurrency: worker.concurrency(), streaming: worker.streaming(), - remote: worker.remote(), + supports_bazel_remote_persistent_worker_protocol: worker.supports_bazel_remote_persistent_worker_protocol(), }); Ok(UnpackedRunActionValues { @@ -355,7 +355,7 @@ impl RunAction { .exe .add_to_command_line(&mut worker_rendered, &mut cli_ctx)?; worker.exe.visit_artifacts(artifact_visitor)?; - let worker_key = if worker.remote { + let worker_key = if worker.supports_bazel_remote_persistent_worker_protocol { let mut worker_visitor = SimpleCommandLineArtifactVisitor::new(); worker.exe.visit_artifacts(&mut worker_visitor)?; if !worker_visitor.outputs.is_empty() { diff --git a/app/buck2_build_api/src/interpreter/rule_defs/provider/builtin/worker_info.rs b/app/buck2_build_api/src/interpreter/rule_defs/provider/builtin/worker_info.rs index 742aab65d9aa..9106c0891e82 100644 --- a/app/buck2_build_api/src/interpreter/rule_defs/provider/builtin/worker_info.rs +++ b/app/buck2_build_api/src/interpreter/rule_defs/provider/builtin/worker_info.rs @@ -50,8 +50,8 @@ pub struct WorkerInfoGen { pub concurrency: ValueOfUncheckedGeneric>, // Whether to always run actions using this worker via the streaming API pub streaming: ValueOfUncheckedGeneric, - // Remote execution capable worker - pub remote: ValueOfUncheckedGeneric, + // Bazel remote persistent worker protocol capable worker. + pub supports_bazel_remote_persistent_worker_protocol: ValueOfUncheckedGeneric, pub id: u64, } @@ -70,7 +70,7 @@ fn worker_info_creator(globals: &mut GlobalsBuilder) { ValueOf<'v, usize>, >, #[starlark(require = named, default = NoneType)] streaming: Value<'v>, - #[starlark(require = named, default = false)] remote: bool, + #[starlark(require = named, default = false)] supports_bazel_remote_persistent_worker_protocol: bool, eval: &mut Evaluator<'v, '_, '_>, ) -> starlark::Result> { let heap = eval.heap(); @@ -82,7 +82,7 @@ fn worker_info_creator(globals: &mut GlobalsBuilder) { id, concurrency: heap.alloc_typed_unchecked(concurrency).cast(), streaming: ValueOfUnchecked::new(streaming), - remote: heap.alloc_typed_unchecked(remote).cast(), + supports_bazel_remote_persistent_worker_protocol: heap.alloc_typed_unchecked(supports_bazel_remote_persistent_worker_protocol).cast(), }) } } @@ -110,8 +110,8 @@ impl<'v, V: ValueLike<'v>> WorkerInfoGen { .unwrap_or(false) } - pub fn remote(&self) -> bool { - self.remote + pub fn supports_bazel_remote_persistent_worker_protocol(&self) -> bool { + self.supports_bazel_remote_persistent_worker_protocol .to_value() .unpack() .expect("validated at construction") From 6a3e3dd0bb1160858f458cbd7d99cc790978e8a2 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 20 Jan 2025 15:00:46 +0100 Subject: [PATCH 33/36] Explicit Bazel remote persistent worker support Addresses https://github.com/facebook/buck2/pull/787#discussion_r1922141887 --- .../src/interpreter/rule_defs/command_executor_config.rs | 6 +++--- app/buck2_core/src/execution_types/executor_config.rs | 4 ++-- app/buck2_execute/src/execute/command_executor.rs | 2 +- app/buck2_server/src/daemon/common.rs | 2 +- app/buck2_test/src/orchestrator.rs | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/buck2_build_api/src/interpreter/rule_defs/command_executor_config.rs b/app/buck2_build_api/src/interpreter/rule_defs/command_executor_config.rs index 25c93604fcd8..51a1ea8f38fd 100644 --- a/app/buck2_build_api/src/interpreter/rule_defs/command_executor_config.rs +++ b/app/buck2_build_api/src/interpreter/rule_defs/command_executor_config.rs @@ -86,7 +86,7 @@ pub fn register_command_executor_config(builder: &mut GlobalsBuilder) { /// * `allow_hybrid_fallbacks_on_failure`: Whether to allow fallbacks when the result is failure (i.e. the command failed on the primary, but the infra worked) /// * `use_windows_path_separators`: Whether to use Windows path separators in command line arguments /// * `use_persistent workers`: Whether to use persistent workers for local execution if they are available - /// * `use_remote_persistent_workers`: Whether to use persistent workers for remote execution if they are available + /// * `use_bazel_protocol_remote_persistent_workers`: Whether to use persistent workers for remote execution via the Bazel remote persistent worker protocol if they are available /// * `allow_cache_uploads`: Whether to upload local actions to the RE cache /// * `max_cache_upload_mebibytes`: Maximum size to upload in cache uploads /// * `experimental_low_pass_filter`: Whether to use the experimental low pass filter @@ -112,7 +112,7 @@ pub fn register_command_executor_config(builder: &mut GlobalsBuilder) { #[starlark(default = false, require = named)] allow_hybrid_fallbacks_on_failure: bool, #[starlark(default = false, require = named)] use_windows_path_separators: bool, #[starlark(default = false, require = named)] use_persistent_workers: bool, - #[starlark(default = false, require = named)] use_remote_persistent_workers: bool, + #[starlark(default = false, require = named)] use_bazel_protocol_remote_persistent_workers: bool, #[starlark(default = false, require = named)] allow_cache_uploads: bool, #[starlark(default = NoneOr::None, require = named)] max_cache_upload_mebibytes: NoneOr< i32, @@ -325,7 +325,7 @@ pub fn register_command_executor_config(builder: &mut GlobalsBuilder) { PathSeparatorKind::Unix }, output_paths_behavior, - use_remote_persistent_workers, + use_bazel_protocol_remote_persistent_workers, }, } }; diff --git a/app/buck2_core/src/execution_types/executor_config.rs b/app/buck2_core/src/execution_types/executor_config.rs index 03689701c901..48c9406ea195 100644 --- a/app/buck2_core/src/execution_types/executor_config.rs +++ b/app/buck2_core/src/execution_types/executor_config.rs @@ -283,7 +283,7 @@ impl Default for CacheUploadBehavior { pub struct CommandGenerationOptions { pub path_separator: PathSeparatorKind, pub output_paths_behavior: OutputPathsBehavior, - pub use_remote_persistent_workers: bool, + pub use_bazel_protocol_remote_persistent_workers: bool, } #[derive(Debug, Eq, PartialEq, Hash, Allocative, Clone)] @@ -315,7 +315,7 @@ impl CommandExecutorConfig { options: CommandGenerationOptions { path_separator: PathSeparatorKind::system_default(), output_paths_behavior: Default::default(), - use_remote_persistent_workers: false, + use_bazel_protocol_remote_persistent_workers: false, }, }) } diff --git a/app/buck2_execute/src/execute/command_executor.rs b/app/buck2_execute/src/execute/command_executor.rs index 657383d7d8d8..d8bed0d28dbc 100644 --- a/app/buck2_execute/src/execute/command_executor.rs +++ b/app/buck2_execute/src/execute/command_executor.rs @@ -187,7 +187,7 @@ impl CommandExecutor { CommandExecutionInput::ScratchPath(_) => None, }); let mut platform = self.0.re_platform.clone(); - let args = if self.0.options.use_remote_persistent_workers + let args = if self.0.options.use_bazel_protocol_remote_persistent_workers && let Some(worker) = request.worker() && let Some(key) = worker.remote_key.as_ref() { diff --git a/app/buck2_server/src/daemon/common.rs b/app/buck2_server/src/daemon/common.rs index 61202c8725ce..b39bfe0394a7 100644 --- a/app/buck2_server/src/daemon/common.rs +++ b/app/buck2_server/src/daemon/common.rs @@ -500,7 +500,7 @@ pub fn get_default_executor_config(host_platform: HostPlatformOverride) -> Comma options: CommandGenerationOptions { path_separator: get_default_path_separator(host_platform), output_paths_behavior: Default::default(), - use_remote_persistent_workers: false, + use_bazel_protocol_remote_persistent_workers: false, }, } } diff --git a/app/buck2_test/src/orchestrator.rs b/app/buck2_test/src/orchestrator.rs index 990e7a87c2fc..5c1184901b53 100644 --- a/app/buck2_test/src/orchestrator.rs +++ b/app/buck2_test/src/orchestrator.rs @@ -1191,7 +1191,7 @@ impl<'b> BuckTestOrchestrator<'b> { options: CommandGenerationOptions { path_separator: PathSeparatorKind::system_default(), output_paths_behavior: Default::default(), - use_remote_persistent_workers: false, + use_bazel_protocol_remote_persistent_workers: false, }, }; let CommandExecutorResponse { From 8204f04edb510566dab6c8906c3a0290ebd551ea Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 20 Jan 2025 15:03:56 +0100 Subject: [PATCH 34/36] Update remote persistent worker example --- examples/persistent_worker/defs.bzl | 2 +- examples/persistent_worker/platforms/buildbuddy.bzl | 2 +- examples/persistent_worker/platforms/local.bzl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/persistent_worker/defs.bzl b/examples/persistent_worker/defs.bzl index 32f976b7154e..55e978a2f3d1 100644 --- a/examples/persistent_worker/defs.bzl +++ b/examples/persistent_worker/defs.bzl @@ -13,7 +13,7 @@ def _worker_impl(ctx: AnalysisContext) -> list[Provider]: WorkerInfo( exe = ctx.attrs.worker[RunInfo].args, concurrency = None, - remote = True, + supports_bazel_remote_persistent_worker_protocol = True, ), ] diff --git a/examples/persistent_worker/platforms/buildbuddy.bzl b/examples/persistent_worker/platforms/buildbuddy.bzl index 2d6db748473c..a913ea1aae16 100644 --- a/examples/persistent_worker/platforms/buildbuddy.bzl +++ b/examples/persistent_worker/platforms/buildbuddy.bzl @@ -21,7 +21,7 @@ def _platforms(ctx): allow_cache_uploads = True, use_limited_hybrid = True, use_persistent_workers = ctx.attrs.use_persistent_workers, - use_remote_persistent_workers = ctx.attrs.use_persistent_workers, + use_bazel_protocol_remote_persistent_workers = ctx.attrs.use_persistent_workers, remote_execution_properties = { "OSFamily": "Linux", "nonroot-workspace": True, diff --git a/examples/persistent_worker/platforms/local.bzl b/examples/persistent_worker/platforms/local.bzl index f09c0f61bc58..15e43a3d5778 100644 --- a/examples/persistent_worker/platforms/local.bzl +++ b/examples/persistent_worker/platforms/local.bzl @@ -20,7 +20,7 @@ def _platforms(ctx): remote_cache_enabled = False, allow_cache_uploads = False, use_persistent_workers = ctx.attrs.use_persistent_workers, - use_remote_persistent_workers = False, + use_bazel_protocol_remote_persistent_workers = False, ), ) From 6cee3245d6e08b6de8a82965319dcd222a7e9998 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 20 Jan 2025 15:34:44 +0100 Subject: [PATCH 35/36] fix missing field update --- app/buck2_build_api/src/actions/execute/action_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/buck2_build_api/src/actions/execute/action_executor.rs b/app/buck2_build_api/src/actions/execute/action_executor.rs index e0fd9256fb9f..c59a496ee874 100644 --- a/app/buck2_build_api/src/actions/execute/action_executor.rs +++ b/app/buck2_build_api/src/actions/execute/action_executor.rs @@ -761,7 +761,7 @@ mod tests { CommandGenerationOptions { path_separator: PathSeparatorKind::Unix, output_paths_behavior: Default::default(), - use_remote_persistent_workers: false, + use_bazel_protocol_remote_persistent_workers: false, }, Default::default(), ), From 61b95610a39d7f71998bbf5431edb4ae46b7392b Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 20 Jan 2025 16:19:49 +0100 Subject: [PATCH 36/36] update test-case --- .../src/interpreter/rule_defs/provider/builtin/worker_info.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/buck2_build_api_tests/src/interpreter/rule_defs/provider/builtin/worker_info.rs b/app/buck2_build_api_tests/src/interpreter/rule_defs/provider/builtin/worker_info.rs index 3fd51e7dae70..97b5ff92aaf4 100644 --- a/app/buck2_build_api_tests/src/interpreter/rule_defs/provider/builtin/worker_info.rs +++ b/app/buck2_build_api_tests/src/interpreter/rule_defs/provider/builtin/worker_info.rs @@ -24,7 +24,7 @@ fn run_display() { .run_starlark_bzl_test( r#" def test(): - assert_eq('WorkerInfo(exe=cmd_args("x"), concurrency=None, streaming=None, remote=False)', str(WorkerInfo(exe="x"))) + assert_eq('WorkerInfo(exe=cmd_args("x"), concurrency=None, streaming=None, supports_bazel_remote_persistent_worker_protocol=False)', str(WorkerInfo(exe="x"))) "#, ) .unwrap();