From 59e95795a157df054f03609b1b1a20380afb1aaa Mon Sep 17 00:00:00 2001 From: vmagro Date: Tue, 27 Aug 2024 16:09:36 -0700 Subject: [PATCH] configure buck2 for localhost nativelink for caching --- .buckconfig | 14 ++++ .github/workflows/ci.yml | 10 +++ ci/nativelink.json | 164 +++++++++++++++++++++++++++++++++++++++ platforms/BUCK | 26 +++++++ platforms/defs.bzl | 44 +++++++++++ 5 files changed, 258 insertions(+) create mode 100644 ci/nativelink.json create mode 100644 platforms/BUCK create mode 100644 platforms/defs.bzl diff --git a/.buckconfig b/.buckconfig index 379b397f8c..682fd2dde2 100644 --- a/.buckconfig +++ b/.buckconfig @@ -23,5 +23,19 @@ [buck2] file_watcher = watchman + materializations = deferred + sqlite_materializer_state = true + defer_write_actions = true + hash_all_commands = true + +[buck2_re_client] + action_cache_address = grpcs://127.0.0.1:50051 + engine_address = grpcs://127.0.0.1:50051 + cas_address = grpcs://127.0.0.1:50051 + tls = false + instance_name = main + +[build] + execution_platforms = antlir//platforms:exec-platforms diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88b6ee1c26..3cdcbc21bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,10 +11,20 @@ jobs: strategy: matrix: shard: [1/3, 2/3, 3/3] + + services: + nativelink: + image: ghcr.io/tracemachina/nativelink:v0.5.1 + options: /config + volumes: + - ${{ github.workspace }}/ci/nativelink.json:/config + steps: - uses: actions/checkout@v4 with: submodules: true + - name: Restart NativeLink + run: docker restart nativelink - uses: facebook/install-dotslash@latest - uses: dtolnay/rust-toolchain@stable with: diff --git a/ci/nativelink.json b/ci/nativelink.json new file mode 100644 index 0000000000..173951deb9 --- /dev/null +++ b/ci/nativelink.json @@ -0,0 +1,164 @@ +{ + "stores": { + "AC_MAIN_STORE": { + "filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-ac", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac", + "eviction_policy": { + // 1gb. + "max_bytes": 1000000000, + } + } + }, + "WORKER_FAST_SLOW_STORE": { + "fast_slow": { + // "fast" must be a "filesystem" store because the worker uses it to make + // hardlinks on disk to a directory where the jobs are running. + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + // 10gb. + "max_bytes": 10000000000, + } + } + }, + "slow": { + /// Discard data. + /// This example usage has the CAS and the Worker live in the same place, + /// so they share the same underlying CAS. Since workers require a fast_slow + /// store, we use the fast store as our primary data store, and the slow store + /// is just a noop, since there's no shared storage in this config. + "noop": {} + } + } + } + }, + "schedulers": { + "MAIN_SCHEDULER": { + "simple": { + "supported_platform_properties": { + "cpu_count": "minimum", + "memory_kb": "minimum", + "network_kbps": "minimum", + "disk_read_iops": "minimum", + "disk_read_bps": "minimum", + "disk_write_iops": "minimum", + "disk_write_bps": "minimum", + "shm_size": "minimum", + "gpu_count": "minimum", + "gpu_model": "exact", + "cpu_vendor": "exact", + "cpu_arch": "exact", + "cpu_model": "exact", + "kernel_version": "exact", + "OSFamily": "priority", + "container-image": "priority", + // Example of how to set which docker images are available and set + // them in the platform properties. + // "docker_image": "priority", + } + } + } + }, + "workers": [{ + "local": { + "worker_api_endpoint": { + "uri": "grpc://127.0.0.1:50061", + }, + "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE", + "upload_action_result": { + "ac_store": "AC_MAIN_STORE", + }, + "work_directory": "/tmp/nativelink/work", + "platform_properties": { + "cpu_count": { + "values": ["16"], + }, + "memory_kb": { + "values": ["500000"], + }, + "network_kbps": { + "values": ["100000"], + }, + "cpu_arch": { + "values": ["x86_64"], + }, + "OSFamily": { + "values": [""] + }, + "container-image": { + "values": [""] + }, + // Example of how to set which docker images are available and set + // them in the platform properties. + // "docker_image": { + // "query_cmd": "docker images --format {{.Repository}}:{{.Tag}}", + // } + } + } + }], + "servers": [{ + "name": "public", + "listener": { + "http": { + "socket_address": "0.0.0.0:50051" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "execution": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE", + "scheduler": "MAIN_SCHEDULER", + } + }, + "capabilities": { + "main": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER", + } + } + }, + "bytestream": { + "cas_stores": { + "main": "WORKER_FAST_SLOW_STORE", + } + } + } + }, { + "name": "private_workers_servers", + "listener": { + "http": { + "socket_address": "0.0.0.0:50061" + } + }, + "services": { + "experimental_prometheus": { + "path": "/metrics" + }, + // Note: This should be served on a different port, because it has + // a different permission set than the other services. + // In other words, this service is a backend api. The ones above + // are a frontend api. + "worker_api": { + "scheduler": "MAIN_SCHEDULER", + }, + "admin": {}, + "health": {}, + } + }], + "global": { + "max_open_files": 512 + } +} diff --git a/platforms/BUCK b/platforms/BUCK new file mode 100644 index 0000000000..c32c75f92a --- /dev/null +++ b/platforms/BUCK @@ -0,0 +1,26 @@ +load("@prelude//:rules.bzl", "platform") +load("@prelude//platforms:defs.bzl", "execution_platform", "host_configuration") +load(":defs.bzl", "nativelink_platform", "exec_platforms") + +nativelink_platform( + name = "nativelink", + cpu_configuration = "config//cpu:x86_64", + os_configuration = "config//os:linux", + visibility = ["PUBLIC"], +) + +execution_platform( + name = "host", + cpu_configuration = host_configuration.cpu, + os_configuration = host_configuration.os, + use_windows_path_separators = host_info().os.is_windows, + visibility = ["PUBLIC"], +) + +exec_platforms( + name = "exec-platforms", + platforms = [ + ":nativelink", + ":host", + ] +) diff --git a/platforms/defs.bzl b/platforms/defs.bzl new file mode 100644 index 0000000000..beadb8fda0 --- /dev/null +++ b/platforms/defs.bzl @@ -0,0 +1,44 @@ +def _nativelink_platform(ctx): + constraints = {} + constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints) + constraints.update(ctx.attrs.os_configuration[ConfigurationInfo].constraints) + + configuration = ConfigurationInfo( + constraints = constraints, + values = {}, + ) + + platform = ExecutionPlatformInfo( + label = ctx.label.raw_target(), + configuration = configuration, + executor_config = CommandExecutorConfig( + local_enabled = True, + remote_enabled = True, + use_limited_hybrid = True, + remote_execution_properties = { + # "arch": ctx.attrs.cpu_configuration.label.name, + # "os": ctx.attrs.os_configuration.label.name, + }, + remote_execution_use_case = "antlir", + remote_output_paths = "output_paths", + ), + ) + + return [DefaultInfo(), platform] + +nativelink_platform = rule(attrs = { + "cpu_configuration": attrs.dep(providers = [ConfigurationInfo]), + "os_configuration": attrs.dep(providers = [ConfigurationInfo]), +}, impl = _nativelink_platform) + +def _exec_platforms(ctx): + return [ + DefaultInfo(), + ExecutionPlatformRegistrationInfo( + platforms = [p[ExecutionPlatformInfo] for p in ctx.attrs.platforms], + ), + ] + +exec_platforms = rule(attrs = { + "platforms": attrs.list(attrs.dep(providers = [ExecutionPlatformInfo])), +}, impl = _exec_platforms)