add Buck build

Change-Id: Ia3d5cca3998af155bcdc78e72effdb6dcdc21af0

Author: tek

.buckconfig

@@ -0,0 +1,35 @@
+[cells]
+root = .
+prelude = prelude
+toolchains = toolchains
+none = none
+ghc_persistent_worker = ghc_persistent_worker
+
+[cell_aliases]
+config = prelude
+ovr_config = prelude
+fbcode = none
+fbsource = none
+fbcode_macros = none
+buck = none
+
+[external_cells]
+prelude = git
+
+[external_cell_prelude]
+git_origin = https://github.com/MercuryTechnologies/buck2-prelude.git
+# branch: tek/only-spawn-make
+commit_hash = 97ee3e1e62605ef78b49cb8a1362634d96119a4a
+
+[parser]
+target_platform_detector_spec = target:root//...->prelude//platforms:default
+
+[project]
+ignore = .git, tmp, ghcHEAD-toolchains
+
+[buck2]
+materializations = deferred
+hash_all_commands = true
+
+[build]
+execution_platforms = root//platforms:host-persistent-worker

.buckconfig.d/01-default-platform

@@ -0,0 +1,9 @@
+# configure the default execution platform here
+#
+# Note: this file has a lower precedence than .buckconfig and .buckconfig.local (which has the highest)
+
+[build]
+execution_platforms = prelude//platforms:default
+
+[test]
+re_profile = default

.github/workflows/buck-test.yaml

@@ -0,0 +1,26 @@
+name: Buck Tests
+
+on:
+  push:
+    branches:
+      - tek/debug
+  workflow_dispatch:
+
+jobs:
+
+  tests:
+    name: Run tests
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@v16
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: cachix/cachix-action@v16
+        with:
+          name: tek
+      - run: nix run .#hybrid
+      - run: nix run .#three-layers

BUCK

@@ -0,0 +1,15 @@
+filegroup(
+    name = "nix_overlays",
+    srcs = glob([
+        "flake.lock",
+        "**/*.nix",
+        "buck-proxy/**",
+        "ghc-worker/**",
+        "grpc/**",
+        "instrument/**",
+        "internal/**",
+        "proto/**",
+        "types/**",
+   ]),
+   visibility = ["PUBLIC"],
+)

README.md

@@ -30,3 +30,15 @@ $ cabal run ghc-persistent-worker-server -- --ghc ~/repo/srcc/ghcHEAD/_build/sta
 $ GHC_PERSISTENT_WORKER_SOCKET=/tmp/ghc_server.ipc cabal run ghc-persistent-worker-client -- test/A.hs
 ```
 (optionally, one can set `--package-db (pkg_db_path)`. One can have multiple `--package-db`.)
+
+Buck
+====
+
+The flake provides a test environment for Buck with a locally Nix-built worker.
+Enter the shell `buck` to use it:
+
+```
+$ nix develop .#buck
+$ buck build //test/three-layers/project/...
+
+```

flake.lock

@@ -6,8 +6,12 @@
     url = "github:tek/ghc-debug/59198910da4573612bf5c6f9969c64fa620db396";
     flake = false;
   };
+  inputs.fenix = {
+    url = "github:nix-community/fenix/9d17341a4f227fe15a0bca44655736b3808e6a03";
+    inputs.nixpkgs.follows = "hix/nixpkgs";
+  };
 
-  outputs = {hix, ghc-debug, ...}: let
+  outputs = {hix, ghc-debug, fenix, ...}: let
 
     testEnv = config: {
       ghc_dir = "${config.toolchain.vanilla.ghc}";
@@ -38,7 +42,15 @@
       ghc-worker = notest;
     };
 
-  in hix ({config, build, lib, ...}: {
+  in hix ({config, build, lib, util, ...}: let
+
+    buckTest = dir: {
+      expose = true;
+      env = "buck";
+      command = "${import ./ops/buck-test/${dir} { inherit util; }}";
+    };
+
+  in {
 
     compiler = "ghc910";
     ghcVersions = [];
@@ -64,6 +76,14 @@
         source.build.flavour = "release+split_sections+ipe";
       };
 
+      mwb-hybrid.source.build = {
+        version = "9.10.1";
+        url = "https://gitlab.haskell.org/ghc/ghc";
+        rev = "f9587b0fcc14565953326d0032a3e92a871d0468";
+        hash = "sha256-eSFKCb2jgcxdNuy1NaLgkmjkZrr1FD09RbNm+c8FUW4=";
+        flavour = "release+split_sections+ipe";
+      };
+
     };
 
     envs.dev = args: {
@@ -100,6 +120,18 @@
       ];
     };
 
+    envs.hybrid = args: {
+      expose.scoped = true;
+      env = testEnv args.config;
+      package-set.compiler = "mwb-hybrid";
+      overrides = [
+        sharedExeOverrides
+        ipeOverrides
+        (envOverrides args.config)
+        buckBinOverrides
+      ];
+    };
+
     envs.profiled = args: {
       env = testEnv args.config;
       overrides = [
@@ -109,12 +141,63 @@
       package-set.compiler = "mwb-ipe";
     };
 
+    # ------------------------------------------------------------------------------------------------------------------
+    # Buck
+
+    # The environment for the CLI tool `buck`, using the Buck overlay extracted from MWB.
+    # `fenix` is a dep of Buck.
+    # Exposes a devShell named `buck` that should be used to gain access to the CLI tool.
+    envs.buck = {
+      expose.shell = true;
+      packages = [];
+      buildInputs = pkgs: [pkgs.buck2-source];
+
+      package-set.compiler.nixpkgs.overlays = [
+        fenix.overlays.default
+        (import ./ops/buck/overlay.nix)
+      ];
+    };
+
+    # The environment for our Buck nixpkgs integration, from which GHC and the package set are taken when exposing them
+    # in `outputs.packages` below.
+    # Uses our custom GHC build and injects a hook into all Haskell derivations that creates `package.cache` in the
+    # store dir, which is needed because Buck supplies individual package DBs to GHC.
+    envs.buck-build = {config, ...}: {
+      packages = [];
+      package-set.compiler = "mwb-ipe";
+      env = testEnv config;
+
+      overrides = api@{override, ...}: let
+        testDeps = import ./ops/test-deps.nix { inherit util; };
+      in testDeps.overrides api // {
+        __all = override (drv: {
+          postInstall = (drv.postInstall or "") + ''
+            ghc-pkg recache --package-db $packageConfDir
+          '';
+        });
+      };
+    };
+
+    # The interface that Buck expects when loading Nix packages in `toolchains/BUCK` using those `nix.rules.flake`
+    # rules.
+    # Exposes the toolchain Haskell packages listed in `./ops/ghc-toolchain-libraries.nix` in the attribute
+    # `haskellPackages.libs` as well as Python and the GHC compiler derivation.
+    outputs.packages =
+      import ./ops/buck/packages.nix { inherit config lib; };
+
+    # ------------------------------------------------------------------------------------------------------------------
+
     envs.hls-db = {};
 
     commands.hls.env = "hls-db";
 
     output.extraPackages = ["ghc-debug-brick" "eventlog2html" "hp2pretty" "ghc-events"];
 
+    commands.comparison-1 = buckTest "comparison1";
+    commands.comparison-2 = buckTest "comparison2";
+    commands.three-layers = buckTest "three-layers";
+    commands.hybrid = buckTest "hybrid";
+
     packages = {
 
       ghc-worker = {
@@ -170,6 +253,20 @@
           source-dirs = "test";
           dependOnLibrary = false;
         };
+
+        executables.batch-worker = {};
+
+        executables.ghc-bin = {
+          dependencies = [
+            "containers"
+            "filepath"
+            "ghc"
+            "ghc-boot"
+            "transformers"
+          ];
+          dependOnLibrary = false;
+        };
+
       };
 
       debug = {
@@ -349,4 +446,5 @@
     };
 
   });
+
 }

flake.nix

@@ -0,0 +1,12 @@
+module Main where
+
+import GhcWorker.Batch (batchCompile)
+import System.Environment (getArgs)
+import System.IO (BufferMode (..), hSetBuffering, stderr, stdout)
+
+main :: IO ()
+main = do
+  hSetBuffering stdout LineBuffering
+  hSetBuffering stderr LineBuffering
+  paths <- getArgs
+  batchCompile paths