bazel: run Python script with runfiles access

Author: AlexTereshenkov

.github/workflows/build.yml

@@ -33,6 +33,9 @@ jobs:
       - name: Test
         run: bazel test //...
 
+      - name: Runfiles
+        run: bazel run //tools:archiver
+
   lint-host:
     runs-on: ubuntu-24.04
 

BUILD.bazel

@@ -38,3 +38,6 @@ tar(
     name = "archive-lean",
     srcs = ["//:LICENSE"],
 )
+
+# to access by a `py_binary` target under `data`
+exports_files(["adhoc/data.json"])

MODULE.bazel

@@ -5,6 +5,8 @@ module(
 
 http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
+http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
+
 # Bazel specific
 bazel_dep(name = "platforms", version = "0.0.11")
 bazel_dep(name = "rules_shell", version = "0.5.0")
@@ -66,6 +68,20 @@ http_archive(
     urls = ["https://github.com/kthohr/gcem/archive/v1.18.0.tar.gz"],
 )
 
+http_file(
+    name = "zstd_cli",
+    executable = True,
+    sha256 = "190c196b46883a234238d9c52a211c1494df79bb170fbe30c6fc22639d5d4731",
+    url = "https://github.com/aspect-build/zstd-prebuilt/releases/download/v1.5.6-bcr2/zstd_linux_amd64",
+)
+
+http_archive(
+    name = "yq_cli",
+    build_file = "@//tools/yq:BUILD.bazel.gen",
+    sha256 = "09a40787e527005fce869dac01146fcc2473d4e24074a841baf502b8c87fb6e2",
+    url = "https://github.com/mikefarah/yq/releases/download/v4.48.1/yq_linux_amd64.tar.gz",
+)
+
 toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains")
 
 # cannot use this due to an error

MODULE.bazel.lock

@@ -0,0 +1 @@
+{"a": "A", "b":  "B"}

adhoc/data.json

@@ -17,3 +17,6 @@ cc_import(
     # to be linked into any binary that depends on it
     alwayslink = True,
 )
+
+# to access by a `py_binary` target under `data`
+exports_files(["dates.so"])

compiled/BUILD.bazel

@@ -1,3 +1,4 @@
+load("@rules_python//python:defs.bzl", "py_binary")
 load("@rules_shell//shell:sh_binary.bzl", "sh_binary")
 
 package(default_visibility = ["//visibility:public"])
@@ -19,3 +20,25 @@ sh_binary(
         "@oclint_prebuilt//:oclint_bundle",
     ],
 )
+
+py_binary(
+    name = "archiver",
+    srcs = ["archiver.py"],
+    data = [
+        # http_file
+        "@zstd_cli//file",
+        # http_archive
+        "@yq_cli//:yq_linux_amd64",
+        # local file (in a Bazel package)
+        "//compiled:dates.so",
+        # local file (not in a Bazel package)
+        "//:adhoc/data.json",
+    ],
+    env = {
+        "ZSTD_COMPRESSION_LEVEL": "10",
+    },
+    tags = ["manual"],
+    deps = [
+        "@rules_python//python/runfiles",
+    ],
+)

tools/BUILD.bazel

@@ -0,0 +1,58 @@
+"""
+Use `import pdb; pdb.set_trace()` to pause and inspect the runfiles directory
+"""
+
+import os
+import shutil
+import subprocess
+
+from python.runfiles import Runfiles
+
+runfiles = Runfiles.Create()
+
+# http_file location
+ZSTD_CLI = runfiles.Rlocation("zstd_cli/file/downloaded")
+
+# http_archive location
+YQ_CLI = runfiles.Rlocation("yq_cli/yq_linux_amd64")
+
+# local workspace file location (already in a Bazel package)
+dates_so = runfiles.Rlocation("_main/compiled/dates.so")
+
+# local workspace file location (not in a Bazel package)
+data_json = runfiles.Rlocation("_main/adhoc/data.json")
+
+pairs = [
+    ("zstd", ZSTD_CLI),
+    ("yq", YQ_CLI),
+    ("Data file tracked by Bazel", dates_so),
+    ("Data file not tracked by Bazel", data_json),
+]
+for pair in pairs:
+    print(pair)
+
+print("Pretty print of JSON with yq:")
+_ = subprocess.run(
+    args=[YQ_CLI, data_json],
+    check=True,
+    text=True,
+)
+
+print(f"Zstd compression level from env var: {os.getenv('ZSTD_COMPRESSION_LEVEL')}")
+_ = subprocess.run(
+    args=[
+        ZSTD_CLI,
+        dates_so,
+        "-o",
+        "archive.zst",
+        "--force",
+    ],
+    check=True,
+    text=True,
+)
+
+# need to copy the archive from `/home/%username%/.cache/bazel/_bazel_%username%/checksum/execroot/_main/bazel-out/k8-fastbuild/bin/tools/archiver.runfiles/_main`
+shutil.copyfile(
+    "archive.zst",
+    f"{os.getenv('BUILD_WORKSPACE_DIRECTORY')}/archive.zst",
+)

tools/archiver.py

@@ -0,0 +1 @@
+# this file needs to exist to mark `tools/yq` as a package

tools/yq/BUILD.bazel

@@ -0,0 +1,3 @@
+package(default_visibility = ["//visibility:public"])
+
+exports_files(["yq_linux_amd64"])