Switch to JSON api and allow pullling a specific generation (#14)

Author: leoconco

.gitignore

@@ -1,2 +1,3 @@
 bazel-*
 MODULE.bazel.lock
+.ijwb/

README.md

@@ -215,6 +215,11 @@ Examples:
         "trainingdata/model/very_large.bin": {
             "remote_path": "weird/nested/path/extra/model/very_large.bin",
             "integrity": "sha256-Oibw8PV3cDY84HKv3sAWIEuk+R2s8Hwhvlg6qg4H7uY="
+        },
+        "trainingdata/model/very_large.bin.old": {
+            "remote_path": "weird/nested/path/extra/model/very_large.bin",
+            "generation": "1721216992694798",
+            "integrity": "sha256-Oibw8PV3cDY84HKv3sAWIEuk+R2s8Hwhvlg6qg4H7uY="
         }
     }
   ```
@@ -224,6 +229,7 @@ Examples:
   - `sha256`: the expected sha256 hash of the file. Required unless `integrity` is used.
   - `integrity`: the expected SRI value of the file. Required unless `sha256` is used.
   - `remote_path`: name of the object within the bucket. If not set, the local path is used.
+  - `generation`: generation number to a specific version of the object in the bucket. If not set the `live` generation is used.
 
   Targets in the main repository can depend on this target if the
   following lines are added to `MODULE.bazel`:

examples/full/gcs_lock.json

@@ -13,6 +13,7 @@
     },
     "custom/local/path/SHA256SUMS": {
       "remote_path": "release/v1.30.3/SHA256SUMS",
+      "generation": "1721216992694798",
       "integrity": "sha256-Oibw8PV3cDY84HKv3sAWIEuk+R2s8Hwhvlg6qg4H7uY="
     }
 }

gcs/private/extensions/gcs_bucket.bzl

@@ -67,6 +67,7 @@ def dep_to_blob_repo(bucket_name, local_path, info):
         "name": object_repo_name(bucket_name, info["remote_path"]),
         "downloaded_file_path": local_path,
         "executable": True,
+        "generation": info["generation"] if "generation" in info else None,
         "sha256": info["sha256"] if "sha256" in info else None,
         "integrity": info["integrity"] if "integrity" in info else None,
         "url": gs_url,
@@ -120,6 +121,7 @@ Examples:
   - `sha256`: the expected sha256 hash of the file. Required unless `integrity` is used.
   - `integrity`: the expected SRI value of the file. Required unless `sha256` is used.
   - `remote_path`: name of the object within the bucket. If not set, the local path is used.
+  - `generation`: generation number to a specific version of the object in the bucket. If not set the `live` generation is used.
 
   Targets in the main repository can depend on this target if the
   following lines are added to `MODULE.bazel`:

gcs/private/repo_rules/eager.bzl

@@ -45,7 +45,11 @@ def _eager_impl(repository_ctx):
 
 def info_to_download_args(bucket_name, local_path, info):
     args = {
-        "url": bucket_url(bucket_name, info["remote_path"]),
+        "url": bucket_url(
+            bucket_name,
+            info["remote_path"],
+            info["generation"] if "generation" in info else None,
+        ),
         "output": local_path,
         "executable": True,
         "block": False,

gcs/private/repo_rules/gcs_archive.bzl

@@ -255,6 +255,11 @@ By default, the archive type is determined from the file extension of the
 URL. If the file has no extension, you can explicitly specify one of the
 following: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`, `"tar.gz"`, `"tgz"`,
 `"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `"tar.bz2"`, `"ar"`, or `"deb"`.""",
+    ),
+    "generation": attr.string(
+        doc = """A specific generation number to pull from gcs.
+This must be an integer, live generation is assumed if empty.
+""",
     ),
     "url": attr.string(
         mandatory = True,

gcs/private/repo_rules/gcs_file.bzl

@@ -109,6 +109,11 @@ This must match the SHA-256 of the file downloaded. _It is a security risk
 to omit the SHA-256 as remote files can change._ At best omitting this
 field will make your build non-hermetic. It is optional to make development
 easier but should be set before shipping.""",
+    ),
+    "generation": attr.string(
+        doc = """A specific generation number to pull from gcs.
+This must be an integer, live generation is assumed if empty.
+""",
     ),
     "url": attr.string(
         mandatory = True,

gcs/private/util.bzl

@@ -46,8 +46,13 @@ def repository_ctx_download_common_args(attr, bucket_name, remote_path):
     has_sha256 = len(attr.sha256) > 0
     if has_integrity == has_sha256:
         fail("expected exactly one of \"integrity\" and \"sha256\"")
+    has_generation = len(attr.generation) > 0
     args = {
-        "url": bucket_url(bucket_name, remote_path),
+        "url": bucket_url(
+            bucket_name,
+            remote_path,
+            attr.generation if has_generation else None,
+        ),
         "sha256": attr.sha256,
         "integrity": attr.integrity,
     }
@@ -79,8 +84,14 @@ def download_and_extract_args(attr, bucket_name, remote_path):
         args.pop("rename_files")
     return args
 
-def bucket_url(bucket, object_path):
-    return "https://storage.googleapis.com/{}/{}".format(bucket, url_encode(object_path))
+# Use the JSON API, which allows for more options https://cloud.google.com/storage/docs/json_api/v1/objects/get
+def bucket_url(bucket, object_path, generation):
+    return "https://storage.googleapis.com/storage/v1/b/{}/o/{}?alt=media".format(
+        bucket,
+        url_encode(object_path),
+    ) + ("&generation={}".format(
+        generation,
+    ) if generation else "")
 
 def deps_from_file(module_ctx, lockfile_label, jsonpath):
     lockfile_path = module_ctx.path(lockfile_label)
@@ -99,6 +110,7 @@ def parse_lockfile(lockfile_content, jsonpath):
         # we expect the following schema:
         # - exactly one of sha256 or integrity
         # - optionally a remote_path (if not, we populate it with the local_path instead)
+        has_generation = "generation" in v
         has_remote_path = "remote_path" in v
         has_integrity = "integrity" in v
         has_sha256 = "sha256" in v
@@ -109,6 +121,8 @@ def parse_lockfile(lockfile_content, jsonpath):
         }
         if has_integrity:
             info["integrity"] = v["integrity"]
+        if has_generation:
+            info["generation"] = v["generation"]
         if has_sha256:
             info["sha256"] = v["sha256"]
         processed_lockfile[local_path] = info