Vend in a versioned protosol via Crate instead of fetch script (#376)

* Changes

* leader_schedule

Author: cmoyes-jump

.github/workflows/build-common.yml

@@ -30,9 +30,6 @@ jobs:
           sudo dnf install -y protobuf-compiler
           sudo dnf install -y systemd-devel
 
-      - name: Clone protosol
-        run: make fetch_proto
-
       - name: Check lints and clippy
         run: |
           cargo fmt --all -- --check

Cargo.lock

@@ -518,6 +518,7 @@ winapi = "0.3.8"
 x509-parser = "0.14.0"
 zeroize = { version = "1.7", default-features = false }
 zstd = "0.13.3"
+protosol = {git = "https://github.com/firedancer-io/protosol", tag = "v1.0.4"}
 solfuzz-agave-macro = { path = "macro" }
 once_cell = "1.21.3"
 lazy_static = "1.5.0"

Cargo.toml

@@ -17,17 +17,14 @@ CC:=clang
 
 CARGO?=cargo
 
-.PHONY: build clean binaries shared_obj fetch_proto
+.PHONY: build clean binaries shared_obj
 
-all: | fetch_proto shared_obj binaries
+all: | shared_obj binaries
 
 # Alias for backwards compatibility
-build: | fetch_proto shared_obj
+build: | shared_obj
 
-conformance: | fetch_proto shared_obj_debug
-
-fetch_proto:
-	./scripts/fetch_proto.sh
+conformance: | shared_obj_debug
 
 shared_obj:
 	RUSTFLAGS="$(RUSTFLAGS)" $(CARGO) build --target x86_64-unknown-linux-gnu --release --lib

Makefile

@@ -25,10 +25,19 @@ apt install libudev-dev protobuf-compiler pkg-config
 
 Running with a local copy of Agave (for easily adding print statements):
 ```sh
-pip3 install tomlkit
-python3 scripts/generate_cargo.py -p <your_local_agave_path> -o Cargo.toml
+./scripts/generate_cargo.py -p <your_local_agave_path> -o Cargo.toml
 ```
 
+Running with a specific Agave commit:
+```sh
+./scripts/generate_cargo.py -c <commit_hash> -o Cargo.toml
+```
+
+Running with a specific protosol version:
+```sh
+./scripts/generate_cargo.py -c <commit_hash> -v <version> -o Cargo.toml
+# Example: ./scripts/generate_cargo.py -c <commit_hash> -v 1.0.4 -o Cargo.toml
+```
 
 Check and test:
 
@@ -64,7 +73,7 @@ $ nm -D target/x86_64-unknown-linux-gnu/release/libsolfuzz_agave.so | grep '__sa
                  U __sanitizer_cov_trace_pc_indir
 ```
 
-**Note:** You may have to periodically run `make build` to ensure that Protobuf definitions stay in sync with [Protosol](https://github.com/firedancer-io/protosol/). Alternatively, you can run `./scripts/fetch_proto.sh` to keep Protosol up to date. Maintainers are expected to bump the `PROTO_VERSION` corresponding to the versioned git tag of the protosol repository when staging new protobuf changes.
+**Note:** Protobuf definitions are now managed through the `protosol` Rust crate dependency. The `protosol` crate is automatically included in the generated Cargo.toml with a specific version tag. When protobuf schema changes are needed, maintainers should update the `PROTOSOL_VERSION_TAG` in `scripts/generate_cargo.py` to the appropriate version tag from the [protosol repository](https://github.com/firedancer-io/protosol/).
 
 ## Building Targets with Core BPF Programs
 

README.md

@@ -1,6 +1,6 @@
-use std::io::Result;
+use std::{env, fs, path::PathBuf};
 
-fn main() -> Result<()> {
+fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Tells rustc to recompile the `load_core_bpf_program!` macro if either
     // of the required environment variables has changed.
     println!("cargo:rerun-if-env-changed=CORE_BPF_PROGRAM_ID");
@@ -12,36 +12,40 @@ fn main() -> Result<()> {
         println!("cargo:rerun-if-changed=force_rebuild");
     }
 
-    let proto_base_path = std::path::PathBuf::from("protosol/proto");
-
-    let protos = &[
-        proto_base_path.join("invoke.proto"),
-        proto_base_path.join("vm.proto"),
-        proto_base_path.join("txn.proto"),
-        proto_base_path.join("elf.proto"),
-        proto_base_path.join("shred.proto"),
-        proto_base_path.join("pack.proto"),
-        proto_base_path.join("block.proto"),
-        proto_base_path.join("type.proto"),
-    ];
-
-    protos
-        .iter()
-        .for_each(|proto| println!("cargo:rerun-if-changed={}", proto.display()));
-
-    for proto in protos {
-        if !proto.exists() {
-            return Err(std::io::Error::new(
-                std::io::ErrorKind::NotFound,
-                format!(
-                    "Proto file does not exist: {}. Run ./scripts/fetch_proto.sh",
-                    proto.display()
-                ),
-            ));
+    // Get absolute proto dir from producer
+    let proto_dir = PathBuf::from(
+        env::var("DEP_PROTOSOL_PROTO_DIR")
+            .expect("protosol did not expose PROTO_DIR, did protosol build.rs run first?"),
+    );
+
+    println!("cargo:rerun-if-env-changed=DEP_PROTOSOL_PROTO_DIR");
+    println!("cargo:rerun-if-changed={}", proto_dir.display());
+
+    // Collect absolute .proto paths
+    let mut proto_files = vec![];
+    for entry in fs::read_dir(&proto_dir)? {
+        let path = entry?.path();
+        if path.extension().and_then(|e| e.to_str()) == Some("proto") {
+            println!("cargo:rerun-if-changed={}", path.display());
+            proto_files.push(path);
         }
     }
 
-    prost_build::compile_protos(protos, &[proto_base_path])?;
+    // Ensure deterministic order for rebuilds
+    proto_files.sort();
+
+    // Compile protos into Rust
+    let out_dir = PathBuf::from(env::var("OUT_DIR")?);
+    let mut config = prost_build::Config::new();
+    config.out_dir(&out_dir);
+
+    config.compile_protos(
+        &proto_files
+            .iter()
+            .map(|p| p.display().to_string())
+            .collect::<Vec<_>>(),
+        &[proto_dir.to_str().unwrap()],
+    )?;
 
     Ok(())
 }

build.rs

@@ -11,6 +11,9 @@
 import os
 import subprocess
 
+# NOTE: this needs bumped with schema version upgrades of the protocol
+PROTOSOL_VERSION_TAG = "v1.0.4"
+
 def replace_path_with_git_rev(toml_data, git_url, rev):
     """
     Recursively process the TOML data to replace path with git and rev,
@@ -78,15 +81,23 @@ def parse_toml_file(file_path):
         return parse(f.read())
 
 def main():
+    global PROTOSOL_VERSION_TAG
 
     parser = argparse.ArgumentParser(description="Process input files.")
 
     # # Add flags
     parser.add_argument("--commit", "-c", help="Commit in firedancer-io/agave to use")
     parser.add_argument("--agave-path", "-p", help="Commit in firedancer-io/agave to use")
     parser.add_argument("--output", "-o", help="Path to the output file")
+    parser.add_argument("--version", "-v", help=f"Protosol version to use (e.g. \"{PROTOSOL_VERSION_TAG}\")")
 
     args = parser.parse_args()
+    if args.version:
+        PROTOSOL_VERSION_TAG = args.version
+        # Prepend 'v' if not already present
+        if not PROTOSOL_VERSION_TAG.startswith('v'):
+            PROTOSOL_VERSION_TAG = 'v' + PROTOSOL_VERSION_TAG
+    print(f"Using protosol version: {PROTOSOL_VERSION_TAG}")
 
     if args.agave_path:
         toml_data = parse_toml_file(args.agave_path + "/Cargo.toml")
@@ -117,6 +128,20 @@ def main():
         if patch_to_remove in toml_data.get("patch", {}).get("crates-io", {}):
             del toml_data["patch"]["crates-io"][patch_to_remove]
 
+    # Add protosol dependency (deduped if already present)
+    if "dependencies" not in toml_data:
+        toml_data["dependencies"] = table()
+    
+    # Remove existing protosol if present to ensure we use the correct version
+    if "protosol" in toml_data["dependencies"]:
+        del toml_data["dependencies"]["protosol"]
+    
+    # Add the required protosol dependency
+    protosol_dep = inline_table()
+    protosol_dep["git"] = "https://github.com/firedancer-io/protosol"
+    protosol_dep["tag"] = PROTOSOL_VERSION_TAG
+    toml_data["dependencies"]["protosol"] = protosol_dep
+
     # add required solfuzz-agave added configurations
     solfuzz_agave_config = parse_toml_file("solfuzz_agave.toml")
     for section, values in solfuzz_agave_config.items():
@@ -141,4 +166,4 @@ def main():
         f.write(toml_data.as_string())
 
 if __name__ == "__main__":
-    main()
+    main()

scripts/fetch_proto.sh

@@ -20,9 +20,6 @@ mkdir -p dump
 # Show the commit hashes being used
 repo_commit=$(git rev-parse HEAD)
 echo "Using repo commit: $repo_commit"
-protosol_commit=$(cd protosol && git rev-parse HEAD)
-protosol_tag=$(cd protosol && git describe --tags --exact-match 2>/dev/null || echo "<no tag>")
-echo "Using protosol commit: $protosol_commit (tag: $protosol_tag)"
 
 # Fetch/update test-vectors repo
 if [ ! -d dump/test-vectors ]; then