feat: TipRouter CLI separate stages (#87)

### Change notes
* New top-level environment variable/CLI arg
* SAVE_PATH: renaming of `META_MERKLE_TREE_DIR`. Path where all
generated operator files are stored
* Separates each stage of the process into more coherent functions. 
* `run` CLI command loops through each stage, saving the outputs to
_save-path_ CLI argument. The output from each stage is held in memory
to reduce the processing.
        * New Environment Variables/CLI Args for `run`
* STARTING_STAGE: Ability to start the operator from a specific stage
before it continues
* one of: `load-bank-from-snapshot, create-stake-meta,
create-merkle-tree-collection, create-meta-merkle-tree, cast-vote,
wait-for-next-epoch`
* SAVE_STAGES: true/false - whether to save the state of intermediate
stages in files. Needed for CLAIM_TIPS and SET_MERKLE_ROOT
* SAVE_SNAPSHOTS: true/false - whether to save the bank loaded in the
target as a full snapshot (~100GB).
* Create CLI commands for each stage
* _snapshot-slot_ - Create and store a full snapshot for a given slot
* _create-stake-meta_ - For a given epoch and slot, load the bank and
create the `StakeMetaCollection` file which contains validator vote
accounts and their staking delegations.
* _create-merkle-tree-collection_ - For a given epoch, load the
`StakeMetaCollection` from disk at _save-path_, generate and save the
`GeneratedMerkleTreeCollection`.
* _create-meta-merkle-tree_ - For a given epoch, load the
`GeneratedMerkleTreeCollection` from disk at _save-path_, generate and
save the`MetaMerkleTree`.
* _submit-epoch_ - For a given epoch, load the MetaMerkle file and cast
a vote to the NCN's ballot box. Optionally set the merkle root on the
Tip Distribution account if the NCN has consensus for the epoch.
* _claim-tips_ - For a given epoch, generate and execute Tip
Distribution Claim instructions for any unclaimed tips.
* TipRouter saved files
    * generated files are now saved under _save-path_
* Files _<= current_epoch - num_monitored_epochs_ are purged for
anything
* NOTE: file names have change to a _{epoch}_file_name.json_ format. All
previous files will not be removed. Operators should manually delete old
files.

---------

Co-authored-by: Evan Batsell <[email protected]>

Author: tomjohn1028

.github/workflows/ci.yaml

@@ -200,7 +200,7 @@ jobs:
           name: jito_tip_router_program.so
           path: integration_tests/tests/fixtures/
       - uses: taiki-e/install-action@nextest
-      - run: cargo nextest run --all-features -E 'not test(ledger_utils::tests::test_get_bank_from_ledger_success) and not test(test_meta_merkle_creation_from_ledger)'
+      - run: cargo nextest run --all-features -E 'not test(ledger_utils::tests)'
         env:
           SBF_OUT_DIR: ${{ github.workspace }}/integration_tests/tests/fixtures
 

Cargo.lock

@@ -14,7 +14,7 @@ cargo fmt --all
 
 print_executing "cargo nextest run --all-features"
 cargo build-sbf --sbf-out-dir integration_tests/tests/fixtures
-SBF_OUT_DIR=integration_tests/tests/fixtures cargo nextest run --all-features -E 'not test(ledger_utils::tests::test_get_bank_from_ledger_success) and not test(test_meta_merkle_creation_from_ledger)'
+SBF_OUT_DIR=integration_tests/tests/fixtures cargo nextest run --all-features -E 'not test(ledger_utils::tests::test_get_bank_from_ledger_success)'
 
 # Code coverage only runs with flag
 if [[ "$*" == *"--code-coverage"* ]]; then

format.sh

@@ -1,4 +1,8 @@
-use std::{fs::File, io::BufReader, path::PathBuf};
+use std::{
+    fs::File,
+    io::{BufReader, Write},
+    path::PathBuf,
+};
 
 use jito_tip_distribution_sdk::{
     jito_tip_distribution::ID as TIP_DISTRIBUTION_ID, CLAIM_STATUS_SEED,
@@ -119,6 +123,14 @@ impl GeneratedMerkleTreeCollection {
 
         Ok(tree)
     }
+
+    /// Write a GeneratedMerkleTreeCollection to a filepath
+    pub fn write_to_file(&self, path: &PathBuf) -> Result<(), MerkleRootGeneratorError> {
+        let serialized = serde_json::to_string_pretty(&self)?;
+        let mut file = File::create(path)?;
+        file.write_all(serialized.as_bytes())?;
+        Ok(())
+    }
 }
 
 #[derive(Clone, Eq, Debug, Hash, PartialEq, Deserialize, Serialize)]
@@ -339,6 +351,24 @@ pub struct StakeMetaCollection {
     pub slot: Slot,
 }
 
+impl StakeMetaCollection {
+    /// Load a serialized merkle tree from file path
+    pub fn new_from_file(path: &PathBuf) -> Result<Self, MerkleRootGeneratorError> {
+        let file = File::open(path)?;
+        let reader = BufReader::new(file);
+        let tree: Self = serde_json::from_reader(reader)?;
+
+        Ok(tree)
+    }
+
+    /// Write a merkle tree to a filepath
+    pub fn write_to_file(&self, path: &PathBuf) {
+        let serialized = serde_json::to_string_pretty(&self).unwrap();
+        let mut file = File::create(path).unwrap();
+        file.write_all(serialized.as_bytes()).unwrap();
+    }
+}
+
 #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)]
 pub struct StakeMeta {
     #[serde(with = "pubkey_string_conversion")]

meta_merkle_tree/src/generated_merkle_tree.rs

@@ -90,10 +90,11 @@ impl MetaMerkleTree {
     }
 
     /// Write a merkle tree to a filepath
-    pub fn write_to_file(&self, path: &PathBuf) {
-        let serialized = serde_json::to_string_pretty(&self).unwrap();
-        let mut file = File::create(path).unwrap();
-        file.write_all(serialized.as_bytes()).unwrap();
+    pub fn write_to_file(&self, path: &PathBuf) -> Result<()> {
+        let serialized = serde_json::to_string_pretty(&self)?;
+        let mut file = File::create(path)?;
+        file.write_all(serialized.as_bytes())?;
+        Ok(())
     }
 
     pub fn get_node(&self, tip_distribution_account: &Pubkey) -> TreeNode {
@@ -241,7 +242,7 @@ mod tests {
         let path = PathBuf::from("merkle_tree.json");
 
         // serialize merkle distributor to file
-        merkle_distributor_info.write_to_file(&path);
+        merkle_distributor_info.write_to_file(&path).unwrap();
         // now test we can successfully read from file
         let merkle_distributor_read: MetaMerkleTree = MetaMerkleTree::new_from_file(&path).unwrap();
 

meta_merkle_tree/src/meta_merkle_tree.rs

@@ -17,7 +17,6 @@ hex = "0.4"
 im = "15.1"
 itertools = "0.11"
 jito-bytemuck = { workspace = true }
-jito-restaking-program = { workspace = true }
 jito-tip-distribution-sdk = { workspace = true }
 jito-tip-payment-sdk = { workspace = true }
 jito-tip-router-client = { workspace = true }