diff --git a/Makefile b/Makefile index 2f15ccd714..42007d6197 100644 --- a/Makefile +++ b/Makefile @@ -183,7 +183,7 @@ nimbus-build-system-nimble-dir: .PHONY: librln LIBRLN_BUILDDIR := $(CURDIR)/vendor/zerokit -LIBRLN_VERSION := v0.9.0 +LIBRLN_VERSION := master ifeq ($(detected_OS),Windows) LIBRLN_FILE := rln.lib diff --git a/RLN_FFI_MIGRATION_PLAN.md b/RLN_FFI_MIGRATION_PLAN.md new file mode 100644 index 0000000000..ba4da2b993 --- /dev/null +++ b/RLN_FFI_MIGRATION_PLAN.md @@ -0,0 +1,799 @@ +# RLN FFI Migration Plan: v0.8.0 → master (v0.9.0+) + +## Overview +This document outlines the complete migration plan for upgrading nwaku's RLN (Rate Limiting Nullifier) FFI from zerokit v0.8.0 to the master branch (post v0.9.0). The migration focuses on transitioning to a **stateless RLN** implementation that relies on smart contract-based Merkle trees instead of local tree management. + +**Current Status:** +- **Current Version:** v0.8.0 (submodule at `a4bb3feb` which is v0.9.0 tag) +- **Target:** master branch (latest stateless features) +- **Migration Type:** Stateless (smart contract-based Merkle tree) +- **Build Feature:** `stateless,parallel` (no local tree) + +--- + +## Key Changes Summary + +### 1. **Keygen Functions - Now Independent from RLN Instance** +- **Old:** Required RLN instance to generate keys +- **New:** Standalone functions with endianness flag +- **Impact:** Functions can be called without creating RLN instance first + +### 2. **Circuit Format - arkzkey Only** +- **Old:** Multiple formats with flags +- **New:** arkzkey is the only format (no flag needed) +- **Impact:** Simplified initialization + +### 3. **Build Configuration - Stateless + Parallel** +- **Old:** `cargo build -p rln --release` +- **New:** `cargo build -p rln --release --no-default-features --features stateless,parallel` +- **Impact:** Must use `--no-default-features` to avoid default tree features + +### 4. **Binary Assets - New Naming Convention** +- **Old:** `*-rln.tar.gz` +- **New:** `*-stateless-parallel-rln.*` +- **Impact:** Download URLs and filenames need updating + +### 5. **Proof Functions - Stateless Variants Only** +- **Old:** Multiple proof generation/verification functions +- **New:** Only `generate_rln_proof_with_witness` and `verify_with_roots` for stateless +- **Impact:** Must provide Merkle path and roots from smart contract + +### 6. **Tree Functions - Not Available in Stateless** +- **Old:** Local tree operations (set_leaf, get_leaf, etc.) +- **New:** Not available in stateless mode +- **Impact:** All tree data comes from smart contract + +### 7. **Function Naming - New Prefix Convention** +- **Old:** `new`, `new_with_params` +- **New:** `ffi_rln_new`, `ffi_rln_new_with_params` +- **Impact:** All FFI function imports need updating + +### 8. **Return Types - Enhanced Error Handling** +- **Old:** Simple bool returns +- **New:** Result types: `CResultVecCFrVecU8`, `CResultVecU8VecU8`, `CBoolResult` +- **Impact:** Better error handling, memory leak fixes + +--- + +## Migration Phases + +### **Phase 1: Preparation & Analysis** ⏱️ 2-3 days + +#### 1.1 Understand Current Implementation +- [x] Review current `rln_interface.nim` (169 lines) +- [x] Review current `wrappers.nim` (194 lines) +- [x] Review `build_rln.sh` script +- [x] Document all current FFI function usages +- [ ] Map current functions to new FFI equivalents +- [ ] Identify functions that won't be available in stateless mode + +#### 1.2 Study New FFI API +- [ ] Read zerokit master branch FFI documentation +- [ ] Review Nim example in zerokit repo +- [ ] Understand new return types and error handling +- [ ] Document memory management changes +- [ ] Understand endianness handling (use little-endian for now) + +#### 1.3 Create Function Mapping Document +Create a detailed mapping of: +``` +OLD FUNCTION → NEW FUNCTION → CHANGES REQUIRED +``` + +**Example Mappings:** + +| Old Function | New Function | Status | Changes | +|--------------|--------------|--------|---------| +| `key_gen` | `extended_key_gen` | ✅ Available | Add endianness param (use `true` for little-endian) | +| `seeded_key_gen` | `seeded_extended_key_gen` | ✅ Available | Add endianness param | +| `new_circuit(tree_depth, buffer, ctx)` | `ffi_rln_new(ctx)` | ⚠️ Changed | No tree_depth or buffer needed in stateless | +| `new_circuit_from_data(zkey, vk, ctx)` | `ffi_rln_new_with_params(zkey, graph, ctx)` | ⚠️ Changed | vk_buffer removed, circom_buffer → graph_buffer | +| `generate_proof` | N/A | ❌ Not for stateless | Use `generate_rln_proof_with_witness` | +| `verify` | N/A | ❌ Not for stateless | Use `verify_with_roots` | +| `generate_proof_with_witness` | `generate_rln_proof_with_witness` | ✅ Available | Same name, check signature | +| `verify_with_roots` | `verify_with_roots` | ✅ Available | Now returns `CBoolResult` | +| `set_leaf`, `get_leaf`, etc. | N/A | ❌ Removed | Not available in stateless mode | + +--- + +### **Phase 2: Build System Updates** ⏱️ 1 day + +#### 2.1 Update Makefile +**File:** `/Users/darshan/work/nwaku/Makefile` + +**Current (line 186):** +```makefile +LIBRLN_VERSION := v0.9.0 +``` + +**Action:** Update to master branch or specific commit +```makefile +LIBRLN_VERSION := master # or specific commit hash +``` + +#### 2.2 Update Build Script +**File:** `/Users/darshan/work/nwaku/scripts/build_rln.sh` + +**Current (line 52):** +```bash +cargo build --release -p rln --manifest-path "${build_dir}/rln/Cargo.toml" +``` + +**Required Change:** +```bash +cargo build -p rln --release --no-default-features --features stateless,parallel --manifest-path "${build_dir}/rln/Cargo.toml" +``` + +**Critical:** The `--no-default-features` flag is essential to avoid including local tree features. + +#### 2.3 Update Binary Download Logic +**File:** `/Users/darshan/work/nwaku/scripts/build_rln.sh` + +**Current (line 22):** +```bash +tarball+="-rln.tar.gz" +``` + +**Required Change:** +```bash +tarball+="-stateless-parallel-rln.tar.gz" +``` + +#### 2.4 Update Submodule +```bash +cd vendor/zerokit +git checkout master +git pull origin master +cd ../.. +git add vendor/zerokit +git commit -m "chore: update zerokit to master for new stateless FFI" +``` + +--- + +### **Phase 3: FFI Interface Updates** ⏱️ 3-4 days + +#### 3.1 Update `rln_interface.nim` +**File:** `/Users/darshan/work/nwaku/waku/waku_rln_relay/rln/rln_interface.nim` + +**Changes Required:** + +##### A. Update Key Generation Functions (Lines 25-44) +```nim +# OLD +proc key_gen*( + output_buffer: ptr Buffer, is_little_endian: bool +): bool {.importc: "extended_key_gen".} + +proc seeded_key_gen*( + input_buffer: ptr Buffer, output_buffer: ptr Buffer, is_little_endian: bool +): bool {.importc: "seeded_extended_key_gen".} +``` + +**Status:** ✅ These are already correct! Just verify they work with new FFI. + +##### B. Update RLN Instance Creation (Lines 126-146) + +**OLD - Stateful Version:** +```nim +proc new_circuit*( + tree_depth: uint, input_buffer: ptr Buffer, ctx: ptr (ptr RLN) +): bool {.importc: "new".} +``` + +**NEW - Stateless Version:** +```nim +proc new_circuit*(ctx: ptr (ptr RLN)): bool {.importc: "ffi_rln_new".} +``` + +**OLD - With Parameters:** +```nim +proc new_circuit_from_data*( + zkey_buffer: ptr Buffer, graph_buffer: ptr Buffer, ctx: ptr (ptr RLN) +): bool {.importc: "new_with_params".} +``` + +**NEW - With Parameters (Stateless):** +```nim +proc new_circuit_from_data*( + zkey_buffer: ptr Buffer, graph_buffer: ptr Buffer, ctx: ptr (ptr RLN) +): bool {.importc: "ffi_rln_new_with_params".} +``` + +**Key Changes:** +- Function name: `new` → `ffi_rln_new` +- Function name: `new_with_params` → `ffi_rln_new_with_params` +- Removed: `tree_depth` parameter (not needed in stateless) +- Removed: `vk_buffer` parameter (zerokit extracts it from zkey) +- Changed: `circom_buffer` → `graph_buffer` (graph.bin file) + +##### C. Update Proof Generation (Lines 46-72) + +**Remove or Mark as Unavailable:** +```nim +# This function is NOT available in stateless mode +# proc generate_proof*( +# ctx: ptr RLN, input_buffer: ptr Buffer, output_buffer: ptr Buffer +# ): bool {.importc: "generate_rln_proof".} +``` + +**Keep Only Stateless Version:** +```nim +proc generate_proof_with_witness*( + ctx: ptr RLN, input_buffer: ptr Buffer, output_buffer: ptr Buffer +): bool {.importc: "generate_rln_proof_with_witness".} +``` + +**Input Buffer Format (RLN-v2):** +``` +[ identity_secret<32> | user_message_limit<32> | message_id<32> | + path_elements> | identity_path_index> | + x<32> | external_nullifier<32> ] +``` + +**Output Buffer Format:** +``` +[ proof<128> | root<32> | external_nullifier<32> | + share_x<32> | share_y<32> | nullifier<32> ] +``` + +##### D. Update Proof Verification (Lines 74-98) + +**Remove or Mark as Unavailable:** +```nim +# This function is NOT available in stateless mode +# proc verify*( +# ctx: ptr RLN, proof_buffer: ptr Buffer, proof_is_valid_ptr: ptr bool +# ): bool {.importc: "verify_rln_proof".} +``` + +**Update Return Type for Stateless Version:** +```nim +# OLD +proc verify_with_roots*( + ctx: ptr RLN, + proof_buffer: ptr Buffer, + roots_buffer: ptr Buffer, + proof_is_valid_ptr: ptr bool, +): bool {.importc: "verify_with_roots".} + +# NEW - Returns CBoolResult instead of bool +# Need to define CBoolResult type first +type CBoolResult* = object + value*: bool + error*: cstring + +proc verify_with_roots*( + ctx: ptr RLN, + proof_buffer: ptr Buffer, + roots_buffer: ptr Buffer, +): CBoolResult {.importc: "ffi_verify_with_roots".} +``` + +**Input Format:** +``` +proof_buffer: [ proof<128> | root<32> | external_nullifier<32> | + share_x<32> | share_y<32> | nullifier<32> | + signal_len<8> | signal ] +roots_buffer: [ root1<32> | root2<32> | ... | rootN<32> ] +``` + +##### E. Remove Tree Functions (Lines 176-191) + +**These are NOT available in stateless mode:** +```nim +# REMOVE THESE - Not available in stateless mode +# - set_leaf +# - get_leaf +# - delete_leaf +# - set_tree +# - leaves_set +# - get_root +# - set_metadata +# - get_metadata +# - flush +``` + +##### F. Update Hash Functions (Lines 148-168) + +**Verify these still work (should be unchanged):** +```nim +proc sha256*( + input_buffer: ptr Buffer, output_buffer: ptr Buffer, is_little_endian: bool +): bool {.importc: "hash".} + +proc poseidon*( + input_buffer: ptr Buffer, output_buffer: ptr Buffer, is_little_endian: bool +): bool {.importc: "poseidon_hash".} +``` + +#### 3.2 Add New Result Types + +**Add to `rln_interface.nim` after Buffer definition:** + +```nim +# New result types for better error handling +type CResultVecCFrVecU8* = object + value*: ptr UncheckedArray[byte] + len*: uint + error*: cstring + +type CResultVecU8VecU8* = object + value*: ptr UncheckedArray[byte] + len*: uint + error*: cstring + +type CBoolResult* = object + value*: bool + error*: cstring + +# Helper functions for working with vecFr +proc bytes_le_to_vec_cfr*( + input_buffer: ptr Buffer +): CResultVecCFrVecU8 {.importc: "ffi_bytes_le_to_vec_cfr".} + +proc bytes_be_to_vec_cfr*( + input_buffer: ptr Buffer +): CResultVecCFrVecU8 {.importc: "ffi_bytes_be_to_vec_cfr".} + +proc bytes_le_to_vec_u8*( + input_buffer: ptr Buffer +): CResultVecU8VecU8 {.importc: "ffi_bytes_le_to_vec_u8".} + +proc bytes_be_to_vec_u8*( + input_buffer: ptr Buffer +): CResultVecU8VecU8 {.importc: "ffi_bytes_be_to_vec_u8".} +``` + +--- + +### **Phase 4: Wrapper Updates** ⏱️ 3-4 days + +#### 4.1 Update `wrappers.nim` +**File:** `/Users/darshan/work/nwaku/waku/waku_rln_relay/rln/wrappers.nim` + +##### A. Update `membershipKeyGen` (Lines 17-59) +**Status:** ✅ Should work as-is, but verify endianness flag is correct + +**Current:** +```nim +var done = key_gen(keysBufferPtr, true) # true = little endian +``` + +**Verify:** The `true` parameter means little-endian, which is correct per Ekaterina's recommendation. + +##### B. Update `createRLNInstanceLocal` (Lines 83-112) + +**OLD - Stateful:** +```nim +proc createRLNInstanceLocal(): RLNResult = + let rln_config = RlnConfig( + resources_folder: "tree_height_/", + tree_config: RlnTreeConfig(...) + ) + + var + rlnInstance: ptr RLN + merkleDepth: csize_t = uint(20) + configBuffer = serialized_rln_config.toOpenArrayByte(...).toBuffer() + + let res = new_circuit(merkleDepth, addr configBuffer, addr rlnInstance) + ... +``` + +**NEW - Stateless:** +```nim +proc createRLNInstanceLocal(): RLNResult = + # No config needed for stateless mode + var rlnInstance: ptr RLN + + # Simple initialization without tree config + let res = new_circuit(addr rlnInstance) + + if (res == false): + info "error in RLN instance creation" + return err("error in RLN instance creation") + return ok(rlnInstance) +``` + +**Key Changes:** +- Removed: `RlnConfig` and `RlnTreeConfig` (not needed) +- Removed: `merkleDepth` parameter +- Removed: `configBuffer` parameter +- Simplified: Direct call to `new_circuit` with only context pointer + +**Alternative - With Parameters (if using embedded circuit data):** +```nim +proc createRLNInstanceWithParams*( + zkeyData: seq[byte], graphData: seq[byte] +): RLNResult = + var + rlnInstance: ptr RLN + zkeyBuffer = zkeyData.toBuffer() + graphBuffer = graphData.toBuffer() + + let res = new_circuit_from_data( + addr zkeyBuffer, + addr graphBuffer, + addr rlnInstance + ) + + if (res == false): + info "error in RLN instance creation with params" + return err("error in RLN instance creation with params") + return ok(rlnInstance) +``` + +##### C. Update Hash Functions (Lines 122-155) + +**Status:** ✅ Should work as-is, already using endianness flag correctly + +--- + +### **Phase 5: Usage Pattern Updates** ⏱️ 4-5 days + +#### 5.1 Identify All RLN Usage Points + +**Search for:** +```bash +grep -r "generate_proof\|verify\|new_circuit" waku/ +``` + +**Common locations:** +- `waku/waku_rln_relay/protocol.nim` +- `waku/waku_rln_relay/group_manager/` +- Test files in `tests/waku_rln_relay/` + +#### 5.2 Update Proof Generation Calls + +**OLD Pattern:** +```nim +# This won't work in stateless mode +let proofGenSuccess = generate_proof( + rlnInstance, + addr inputBuffer, + addr outputBuffer +) +``` + +**NEW Pattern - Must Provide Merkle Path:** +```nim +# Must construct witness with Merkle path from smart contract +let witness = constructWitness( + identitySecret, + userMessageLimit, + messageId, + merklePathFromContract, # Get from smart contract + identityPathIndex, # Get from smart contract + x, + externalNullifier +) + +let proofGenSuccess = generate_proof_with_witness( + rlnInstance, + addr witnessBuffer, + addr outputBuffer +) +``` + +**Key Requirement:** You MUST fetch the Merkle path and indices from the smart contract before generating proofs. + +#### 5.3 Update Verification Calls + +**OLD Pattern:** +```nim +var proofIsValid: bool +let verifySuccess = verify( + rlnInstance, + addr proofBuffer, + addr proofIsValid +) +``` + +**NEW Pattern - Must Provide Roots:** +```nim +# Get acceptable roots from smart contract +let rootsFromContract = fetchAcceptableRoots() # Your implementation +let rootsBuffer = rootsFromContract.toBuffer() + +let result = verify_with_roots( + rlnInstance, + addr proofBuffer, + addr rootsBuffer +) + +# Check result +if result.error != nil: + # Handle error + error "Verification failed", error = $result.error +else: + let proofIsValid = result.value + # Use proofIsValid +``` + +**Key Changes:** +- Return type is now `CBoolResult` instead of bool +- Must provide roots buffer with acceptable roots from smart contract +- Better error handling with error messages + +--- + +### **Phase 6: Smart Contract Integration** ⏱️ 3-4 days + +#### 6.1 Merkle Path Retrieval + +**Required:** Implement functions to fetch Merkle paths from smart contract + +```nim +proc getMerklePathForMember*( + membershipIndex: uint +): RlnRelayResult[MerklePath] = + # Call smart contract to get: + # - path_elements: Vec<32 bytes> + # - path_indices: Vec<1 byte> + # Return as MerklePath object + ... +``` + +#### 6.2 Root Management + +**Required:** Implement root validation against smart contract + +```nim +proc getAcceptableRoots*(): RlnRelayResult[seq[MerkleNode]] = + # Fetch current and recent roots from smart contract + # Typically: current root + last N roots for flexibility + ... + +proc isRootValid*(root: MerkleNode): bool = + # Check if root exists in smart contract + ... +``` + +#### 6.3 Update Group Manager + +**Files to update:** +- `waku/waku_rln_relay/group_manager/on_chain/group_manager_onchain.nim` +- `waku/waku_rln_relay/group_manager/group_manager_base.nim` + +**Changes:** +- Remove local tree management +- Add Merkle path fetching from contract +- Add root validation logic +- Update member registration flow + +--- + +### **Phase 7: Testing Strategy** ⏱️ 5-6 days + +#### 7.1 Unit Tests + +**Create new test file:** `tests/waku_rln_relay/test_rln_stateless.nim` + +**Test cases:** +1. ✅ Key generation (with and without seed) +2. ✅ RLN instance creation (simple and with params) +3. ✅ Hash functions (sha256, poseidon) +4. ✅ Proof generation with witness +5. ✅ Proof verification with roots +6. ✅ Error handling for invalid inputs +7. ✅ Memory management (no leaks) + +#### 7.2 Integration Tests + +**Update existing tests:** +- `tests/waku_rln_relay/test_rln_relay.nim` +- `tests/waku_rln_relay/test_waku_rln_relay.nim` + +**Focus areas:** +1. End-to-end proof generation and verification +2. Smart contract interaction +3. Multiple roots handling +4. Concurrent operations + +#### 7.3 Performance Tests + +**Benchmark:** +- Proof generation time (with witness) +- Verification time (with multiple roots) +- Memory usage comparison +- Smart contract call overhead + +#### 7.4 Compatibility Tests + +**Verify:** +- Cross-platform builds (Linux, macOS, Windows) +- Android builds (if applicable) +- Binary downloads work correctly + +--- + +### **Phase 8: Documentation & Cleanup** ⏱️ 2-3 days + +#### 8.1 Update Code Documentation + +**Files to update:** +- `docs/operators/how-to/configure-rln.md` +- `README.md` (if RLN mentioned) +- Inline code comments + +**Key points to document:** +- Stateless mode requirements +- Smart contract dependency +- Merkle path handling +- Root validation + +#### 8.2 Update Build Documentation + +**Document:** +- New build flags +- Binary naming convention +- Troubleshooting common issues + +#### 8.3 Create Migration Guide + +**For users:** +- What changed +- How to update their setup +- Breaking changes +- Migration checklist + +#### 8.4 Cleanup + +**Remove:** +- Unused tree management code +- Old stateful function references +- Deprecated configuration options + +--- + +## Critical Considerations + +### 🔴 Breaking Changes + +1. **No Local Tree:** All tree data must come from smart contract +2. **Merkle Path Required:** Cannot generate proofs without path from contract +3. **Roots Required:** Cannot verify without roots from contract +4. **Function Signatures Changed:** Many functions have different parameters +5. **Return Types Changed:** Better error handling but requires code updates + +### ⚠️ Performance Implications + +1. **Network Dependency:** Every proof operation requires smart contract data +2. **Latency:** Added latency from contract calls +3. **Caching Strategy:** Need to cache Merkle paths and roots efficiently +4. **Memory:** Stateless uses more RAM during initialization (per Ekaterina) + +### 🔒 Security Considerations + +1. **Root Validation:** Must validate roots against smart contract +2. **Path Verification:** Must verify Merkle paths are correct +3. **Replay Protection:** Ensure nullifiers are checked properly +4. **Contract Trust:** Fully dependent on smart contract integrity + +### 🐛 Common Pitfalls + +1. **Endianness:** Use little-endian (`true`) for all operations +2. **Buffer Management:** Proper memory cleanup to avoid leaks +3. **Error Handling:** Check `CBoolResult.error` for detailed errors +4. **Build Flags:** Must use `--no-default-features` in cargo build +5. **Binary Names:** Must use correct binary name for downloads + +--- + +## Rollback Plan + +### If Migration Fails + +1. **Revert Submodule:** + ```bash + cd vendor/zerokit + git checkout a4bb3feb # v0.9.0 tag + cd ../.. + git add vendor/zerokit + ``` + +2. **Revert Build Scripts:** + - Restore old `build_rln.sh` + - Restore old Makefile + +3. **Revert Code Changes:** + ```bash + git checkout HEAD -- waku/waku_rln_relay/rln/ + ``` + +4. **Keep Changes in Branch:** + - Don't force push + - Keep migration branch for future retry + +--- + +## Timeline Estimate + +| Phase | Duration | Dependencies | +|-------|----------|--------------| +| Phase 1: Preparation | 2-3 days | None | +| Phase 2: Build System | 1 day | Phase 1 | +| Phase 3: FFI Interface | 3-4 days | Phase 1, 2 | +| Phase 4: Wrappers | 3-4 days | Phase 3 | +| Phase 5: Usage Patterns | 4-5 days | Phase 4 | +| Phase 6: Smart Contract | 3-4 days | Phase 5 | +| Phase 7: Testing | 5-6 days | Phase 6 | +| Phase 8: Documentation | 2-3 days | Phase 7 | +| **Total** | **23-34 days** | **~5-7 weeks** | + +--- + +## Success Criteria + +### ✅ Migration Complete When: + +1. All tests pass with new FFI +2. Proof generation works with smart contract paths +3. Proof verification works with smart contract roots +4. No memory leaks detected +5. Performance is acceptable +6. Documentation is updated +7. CI/CD pipeline passes +8. Code review approved + +--- + +## Questions for Clarification + +### Before Starting: + +1. ❓ Which smart contract are we using for the Merkle tree? +2. ❓ What's the expected latency for smart contract calls? +3. ❓ How many roots should we keep for verification? +4. ❓ Do we need to support both stateful and stateless modes? +5. ❓ What's the caching strategy for Merkle paths? +6. ❓ Are there any backward compatibility requirements? +7. ❓ What's the testing environment for smart contract integration? + +--- + +## Resources & References + +### Zerokit Documentation +- **FFI Source:** `vendor/zerokit/rln/src/ffi.rs` +- **Nim Example:** Check zerokit repo for Nim examples +- **Release Notes:** https://github.com/vacp2p/zerokit/releases/tag/v0.9.0 + +### nwaku Current Implementation +- **RLN Interface:** `waku/waku_rln_relay/rln/rln_interface.nim` +- **Wrappers:** `waku/waku_rln_relay/rln/wrappers.nim` +- **Build Script:** `scripts/build_rln.sh` +- **Tests:** `tests/waku_rln_relay/` + +### Conversation References +- Ekaterina's guidance on stateless features +- Discussion about memory usage +- Clarification on Merkle proof verification + +--- + +## Next Steps + +1. **Review this plan** with the team +2. **Get approval** for timeline and approach +3. **Set up development environment** with master branch +4. **Create feature branch** for migration +5. **Start Phase 1** - Preparation & Analysis + +--- + +## Notes + +- This is a **major migration** requiring careful testing +- **Stateless mode** is fundamentally different from stateful +- **Smart contract integration** is critical for success +- **Performance testing** is essential before production +- **Rollback plan** must be ready before deployment + +--- + +**Document Version:** 1.0 +**Created:** 2024-12-01 +**Author:** Migration Planning +**Status:** Ready for Review diff --git a/RLN_FFI_QUICK_REFERENCE.md b/RLN_FFI_QUICK_REFERENCE.md new file mode 100644 index 0000000000..355c583f37 --- /dev/null +++ b/RLN_FFI_QUICK_REFERENCE.md @@ -0,0 +1,476 @@ +# RLN FFI Quick Reference Guide + +## 🚀 Quick Start + +### Current Status +- **From:** v0.8.0 (stateful with local tree) +- **To:** master (stateless with smart contract tree) +- **Mode:** Stateless + Parallel + +--- + +## 📋 Critical Changes Checklist + +### Build Changes +```bash +# OLD +cargo build --release -p rln + +# NEW +cargo build -p rln --release --no-default-features --features stateless,parallel +``` + +### Binary Names +```bash +# OLD +x86_64-unknown-linux-gnu-rln.tar.gz + +# NEW +x86_64-unknown-linux-gnu-stateless-parallel-rln.tar.gz +``` + +--- + +## 🔄 Function Migration Map + +### Instance Creation + +```nim +# OLD - Stateful +proc new_circuit*( + tree_depth: uint, + input_buffer: ptr Buffer, + ctx: ptr (ptr RLN) +): bool {.importc: "new".} + +# NEW - Stateless +proc new_circuit*( + ctx: ptr (ptr RLN) +): bool {.importc: "ffi_rln_new".} +``` + +**Changes:** +- ❌ Remove `tree_depth` parameter +- ❌ Remove `input_buffer` parameter +- ✅ Change import name: `"new"` → `"ffi_rln_new"` + +--- + +### Instance Creation with Parameters + +```nim +# OLD +proc new_circuit_from_data*( + zkey_buffer: ptr Buffer, + vk_buffer: ptr Buffer, # ❌ REMOVED + circom_buffer: ptr Buffer, # ❌ RENAMED + ctx: ptr (ptr RLN) +): bool {.importc: "new_with_params".} + +# NEW +proc new_circuit_from_data*( + zkey_buffer: ptr Buffer, + graph_buffer: ptr Buffer, # ✅ NEW NAME + ctx: ptr (ptr RLN) +): bool {.importc: "ffi_rln_new_with_params".} +``` + +**Changes:** +- ❌ Remove `vk_buffer` (extracted from zkey automatically) +- ✅ Rename `circom_buffer` → `graph_buffer` (graph.bin file) +- ✅ Change import: `"new_with_params"` → `"ffi_rln_new_with_params"` + +--- + +### Key Generation + +```nim +# UNCHANGED - Already correct! +proc key_gen*( + output_buffer: ptr Buffer, + is_little_endian: bool # Use true for little-endian +): bool {.importc: "extended_key_gen".} + +proc seeded_key_gen*( + input_buffer: ptr Buffer, + output_buffer: ptr Buffer, + is_little_endian: bool # Use true for little-endian +): bool {.importc: "seeded_extended_key_gen".} +``` + +**Status:** ✅ No changes needed, already using new names + +--- + +### Proof Generation + +```nim +# OLD - NOT AVAILABLE IN STATELESS +proc generate_proof*( + ctx: ptr RLN, + input_buffer: ptr Buffer, + output_buffer: ptr Buffer +): bool {.importc: "generate_rln_proof".} + +# NEW - ONLY THIS ONE FOR STATELESS +proc generate_proof_with_witness*( + ctx: ptr RLN, + input_buffer: ptr Buffer, + output_buffer: ptr Buffer +): bool {.importc: "generate_rln_proof_with_witness".} +``` + +**Input Format (RLN-v2):** +``` +[ identity_secret<32> +| user_message_limit<32> +| message_id<32> +| path_elements> ⬅️ FROM SMART CONTRACT +| identity_path_index> ⬅️ FROM SMART CONTRACT +| x<32> +| external_nullifier<32> ] +``` + +**Key Requirement:** Must fetch `path_elements` and `identity_path_index` from smart contract! + +--- + +### Proof Verification + +```nim +# OLD - NOT AVAILABLE IN STATELESS +proc verify*( + ctx: ptr RLN, + proof_buffer: ptr Buffer, + proof_is_valid_ptr: ptr bool +): bool {.importc: "verify_rln_proof".} + +# NEW - ONLY THIS ONE FOR STATELESS +type CBoolResult* = object + value*: bool + error*: cstring + +proc verify_with_roots*( + ctx: ptr RLN, + proof_buffer: ptr Buffer, + roots_buffer: ptr Buffer # ⬅️ FROM SMART CONTRACT +): CBoolResult {.importc: "ffi_verify_with_roots".} +``` + +**Changes:** +- ✅ Return type: `bool` → `CBoolResult` +- ✅ Must provide `roots_buffer` from smart contract +- ✅ Better error handling via `result.error` + +**Usage:** +```nim +let result = verify_with_roots(ctx, addr proofBuffer, addr rootsBuffer) +if result.error != nil: + error "Verification failed", msg = $result.error +else: + let isValid = result.value +``` + +--- + +### Hash Functions + +```nim +# UNCHANGED +proc sha256*( + input_buffer: ptr Buffer, + output_buffer: ptr Buffer, + is_little_endian: bool # Use true +): bool {.importc: "hash".} + +proc poseidon*( + input_buffer: ptr Buffer, + output_buffer: ptr Buffer, + is_little_endian: bool # Use true +): bool {.importc: "poseidon_hash".} +``` + +**Status:** ✅ No changes needed + +--- + +## ❌ Functions NOT Available in Stateless + +These functions are **removed** in stateless mode: + +```nim +# Tree operations - ALL REMOVED +- set_leaf +- get_leaf +- delete_leaf +- set_tree +- leaves_set +- get_root + +# Metadata operations - ALL REMOVED +- set_metadata +- get_metadata +- flush + +# Proof operations - REPLACED +- generate_proof (use generate_proof_with_witness) +- verify (use verify_with_roots) +``` + +--- + +## 🆕 New Result Types + +Add these to `rln_interface.nim`: + +```nim +type CResultVecCFrVecU8* = object + value*: ptr UncheckedArray[byte] + len*: uint + error*: cstring + +type CResultVecU8VecU8* = object + value*: ptr UncheckedArray[byte] + len*: uint + error*: cstring + +type CBoolResult* = object + value*: bool + error*: cstring +``` + +--- + +## 🔧 Wrapper Updates + +### Old RLN Instance Creation + +```nim +# OLD - wrappers.nim +proc createRLNInstanceLocal(): RLNResult = + let rln_config = RlnConfig( + resources_folder: "tree_height_/", + tree_config: RlnTreeConfig( + cache_capacity: 15_000, + mode: "high_throughput", + compression: false, + flush_every_ms: 500, + ), + ) + + var serialized_rln_config = $(%rln_config) + var + rlnInstance: ptr RLN + merkleDepth: csize_t = uint(20) + configBuffer = serialized_rln_config.toOpenArrayByte(...).toBuffer() + + let res = new_circuit(merkleDepth, addr configBuffer, addr rlnInstance) + if (res == false): + return err("error in parameters generation") + return ok(rlnInstance) +``` + +### New RLN Instance Creation + +```nim +# NEW - wrappers.nim +proc createRLNInstanceLocal(): RLNResult = + # No config needed for stateless! + var rlnInstance: ptr RLN + + let res = new_circuit(addr rlnInstance) + if (res == false): + return err("error in RLN instance creation") + return ok(rlnInstance) +``` + +**Changes:** +- ❌ Remove `RlnConfig` and `RlnTreeConfig` +- ❌ Remove `merkleDepth` +- ❌ Remove `configBuffer` +- ✅ Simple one-line call + +--- + +## 🌐 Smart Contract Integration + +### Required Functions + +You **must** implement these to work with stateless RLN: + +```nim +# 1. Get Merkle path for a member +proc getMerklePathForMember*( + memberIndex: uint +): RlnRelayResult[MerklePath] = + # Call smart contract + # Return path_elements and path_indices + ... + +# 2. Get acceptable roots +proc getAcceptableRoots*(): RlnRelayResult[seq[MerkleNode]] = + # Get current root + recent roots from contract + # Typically current + last 5-10 roots + ... + +# 3. Validate root +proc isRootValid*(root: MerkleNode): bool = + # Check if root exists in contract + ... +``` + +--- + +## 📊 Proof Generation Flow + +### Old Flow (Stateful) +``` +1. Get identity credentials +2. Call generate_proof() + ↳ RLN uses local tree +3. Done! +``` + +### New Flow (Stateless) +``` +1. Get identity credentials +2. Fetch Merkle path from smart contract ⬅️ NEW! +3. Construct witness with path +4. Call generate_proof_with_witness() +5. Done! +``` + +--- + +## 📊 Verification Flow + +### Old Flow (Stateful) +``` +1. Receive proof +2. Call verify() + ↳ RLN checks against local tree +3. Done! +``` + +### New Flow (Stateless) +``` +1. Receive proof +2. Fetch acceptable roots from smart contract ⬅️ NEW! +3. Call verify_with_roots() + ↳ Checks if proof root matches any acceptable root +4. Check result.error for errors +5. Done! +``` + +--- + +## ⚠️ Common Mistakes + +### 1. Wrong Build Flags +```bash +# ❌ WRONG - Will include default tree features +cargo build -p rln --release --features stateless,parallel + +# ✅ CORRECT - Must use --no-default-features +cargo build -p rln --release --no-default-features --features stateless,parallel +``` + +### 2. Wrong Endianness +```nim +# ❌ WRONG +key_gen(keysBufferPtr, false) # big-endian not fully supported + +# ✅ CORRECT +key_gen(keysBufferPtr, true) # little-endian recommended +``` + +### 3. Missing Merkle Path +```nim +# ❌ WRONG - Can't generate proof without path +let input = serializeIdentityOnly(identity) +generate_proof_with_witness(ctx, addr input, addr output) + +# ✅ CORRECT - Must include path from contract +let merklePath = getMerklePathForMember(memberIndex) +let input = serializeWithWitness(identity, merklePath) +generate_proof_with_witness(ctx, addr input, addr output) +``` + +### 4. Not Checking Errors +```nim +# ❌ WRONG - Ignoring error information +let result = verify_with_roots(ctx, addr proof, addr roots) +let isValid = result.value # Might be false due to error! + +# ✅ CORRECT - Check error first +let result = verify_with_roots(ctx, addr proof, addr roots) +if result.error != nil: + error "Verification error", msg = $result.error + return err($result.error) +let isValid = result.value +``` + +### 5. Using Removed Functions +```nim +# ❌ WRONG - These don't exist in stateless +set_leaf(ctx, index, addr leafBuffer) +get_root(ctx, addr rootBuffer) + +# ✅ CORRECT - Get data from smart contract +let root = fetchRootFromContract() +``` + +--- + +## 🎯 Testing Checklist + +Before considering migration complete: + +- [ ] Key generation works (with and without seed) +- [ ] RLN instance creation works +- [ ] Can fetch Merkle paths from smart contract +- [ ] Proof generation works with witness +- [ ] Can fetch roots from smart contract +- [ ] Proof verification works with roots +- [ ] Error handling works correctly +- [ ] No memory leaks detected +- [ ] Performance is acceptable +- [ ] All existing tests pass +- [ ] Cross-platform builds work +- [ ] Binary downloads work + +--- + +## 🔗 Key Files to Modify + +1. **`scripts/build_rln.sh`** - Build flags and binary names +2. **`Makefile`** - Version and build targets +3. **`waku/waku_rln_relay/rln/rln_interface.nim`** - FFI declarations +4. **`waku/waku_rln_relay/rln/wrappers.nim`** - High-level wrappers +5. **`waku/waku_rln_relay/protocol.nim`** - Usage patterns +6. **`waku/waku_rln_relay/group_manager/`** - Smart contract integration + +--- + +## 📚 References + +- **Zerokit FFI:** `vendor/zerokit/rln/src/ffi.rs` +- **Full Migration Plan:** `RLN_FFI_MIGRATION_PLAN.md` +- **Zerokit Releases:** https://github.com/vacp2p/zerokit/releases + +--- + +## 💡 Pro Tips + +1. **Start Simple:** Get basic instance creation working first +2. **Test Incrementally:** Don't change everything at once +3. **Use Little-Endian:** Recommended by Ekaterina +4. **Cache Smart Contract Data:** Reduce latency +5. **Check Errors:** New result types provide better error info +6. **Keep Rollback Ready:** Don't delete old code immediately + +--- + +**Last Updated:** 2024-12-01 +**For:** nwaku RLN FFI Migration +**Version:** 1.0 diff --git a/RLN_MIGRATION_README.md b/RLN_MIGRATION_README.md new file mode 100644 index 0000000000..035d1a7a90 --- /dev/null +++ b/RLN_MIGRATION_README.md @@ -0,0 +1,452 @@ +# RLN FFI Migration Documentation + +## 📚 Document Overview + +This directory contains comprehensive documentation for migrating nwaku's RLN (Rate Limiting Nullifier) FFI from zerokit v0.8.0 to the master branch with stateless features. + +--- + +## 📄 Available Documents + +### 1. **RLN_FFI_MIGRATION_PLAN.md** (Main Document) +**Purpose:** Complete migration plan with all 8 phases +**Use when:** Planning the entire migration, understanding scope +**Length:** ~1000 lines, comprehensive +**Key sections:** +- Overview and key changes +- 8 detailed phases with timelines +- Critical considerations +- Rollback plan +- Success criteria + +**Start here if:** You need the full picture and detailed timeline. + +--- + +### 2. **RLN_FFI_QUICK_REFERENCE.md** (Cheat Sheet) +**Purpose:** Quick lookup for common changes +**Use when:** Coding and need quick answers +**Length:** ~500 lines, focused +**Key sections:** +- Function migration map (old → new) +- Common mistakes and fixes +- Code snippets for each change +- Testing checklist + +**Start here if:** You know what you're doing and need quick reference. + +--- + +### 3. **RLN_PHASE1_IMPLEMENTATION.md** (Getting Started) +**Purpose:** Step-by-step guide for Phase 1 +**Use when:** Starting the migration +**Length:** ~600 lines, actionable +**Key sections:** +- Day-by-day breakdown +- Specific tasks and commands +- Self-assessment questions +- Deliverables checklist + +**Start here if:** You're ready to begin and want a structured approach. + +--- + +## 🚀 Getting Started + +### For First-Time Readers + +**Recommended Reading Order:** + +1. **Start:** Read this README (you are here!) +2. **Understand:** Read "Key Changes Summary" below +3. **Plan:** Skim `RLN_FFI_MIGRATION_PLAN.md` (focus on Phases 1-3) +4. **Learn:** Read `RLN_FFI_QUICK_REFERENCE.md` completely +5. **Execute:** Follow `RLN_PHASE1_IMPLEMENTATION.md` day by day + +**Time Investment:** +- Initial reading: 2-3 hours +- Phase 1 execution: 2-3 days +- Total migration: 5-7 weeks + +--- + +## 🔑 Key Changes Summary + +### What's Changing? + +#### 1. **Mode: Stateful → Stateless** +- **Before:** Local Merkle tree in RLN instance +- **After:** Merkle tree in smart contract only +- **Impact:** Must fetch tree data from contract + +#### 2. **Build: Default → Stateless+Parallel** +- **Before:** `cargo build -p rln --release` +- **After:** `cargo build -p rln --release --no-default-features --features stateless,parallel` +- **Impact:** Different binary, different features + +#### 3. **Functions: Many → Fewer** +- **Before:** ~20 FFI functions +- **After:** ~12 FFI functions (tree functions removed) +- **Impact:** Some code won't work anymore + +#### 4. **API: Simple → Enhanced** +- **Before:** Functions return `bool` +- **After:** Functions return `CBoolResult` with error info +- **Impact:** Better error handling, code changes needed + +--- + +## ⚠️ Critical Breaking Changes + +### 🔴 High Impact + +1. **No Local Tree Operations** + - Removed: `set_leaf`, `get_leaf`, `delete_leaf`, `get_root`, etc. + - Reason: Stateless mode has no local tree + - Solution: Get all tree data from smart contract + +2. **Proof Generation Requires Merkle Path** + - Old: `generate_proof(ctx, input, output)` + - New: `generate_proof_with_witness(ctx, input_with_path, output)` + - Reason: Need path to recalculate root + - Solution: Fetch path from smart contract before generating proof + +3. **Verification Requires Roots** + - Old: `verify(ctx, proof, &isValid)` + - New: `verify_with_roots(ctx, proof, roots) -> CBoolResult` + - Reason: Need to check against acceptable roots + - Solution: Fetch roots from smart contract before verifying + +### ⚠️ Medium Impact + +4. **Instance Creation Simplified** + - Old: `new_circuit(tree_depth, config_buffer, ctx)` + - New: `new_circuit(ctx)` + - Reason: No tree configuration needed + - Solution: Remove tree config code + +5. **Function Names Changed** + - Old: `new`, `new_with_params` + - New: `ffi_rln_new`, `ffi_rln_new_with_params` + - Reason: Better naming convention + - Solution: Update all import declarations + +6. **Binary Names Changed** + - Old: `*-rln.tar.gz` + - New: `*-stateless-parallel-rln.tar.gz` + - Reason: Different build features + - Solution: Update download scripts + +--- + +## 📋 Migration Phases Overview + +### Phase 1: Preparation (2-3 days) 📖 +- Understand current implementation +- Study new FFI +- Create function mapping +- **Deliverable:** Complete understanding and plan + +### Phase 2: Build System (1 day) 🔧 +- Update Makefile +- Update build scripts +- Update submodule +- **Deliverable:** New FFI builds successfully + +### Phase 3: FFI Interface (3-4 days) 💻 +- Update `rln_interface.nim` +- Add new result types +- Remove unavailable functions +- **Deliverable:** Interface matches new FFI + +### Phase 4: Wrappers (3-4 days) 🎁 +- Update `wrappers.nim` +- Simplify instance creation +- Update error handling +- **Deliverable:** High-level API works + +### Phase 5: Usage Patterns (4-5 days) 🔄 +- Update proof generation +- Update verification +- Update all call sites +- **Deliverable:** All code uses new patterns + +### Phase 6: Smart Contract (3-4 days) 🌐 +- Add Merkle path fetching +- Add root management +- Update group manager +- **Deliverable:** Smart contract integration complete + +### Phase 7: Testing (5-6 days) ✅ +- Unit tests +- Integration tests +- Performance tests +- **Deliverable:** All tests pass + +### Phase 8: Documentation (2-3 days) 📝 +- Update docs +- Create migration guide +- Cleanup old code +- **Deliverable:** Ready for production + +**Total Timeline:** 23-34 days (5-7 weeks) + +--- + +## 🎯 Quick Decision Tree + +**"Where should I start?"** + +``` +Are you just exploring? +├─ Yes → Read RLN_FFI_MIGRATION_PLAN.md (Overview section) +└─ No ↓ + +Do you understand the changes? +├─ No → Read RLN_FFI_QUICK_REFERENCE.md (full document) +└─ Yes ↓ + +Ready to start coding? +├─ No → Follow RLN_PHASE1_IMPLEMENTATION.md +└─ Yes ↓ + +Need quick lookup while coding? +└─ Use RLN_FFI_QUICK_REFERENCE.md (as reference) +``` + +--- + +## 🔍 Finding Information + +### "How do I...?" + +**...understand what changed?** +→ `RLN_FFI_QUICK_REFERENCE.md` - Function Migration Map + +**...know what to do first?** +→ `RLN_PHASE1_IMPLEMENTATION.md` - Day 1 tasks + +**...update a specific function?** +→ `RLN_FFI_QUICK_REFERENCE.md` - Search for function name + +**...understand the timeline?** +→ `RLN_FFI_MIGRATION_PLAN.md` - Timeline Estimate section + +**...know what's risky?** +→ `RLN_FFI_MIGRATION_PLAN.md` - Critical Considerations + +**...test my changes?** +→ `RLN_FFI_QUICK_REFERENCE.md` - Testing Checklist + +**...roll back if needed?** +→ `RLN_FFI_MIGRATION_PLAN.md` - Rollback Plan + +--- + +## 💡 Pro Tips + +### Before You Start +1. ✅ Read all three documents (at least skim) +2. ✅ Understand stateless vs stateful +3. ✅ Set up test environment +4. ✅ Create feature branch +5. ✅ Have rollback plan ready + +### While Working +1. 📝 Keep notes of issues encountered +2. ✅ Test after each phase +3. 💬 Ask questions early +4. 🔄 Commit frequently +5. 📊 Track progress + +### Best Practices +1. **Don't rush Phase 1** - Understanding saves time later +2. **Test incrementally** - Don't change everything at once +3. **Use little-endian** - Recommended by zerokit team +4. **Check errors** - New result types provide better info +5. **Keep old code** - Don't delete until migration complete + +--- + +## 🆘 Common Questions + +### Q: Do we need to support both stateful and stateless? +**A:** No, we're fully migrating to stateless (smart contract-based). + +### Q: Can I use the old `generate_proof` function? +**A:** No, it's not available in stateless mode. Use `generate_proof_with_witness`. + +### Q: Why does stateless use more RAM? +**A:** During initialization, it loads more data. This is a known trade-off. + +### Q: How many roots should we keep for verification? +**A:** Typically current root + last 5-10 roots. Discuss with team. + +### Q: What if smart contract calls are too slow? +**A:** Implement caching layer for Merkle paths and roots. + +### Q: Can I skip Phase 1? +**A:** Not recommended. Phase 1 builds understanding needed for later phases. + +--- + +## 📞 Getting Help + +### If You're Stuck + +1. **Check Quick Reference** - Most common issues covered +2. **Review Phase 1 Guide** - Might have missed something +3. **Search Zerokit Issues** - Others might have same problem +4. **Ask Team** - Don't struggle alone +5. **Check Ekaterina's Messages** - Included in main plan + +### Useful Commands + +```bash +# Search for function usage +grep -r "function_name" waku/ --include="*.nim" + +# Find all RLN-related files +find waku/waku_rln_relay -name "*.nim" -type f + +# Check current zerokit version +git submodule status vendor/zerokit + +# Test build +make clean-librln && make librln + +# Run specific test +make test tests/waku_rln_relay/test_file.nim +``` + +--- + +## ✅ Success Criteria + +### Migration Complete When: + +- [ ] All 8 phases completed +- [ ] All tests pass +- [ ] No memory leaks +- [ ] Performance acceptable +- [ ] Documentation updated +- [ ] Code reviewed +- [ ] CI/CD passes +- [ ] Team approval + +--- + +## 📊 Progress Tracking + +### Suggested Format + +Create a file `MIGRATION_PROGRESS.md`: + +```markdown +# RLN FFI Migration Progress + +## Phase 1: Preparation ⏳ +- [x] Task 1.1: Map current FFI functions +- [x] Task 1.2: Identify usage patterns +- [ ] Task 1.3: Understand build process +- [ ] Task 1.4: Study instance creation +... + +## Blockers +- None yet + +## Questions +1. Which smart contract to use? +2. Performance requirements? + +## Notes +- Stateless uses more RAM during init +- Must use little-endian +``` + +--- + +## 🔗 External Resources + +### Zerokit +- **Repository:** https://github.com/vacp2p/zerokit +- **FFI Source:** `vendor/zerokit/rln/src/ffi.rs` +- **Releases:** https://github.com/vacp2p/zerokit/releases +- **v0.9.0 Release:** https://github.com/vacp2p/zerokit/releases/tag/v0.9.0 + +### nwaku +- **RLN Interface:** `waku/waku_rln_relay/rln/rln_interface.nim` +- **Wrappers:** `waku/waku_rln_relay/rln/wrappers.nim` +- **Tests:** `tests/waku_rln_relay/` +- **Build Script:** `scripts/build_rln.sh` + +--- + +## 📝 Document Maintenance + +### Keeping Docs Updated + +As you progress: +1. Update progress tracking +2. Add new findings to notes +3. Document issues and solutions +4. Update timelines if needed +5. Add new questions + +### After Completion + +1. Archive these docs (don't delete) +2. Create final migration report +3. Update main project docs +4. Share learnings with team + +--- + +## 🎓 Learning Outcomes + +After completing this migration, you'll understand: + +- ✅ FFI (Foreign Function Interface) in Nim +- ✅ Zero-knowledge proofs (RLN, Groth16) +- ✅ Merkle trees and proofs +- ✅ Stateful vs stateless architectures +- ✅ Smart contract integration +- ✅ Rust-Nim interop +- ✅ Build system configuration +- ✅ Large-scale refactoring + +--- + +## 🙏 Acknowledgments + +- **Ekaterina** - For detailed explanations and guidance +- **Zerokit Team** - For the new FFI implementation +- **nwaku Team** - For existing codebase + +--- + +## 📅 Document History + +- **2024-12-01:** Initial creation +- **Version:** 1.0 +- **Status:** Ready for use + +--- + +## 🚀 Ready to Start? + +1. ✅ Read this README completely +2. ✅ Skim all three documents +3. ✅ Create feature branch: `git checkout -b zerokit_ffi_upgrade` +4. ✅ Open `RLN_PHASE1_IMPLEMENTATION.md` +5. ✅ Start Day 1, Task 1.1 + +**Good luck with the migration! 🎉** + +--- + +**Questions?** Review the "Getting Help" section above. +**Stuck?** Check the "Common Questions" section. +**Need quick info?** Use the "Finding Information" section. + +**Remember:** Take your time with Phase 1. Understanding now = confidence later! 💪 diff --git a/RLN_PHASE1_IMPLEMENTATION.md b/RLN_PHASE1_IMPLEMENTATION.md new file mode 100644 index 0000000000..cb35e513f7 --- /dev/null +++ b/RLN_PHASE1_IMPLEMENTATION.md @@ -0,0 +1,445 @@ +# Phase 1 Implementation Guide: Preparation & Analysis + +## Overview +This guide walks you through Phase 1 of the RLN FFI migration. Complete this phase before making any code changes. This will give you confidence to answer reviewer questions. + +**Duration:** 2-3 days +**Goal:** Understand current implementation and new FFI completely + +--- + +## Day 1: Current Implementation Analysis + +### Morning Session (3-4 hours) + +#### Task 1.1: Map Current FFI Functions +Create a spreadsheet or document with all current FFI functions. + +**Action:** +```bash +cd /Users/darshan/work/nwaku +grep -n "importc:" waku/waku_rln_relay/rln/rln_interface.nim > current_ffi_functions.txt +``` + +**Expected Output:** List of ~15-20 function declarations + +**Document for each function:** +1. Function name +2. Parameters +3. Return type +4. Where it's used (search in codebase) +5. Purpose/what it does + +**Template:** +``` +Function: key_gen +Import Name: "extended_key_gen" +Parameters: output_buffer (ptr Buffer), is_little_endian (bool) +Returns: bool +Used In: wrappers.nim (line 25) +Purpose: Generate identity credentials +Status: ✅ Already compatible with new FFI +``` + +#### Task 1.2: Identify Usage Patterns + +**Search for RLN usage:** +```bash +# Find all files using RLN functions +grep -r "generate_proof\|verify\|new_circuit\|key_gen" waku/ --include="*.nim" | cut -d: -f1 | sort -u + +# Count usages +grep -r "generate_proof" waku/ --include="*.nim" | wc -l +grep -r "verify" waku/ --include="*.nim" | wc -l +grep -r "new_circuit" waku/ --include="*.nim" | wc -l +``` + +**Create a usage map:** +``` +File: waku/waku_rln_relay/protocol.nim +- Uses: generate_proof (line X) +- Uses: verify (line Y) +- Context: Main protocol implementation +- Complexity: High (core functionality) +``` + +### Afternoon Session (3-4 hours) + +#### Task 1.3: Understand Current Build Process + +**Study build script:** +```bash +cat scripts/build_rln.sh +``` + +**Document:** +1. What cargo command is used? (line 52) +2. What features are enabled? +3. How are binaries downloaded? +4. What's the fallback if download fails? + +**Test current build:** +```bash +# Try building current version +make clean-librln +make librln + +# Check what was built +ls -lh librln_*.a +file librln_*.a +``` + +**Document:** +- Build time +- Output file size +- Any warnings/errors + +#### Task 1.4: Study Current RLN Instance Creation + +**Read carefully:** +```bash +# Study the wrapper +cat waku/waku_rln_relay/rln/wrappers.nim | grep -A 30 "proc createRLNInstanceLocal" +``` + +**Understand:** +1. What is `RlnConfig`? +2. What is `RlnTreeConfig`? +3. Why is `tree_height_/` used? +4. What does each config parameter do? +5. Where is this function called from? + +**Trace the call chain:** +```bash +grep -r "createRLNInstance" waku/ --include="*.nim" +``` + +--- + +## Day 2: New FFI Study + +### Morning Session (3-4 hours) + +#### Task 2.1: Read New FFI Source Code + +**Open and study:** +```bash +cd vendor/zerokit +git checkout master +git pull origin master + +# Read the FFI +less rln/src/ffi.rs +``` + +**Focus on these sections:** +1. Lines 1-100: Macros and helper functions +2. Lines 200-350: RLN instance creation +3. Lines 450-550: Proof generation/verification +4. Lines 590-650: Key generation functions + +**For each function, note:** +- Function signature +- Parameters (especially new ones) +- Return type (especially `CBoolResult`) +- `#[cfg(feature = "stateless")]` annotations +- Comments explaining behavior + +#### Task 2.2: Compare Old vs New + +**Create comparison table:** + +| Function | Old Signature | New Signature | Changes | Breaking? | +|----------|---------------|---------------|---------|-----------| +| new | `new(depth, buffer, ctx)` | `ffi_rln_new(ctx)` | Removed params | ✅ Yes | +| ... | ... | ... | ... | ... | + +**Identify:** +- ✅ Functions that are the same +- ⚠️ Functions with minor changes +- ❌ Functions that are removed +- 🆕 Functions that are new + +### Afternoon Session (3-4 hours) + +#### Task 2.3: Study Stateless Architecture + +**Read Ekaterina's explanation again:** + +Key concepts to understand: +1. **Two types of proofs:** + - Merkle proof (tree-based) + - ZK proof (Groth16) + +2. **Stateless verification:** + - Recalculate root from Merkle path + - Compare with root from smart contract + - Verify ZK proof + +3. **Why it's slower:** + - Must recalculate root each time + - No local tree cache + - Network calls to smart contract + +**Draw a diagram:** +``` +Stateful Flow: +User → RLN (with local tree) → Proof + ↓ + Local tree lookup + +Stateless Flow: +User → Smart Contract (get path) → RLN → Proof + ↓ + Recalculate root +``` + +#### Task 2.4: Understand Memory Implications + +**From Ekaterina:** +> "new_circuit stateless function uses much more RAM compared to the stateful function" + +**Research:** +1. Why does stateless use more RAM? +2. How much more? (test if possible) +3. Is this during initialization only? +4. Can we optimize? + +**Test (if possible):** +```bash +# Build both versions and compare +cargo build -p rln --release --features stateful +ls -lh target/release/librln.a + +cargo build -p rln --release --no-default-features --features stateless,parallel +ls -lh target/release/librln.a +``` + +--- + +## Day 3: Integration Planning + +### Morning Session (3-4 hours) + +#### Task 3.1: Smart Contract Interface Study + +**Find current smart contract code:** +```bash +grep -r "contract\|ethereum\|web3" waku/waku_rln_relay/ --include="*.nim" +``` + +**Understand:** +1. Which smart contract is used? +2. How are members registered? +3. How is the tree stored? +4. What functions are available? + +**Document required additions:** +```nim +# Need to add: +proc getMerklePathForMember*(index: uint): MerklePath +proc getAcceptableRoots*(): seq[MerkleNode] +proc getCurrentRoot*(): MerkleNode +``` + +#### Task 3.2: Create Detailed Function Mapping + +**For each function in `rln_interface.nim`, create:** + +```markdown +### Function: generate_proof + +**Current Implementation:** +- Name: `generate_proof` +- Import: `"generate_rln_proof"` +- Parameters: ctx, input_buffer, output_buffer +- Returns: bool +- Used in: protocol.nim (line 123), group_manager.nim (line 456) + +**New Implementation:** +- Name: `generate_proof_with_witness` +- Import: `"generate_rln_proof_with_witness"` +- Parameters: ctx, input_buffer (with witness), output_buffer +- Returns: bool +- Changes needed: + 1. Input buffer must include Merkle path + 2. Must fetch path from smart contract first + 3. Serialization format changes + +**Migration Steps:** +1. Add getMerklePathForMember function +2. Update input buffer construction +3. Update all call sites +4. Add error handling +5. Add tests + +**Risk Level:** 🔴 High (core functionality) +**Estimated Effort:** 2 days +``` + +### Afternoon Session (3-4 hours) + +#### Task 3.3: Identify Risks and Challenges + +**Create risk matrix:** + +| Risk | Probability | Impact | Mitigation | +|------|-------------|--------|------------| +| Smart contract calls too slow | Medium | High | Add caching layer | +| Memory usage too high | Low | Medium | Profile and optimize | +| Breaking changes in tests | High | Medium | Update tests incrementally | +| Compatibility issues | Medium | High | Thorough testing | + +#### Task 3.4: Create Test Strategy + +**List all test files:** +```bash +find tests/waku_rln_relay -name "*.nim" -type f +``` + +**For each test file:** +1. What does it test? +2. Will it break with new FFI? +3. How to update it? +4. Priority (high/medium/low)? + +**Plan new tests:** +``` +New Test: test_stateless_proof_generation.nim +- Test proof generation with Merkle path +- Test with invalid path +- Test with multiple roots +- Test error handling +``` + +#### Task 3.5: Review with Team (if possible) + +**Prepare presentation:** +1. Current state summary +2. New FFI overview +3. Key changes +4. Migration approach +5. Risks and mitigations +6. Timeline + +**Questions to ask:** +- Is stateless-only acceptable? +- Performance requirements? +- Backward compatibility needs? +- Testing environment for smart contract? + +--- + +## Deliverables Checklist + +By end of Phase 1, you should have: + +### Documentation +- [ ] Complete function mapping (old → new) +- [ ] Usage pattern analysis +- [ ] Risk assessment +- [ ] Test strategy +- [ ] Smart contract integration plan + +### Understanding +- [ ] Can explain stateless vs stateful +- [ ] Understand all breaking changes +- [ ] Know which functions are removed +- [ ] Understand new return types +- [ ] Know build flag changes + +### Technical +- [ ] Built current version successfully +- [ ] Checked out new zerokit master +- [ ] Read new FFI source code +- [ ] Identified all usage points +- [ ] Created migration checklist + +### Confidence +- [ ] Can explain changes to reviewer +- [ ] Know what questions to ask +- [ ] Understand risks +- [ ] Have rollback plan +- [ ] Ready to start coding + +--- + +## Self-Assessment Questions + +Test your understanding: + +### Basic Understanding +1. What's the main difference between stateful and stateless RLN? +2. Why was `tree_depth` removed from `new_circuit`? +3. What does `--no-default-features` do in the build command? +4. What's the new binary naming convention? + +### Intermediate +5. Why can't we use `generate_proof` in stateless mode? +6. What's the difference between Merkle proof and ZK proof? +7. Why does `verify_with_roots` return `CBoolResult` instead of `bool`? +8. What data must come from the smart contract? + +### Advanced +9. How does stateless verification work without a local tree? +10. Why does stateless use more RAM during initialization? +11. What happens if the smart contract root changes during verification? +12. How do we handle multiple acceptable roots? + +**If you can answer all these confidently, you're ready for Phase 2!** + +--- + +## Phase 1 Completion Criteria + +✅ **Ready to proceed when:** +1. All deliverables completed +2. All self-assessment questions answered +3. Team review done (if applicable) +4. No blocking questions remaining +5. Confident in migration approach + +--- + +## Next Steps + +After completing Phase 1: +1. Review findings with team +2. Get approval to proceed +3. Create feature branch +4. Start Phase 2: Build System Updates + +--- + +## Resources + +### Reading Material +- Zerokit FFI: `vendor/zerokit/rln/src/ffi.rs` +- Current interface: `waku/waku_rln_relay/rln/rln_interface.nim` +- Ekaterina's messages (saved in main plan) + +### Tools +```bash +# Search codebase +grep -r "pattern" waku/ --include="*.nim" + +# Count occurrences +grep -r "pattern" waku/ --include="*.nim" | wc -l + +# Find files +find waku/ -name "*.nim" -type f + +# Check git history +git log --oneline -- waku/waku_rln_relay/rln/ +``` + +### Documentation +- Main migration plan: `RLN_FFI_MIGRATION_PLAN.md` +- Quick reference: `RLN_FFI_QUICK_REFERENCE.md` +- This guide: `RLN_PHASE1_IMPLEMENTATION.md` + +--- + +**Remember:** Phase 1 is about understanding, not coding. Take your time to understand everything thoroughly. This will save time in later phases and help you answer reviewer questions confidently. + +**Good luck! 🚀** diff --git a/scripts/build_rln.sh b/scripts/build_rln.sh index 5e1b0caa5c..dad9fd2558 100755 --- a/scripts/build_rln.sh +++ b/scripts/build_rln.sh @@ -19,7 +19,7 @@ host_triplet=$(rustc --version --verbose | awk '/host:/{print $2}') tarball="${host_triplet}" -tarball+="-rln.tar.gz" +tarball+="-stateless-parallel-rln.tar.gz" # Download the prebuilt rln library if it is available if curl --silent --fail-with-body -L \ @@ -49,6 +49,6 @@ else exit 1 fi # if submodule version = version in Makefile, build rln - cargo build --release -p rln --manifest-path "${build_dir}/rln/Cargo.toml" + cargo build -p rln --release --no-default-features --features stateless,parallel --manifest-path "${build_dir}/rln/Cargo.toml" cp "${build_dir}/target/release/librln.a" "${output_filename}" fi diff --git a/vendor/zerokit b/vendor/zerokit index a4bb3feb50..c74ab11c82 160000 --- a/vendor/zerokit +++ b/vendor/zerokit @@ -1 +1 @@ -Subproject commit a4bb3feb5054e6fd24827adf204493e6e173437b +Subproject commit c74ab11c82db10c765d1073087619c53294aab7f