Update to df03d866237f44c6d0f81d05783da493654f0f8c (#66)

Author: patrick-ogrady

Cargo.lock

@@ -33,14 +33,18 @@ axum = "0.8.1"
 base64 = "0.22.1"
 hex = "0.4"
 regex = "1"
-commonware-codec = { git = "https://github.com/commonwarexyz/monorepo", rev = "2cc941653cb936952c00587f8cb037c8021241a5", package = "commonware-codec" }
-commonware-consensus = { git = "https://github.com/commonwarexyz/monorepo", rev = "2cc941653cb936952c00587f8cb037c8021241a5", package = "commonware-consensus" }
-commonware-cryptography = { git = "https://github.com/commonwarexyz/monorepo", rev = "2cc941653cb936952c00587f8cb037c8021241a5", package = "commonware-cryptography" }
-commonware-math = { git = "https://github.com/commonwarexyz/monorepo", rev = "2cc941653cb936952c00587f8cb037c8021241a5", package = "commonware-math" }
-commonware-parallel = { git = "https://github.com/commonwarexyz/monorepo", rev = "2cc941653cb936952c00587f8cb037c8021241a5", package = "commonware-parallel" }
-commonware-runtime = { git = "https://github.com/commonwarexyz/monorepo", rev = "2cc941653cb936952c00587f8cb037c8021241a5", package = "commonware-runtime" }
-commonware-storage = { git = "https://github.com/commonwarexyz/monorepo", rev = "2cc941653cb936952c00587f8cb037c8021241a5", package = "commonware-storage" }
-commonware-utils = { git = "https://github.com/commonwarexyz/monorepo", rev = "2cc941653cb936952c00587f8cb037c8021241a5", package = "commonware-utils" }
+commonware-codec = { git = "https://github.com/commonwarexyz/monorepo", rev = "df03d866237f44c6d0f81d05783da493654f0f8c", package = "commonware-codec" }
+commonware-actor = { git = "https://github.com/commonwarexyz/monorepo", rev = "df03d866237f44c6d0f81d05783da493654f0f8c", package = "commonware-actor" }
+commonware-consensus = { git = "https://github.com/commonwarexyz/monorepo", rev = "df03d866237f44c6d0f81d05783da493654f0f8c", package = "commonware-consensus" }
+commonware-cryptography = { git = "https://github.com/commonwarexyz/monorepo", rev = "df03d866237f44c6d0f81d05783da493654f0f8c", package = "commonware-cryptography" }
+commonware-glue = { git = "https://github.com/commonwarexyz/monorepo", rev = "df03d866237f44c6d0f81d05783da493654f0f8c", package = "commonware-glue" }
+commonware-math = { git = "https://github.com/commonwarexyz/monorepo", rev = "df03d866237f44c6d0f81d05783da493654f0f8c", package = "commonware-math" }
+commonware-parallel = { git = "https://github.com/commonwarexyz/monorepo", rev = "df03d866237f44c6d0f81d05783da493654f0f8c", package = "commonware-parallel" }
+commonware-p2p = { git = "https://github.com/commonwarexyz/monorepo", rev = "df03d866237f44c6d0f81d05783da493654f0f8c", package = "commonware-p2p" }
+commonware-resolver = { git = "https://github.com/commonwarexyz/monorepo", rev = "df03d866237f44c6d0f81d05783da493654f0f8c", package = "commonware-resolver" }
+commonware-runtime = { git = "https://github.com/commonwarexyz/monorepo", rev = "df03d866237f44c6d0f81d05783da493654f0f8c", package = "commonware-runtime" }
+commonware-storage = { git = "https://github.com/commonwarexyz/monorepo", rev = "df03d866237f44c6d0f81d05783da493654f0f8c", package = "commonware-storage" }
+commonware-utils = { git = "https://github.com/commonwarexyz/monorepo", rev = "df03d866237f44c6d0f81d05783da493654f0f8c", package = "commonware-utils" }
 rocksdb = "0.22.0"
 rand = "0.8.5"
 bincode = "1.3.3"
@@ -55,10 +59,10 @@ portpicker = "0.1.1"
 http = "1.3.1"
 tower-http = { version = "0.5.2", features = ["cors"] }
 bytes = "1"
-connectrpc = { version = "0.3.1", features = ["client", "axum"] }
+connectrpc = { version = "0.6.0", features = ["client", "axum"] }
 hyper = { version = "1", features = ["full"] }
-buffa = { version = "0.3", features = ["json"] }
-buffa-types = { version = "0.3", features = ["json"] }
+buffa = { version = "0.6", features = ["json"] }
+buffa-types = { version = "0.6", features = ["json"] }
 http-body-util = "0.1"
 tower = { version = "0.5", features = ["util"] }
 anyhow = "1"

Cargo.toml

@@ -104,6 +104,8 @@ In addition to the simulator running above:
    - **Get Many** verifies current hit/miss lookup proofs against the same root and reports proof size.
    - **Get Range** verifies an ordered current range plus boundary proofs
      against the same root and reports proof size.
+   - **Get Historical Operation Range** verifies a contiguous historical
+     operation-log range against the same trusted root and reports proof size.
    - **Subscribe** verifies each emitted historical proof from the trusted
      current root for that emitted tip and reports proof size. Paste seed
      output lines into Trusted Roots when replaying or following multiple tips.

examples/sandbox/README.md

@@ -4,10 +4,12 @@ import {
   matchPrefix,
   matchRegex,
   OrderedQmdbClient,
+  type OrderedOperation,
   type OrderedSubscribeProof,
   type VerifiedCurrentKeyLookupProof,
   type VerifiedCurrentKeyRangeProof,
   type VerifiedCurrentKeyValueProof,
+  type VerifiedHistoricalMultiProof,
 } from '@qmdb-ts';
 
 export const QMDB_URL = import.meta.env.VITE_QMDB_URL as string | undefined;
@@ -56,21 +58,23 @@ function formatProofSize(bytes: number): string {
   return `${(bytes / 1024).toFixed(1)} KiB`;
 }
 
-function parseTip(value: string): bigint {
+function parseNonNegativeBigInt(value: string, label: string): bigint {
   const trimmed = value.trim();
   if (!trimmed) {
-    throw new Error('Tip is required');
+    throw new Error(`${label} is required`);
   }
-  const tip = BigInt(trimmed);
-  if (tip < 0n) {
-    throw new Error('Tip must be non-negative');
+  const parsed = BigInt(trimmed);
+  if (parsed < 0n) {
+    throw new Error(`${label} must be non-negative`);
   }
-  return tip;
+  return parsed;
+}
+
+function parseTip(value: string): bigint {
+  return parseNonNegativeBigInt(value, 'Tip');
 }
 
-function renderOperation(
-  proofOperation: OrderedSubscribeProof['proof']['operations'][number]['operation'],
-) {
+function renderOperation(proofOperation: OrderedOperation) {
   switch (proofOperation.type) {
     case 'update':
       return (
@@ -132,6 +136,11 @@ export function QmdbPanel({
   const [rangeProof, setRangeProof] = useState<VerifiedCurrentKeyRangeProof | null>(null);
   const [isGettingRange, setIsGettingRange] = useState(false);
 
+  const [historyStartLocation, setHistoryStartLocation] = useState('0');
+  const [historyMaxLocations, setHistoryMaxLocations] = useState('5');
+  const [historyProof, setHistoryProof] = useState<VerifiedHistoricalMultiProof | null>(null);
+  const [isGettingHistory, setIsGettingHistory] = useState(false);
+
   const [keyMatcherKind, setKeyMatcherKind] = useState<'exact' | 'prefix' | 'regex' | 'none'>(
     'none',
   );
@@ -245,6 +254,35 @@ export function QmdbPanel({
     }
   };
 
+  const handleGetOperationRange = async () => {
+    setIsGettingHistory(true);
+    setHistoryProof(null);
+    try {
+      const maxLocations = Number(historyMaxLocations);
+      if (!Number.isInteger(maxLocations) || maxLocations <= 0) {
+        throw new Error('Max Locations must be a positive integer');
+      }
+      const proof = await client.getOperationRange(
+        {
+          tip: parseTip(tip),
+          startLocation: parseNonNegativeBigInt(historyStartLocation, 'Start Location'),
+          maxLocations,
+        },
+        parseHexRoot(expectedCurrentRoot),
+      );
+      setHistoryProof(proof);
+      showNotification(
+        'success',
+        'QMDB Historical Range',
+        `Verified ${proof.operations.length} historical operations against expected root (${formatProofSize(proof.proofSizeBytes)})`,
+      );
+    } catch (error) {
+      showNotification('error', 'QMDB Historical Range Failed', String(error));
+    } finally {
+      setIsGettingHistory(false);
+    }
+  };
+
   function buildFilter(
     kind: 'exact' | 'prefix' | 'regex' | 'none',
     value: string,
@@ -310,9 +348,9 @@ export function QmdbPanel({
         <p className="section-note">
           Proofs are anchored to roots the writer emits per batch. Run `qmdb run`
           locally and `qmdb seed` to stream fresh tips; each line prints
-          `tip=N root=0x..`. Get Proof, Get Many, and Get Range verify
-          against that current root. Subscribe streams each proof with its tip
-          and included operations.
+          `tip=N root=0x..`. Get Proof, Get Many, Get Range, and historical
+          operation ranges verify against that current root. Subscribe streams
+          each proof with its tip and included operations.
         </p>
         <p><strong>Server:</strong> {qmdbUrl}</p>
         <p><strong>Merkle Family:</strong> MMB</p>
@@ -645,6 +683,67 @@ export function QmdbPanel({
           )}
         </div>
       </div>
+
+      <div className="form-section">
+        <h3>Get Historical Operation Range</h3>
+        <p className="section-note">
+          Fetches a contiguous historical operation proof for the operation log and verifies it
+          against the expected root for the selected tip.
+        </p>
+        <div className="form-row">
+          <div className="form-group">
+            <label htmlFor="qmdb-history-start">Start Location</label>
+            <input
+              id="qmdb-history-start"
+              type="number"
+              min="0"
+              value={historyStartLocation}
+              onChange={(event) => setHistoryStartLocation(event.target.value)}
+            />
+          </div>
+          <div className="form-group">
+            <label htmlFor="qmdb-history-max">Max Locations</label>
+            <input
+              id="qmdb-history-max"
+              type="number"
+              min="1"
+              value={historyMaxLocations}
+              onChange={(event) => setHistoryMaxLocations(event.target.value)}
+            />
+          </div>
+        </div>
+        <button
+          className={`btn-primary ${isGettingHistory ? 'loading' : ''}`}
+          onClick={handleGetOperationRange}
+          disabled={
+            isGettingHistory ||
+            !historyStartLocation.trim() ||
+            !historyMaxLocations.trim() ||
+            !tip.trim() ||
+            !expectedCurrentRoot.trim()
+          }
+        >
+          {isGettingHistory ? 'Verifying...' : 'Get Historical Range'}
+        </button>
+        {historyProof && (
+          <div className="result fade-in">
+            <h4>Verified Historical Operations</h4>
+            <p><strong>Proof Size:</strong> {formatProofSize(historyProof.proofSizeBytes)}</p>
+            <p><strong>Operations:</strong> {historyProof.operations.length}</p>
+            <div className="result-list">
+              {historyProof.operations.map((op, index) => (
+                <div
+                  key={`${op.location.toString()}-${index}`}
+                  className="result-row-block"
+                >
+                  <p><strong>Location:</strong> {op.location.toString()}</p>
+                  {renderOperation(op.operation)}
+                </div>
+              ))}
+            </div>
+          </div>
+        )}
+      </div>
     </div>
   );
 }

examples/sandbox/src/QmdbPanel.tsx

@@ -482,11 +482,11 @@ impl Ingest for RocksStore {
 impl Query for RocksStore {
     type RangeScan = RocksRangeScanCursor;
 
-    async fn get(&self, key: Bytes) -> Result<(Option<Vec<u8>>, QueryExtra), String> {
+    async fn get(&self, key: Bytes) -> Result<(Option<Bytes>, QueryExtra), String> {
         let store = self.clone();
         store
             .get_rocksdb(&key)
-            .map(|value| (value, QueryExtra::default()))
+            .map(|value| (value.map(Bytes::from), QueryExtra::default()))
             .map_err(|e| e.to_string())
     }
 
@@ -504,15 +504,15 @@ impl Query for RocksStore {
     async fn get_many(
         &self,
         keys: Vec<Bytes>,
-    ) -> Result<(Vec<(Vec<u8>, Option<Vec<u8>>)>, QueryExtra), String> {
+    ) -> Result<(Vec<(Bytes, Option<Bytes>)>, QueryExtra), String> {
         let store = self.clone();
         let results = store.db.multi_get(keys.iter().map(|key| key.as_ref()));
         let entries = keys
             .into_iter()
             .zip(results)
             .map(|(k, r)| {
                 let value = r.map_err(|e| e.to_string())?;
-                Ok((k.to_vec(), value))
+                Ok((k, value.map(Bytes::from)))
             })
             .collect::<Result<Vec<_>, String>>()?;
         Ok((entries, QueryExtra::default()))

examples/simulator/src/rocks.rs

@@ -272,7 +272,7 @@ async fn prune_drop_all_removes_keys() {
     assert!(client.query().get(&ka).await.expect("get a").is_some());
 
     let compact_config = ClientConfig::new(url.parse::<http::Uri>().unwrap())
-        .compression(connect_compression_registry());
+        .with_compression(connect_compression_registry());
     let compact_client =
         CompactServiceClient::new(PreferZstdHttpClient::plaintext(), compact_config);
     compact_client
@@ -439,7 +439,7 @@ async fn prune_keep_latest_retains_newest() {
         .expect("put");
 
     let compact_config = ClientConfig::new(url.parse::<http::Uri>().unwrap())
-        .compression(connect_compression_registry());
+        .with_compression(connect_compression_registry());
     let compact_client =
         CompactServiceClient::new(PreferZstdHttpClient::plaintext(), compact_config);
     compact_client
@@ -662,7 +662,7 @@ async fn prune_greater_than_retains_above_threshold() {
         .expect("put");
 
     let compact_config = ClientConfig::new(url.parse::<http::Uri>().unwrap())
-        .compression(connect_compression_registry());
+        .with_compression(connect_compression_registry());
     let compact_client =
         CompactServiceClient::new(PreferZstdHttpClient::plaintext(), compact_config);
     compact_client

examples/simulator/tests/e2e_connect.rs

@@ -41,7 +41,7 @@ fn put_batch(store: &RocksStore, kvs: Vec<(Bytes, Bytes)>) -> u64 {
     block_on(store.put_batch(kvs)).expect("put_batch")
 }
 
-fn get_value(store: &RocksStore, key: &Bytes) -> Option<Vec<u8>> {
+fn get_value(store: &RocksStore, key: &Bytes) -> Option<Bytes> {
     block_on(store.get(key.clone())).expect("get").0
 }
 

examples/simulator/tests/prune_contract.rs

@@ -59,13 +59,13 @@ fn scan(
     rows
 }
 
-fn get_value(store: &RocksStore, key: &[u8]) -> Option<Vec<u8>> {
+fn get_value(store: &RocksStore, key: &[u8]) -> Option<Bytes> {
     block_on(store.get(Bytes::copy_from_slice(key)))
         .expect("get")
         .0
 }
 
-fn get_many_values(store: &RocksStore, keys: &[&[u8]]) -> Vec<(Vec<u8>, Option<Vec<u8>>)> {
+fn get_many_values(store: &RocksStore, keys: &[&[u8]]) -> Vec<(Bytes, Option<Bytes>)> {
     let keys = keys.iter().map(|key| Bytes::copy_from_slice(key)).collect();
     block_on(store.get_many(keys)).expect("get_many").0
 }
@@ -242,7 +242,13 @@ fn get_many_returns_found_and_missing() {
 
     let results = get_many_values(&store, &[b"a", b"missing", b"c"]);
     assert_eq!(results.len(), 3);
-    assert_eq!(results[0], (b"a".to_vec(), Some(b"1".to_vec())));
-    assert_eq!(results[1], (b"missing".to_vec(), None));
-    assert_eq!(results[2], (b"c".to_vec(), Some(b"3".to_vec())));
+    assert_eq!(
+        results[0],
+        (Bytes::from_static(b"a"), Some(Bytes::from_static(b"1")))
+    );
+    assert_eq!(results[1], (Bytes::from_static(b"missing"), None));
+    assert_eq!(
+        results[2],
+        (Bytes::from_static(b"c"), Some(Bytes::from_static(b"3")))
+    );
 }

examples/simulator/tests/range_scan_contract.rs

@@ -19,7 +19,7 @@ service KeyLookupService {
 
 // Current key proof request.
 message GetRequest {
-  // Raw logical QMDB key (`K::as_ref()` bytes), not a store row key.
+  // Codec-encoded logical QMDB key (`K::encode()` bytes), not a store row key.
   bytes key = 1;
 
   // Published ordered-QMDB batch-boundary to prove against. The client must
@@ -29,7 +29,7 @@ message GetRequest {
 
 // Current key proof request for one or more logical keys.
 message GetManyRequest {
-  // Raw logical QMDB keys (`K::as_ref()` bytes), not store row keys.
+  // Codec-encoded logical QMDB keys (`K::encode()` bytes), not store row keys.
   repeated bytes keys = 1 [
     (buf.validate.field).repeated.min_items = 1,
     (buf.validate.field).repeated.max_items = 1024

proto/qmdb/v1/key_lookup.proto

@@ -12,17 +12,20 @@ service OrderedKeyRangeService {
   rpc GetRange(GetRangeRequest) returns (GetRangeResponse);
 }
 
-// Current ordered key range proof request. The range is half-open:
-// `start_key <= key < end_key` when `end_key` is set; otherwise it scans to the
-// end of the ordered keyspace. `limit` must be non-zero.
+// Current ordered key range proof request. Key fields are codec-encoded logical
+// QMDB keys (`K::encode()` bytes), not store row keys. The range is half-open:
+// `start_key <= key < end_key` after decoding `K` when `end_key` is set;
+// otherwise it scans to the end of the ordered keyspace. `limit` must be
+// non-zero.
 message GetRangeRequest {
   bytes start_key = 1;
   optional bytes end_key = 2;
   uint32 limit = 3 [(buf.validate.field).uint32.gt = 0];
   uint64 tip = 4;
 }
 
-// Ordered current key-range proof response. `start_proof`, when present,
+// Ordered current key-range proof response. Key fields are codec-encoded
+// logical QMDB keys (`K::encode()` bytes). `start_proof`, when present,
 // authenticates the boundary before the first returned key (or the entire empty
 // range). `end_proof`, when present, authenticates the exclusive end boundary.
 message GetRangeResponse {