[coding] Introduce commonware-coding (#1272)

Author: patrick-ogrady

.github/workflows/publish.yml

@@ -75,6 +75,11 @@ jobs:
       continue-on-error: true
       env:
         CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+    - name: Publish coding
+      run: cargo publish --manifest-path coding/Cargo.toml
+      continue-on-error: true
+      env:
+        CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
     - name: Publish deployer
       run: cargo publish --manifest-path deployer/Cargo.toml
       continue-on-error: true

.github/workflows/slow.yml

@@ -62,6 +62,10 @@ jobs:
             cargo_flags: ""
             file_suffix: ""
             benchmark_name: "commonware-stream"
+          - package: commonware-coding
+            cargo_flags: ""
+            file_suffix: ""
+            benchmark_name: "commonware-coding"
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4

Cargo.lock

@@ -2,6 +2,7 @@
 members = [
     "broadcast",
     "codec",
+    "coding",
     "collector",
     "consensus",
     "cryptography",

Cargo.toml

@@ -13,6 +13,7 @@ _Primitives are designed for deployment in adversarial environments. If you find
 
 * [broadcast](./broadcast/README.md): Disseminate data over a wide-area network.
 * [codec](./codec/README.md): Serialize structured data.
+* [coding](./coding/README.md): Encode data to enable recovery from a subset of fragments.
 * [collector](./collector/README.md): Collect responses to committable requests.
 * [consensus](./consensus/README.md): Order opaque messages in a Byzantine environment.
 * [cryptography](./cryptography/README.md): Generate keys, sign arbitrary messages, and deterministically verify signatures.

README.md

@@ -0,0 +1,31 @@
+[package]
+name = "commonware-coding"
+edition = "2021"
+publish = true
+version = "0.0.55"
+license = "MIT OR Apache-2.0"
+description = "Encode data to enable recovery from a subset of fragments."
+readme = "README.md"
+homepage = "https://commonware.xyz"
+repository = "https://github.com/commonwarexyz/monorepo/tree/main/coding"
+documentation = "https://docs.rs/commonware-coding"
+
+[dependencies]
+commonware-codec = { workspace = true }
+commonware-cryptography = { workspace = true }
+commonware-storage = { workspace = true }
+bytes = { workspace = true }
+thiserror = { workspace = true }
+reed-solomon-simd = "3.0.1"
+
+[lib]
+bench = false
+
+[dev-dependencies]
+criterion = { workspace = true }
+rand = { workspace = true }
+
+[[bench]]
+name = "reed_solomon"
+harness = false
+path = "src/reed_solomon/benches/bench.rs"

coding/Cargo.toml

@@ -0,0 +1,10 @@
+# commonware-coding
+
+[![Crates.io](https://img.shields.io/crates/v/commonware-coding.svg)](https://crates.io/crates/commonware-coding)
+[![Docs.rs](https://docs.rs/commonware-coding/badge.svg)](https://docs.rs/commonware-coding)
+
+Encode data to enable recovery from a subset of fragments.
+
+## Status
+
+`commonware-coding` is **ALPHA** software and is not yet recommended for production use. Developers should expect breaking changes and occasional instability.

coding/README.md

@@ -0,0 +1,8 @@
+//! Encode data to enable recovery from a subset of fragments.
+//!
+//! # Status
+//!
+//! `commonware-coding` is **ALPHA** software and is not yet recommended for production use. Developers should
+//! expect breaking changes and occasional instability.
+
+pub mod reed_solomon;

coding/src/lib.rs

@@ -0,0 +1,6 @@
+use criterion::criterion_main;
+
+mod decode;
+mod encode;
+
+criterion_main!(encode::benches, decode::benches);

coding/src/reed_solomon/benches/bench.rs

@@ -0,0 +1,54 @@
+use commonware_coding::reed_solomon::{decode, encode};
+use commonware_cryptography::Sha256;
+use criterion::{criterion_group, BatchSize, Criterion};
+use rand::{rngs::StdRng, seq::SliceRandom, RngCore, SeedableRng};
+
+fn benchmark_decode(c: &mut Criterion) {
+    let mut sampler = StdRng::seed_from_u64(0);
+    let cases = [8, 12, 16, 19, 20, 24].map(|i| 2usize.pow(i));
+    for data_length in cases.into_iter() {
+        for chunks in [10, 25, 50, 100, 250] {
+            let min = chunks / 3;
+            c.bench_function(
+                &format!(
+                    "{}/msg_len={} chunks={}",
+                    module_path!(),
+                    data_length,
+                    chunks
+                ),
+                |b| {
+                    b.iter_batched(
+                        || {
+                            // Generate random data
+                            let mut data = vec![0u8; data_length];
+                            sampler.fill_bytes(&mut data);
+
+                            // Encode data
+                            let (root, proofs) = encode::<Sha256>(chunks, min, data).unwrap();
+
+                            // Select min random chunks
+                            let mut shuffled = Vec::with_capacity(min as usize);
+                            let mut indices: Vec<u16> = (0..chunks).collect();
+                            indices.shuffle(&mut sampler);
+                            for &i in indices.iter().take(min as usize) {
+                                shuffled.push(proofs[i as usize].clone());
+                            }
+
+                            (root, shuffled)
+                        },
+                        |(root, shuffled)| {
+                            decode::<Sha256>(chunks, min, &root, shuffled).unwrap();
+                        },
+                        BatchSize::SmallInput,
+                    );
+                },
+            );
+        }
+    }
+}
+
+criterion_group! {
+    name = benches;
+    config = Criterion::default().sample_size(10);
+    targets = benchmark_decode
+}