diff --git a/.github/workflows/ci-cloud.yaml b/.github/workflows/ci-cloud.yaml index 976a20f3..5ed78bbe 100644 --- a/.github/workflows/ci-cloud.yaml +++ b/.github/workflows/ci-cloud.yaml @@ -58,6 +58,11 @@ jobs: with: tool: just + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: Swatinem/rust-cache@v2 - name: just check @@ -125,6 +130,11 @@ jobs: with: tool: just + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: Swatinem/rust-cache@v2 - name: just lint @@ -172,6 +182,11 @@ jobs: with: tool: just + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: Swatinem/rust-cache@v2 - name: just test @@ -216,6 +231,11 @@ jobs: with: tool: just + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: Swatinem/rust-cache@v2 - name: just doc diff --git a/.github/workflows/ci-standalone.yaml b/.github/workflows/ci-standalone.yaml index 13db0d03..34cea741 100644 --- a/.github/workflows/ci-standalone.yaml +++ b/.github/workflows/ci-standalone.yaml @@ -58,6 +58,11 @@ jobs: with: tool: just + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: Swatinem/rust-cache@v2 - name: just check @@ -122,6 +127,11 @@ jobs: with: tool: just + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: Swatinem/rust-cache@v2 - name: just lint @@ -166,6 +176,11 @@ jobs: with: tool: just + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: Swatinem/rust-cache@v2 - name: just test @@ -207,6 +222,11 @@ jobs: with: tool: just + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: Swatinem/rust-cache@v2 - name: just doc diff --git a/Cargo.lock b/Cargo.lock index 8e55736d..309c137f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -653,9 +653,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" dependencies = [ "serde", ] @@ -979,9 +979,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.22" +version = "1.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32db95edf998450acc7881c932f94cd9b05c87b4b2599e8bab064753da4acfd1" +checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766" dependencies = [ "shlex", ] @@ -1744,9 +1744,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" dependencies = [ "libc", "windows-sys 0.59.0", @@ -1937,6 +1937,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.1" @@ -2967,16 +2973,21 @@ dependencies = [ "log", "metrics", "midnight-ledger", + "prost", "secrecy", "serde", "sqlx", "thiserror 2.0.12", "tokio", "tokio-stream", + "tonic 0.13.1", + "tonic-build", + "tonic-reflection", "tower 0.5.2", "tower-http", "trait-variant", "uuid", + "walkdir", ] [[package]] @@ -3053,6 +3064,7 @@ dependencies = [ "indexer-common", "itertools 0.14.0", "midnight-ledger", + "prost", "reqwest 0.12.15", "serde", "serde_json", @@ -3063,6 +3075,9 @@ dependencies = [ "testcontainers-modules", "tokio", "tokio-tungstenite", + "tonic 0.13.1", + "tonic-build", + "walkdir", ] [[package]] @@ -3417,7 +3432,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "libc", "redox_syscall 0.5.12", ] @@ -3543,6 +3558,12 @@ dependencies = [ "hashbrown 0.15.3", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "manyhow" version = "0.11.4" @@ -3962,6 +3983,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "685a9ac4b61f4e728e1d2c6a7844609c16527aeb5e6c865915c08e619c16410f" +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "nkeys" version = "0.4.4" @@ -4178,7 +4205,7 @@ dependencies = [ "reqwest 0.12.15", "thiserror 2.0.12", "tokio", - "tonic", + "tonic 0.12.3", "tracing", ] @@ -4191,7 +4218,7 @@ dependencies = [ "opentelemetry", "opentelemetry_sdk", "prost", - "tonic", + "tonic 0.12.3", ] [[package]] @@ -4424,6 +4451,16 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap 2.9.0", +] + [[package]] name = "pin-project" version = "1.1.10" @@ -4565,6 +4602,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" +dependencies = [ + "proc-macro2", + "syn 2.0.101", +] + [[package]] name = "primitive-types" version = "0.13.1" @@ -4657,7 +4704,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "lazy_static", "num-traits", "rand 0.8.5", @@ -4677,6 +4724,26 @@ dependencies = [ "prost-derive", ] +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.101", + "tempfile", +] + [[package]] name = "prost-derive" version = "0.13.5" @@ -4690,6 +4757,15 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + [[package]] name = "ptr_meta" version = "0.1.4" @@ -4727,9 +4803,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" +checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" dependencies = [ "bytes", "cfg_aliases", @@ -4747,12 +4823,13 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.11" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcbafbbdbb0f638fe3f35f3c56739f77a8a1d070cb25603226c83339b391472b" +checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" dependencies = [ "bytes", "getrandom 0.3.3", + "lru-slab", "rand 0.9.1", "ring", "rustc-hash", @@ -4767,16 +4844,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" +checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4889,7 +4966,7 @@ version = "11.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] @@ -4927,7 +5004,7 @@ version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] @@ -5193,7 +5270,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -5206,7 +5283,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "errno", "libc", "linux-raw-sys 0.9.4", @@ -5609,7 +5686,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -5622,7 +5699,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "core-foundation 0.10.0", "core-foundation-sys", "libc", @@ -6195,7 +6272,7 @@ checksum = "0afdd3aa7a629683c2d750c2df343025545087081ab5942593a5288855b1b7a7" dependencies = [ "atoi", "base64 0.22.1", - "bitflags 2.9.0", + "bitflags 2.9.1", "byteorder", "bytes", "crc", @@ -6239,7 +6316,7 @@ checksum = "a0bedbe1bbb5e2615ef347a5e9d8cd7680fb63e77d9dafc0f29be15e53f1ebe6" dependencies = [ "atoi", "base64 0.22.1", - "bitflags 2.9.0", + "bitflags 2.9.1", "byteorder", "crc", "dotenvy", @@ -7087,6 +7164,62 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2 0.4.10", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.6.0", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2", + "tokio", + "tokio-stream", + "tower 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote 1.0.40", + "syn 2.0.101", +] + +[[package]] +name = "tonic-reflection" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9687bd5bfeafebdded2356950f278bba8226f0b32109537c4253406e09aafe1" +dependencies = [ + "prost", + "prost-types", + "tokio", + "tokio-stream", + "tonic 0.13.1", +] + [[package]] name = "tower" version = "0.4.13" @@ -7115,9 +7248,12 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", + "indexmap 2.9.0", "pin-project-lite", + "slab", "sync_wrapper 1.0.2", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -7129,7 +7265,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "bytes", "http 1.3.1", "http-body 1.0.1", @@ -7817,15 +7953,15 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.61.0" +version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +checksum = "46ec44dc15085cea82cf9c78f85a9114c463a369786585ad2882d1ff0b0acf40" dependencies = [ "windows-implement", "windows-interface", "windows-link", "windows-result", - "windows-strings 0.4.0", + "windows-strings 0.4.1", ] [[package]] @@ -7869,9 +8005,9 @@ dependencies = [ [[package]] name = "windows-result" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +checksum = "4b895b5356fc36103d0f64dd1e94dfa7ac5633f1c9dd6e80fe9ec4adef69e09d" dependencies = [ "windows-link", ] @@ -7887,9 +8023,9 @@ dependencies = [ [[package]] name = "windows-strings" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +checksum = "2a7ab927b2637c19b3dbe0965e75d8f2d30bdd697a1516191cad2ec4df8fb28a" dependencies = [ "windows-link", ] @@ -8197,7 +8333,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 782dfb18..48595b02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ opentelemetry-otlp = { version = "0.29" } parity-scale-codec = { version = "3.7" } parking_lot = { version = "0.12" } paste = { version = "1.0" } +prost = { version = "0.13" } reqwest = { version = "0.12", default-features = false } secrecy = { version = "0.10" } serde = { version = "1.0" } @@ -72,7 +73,11 @@ tokio = { version = "1" } tokio-stream = { version = "0.1" } tokio-tungstenite = { version = "0.26" } tokio-util = { version = "0.7" } +tonic = { version = "0.13" } +tonic-reflection = { version = "0.13" } +tonic-build = { version = "0.13" } tower = { version = "0.5" } tower-http = { version = "0.6" } trait-variant = { version = "0.1" } uuid = { version = "1.16" } +walkdir = { version = "2.5" } diff --git a/indexer-api/Cargo.toml b/indexer-api/Cargo.toml index 117f4793..a7a62c74 100644 --- a/indexer-api/Cargo.toml +++ b/indexer-api/Cargo.toml @@ -34,12 +34,15 @@ indoc = { workspace = true } log = { workspace = true, features = [ "kv" ] } metrics = { workspace = true } midnight-ledger = { workspace = true } +prost = { workspace = true } secrecy = { workspace = true } serde = { workspace = true, features = [ "derive" ] } sqlx = { workspace = true, features = [ "runtime-tokio", "time" ] } thiserror = { workspace = true } tokio = { workspace = true, features = [ "rt-multi-thread", "time", "signal" ] } tokio-stream = { workspace = true } +tonic = { workspace = true } +tonic-reflection = { workspace = true } tower = { workspace = true } tower-http = { workspace = true, features = [ "cors", "limit" ] } trait-variant = { workspace = true } @@ -48,6 +51,11 @@ uuid = { workspace = true, features = [ "v7" ], optional = true } [dev-dependencies] assert_matches = { workspace = true } +[build-dependencies] +anyhow = { workspace = true } +tonic-build = { workspace = true } +walkdir = { workspace = true } + [features] cloud = [ "indexer-common/cloud", "uuid" ] standalone = [ "indexer-common/standalone", "uuid" ] diff --git a/indexer-api/Dockerfile b/indexer-api/Dockerfile index aeff9543..e4589a8a 100644 --- a/indexer-api/Dockerfile +++ b/indexer-api/Dockerfile @@ -9,6 +9,8 @@ WORKDIR /build RUN git config --global url."https://@github.com".insteadOf "ssh://git@github.com" +RUN apt-get update && apt-get install -y protobuf-compiler + COPY ./Cargo.toml ./Cargo.lock ./rust-toolchain.toml ./ COPY ./.cargo/ ./.cargo/ COPY ./indexer-common/Cargo.toml ./indexer-common/ diff --git a/indexer-api/build.rs b/indexer-api/build.rs new file mode 100644 index 00000000..df20102c --- /dev/null +++ b/indexer-api/build.rs @@ -0,0 +1,34 @@ +use anyhow::{Context, Result}; +use std::{ + env, + ffi::OsStr, + path::{Path, PathBuf}, + vec, +}; +use walkdir::WalkDir; + +const PROTOS: &str = "proto"; + +fn main() -> Result<()> { + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR is defined")); + let protos = list_protos(Path::new(PROTOS))?; + + tonic_build::configure() + .build_client(false) + .file_descriptor_set_path(out_dir.join("midnight_indexer.bin")) + .compile_protos(&protos, &[PROTOS]) + .context("compile protos") +} + +fn list_protos(dir: &Path) -> Result> { + WalkDir::new(dir) + .into_iter() + .try_fold(vec![], |mut protos, entry| { + let entry = entry.context("read proto file")?; + let path = entry.path(); + if path.extension().and_then(OsStr::to_str) == Some("proto") { + protos.push(path.to_path_buf()); + } + Ok(protos) + }) +} diff --git a/indexer-api/proto/midnight_indexer/v1/transaction.proto b/indexer-api/proto/midnight_indexer/v1/transaction.proto new file mode 100644 index 00000000..01a10adc --- /dev/null +++ b/indexer-api/proto/midnight_indexer/v1/transaction.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package midnight_indexer.v1; + +service TransactionService { + rpc Transactions(TransactionsRequest) returns (stream TransactionsResponse); +} + +message TransactionsRequest { + uint64 id = 1; +} + +message TransactionsResponse { + uint64 id = 1; + bytes hash = 2; + uint32 protocol_version = 3; + uint32 apply_stage = 4; + bytes raw = 5; + uint64 start_index = 6; + uint64 end_index = 7; +} diff --git a/indexer-api/src/application.rs b/indexer-api/src/application.rs index 93d6d914..478200ab 100644 --- a/indexer-api/src/application.rs +++ b/indexer-api/src/application.rs @@ -57,7 +57,7 @@ pub async fn run(api: impl Api, subscriber: impl Subscriber) -> anyhow::Result<( }); let serve_api_task = - { task::spawn(async move { api.serve(caught_up).await.context("serving API") }) }; + task::spawn(async move { api.serve(caught_up).await.context("serving API") }); select! { result = block_indexed_task => result, diff --git a/indexer-api/src/domain.rs b/indexer-api/src/domain.rs index 77905c12..f3729639 100644 --- a/indexer-api/src/domain.rs +++ b/indexer-api/src/domain.rs @@ -14,6 +14,7 @@ mod api; mod block; mod contract_action; +mod grpc; mod storage; mod transaction; mod viewing_key; @@ -22,6 +23,7 @@ mod zswap; pub use api::*; pub use block::*; pub use contract_action::*; +pub use grpc::*; pub use storage::*; pub use transaction::*; pub use viewing_key::*; diff --git a/indexer-api/src/domain/grpc.rs b/indexer-api/src/domain/grpc.rs new file mode 100644 index 00000000..6be609cb --- /dev/null +++ b/indexer-api/src/domain/grpc.rs @@ -0,0 +1,26 @@ +// This file is part of midnight-indexer. +// Copyright (C) 2025 Midnight Foundation +// SPDX-License-Identifier: Apache-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::error::Error as StdError; + +/// gRPC abstraction. +#[trait_variant::make(Send)] +pub trait Grpc +where + Self: 'static, +{ + type Error: StdError + Send + Sync + 'static; + + /// Serve gPRC. + async fn serve(self) -> Result<(), Self::Error>; +} diff --git a/indexer-api/src/domain/storage.rs b/indexer-api/src/domain/storage.rs index ebb3a274..00cb80bf 100644 --- a/indexer-api/src/domain/storage.rs +++ b/indexer-api/src/domain/storage.rs @@ -57,6 +57,13 @@ where identifier: &Identifier, ) -> Result, sqlx::Error>; + /// Get a stream of all transactions starting at the given id. + fn get_transactions( + &self, + id: u64, + batch_size: NonZeroU32, + ) -> impl Stream> + Send; + /// Get the contract deploy for the given address. async fn get_contract_deploy_by_address( &self, @@ -202,6 +209,15 @@ impl Storage for NoopStorage { unimplemented!() } + #[cfg_attr(coverage, coverage(off))] + fn get_transactions( + &self, + id: u64, + batch_size: NonZeroU32, + ) -> impl Stream> + Send { + stream::empty() + } + #[cfg_attr(coverage, coverage(off))] async fn get_contract_deploy_by_address( &self, diff --git a/indexer-api/src/infra/api.rs b/indexer-api/src/infra/api.rs index aed92067..80c2aef4 100644 --- a/indexer-api/src/infra/api.rs +++ b/indexer-api/src/infra/api.rs @@ -13,6 +13,8 @@ pub mod v1; +mod grpc; + use crate::domain::{Api, Storage, ZswapStateCache}; use anyhow::Context as _; use async_graphql::Context; @@ -103,7 +105,7 @@ where let listener = TcpListener::bind((address, port)) .await .map_err(AxumApiError::Bind)?; - info!(address:?, port; "listening to TCP connections"); + info!(address:?, port; "API server started"); axum::serve(listener, app) .with_graceful_shutdown(shutdown_signal()) @@ -153,18 +155,20 @@ where let v1_app = v1::make_app( network_id, zswap_state_cache, - storage, + storage.clone(), zswap_state_storage, subscriber, max_complexity, max_depth, ); + let grpc = grpc::routes(storage); + Router::new() .route("/ready", get(ready)) - .route("/health", get(health)) .nest("/api/v1", v1_app) .with_state(caught_up) + .merge(grpc) .layer( ServiceBuilder::new().layer( ServiceBuilder::new() @@ -188,18 +192,6 @@ async fn ready(State(caught_up): State>) -> impl IntoResponse { } } -// TODO: Remove once clients no longer use it! -async fn health(State(caught_up): State>) -> impl IntoResponse { - if !caught_up.load(Ordering::Acquire) { - ( - StatusCode::SERVICE_UNAVAILABLE, - "indexer has not yet caught up with the node; deprecated: use ../ready instead", - ) - } else { - (StatusCode::OK, "OK, deprecated: use ../ready instead") - } -} - /// This is a workaround for async-graphql swallowing `LengthLimitError`s returned by the /// `RequestBodyLimitLayer` for requests that are too large but do not expose that via the /// `Content-Length` header which results in responses with status code 400 instead of 413. diff --git a/indexer-api/src/infra/api/grpc.rs b/indexer-api/src/infra/api/grpc.rs new file mode 100644 index 00000000..e395e16c --- /dev/null +++ b/indexer-api/src/infra/api/grpc.rs @@ -0,0 +1,43 @@ +// This file is part of midnight-indexer. +// Copyright (C) 2025 Midnight Foundation +// SPDX-License-Identifier: Apache-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod v1; + +use crate::domain::Storage; +use axum::{ + Router, + extract::Request, + http::{StatusCode, header::ACCEPT}, + middleware::{self, Next}, + response::{IntoResponse, Response}, +}; +use tonic::{include_file_descriptor_set, service::Routes}; + +const FILE_DESCRIPTOR_SET: &[u8] = include_file_descriptor_set!("midnight_indexer"); + +pub fn routes(storage: S) -> Router +where + S: Storage, +{ + Routes::new(v1::transaction_service(storage)) + .add_service(v1::reflection_service()) + .into_axum_router() + .layer(middleware::from_fn(require_application_grpc)) +} + +async fn require_application_grpc(request: Request, next: Next) -> Response { + match request.headers().get(ACCEPT) { + Some(accept) if accept == "application/grpc" => next.run(request).await, + _ => StatusCode::NOT_ACCEPTABLE.into_response(), + } +} diff --git a/indexer-api/src/infra/api/grpc/v1.rs b/indexer-api/src/infra/api/grpc/v1.rs new file mode 100644 index 00000000..ed9b0106 --- /dev/null +++ b/indexer-api/src/infra/api/grpc/v1.rs @@ -0,0 +1,115 @@ +// This file is part of midnight-indexer. +// Copyright (C) 2025 Midnight Foundation +// SPDX-License-Identifier: Apache-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +tonic::include_proto!("midnight_indexer.v1"); + +use crate::{ + domain::{self, Storage}, + infra::api::grpc::{ + FILE_DESCRIPTOR_SET, + v1::transaction_service_server::{TransactionService, TransactionServiceServer}, + }, +}; +use futures::StreamExt; +use indexer_common::error::StdErrorExt; +use log::error; +use std::{num::NonZeroU32, pin::pin}; +use tokio::{sync::mpsc, task}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; +use tonic_reflection::server::v1::{ServerReflection, ServerReflectionServer}; + +// TODO: Make configurable! +const BATCH_SIZE: NonZeroU32 = NonZeroU32::new(100).unwrap(); + +pub fn transaction_service(storage: S) -> TransactionServiceServer> +where + S: Storage, +{ + TransactionServiceServer::new(TransactionServiceImpl { storage }) +} + +pub fn reflection_service() -> ServerReflectionServer { + tonic_reflection::server::Builder::configure() + .register_encoded_file_descriptor_set(FILE_DESCRIPTOR_SET) + .build_v1() + .expect("v1 reflection can be built") +} + +pub struct TransactionServiceImpl { + storage: S, +} + +#[tonic::async_trait] +impl TransactionService for TransactionServiceImpl +where + S: Storage, +{ + // TODO: Once the Rust async story is better and tonic no longer uses async_trait with a + // requirement for a 'static stream, revisit this indicretion! + type TransactionsStream = ReceiverStream>; + + async fn transactions( + &self, + request: Request, + ) -> Result, Status> { + let id = request.into_inner().id; + + let (sender, receiver) = mpsc::channel(42); + + task::spawn({ + let storage = self.storage.to_owned(); + + async move { + let mut transactions = pin!(storage.get_transactions(id, BATCH_SIZE)); + + while let Some(transaction) = transactions.next().await { + let transaction = transaction + .map_err(|error| { + error!(error = error.as_chain(); "cannot get next transaction"); + Status::internal("internal error") + }) + .map(|transaction| { + let domain::Transaction { + id, + hash, + protocol_version, + apply_stage, + raw, + start_index, + end_index, + .. + } = transaction; + + TransactionsResponse { + id, + hash: hash.into(), + protocol_version: protocol_version.into(), + apply_stage: apply_stage.into(), + raw: raw.0, + start_index, + end_index, + } + }); + + if sender.send(transaction).await.is_err() { + // Receiver has dropped. + break; + } + } + } + }); + + Ok(Response::new(ReceiverStream::new(receiver))) + } +} diff --git a/indexer-api/src/infra/storage/postgres.rs b/indexer-api/src/infra/storage/postgres.rs index f9468352..426066b2 100644 --- a/indexer-api/src/infra/storage/postgres.rs +++ b/indexer-api/src/infra/storage/postgres.rs @@ -99,7 +99,7 @@ impl Storage for PostgresStorage { LIMIT $2 "}; - let blocks = sqlx::query_as::<_, Block>(query) + let blocks = sqlx::query_as(query) .bind(height as i64) .bind(batch_size.get() as i64) .fetch_all(&*self.pool) @@ -220,6 +220,50 @@ impl Storage for PostgresStorage { .await } + fn get_transactions( + &self, + mut id: u64, + batch_size: NonZeroU32, + ) -> impl Stream> { + let chunks = try_stream! { + loop { + let query = indoc! {" + SELECT + transactions.id, + transactions.hash, + blocks.hash AS block_hash, + transactions.protocol_version, + transactions.apply_stage, + transactions.identifiers, + transactions.raw, + transactions.merkle_tree_root, + transactions.start_index, + transactions.end_index + FROM transactions + INNER JOIN blocks ON blocks.id = transactions.block_id + WHERE transactions.id >= $1 + ORDER BY transactions.id + LIMIT $2 + "}; + + let transactions = sqlx::query_as::<_, Transaction>(query) + .bind(id as i64) + .bind(batch_size.get() as i64) + .fetch_all(&*self.pool) + .await?; + + match transactions.last() { + Some(t) => id = t.id + 1, + None => break, + } + + yield transactions; + } + }; + + flatten_chunks(chunks) + } + async fn get_contract_deploy_by_address( &self, address: &ContractAddress, diff --git a/indexer-api/src/infra/storage/sqlite.rs b/indexer-api/src/infra/storage/sqlite.rs index 6e9b45e4..b46a963e 100644 --- a/indexer-api/src/infra/storage/sqlite.rs +++ b/indexer-api/src/infra/storage/sqlite.rs @@ -102,7 +102,7 @@ impl Storage for SqliteStorage { LIMIT $2 "}; - let blocks = sqlx::query_as::<_, Block>(query) + let blocks = sqlx::query_as(query) .bind(height as i64) .bind(batch_size.get() as i64) .fetch_all(&*self.pool) @@ -255,6 +255,49 @@ impl Storage for SqliteStorage { Ok(transactions) } + fn get_transactions( + &self, + mut id: u64, + batch_size: NonZeroU32, + ) -> impl Stream> { + let chunks = try_stream! { + loop { + let query = indoc! {" + SELECT + transactions.id, + transactions.hash, + blocks.hash AS block_hash, + transactions.protocol_version, + transactions.apply_stage, + transactions.raw, + transactions.merkle_tree_root, + transactions.start_index, + transactions.end_index + FROM transactions + INNER JOIN blocks ON blocks.id = transactions.block_id + WHERE transactions.id >= $1 + ORDER BY transactions.id + LIMIT $2 + "}; + + let transactions = sqlx::query_as::<_, Transaction>(query) + .bind(id as i64) + .bind(batch_size.get() as i64) + .fetch_all(&*self.pool) + .await?; + + match transactions.last() { + Some(t) => id = t.id + 1, + None => break, + } + + yield transactions; + } + }; + + flatten_chunks(chunks) + } + async fn get_contract_deploy_by_address( &self, address: &ContractAddress, diff --git a/indexer-api/src/main.rs b/indexer-api/src/main.rs index 6444e893..3c5c0e3a 100644 --- a/indexer-api/src/main.rs +++ b/indexer-api/src/main.rs @@ -91,7 +91,12 @@ async fn run() -> anyhow::Result<()> { let subscriber = pub_sub::nats::subscriber::NatsSubscriber::new(pub_sub_config).await?; - let api = AxumApi::new(api_config, storage, zswap_state_storage, subscriber.clone()); + let api = AxumApi::new( + api_config, + storage.clone(), + zswap_state_storage, + subscriber.clone(), + ); application::run(api, subscriber) .await diff --git a/indexer-common/src/domain.rs b/indexer-common/src/domain.rs index 16c4a2d5..c3809060 100644 --- a/indexer-common/src/domain.rs +++ b/indexer-common/src/domain.rs @@ -55,6 +55,33 @@ pub enum ApplyStage { Failure, } +impl From for u32 { + fn from(apply_stage: ApplyStage) -> Self { + match apply_stage { + ApplyStage::Success => 0, + ApplyStage::PartialSuccess => 1, + ApplyStage::Failure => 2, + } + } +} + +impl TryFrom for ApplyStage { + type Error = ApplyStageTryFromU8Error; + + fn try_from(value: u32) -> Result { + match value { + 0 => Ok(Self::Success), + 1 => Ok(Self::PartialSuccess), + 2 => Ok(Self::Failure), + other => Err(ApplyStageTryFromU8Error(other)), + } + } +} + +#[derive(Debug, Error)] +#[error("discriminant for ApplyStage must be 0..=2, but was {0}")] +pub struct ApplyStageTryFromU8Error(u32); + /// The variant of a contract action. #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)] #[cfg_attr(feature = "cloud", sqlx(type_name = "CONTRACT_ACTION_VARIANT"))] @@ -118,7 +145,8 @@ pub struct UnknownNetworkIdError(String); #[cfg(test)] mod tests { - use crate::domain::NetworkId; + use crate::domain::{ApplyStage, NetworkId}; + use assert_matches::assert_matches; use midnight_ledger::serialize::NetworkId as LedgerNetworkId; #[test] @@ -144,4 +172,16 @@ mod tests { let network_id = serde_json::from_str::("\"FooBarBaz\""); assert!(network_id.is_err()); } + + #[test] + fn test_apply_stage() { + let apply_stage = ApplyStage::try_from(u32::from(ApplyStage::Success)); + assert_matches!(apply_stage, Ok(ApplyStage::Success)); + + let apply_stage = ApplyStage::try_from(u32::from(ApplyStage::PartialSuccess)); + assert_matches!(apply_stage, Ok(ApplyStage::PartialSuccess)); + + let apply_stage = ApplyStage::try_from(u32::from(ApplyStage::Failure)); + assert_matches!(apply_stage, Ok(ApplyStage::Failure)); + } } diff --git a/indexer-common/src/domain/bytes.rs b/indexer-common/src/domain/bytes.rs index e1392d6b..f949c67c 100644 --- a/indexer-common/src/domain/bytes.rs +++ b/indexer-common/src/domain/bytes.rs @@ -55,6 +55,12 @@ impl TryFrom<&[u8]> for ByteArray { } } +impl From> for Vec { + fn from(bytes: ByteArray) -> Self { + bytes.0.into() + } +} + impl TryFrom> for ByteArray { type Error = TryFromForByteArrayError; diff --git a/indexer-common/src/domain/protocol_version.rs b/indexer-common/src/domain/protocol_version.rs index fcfe8b67..b34682c6 100644 --- a/indexer-common/src/domain/protocol_version.rs +++ b/indexer-common/src/domain/protocol_version.rs @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use derive_more::From; +use derive_more::{From, Into}; use parity_scale_codec::Decode; use std::{ fmt::{self, Display}, @@ -22,7 +22,7 @@ use thiserror::Error; pub const PROTOCOL_VERSION_000_012_000: ProtocolVersion = ProtocolVersion(12_000); /// The runtime specification version of the chain; defaults to 1, i.e. 0.0.1. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, From)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, From, Into)] pub struct ProtocolVersion(pub u32); impl ProtocolVersion { diff --git a/indexer-standalone/src/main.rs b/indexer-standalone/src/main.rs index 203cac17..ede514b1 100644 --- a/indexer-standalone/src/main.rs +++ b/indexer-standalone/src/main.rs @@ -117,7 +117,12 @@ async fn run() -> anyhow::Result<()> { let storage = indexer_api::infra::storage::sqlite::SqliteStorage::new(cipher.clone(), pool.clone()); let subscriber = pub_sub.subscriber(); - let api = AxumApi::new(api_config, storage, zswap_state_storage, subscriber.clone()); + let api = AxumApi::new( + api_config, + storage.clone(), + zswap_state_storage, + subscriber.clone(), + ); indexer_api::application::run(api, subscriber) }); diff --git a/indexer-tests/Cargo.toml b/indexer-tests/Cargo.toml index d4bb14a7..8d77544e 100644 --- a/indexer-tests/Cargo.toml +++ b/indexer-tests/Cargo.toml @@ -25,12 +25,14 @@ indexer-api = { path = "../indexer-api" } indexer-common = { path = "../indexer-common" } itertools = { workspace = true } midnight-ledger = { workspace = true, features = [ "fake", "proving", "transaction-construction" ] } +prost = { workspace = true } reqwest = { workspace = true, features = [ "json", "rustls-tls" ] } serde = { workspace = true, features = [ "derive" ] } serde_json = { workspace = true } subxt = { workspace = true } tokio = { workspace = true, features = [ "process", "rt-multi-thread" ] } tokio-tungstenite = { workspace = true, features = [ "rustls-tls-webpki-roots" ] } +tonic = { workspace = true } [dev-dependencies] const-hex = { workspace = true } @@ -41,6 +43,11 @@ tempfile = { workspace = true } testcontainers = { workspace = true } testcontainers-modules = { workspace = true, features = [ "postgres" ] } +[build-dependencies] +anyhow = { workspace = true } +tonic-build = { workspace = true } +walkdir = { workspace = true } + [features] cloud = [ "chain-indexer/cloud", diff --git a/indexer-tests/build.rs b/indexer-tests/build.rs new file mode 100644 index 00000000..9ee565c4 --- /dev/null +++ b/indexer-tests/build.rs @@ -0,0 +1,31 @@ +use anyhow::{Context, Result}; +use std::{ + ffi::OsStr, + path::{Path, PathBuf}, + vec, +}; +use walkdir::WalkDir; + +const PROTOS: &str = "proto"; + +fn main() -> Result<()> { + let protos = list_protos(Path::new(PROTOS))?; + + tonic_build::configure() + .build_server(false) + .compile_protos(&protos, &[PROTOS]) + .context("compile protos") +} + +fn list_protos(dir: &Path) -> Result> { + WalkDir::new(dir) + .into_iter() + .try_fold(vec![], |mut protos, entry| { + let entry = entry.context("read proto file")?; + let path = entry.path(); + if path.extension().and_then(OsStr::to_str) == Some("proto") { + protos.push(path.to_path_buf()); + } + Ok(protos) + }) +} diff --git a/indexer-tests/proto b/indexer-tests/proto new file mode 120000 index 00000000..42d62144 --- /dev/null +++ b/indexer-tests/proto @@ -0,0 +1 @@ +../indexer-api/proto \ No newline at end of file diff --git a/indexer-tests/src/e2e.rs b/indexer-tests/src/e2e.rs index 8cd5c8a5..de87c22e 100644 --- a/indexer-tests/src/e2e.rs +++ b/indexer-tests/src/e2e.rs @@ -13,6 +13,8 @@ //! e2e testing library +tonic::include_proto!("midnight_indexer.v1"); + use crate::{ e2e::{ block_subscription::{ @@ -21,10 +23,11 @@ use crate::{ BlockSubscriptionBlocksTransactionsContractActions as BlockSubscriptionContractAction, }, contract_action_query::ContractActionQueryContractAction, + transaction_service_client::TransactionServiceClient, }, graphql_ws_client, }; -use anyhow::{Context, Ok, bail}; +use anyhow::{Context, bail}; use bech32::{Bech32m, Hrp}; use futures::{StreamExt, TryStreamExt, future::ok}; use graphql_client::{GraphQLQuery, Response}; @@ -41,7 +44,11 @@ use midnight_ledger::{ }; use reqwest::Client; use serde::Serialize; -use std::time::{Duration, Instant}; +use std::{ + str::FromStr, + time::{Duration, Instant}, +}; +use tonic::{metadata::MetadataValue, transport::Endpoint}; const MAX_HEIGHT: usize = 30; @@ -51,13 +58,22 @@ const MAX_HEIGHT: usize = 30; pub async fn run(network_id: NetworkId, host: &str, port: u16, secure: bool) -> anyhow::Result<()> { println!("### starting e2e testing"); - let (api_url, ws_api_url) = { - let core = format!("{host}:{port}/api/v1/graphql"); + let (api_url, ws_api_url, grpc_url) = { + let api_core = format!("{host}:{port}/api/v1/graphql"); + let grpc_core = format!("{host}:{port}"); if secure { - (format!("https://{core}"), format!("wss://{core}/ws")) + ( + format!("https://{api_core}"), + format!("wss://{api_core}/ws"), + format!("https://{grpc_core}"), + ) } else { - (format!("http://{core}"), format!("ws://{core}/ws")) + ( + format!("http://{api_core}"), + format!("ws://{api_core}/ws"), + format!("http://{grpc_core}"), + ) } }; @@ -95,6 +111,11 @@ pub async fn run(network_id: NetworkId, host: &str, port: u16, secure: bool) -> .await .context("test wallet subscription")?; + // Test gRPC. + test_transaction_service(&indexer_data, &grpc_url) + .await + .context("test gRPC")?; + println!("### successfully finished e2e testing"); Ok(()) @@ -661,6 +682,35 @@ async fn test_wallet_subscription(ws_api_url: &str) -> anyhow::Result<()> { Ok(()) } +async fn test_transaction_service(node_data: &IndexerData, grpc_url: &str) -> anyhow::Result<()> { + let channel = Endpoint::from_str(grpc_url) + .context("create endpoint")? + .connect() + .await + .context("connect to endpoint")?; + let mut grpc_client = TransactionServiceClient::with_interceptor(channel, add_accept_header); + + let transaction_hashes = grpc_client + .transactions(TransactionsRequest { id: 0 }) + .await + .context("get transactions")? + .into_inner() + .map_ok(|transaction| transaction.hash) + .try_collect::>() + .await + .context("collect transactions")?; + + let expected_transaction_hashes = node_data + .transactions + .iter() + .flat_map(|t| t.hash.hex_decode::>()) + .collect::>(); + + assert_eq!(transaction_hashes, expected_transaction_hashes); + + Ok(()) +} + trait SerializeExt where Self: Serialize, @@ -841,6 +891,14 @@ fn seed_to_secret_key(seed: &str) -> SecretKey { SecretKeys::from(Seed::from(seed_bytes)).encryption_secret_key } +#[allow(clippy::result_large_err)] +fn add_accept_header(mut request: tonic::Request<()>) -> Result, tonic::Status> { + request + .metadata_mut() + .insert("accept", MetadataValue::from_static("application/grpc")); + Ok(request) +} + #[derive(GraphQLQuery)] #[graphql( schema_path = "../indexer-api/graphql/schema-v1.graphql",