feat(dfir_rs,hydro_lang): add experimental io-uring based networking

Author: shadaj

.github/workflows/ci.yml

@@ -165,6 +165,7 @@ jobs:
           DFIR_EXPECT_WARNINGS: ${{ matrix.rust_release == 'pinned-nightly' && 'noop' || 'ignore' }}
 
       - name: Run doctests
+        # Not supported by nextest: https://github.com/nextest-rs/nextest/issues/16
         run: cargo test --no-fail-fast --features python --features hydro_lang/deploy --doc
 
       - name: Install Python
@@ -197,7 +198,7 @@ jobs:
           source .venv/bin/activate
           cd python_tests
           pip install -r requirements.txt
-          RUST_BACKTRACE=1 pytest
+          pytest
 
       - name: Run Python tests (Windows)
         if: ${{ matrix.os == 'windows-latest' }}
@@ -277,7 +278,7 @@ jobs:
 
       - env:
           RUSTDOCFLAGS: --cfg docsrs -Dwarnings
-        run: cargo doc --no-deps --all-features
+        run: cargo doc --no-deps --all-features --workspace --exclude hydro_cli
 
       - name: Move design docs to output
         shell: bash

Cargo.lock

@@ -15,7 +15,7 @@ RUSTFLAGS="--cfg procmacro2_semver_exempt --cfg super_unstable" CC="$PWD/../LLVM
 
 cd ..
 
-RUSTDOCFLAGS="--cfg docsrs -Dwarnings" cargo doc --no-deps --all-features
+RUSTDOCFLAGS="--cfg docsrs -Dwarnings" cargo doc --no-deps --all-features --workspace --exclude hydro_cli
 
 cp -r target/doc docs/static/rustdoc
 

build_docs.bash

@@ -12,15 +12,16 @@ license = { workspace = true }
 workspace = true
 
 [features]
-default = [ "macros", "debugging", "meta" ]
+default = ["macros", "debugging", "meta"]
 
-meta = [ "dep:dfir_lang" ]
-macros = [ "dfir_macro", "dfir_datalog" ]
-dfir_macro = [ "dep:dfir_macro" ]
-dfir_datalog = [ "dep:dfir_datalog" ]
-deploy_integration = [ "dep:hydro_deploy_integration" ]
-python = [ "dep:pyo3" ]
-debugging = [ "dfir_lang/debugging" ]
+meta = ["dep:dfir_lang"]
+macros = ["dfir_macro", "dfir_datalog"]
+dfir_macro = ["dep:dfir_macro"]
+dfir_datalog = ["dep:dfir_datalog"]
+deploy_integration = ["dep:hydro_deploy_integration"]
+python = ["dep:pyo3"]
+debugging = ["dfir_lang/debugging"]
+io-uring = ["hydro_deploy_integration/io-uring"]
 
 [package.metadata.docs.rs]
 all-features = true
@@ -30,7 +31,7 @@ name = "kvs_bench"
 
 [[example]]
 name = "python_udf"
-required-features = [ "python" ]
+required-features = ["python"]
 
 [dependencies]
 bincode = "1.3.1"
@@ -42,18 +43,22 @@ dfir_macro = { optional = true, path = "../dfir_macro", version = "^0.12.1" }
 futures = "0.3.0"
 hydro_deploy_integration = { optional = true, path = "../hydro_deploy/hydro_deploy_integration", version = "^0.12.1" }
 itertools = "0.13.0"
-lattices = { path = "../lattices", version = "^0.6.1", features = [ "serde" ] }
+lattices = { path = "../lattices", version = "^0.6.1", features = ["serde"] }
 pusherator = { path = "../pusherator", version = "^0.0.12" }
 pyo3 = { optional = true, version = "0.20" }
 ref-cast = "1.0.0"
 regex = "1.10.4"
 rustc-hash = "1.1.0"
 sealed = "0.5.0"
-serde = { version = "1.0.197", features = [ "derive" ] }
+serde = { version = "1.0.197", features = ["derive"] }
 serde_json = "1.0.115"
 slotmap = "1.0.0"
 smallvec = "1.6.1"
-tokio-stream = { version = "0.1.3", default-features = false, features = [ "time", "io-util", "sync" ] }
+tokio-stream = { version = "0.1.3", default-features = false, features = [
+    "time",
+    "io-util",
+    "sync",
+] }
 tracing = "0.1.37"
 variadics = { path = "../variadics", version = "^0.0.9" }
 web-time = "1.0.0"
@@ -63,38 +68,49 @@ multiplatform_test = { path = "../multiplatform_test", version = "^0.5.0", optio
 include_mdtests = { path = "../include_mdtests", version = "^0.0.0", optional = true }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-tokio = { version = "1.29.0", features = [ "full" ] }
-tokio-util = { version = "0.7.5", features = [ "net", "codec" ] }
+tokio = { version = "1.29.0", features = ["full"] }
+tokio-util = { version = "0.7.5", features = ["net", "codec"] }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
-tokio = { version = "1.29.0", features = [ "rt" , "sync", "macros", "io-util", "time" ] }
-tokio-util = { version = "0.7.5", features = [ "codec" ] }
+tokio = { version = "1.29.0", features = [
+    "rt",
+    "sync",
+    "macros",
+    "io-util",
+    "time",
+] }
+tokio-util = { version = "0.7.5", features = ["codec"] }
 # We depend on getrandom transitively through rand. To compile getrandom to
 # WASM, we need to enable its "js" feature. However, rand does not expose a
 # passthrough to enable "js" on getrandom. As a workaround, we enable the
 # getrandom js feature here; when the feature resolver computes the set of
 # features for getrandom (unification), it will include "js".
-getrandom = { version = "0.2.6", features = [ "js" ] }
+getrandom = { version = "0.2.6", features = ["js"] }
 
 [dev-dependencies]
-dfir_lang = { path = "../dfir_lang", version = "^0.12.1", features = [ "clap-derive" ] }
+dfir_lang = { path = "../dfir_lang", version = "^0.12.1", features = [
+    "clap-derive",
+] }
 
-chrono = { version = "0.4.20", features = [ "serde", "clock" ], default-features = false }
-clap = { version = "4.5.4", features = [ "derive" ] }
+chrono = { version = "0.4.20", features = [
+    "serde",
+    "clock",
+], default-features = false }
+clap = { version = "4.5.4", features = ["derive"] }
 colored = "2.0"
 include_mdtests = { path = "../include_mdtests", version = "^0.0.0" }
 insta = "1.39"
 multiplatform_test = { path = "../multiplatform_test", version = "^0.5.0" }
-rand = { version = "0.8.0", features = [ "small_rng" ] }
+rand = { version = "0.8.0", features = ["small_rng"] }
 rand_distr = "0.4.3"
 regex = "1.10.4"
 static_assertions = "1.0.0"
 time = "0.3"
-tracing-subscriber = { version = "0.3", features = [ "env-filter" ] }
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 trybuild = "1"
 wasm-bindgen-test = "0.3.33"
 zipf = "7"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
 # Rayon (rust data-parallelism library) does not compile on WASM.
-criterion = { version = "0.5.0", features = [ "async_tokio", "html_reports" ] }
+criterion = { version = "0.5.0", features = ["async_tokio", "html_reports"] }

dfir_rs/Cargo.toml

@@ -8,6 +8,10 @@ edition = { workspace = true }
 repository = { workspace = true }
 license = { workspace = true }
 
+[features]
+default = []
+io-uring = ["dep:tokio-uring", "dep:libc"]
+
 [lints]
 workspace = true
 
@@ -17,13 +21,17 @@ async-trait = "0.1.54"
 bytes = "1.1.0"
 futures = "0.3.0"
 pin-project = "1.0.0"
-serde = { version = "1.0.197", features = [ "derive" ] }
+serde = { version = "1.0.197", features = ["derive"] }
 tempfile = "3.0.0"
 
 # [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-tokio = { version = "1.29.0", features = [ "full" ] }
-tokio-util = { version = "0.7.5", features = [ "net", "codec" ] }
-tokio-stream = { version = "0.1.3", default-features = false, features = [ "net" ] }
+tokio = { version = "1.29.0", features = ["full"] }
+tokio-util = { version = "0.7.5", features = ["net", "codec"] }
+tokio-stream = { version = "0.1.3", default-features = false, features = [
+    "net",
+] }
+tokio-uring = { version = "0.5.0", features = ["bytes"], optional = true }
+libc = { version = "0.2.77", optional = true }
 
 # [target.'cfg(target_arch = "wasm32")'.dependencies]
 # tokio = { version = "1.29.0", features = [ "rt" , "sync", "macros", "io-util", "time" ] }