diff --git a/.gitignore b/.gitignore index d399d3a6..19281ba7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,16 +5,6 @@ !.gitignore !.gitattributes -# Project files -/.cache/ -/.fleet/ -/.idea/ -/.vscode/ - -# OS specific temporary files -.DS_Store -Thumbs.db - # Added by cargo /target/ @@ -52,7 +42,4 @@ dist/ .env .env.* !.env.example - -# Tools output -tools/adt/samples/ -tools/adt/reports/ \ No newline at end of file +perf.data diff --git a/.mise.toml b/.mise.toml index 66521b17..072a0d93 100644 --- a/.mise.toml +++ b/.mise.toml @@ -1,12 +1,29 @@ -# Unified mise configuration for WoW Emulation Rust projects +# mise configuration for WoW Emulation Rust projects # https://mise.jdx.dev/configuration.html [tools] +# ── Toolchain ────────────────────────────────── rust = "1.92" +node = "22" + +# ── Build tools ──────────────────────────────── +"cargo:cross" = "latest" +"cargo:cargo-binstall" = "latest" +"cargo:wasm-pack" = "latest" + +# ── Testing and coverage ────────────────────── "cargo:cargo-nextest" = "latest" "cargo:cargo-llvm-cov" = "latest" -mdbook = "latest" + +# ── Linting and auditing ───────────────────── +"cargo:cargo-deny" = "latest" "npm:markdownlint-cli2" = "latest" actionlint = "latest" -"cargo:cargo-deny" = "latest" -"cargo:cross" = "latest" + +# ── Documentation ───────────────────────────── +mdbook = "latest" +"cargo:mdbook-mermaid" = "latest" + +# ── Profiling ───────────────────────────────── +"cargo:flamegraph" = "latest" +"cargo:inferno" = "latest" diff --git a/Cargo.lock b/Cargo.lock index 9ffdab44..c634188d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,15 +18,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] -name = "ahash" -version = "0.8.12" +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aegis" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +checksum = "8ae1572243695de9c6c8d16c7889899abac907d14c148f1939d837122bbeca79" +dependencies = [ + "cc", + "softaes", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", - "once_cell", - "version_check", - "zerocopy", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", ] [[package]] @@ -62,12 +95,6 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - [[package]] name = "android_system_properties" version = "0.1.5" @@ -145,6 +172,15 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "arc-swap" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" +dependencies = [ + "rustversion", +] + [[package]] name = "arg_enum_proc_macro" version = "0.3.4" @@ -228,12 +264,41 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn", + "which", +] + [[package]] name = "binrw" version = "0.15.0" @@ -306,6 +371,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "branches" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e426eb5cc1900033930ec955317b302e68f19f326cc7bb0c8a86865a826cdf0c" +dependencies = [ + "rustc_version", +] + [[package]] name = "bstr" version = "1.12.0" @@ -322,6 +396,10 @@ name = "built" version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" +dependencies = [ + "chrono", + "git2", +] [[package]] name = "bumpalo" @@ -334,6 +412,20 @@ name = "bytemuck" version = "1.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "byteorder" @@ -389,9 +481,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.35" +version = "1.2.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "590f9024a68a8c40351881787f1934dc11afd69090f5edb6831464694d836ea3" +checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" dependencies = [ "find-msvc-tools", "jobserver", @@ -399,6 +491,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-expr" version = "0.15.8" @@ -415,18 +516,23 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +[[package]] +name = "cfg_block" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18758054972164c3264f7c8386f5fc6da6114cb46b619fd365d4e3b2dc3ae487" + [[package]] name = "chrono" -version = "0.4.41" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "wasm-bindgen", - "windows-link 0.1.3", + "windows-link 0.2.1", ] [[package]] @@ -456,6 +562,27 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.46" @@ -517,6 +644,15 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "console" version = "0.15.11" @@ -623,6 +759,15 @@ dependencies = [ "itertools 0.13.0", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -642,6 +787,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -661,6 +816,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -685,6 +841,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "der" version = "0.7.10" @@ -696,6 +861,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2163a0e204a148662b6b6816d4b5d5668a5f2f8df498ccbd5cd0e864e78fecba" +dependencies = [ + "powerfmt", +] + [[package]] name = "diff" version = "0.1.13" @@ -761,6 +935,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "doc-comment" version = "0.3.3" @@ -860,10 +1045,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" +name = "fastbloom" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" +dependencies = [ + "getrandom 0.3.3", + "libm", + "rand 0.9.2", + "siphasher", +] [[package]] name = "fastrand" @@ -882,9 +1073,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.0" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e178e4fba8a2726903f6ba98a6d221e76f9c12c650d5dc0e6afdc50677b49650" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flate2" @@ -917,6 +1108,30 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "genawaiter" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0" +dependencies = [ + "genawaiter-macro", +] + +[[package]] +name = "genawaiter-macro" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc" + [[package]] name = "generic-array" version = "0.14.7" @@ -950,6 +1165,29 @@ dependencies = [ "wasi 0.14.3+wasi-0.2.4", ] +[[package]] +name = "getrandom" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gif" version = "0.13.3" @@ -966,6 +1204,19 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "git2" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +dependencies = [ + "bitflags 2.10.0", + "libc", + "libgit2-sys", + "log", + "url", +] + [[package]] name = "glam" version = "0.31.1" @@ -991,15 +1242,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", -] - [[package]] name = "hashbrown" version = "0.15.5" @@ -1011,15 +1253,6 @@ dependencies = [ "foldhash", ] -[[package]] -name = "hashlink" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" -dependencies = [ - "hashbrown 0.14.5", -] - [[package]] name = "heck" version = "0.5.0" @@ -1038,6 +1271,25 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + [[package]] name = "humansize" version = "2.1.3" @@ -1072,72 +1324,181 @@ dependencies = [ ] [[package]] -name = "image" -version = "0.25.6" +name = "icu_collections" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db35664ce6b9810857a38a906215e75a9c879f0696556a39f59c62829710251a" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ - "bytemuck", - "byteorder-lite", - "color_quant", - "exr", - "gif", - "image-webp", - "num-traits", - "png", - "qoi", - "ravif", - "rayon", - "rgb", - "tiff", - "zune-core", - "zune-jpeg", + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "image-webp" -version = "0.2.4" +name = "icu_locale_core" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ - "byteorder-lite", - "quick-error 2.0.1", + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", ] [[package]] -name = "imgref" -version = "1.11.0" +name = "icu_normalizer" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0263a3d970d5c054ed9312c0057b4f3bde9c0b33836d3637361d4a9e6e7a408" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] [[package]] -name = "implode" -version = "0.1.1" +name = "icu_normalizer_data" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be6aed12e8500c350a4009ebdcf0098c89948617e9b846e3c9bbc56a48b56914" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] -name = "indexmap" -version = "2.11.0" +name = "icu_properties" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" dependencies = [ - "equivalent", - "hashbrown 0.15.5", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", ] [[package]] -name = "indicatif" -version = "0.17.11" +name = "icu_properties_data" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" -dependencies = [ - "console 0.15.11", - "number_prefix", - "portable-atomic", - "unicode-width 0.2.1", - "web-time", -] +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "image" +version = "0.25.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db35664ce6b9810857a38a906215e75a9c879f0696556a39f59c62829710251a" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error 2.0.1", +] + +[[package]] +name = "imgref" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0263a3d970d5c054ed9312c0057b4f3bde9c0b33836d3637361d4a9e6e7a408" + +[[package]] +name = "implode" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be6aed12e8500c350a4009ebdcf0098c89948617e9b846e3c9bbc56a48b56914" + +[[package]] +name = "indexmap" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" +dependencies = [ + "equivalent", + "hashbrown", + "serde", +] + +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console 0.15.11", + "number_prefix", + "portable-atomic", + "unicode-width 0.2.1", + "web-time", +] [[package]] name = "indicatif" @@ -1152,6 +1513,15 @@ dependencies = [ "web-time", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "interpolate_name" version = "0.2.4" @@ -1163,6 +1533,15 @@ dependencies = [ "syn", ] +[[package]] +name = "intrusive-collections" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "189d0897e4cbe8c75efedf3502c18c887b05046e59d28404d4d8e46cbc4d1e86" +dependencies = [ + "memoffset", +] + [[package]] name = "io-uring" version = "0.7.10" @@ -1274,6 +1653,18 @@ dependencies = [ "spin", ] +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "lebe" version = "0.5.2" @@ -1302,6 +1693,28 @@ dependencies = [ "cc", ] +[[package]] +name = "libgit2-sys" +version = "0.18.3+1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link 0.2.1", +] + [[package]] name = "libm" version = "0.2.15" @@ -1319,22 +1732,35 @@ dependencies = [ ] [[package]] -name = "libsqlite3-sys" -version = "0.30.1" +name = "libz-sys" +version = "1.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" dependencies = [ "cc", + "libc", "pkg-config", "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + [[package]] name = "lock_api" version = "0.4.13" @@ -1366,7 +1792,7 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.15.5", + "hashbrown", ] [[package]] @@ -1410,9 +1836,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.5" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memmap2" @@ -1423,6 +1849,37 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1514,6 +1971,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" + [[package]] name = "num-derive" version = "0.4.2" @@ -1599,6 +2062,12 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "option-ext" version = "0.2.0" @@ -1611,6 +2080,15 @@ version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e" +[[package]] +name = "pack1" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6e7cd9bd638dc2c831519a0caa1c006cab771a92b1303403a8322773c5b72d6" +dependencies = [ + "bytemuck", +] + [[package]] name = "page_size" version = "0.6.0" @@ -1659,6 +2137,12 @@ dependencies = [ "base64ct", ] +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -1744,6 +2228,32 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 1.1.3", + "windows-sys 0.61.2", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.11.1" @@ -1759,6 +2269,21 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1808,6 +2333,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "prettytable-rs" version = "0.10.0" @@ -1870,6 +2405,29 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools 0.13.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "qoi" version = "0.4.1" @@ -1974,6 +2532,15 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rapidhash" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59" +dependencies = [ + "rustversion", +] + [[package]] name = "rav1e" version = "0.7.1" @@ -2099,6 +2666,16 @@ version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" +[[package]] +name = "roaring" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885" +dependencies = [ + "bytemuck", + "byteorder", +] + [[package]] name = "rsa" version = "0.9.8" @@ -2119,20 +2696,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rusqlite" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" -dependencies = [ - "bitflags 2.10.0", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", -] - [[package]] name = "rustc-demangle" version = "0.1.26" @@ -2140,21 +2703,55 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" [[package]] -name = "rustix" -version = "1.1.3" +name = "rustc-hash" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" -dependencies = [ - "bitflags 2.10.0", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.60.2", -] +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] -name = "rustversion" -version = "1.0.22" +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.10.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags 2.10.0", + "errno", + "libc", + "linux-raw-sys 0.11.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" @@ -2191,6 +2788,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + [[package]] name = "serde" version = "1.0.228" @@ -2266,6 +2869,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2306,6 +2915,21 @@ dependencies = [ "quote", ] +[[package]] +name = "simsimd" +version = "6.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f81af684ca3dc160907f1478d779bdfd8bae52f55f4c58caba0229670a83a0d" +dependencies = [ + "cc", +] + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + [[package]] name = "slab" version = "0.4.11" @@ -2318,6 +2942,12 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "softaes" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fef461faaeb36c340b6c887167a9054a034f6acfc50a014ead26a02b4356b3de" + [[package]] name = "spin" version = "0.9.8" @@ -2334,6 +2964,12 @@ dependencies = [ "der", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "storm-ffi" version = "0.6.4" @@ -2353,6 +2989,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -2370,6 +3028,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "system-deps" version = "6.2.2" @@ -2398,8 +3067,8 @@ dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", - "rustix", - "windows-sys 0.60.2", + "rustix 1.1.3", + "windows-sys 0.61.2", ] [[package]] @@ -2544,6 +3213,47 @@ dependencies = [ "weezl", ] +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -2633,6 +3343,18 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +dependencies = [ + "crossbeam-channel", + "thiserror 2.0.16", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.30" @@ -2683,6 +3405,187 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "turso" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2fe423c2c954948babb36edda12b737e321d8541d4eae519694f7d512ecab6" +dependencies = [ + "thiserror 2.0.16", + "tracing", + "tracing-subscriber", + "turso_sdk_kit", + "turso_sync_sdk_kit", +] + +[[package]] +name = "turso_core" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a8b54994ee025964459322bcdb4f6f78c5dba82643863dabfac680f16c8afa8" +dependencies = [ + "aegis", + "aes", + "aes-gcm", + "arc-swap", + "bitflags 2.10.0", + "branches", + "built", + "bumpalo", + "bytemuck", + "cfg_block", + "chrono", + "crossbeam-skiplist", + "either", + "fallible-iterator", + "fastbloom", + "hex", + "intrusive-collections", + "io-uring", + "libc", + "libloading", + "libm", + "miette", + "pack1", + "parking_lot", + "paste", + "polling", + "rand 0.9.2", + "rapidhash", + "regex", + "regex-syntax", + "roaring", + "rustc-hash 2.1.1", + "rustix 1.1.3", + "ryu", + "simsimd", + "strum", + "strum_macros", + "tempfile", + "thiserror 2.0.16", + "tracing", + "tracing-subscriber", + "turso_ext", + "turso_macros", + "turso_parser", + "twox-hash", + "uncased", + "uuid", +] + +[[package]] +name = "turso_ext" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de917b4c5881bfb34ccbb1dcf4992773bc39853eacf248955f2ece7e3cb3049" +dependencies = [ + "chrono", + "getrandom 0.3.3", + "turso_macros", +] + +[[package]] +name = "turso_macros" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2f62bb271d4cf202bc2acbeb8e2c3f764ec754924f144e704cdcba2e5b0c84" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "turso_parser" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ad89caa1c4888756bd027485499d1dc4c8420d15887ab32aa28b707c411221" +dependencies = [ + "bitflags 2.10.0", + "memchr", + "miette", + "strum", + "strum_macros", + "thiserror 2.0.16", + "turso_macros", +] + +[[package]] +name = "turso_sdk_kit" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00ff5b2cadd6c8b749511648d50c95f69bfa52efc5d88cc2e2deedd0beeb6c89" +dependencies = [ + "bindgen", + "env_logger", + "tracing", + "tracing-appender", + "tracing-subscriber", + "turso_core", + "turso_sdk_kit_macros", +] + +[[package]] +name = "turso_sdk_kit_macros" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "289f7ea7499419e6670363ca18e954ed53397bb1e03ab7eabbb267d9b05ab836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "turso_sync_engine" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea9860c615a7d8df43fc6ac4293636e9d743c1693513c81be22f0e9388624f58" +dependencies = [ + "base64", + "bytes", + "genawaiter", + "http", + "libc", + "prost", + "roaring", + "serde", + "serde_json", + "thiserror 2.0.16", + "tracing", + "turso_core", + "turso_parser", + "uuid", +] + +[[package]] +name = "turso_sync_sdk_kit" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b669b19a5f4bfa7cfdf5045af36ca4a2087431c0d2844ec539ddcf951b5c9d2" +dependencies = [ + "bindgen", + "env_logger", + "genawaiter", + "parking_lot", + "tracing", + "tracing-appender", + "tracing-subscriber", + "turso_core", + "turso_sdk_kit", + "turso_sdk_kit_macros", + "turso_sync_engine", +] + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +dependencies = [ + "rand 0.9.2", +] + [[package]] name = "typenum" version = "1.18.0" @@ -2695,6 +3598,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-ident" version = "1.0.18" @@ -2713,24 +3625,70 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "unit-prefix" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "unsafe-libyaml" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" +dependencies = [ + "getrandom 0.4.1", + "js-sys", + "sha1_smol", + "wasm-bindgen", +] + [[package]] name = "v_frame" version = "0.3.9" @@ -2795,6 +3753,7 @@ dependencies = [ "clap", "clap_complete", "console 0.15.11", + "directories", "env_logger", "glob", "humansize", @@ -2809,6 +3768,8 @@ dependencies = [ "serde_yaml_ng", "tempfile", "thiserror 2.0.16", + "tokio", + "turso", "wow-adt", "wow-blp", "wow-cdbc", @@ -2831,7 +3792,25 @@ version = "0.14.3+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a51ae83037bdd272a9e28ce236db8c07016dd0d50c27038b3f407533c030c95" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.45.0", +] + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", ] [[package]] @@ -2892,6 +3871,40 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.10.0", + "hashbrown", + "indexmap", + "semver", +] + [[package]] name = "web-sys" version = "0.3.77" @@ -2918,6 +3931,18 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3" +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "winapi" version = "0.3.9" @@ -3260,6 +4285,94 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "052283831dbae3d879dc7f51f3d92703a316ca49f91540417d38591826127814" +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.10.0", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "wow-adt" version = "0.6.4" @@ -3345,13 +4458,10 @@ dependencies = [ "byteorder", "bytes", "bzip2", - "chrono", "crc32fast", "criterion", - "directories", "env_logger", "flate2", - "glob", "hex", "implode", "libc", @@ -3370,7 +4480,6 @@ dependencies = [ "rand 0.9.2", "rayon", "rsa", - "rusqlite", "sha1", "tempfile", "thiserror 2.0.16", @@ -3414,12 +4523,41 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + [[package]] name = "yansi" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.26" @@ -3440,12 +4578,66 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zune-core" version = "0.4.12" diff --git a/file-formats/archives/wow-mpq/Cargo.toml b/file-formats/archives/wow-mpq/Cargo.toml index f50c07f5..c927e010 100644 --- a/file-formats/archives/wow-mpq/Cargo.toml +++ b/file-formats/archives/wow-mpq/Cargo.toml @@ -48,11 +48,6 @@ rayon = "1.10" parking_lot = "0.12" lru = "0.12" -# Database support -rusqlite = { version = "0.32", features = ["bundled"] } -directories = "5.0" -chrono = "0.4" -glob = "0.3" # Async I/O support (optional) tokio = { version = "1.0", features = ["io-util", "time", "sync", "rt", "fs", "macros"], optional = true } diff --git a/file-formats/archives/wow-mpq/src/archive.rs b/file-formats/archives/wow-mpq/src/archive.rs index 5b387908..7b7a0c0b 100644 --- a/file-formats/archives/wow-mpq/src/archive.rs +++ b/file-formats/archives/wow-mpq/src/archive.rs @@ -1506,58 +1506,6 @@ impl Archive { Ok(entries) } - /// List files in the archive with database lookup for names - pub fn list_with_db(&mut self, db: &crate::database::Database) -> Result> { - use crate::database::HashLookup; - - // Get all entries with hashes - let mut entries = self.list_all_with_hashes()?; - - // Look up names in database - for entry in &mut entries { - if let Some((hash_a, hash_b)) = entry.hashes - && let Ok(Some(filename)) = db.lookup_filename(hash_a, hash_b) - { - entry.name = filename; - } - } - - Ok(entries) - } - - /// Record all filenames from the archive's listfile to the database - pub fn record_listfile_to_db(&mut self, db: &crate::database::Database) -> Result { - use crate::database::HashLookup; - - // Try to find and read (listfile) - if let Some(_listfile_info) = self.find_file("(listfile)")? - && let Ok(listfile_data) = self.read_file("(listfile)") - && let Ok(filenames) = special_files::parse_listfile(&listfile_data) - { - // Record all filenames from listfile to database - let source = format!("archive:{}", self.path.display()); - let filenames_with_source: Vec<(&str, Option<&str>)> = filenames - .iter() - .map(|f| (f.as_str(), Some(source.as_str()))) - .collect(); - - match db.store_filenames(&filenames_with_source) { - Ok((new_count, updated_count)) => { - log::info!( - "Recorded {new_count} new and {updated_count} updated filenames from listfile to database" - ); - return Ok(new_count + updated_count); - } - Err(e) => { - log::error!("Failed to store filenames in database: {e}"); - return Err(Error::Crypto(format!("Database error: {e}"))); - } - } - } - - Ok(0) - } - /// List all files in the archive by enumerating tables with hash information pub fn list_all_with_hashes(&mut self) -> Result> { let mut entries = Vec::new(); diff --git a/file-formats/archives/wow-mpq/src/crypto/mod.rs b/file-formats/archives/wow-mpq/src/crypto/mod.rs index 0e8294bb..9887ce12 100644 --- a/file-formats/archives/wow-mpq/src/crypto/mod.rs +++ b/file-formats/archives/wow-mpq/src/crypto/mod.rs @@ -93,3 +93,26 @@ pub use types::hash_type; pub use keys::{ASCII_TO_LOWER, ASCII_TO_UPPER, ENCRYPTION_TABLE}; // Internal-only exports + +/// Calculate traditional MPQ hash values for a filename. +/// +/// Returns `(hash_a, hash_b, hash_offset)` after normalizing the filename +/// (converting `/` to `\` and uppercasing). +pub fn calculate_mpq_hashes(filename: &str) -> (u32, u32, u32) { + let normalized = filename.replace('/', "\\").to_uppercase(); + let hash_a = hash_string(&normalized, hash_type::NAME_A); + let hash_b = hash_string(&normalized, hash_type::NAME_B); + let hash_offset = hash_string(&normalized, hash_type::TABLE_OFFSET); + (hash_a, hash_b, hash_offset) +} + +/// Calculate HET (Hash Extended Table) hash values for a filename. +/// +/// Returns `(file_hash, name_hash)` used by HET tables. The filename is +/// normalized (converting `/` to `\`) but not uppercased, as HET hashes +/// are case-sensitive. +pub fn calculate_het_hashes(filename: &str, hash_bits: u8) -> (u64, u64) { + let normalized = filename.replace('/', "\\"); + let (file_hash, name_hash_u8) = het_hash(&normalized, hash_bits as u32); + (file_hash, name_hash_u8 as u64) +} diff --git a/file-formats/archives/wow-mpq/src/database/lookup.rs b/file-formats/archives/wow-mpq/src/database/lookup.rs deleted file mode 100644 index 19e942cd..00000000 --- a/file-formats/archives/wow-mpq/src/database/lookup.rs +++ /dev/null @@ -1,288 +0,0 @@ -//! Hash lookup functionality - -use rusqlite::{OptionalExtension, params}; - -use super::Database; -use crate::database::{calculate_het_hashes, calculate_mpq_hashes}; - -type Result = crate::database::connection::Result; - -/// Trait for traditional MPQ hash lookup operations -pub trait HashLookup { - /// Look up a filename by its traditional MPQ hash values - fn lookup_filename(&self, hash_a: u32, hash_b: u32) -> Result>; - - /// Look up multiple filenames by their hash values - fn lookup_filenames(&self, hashes: &[(u32, u32)]) -> Result)>>; - - /// Store a filename and calculate all its hashes - fn store_filename(&self, filename: &str, source: Option<&str>) -> Result<()>; - - /// Store multiple filenames and return (new_entries, updated_entries) - fn store_filenames(&self, filenames: &[(&str, Option<&str>)]) -> Result<(usize, usize)>; - - /// Check if a filename exists in the database - fn filename_exists(&self, filename: &str) -> Result; - - /// Get statistics about unknown hashes in an archive - fn get_unknown_hashes(&self, hashes: &[(u32, u32)]) -> Result>; -} - -/// Trait for HET hash lookup operations -pub trait HetHashLookup { - /// Look up a filename by its HET hash values - fn lookup_filename_het( - &self, - file_hash: u64, - name_hash: u64, - hash_bits: u8, - ) -> Result>; - - /// Look up multiple filenames by their HET hash values - fn lookup_filenames_het( - &self, - hashes: &[(u64, u64)], - hash_bits: u8, - ) -> Result)>>; -} - -impl HashLookup for Database { - fn lookup_filename(&self, hash_a: u32, hash_b: u32) -> Result> { - let mut stmt = self - .connection() - .prepare("SELECT filename FROM filenames WHERE hash_a = ?1 AND hash_b = ?2 LIMIT 1")?; - - Ok(stmt - .query_row(params![hash_a, hash_b], |row| row.get(0)) - .optional()?) - } - - fn lookup_filenames(&self, hashes: &[(u32, u32)]) -> Result)>> { - if hashes.is_empty() { - return Ok(Vec::new()); - } - - let mut results = Vec::with_capacity(hashes.len()); - let conn = self.connection(); - - // Use a prepared statement for efficiency - let mut stmt = conn - .prepare("SELECT filename FROM filenames WHERE hash_a = ?1 AND hash_b = ?2 LIMIT 1")?; - - for &(hash_a, hash_b) in hashes { - let filename: Option = stmt - .query_row(params![hash_a, hash_b], |row| row.get(0)) - .optional()?; - results.push((hash_a, hash_b, filename)); - } - - Ok(results) - } - - fn store_filename(&self, filename: &str, source: Option<&str>) -> Result<()> { - let (hash_a, hash_b, hash_offset) = calculate_mpq_hashes(filename); - - // Calculate HET hashes for common bit sizes - let het_40 = calculate_het_hashes(filename, 40); - let het_48 = calculate_het_hashes(filename, 48); - let het_56 = calculate_het_hashes(filename, 56); - let het_64 = calculate_het_hashes(filename, 64); - - self.connection().execute( - "INSERT OR REPLACE INTO filenames ( - filename, hash_a, hash_b, hash_offset, - het_hash_40_file, het_hash_40_name, - het_hash_48_file, het_hash_48_name, - het_hash_56_file, het_hash_56_name, - het_hash_64_file, het_hash_64_name, - source - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", - params![ - filename, - hash_a, - hash_b, - hash_offset, - het_40.0 as i64, - het_40.1 as i64, - het_48.0 as i64, - het_48.1 as i64, - het_56.0 as i64, - het_56.1 as i64, - het_64.0 as i64, - het_64.1 as i64, - source - ], - )?; - - Ok(()) - } - - fn store_filenames(&self, filenames: &[(&str, Option<&str>)]) -> Result<(usize, usize)> { - if filenames.is_empty() { - return Ok((0, 0)); - } - - let conn = self.connection(); - let mut new_entries = 0; - let mut updated_entries = 0; - - // Check existing entries first - let mut check_stmt = conn.prepare("SELECT 1 FROM filenames WHERE filename = ?1")?; - - // Use a prepared statement for insert/update - let mut stmt = conn.prepare( - "INSERT OR REPLACE INTO filenames ( - filename, hash_a, hash_b, hash_offset, - het_hash_40_file, het_hash_40_name, - het_hash_48_file, het_hash_48_name, - het_hash_56_file, het_hash_56_name, - het_hash_64_file, het_hash_64_name, - source - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", - )?; - - for (filename, source) in filenames { - // Check if entry exists - let exists: bool = check_stmt - .query_row(params![filename], |_| Ok(true)) - .optional()? - .is_some(); - - let (hash_a, hash_b, hash_offset) = calculate_mpq_hashes(filename); - let het_40 = calculate_het_hashes(filename, 40); - let het_48 = calculate_het_hashes(filename, 48); - let het_56 = calculate_het_hashes(filename, 56); - let het_64 = calculate_het_hashes(filename, 64); - - stmt.execute(params![ - filename, - hash_a, - hash_b, - hash_offset, - het_40.0 as i64, - het_40.1 as i64, - het_48.0 as i64, - het_48.1 as i64, - het_56.0 as i64, - het_56.1 as i64, - het_64.0 as i64, - het_64.1 as i64, - source - ])?; - - if exists { - updated_entries += 1; - } else { - new_entries += 1; - } - } - - Ok((new_entries, updated_entries)) - } - - fn filename_exists(&self, filename: &str) -> Result { - let (hash_a, hash_b, _) = calculate_mpq_hashes(filename); - - let count: i64 = self.connection().query_row( - "SELECT COUNT(*) FROM filenames WHERE hash_a = ?1 AND hash_b = ?2", - params![hash_a, hash_b], - |row| row.get(0), - )?; - - Ok(count > 0) - } - - fn get_unknown_hashes(&self, hashes: &[(u32, u32)]) -> Result> { - let mut unknown = Vec::new(); - - for &(hash_a, hash_b) in hashes { - if self.lookup_filename(hash_a, hash_b)?.is_none() { - unknown.push((hash_a, hash_b)); - } - } - - Ok(unknown) - } -} - -impl HetHashLookup for Database { - fn lookup_filename_het( - &self, - file_hash: u64, - name_hash: u64, - hash_bits: u8, - ) -> Result> { - let column_file = match hash_bits { - 40 => "het_hash_40_file", - 48 => "het_hash_48_file", - 56 => "het_hash_56_file", - 64 => "het_hash_64_file", - _ => return Ok(None), // Unsupported bit size - }; - - let column_name = match hash_bits { - 40 => "het_hash_40_name", - 48 => "het_hash_48_name", - 56 => "het_hash_56_name", - 64 => "het_hash_64_name", - _ => return Ok(None), - }; - - let query = format!( - "SELECT filename FROM filenames WHERE {column_file} = ?1 AND {column_name} = ?2 LIMIT 1" - ); - - let mut stmt = self.connection().prepare(&query)?; - - Ok(stmt - .query_row(params![file_hash as i64, name_hash as i64], |row| { - row.get(0) - }) - .optional()?) - } - - fn lookup_filenames_het( - &self, - hashes: &[(u64, u64)], - hash_bits: u8, - ) -> Result)>> { - if hashes.is_empty() { - return Ok(Vec::new()); - } - - let column_file = match hash_bits { - 40 => "het_hash_40_file", - 48 => "het_hash_48_file", - 56 => "het_hash_56_file", - 64 => "het_hash_64_file", - _ => return Ok(vec![(0, 0, None); hashes.len()]), - }; - - let column_name = match hash_bits { - 40 => "het_hash_40_name", - 48 => "het_hash_48_name", - 56 => "het_hash_56_name", - 64 => "het_hash_64_name", - _ => return Ok(vec![(0, 0, None); hashes.len()]), - }; - - let mut results = Vec::with_capacity(hashes.len()); - let conn = self.connection(); - - let query = format!( - "SELECT filename FROM filenames WHERE {column_file} = ?1 AND {column_name} = ?2 LIMIT 1" - ); - let mut stmt = conn.prepare(&query)?; - - for &(file_hash, name_hash) in hashes { - let filename: Option = stmt - .query_row(params![file_hash as i64, name_hash as i64], |row| { - row.get(0) - }) - .optional()?; - results.push((file_hash, name_hash, filename)); - } - - Ok(results) - } -} diff --git a/file-formats/archives/wow-mpq/src/database/mod.rs b/file-formats/archives/wow-mpq/src/database/mod.rs deleted file mode 100644 index 36dfcd00..00000000 --- a/file-formats/archives/wow-mpq/src/database/mod.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Database functionality for storing and retrieving MPQ filename-to-hash mappings -//! -//! This module provides a persistent SQLite database that records relationships between -//! filenames and their MPQ hash values, enabling file resolution even when archives -//! lack internal listfiles. - -mod connection; -mod import; -mod lookup; -mod models; -mod schema; - -pub use connection::{Database, DatabaseError}; -pub use import::{ImportSource, Importer}; -pub use lookup::{HashLookup, HetHashLookup}; -pub use models::{ArchiveRecord, FileRecord, HashType}; - -use crate::crypto::{hash_string, hash_type, het_hash}; - -/// Calculate traditional MPQ hash values for a filename -pub fn calculate_mpq_hashes(filename: &str) -> (u32, u32, u32) { - let normalized = filename.replace('/', "\\").to_uppercase(); - let hash_a = hash_string(&normalized, hash_type::NAME_A); - let hash_b = hash_string(&normalized, hash_type::NAME_B); - let hash_offset = hash_string(&normalized, hash_type::TABLE_OFFSET); - (hash_a, hash_b, hash_offset) -} - -/// Calculate HET (Hash Extended Table) hash values for a filename -/// Returns (file_hash, name_hash) used by HET tables -pub fn calculate_het_hashes(filename: &str, hash_bits: u8) -> (u64, u64) { - let normalized = filename.replace('/', "\\"); - let (file_hash, name_hash_u8) = het_hash(&normalized, hash_bits as u32); - (file_hash, name_hash_u8 as u64) -} diff --git a/file-formats/archives/wow-mpq/src/lib.rs b/file-formats/archives/wow-mpq/src/lib.rs index 46986956..a47d6c7d 100644 --- a/file-formats/archives/wow-mpq/src/lib.rs +++ b/file-formats/archives/wow-mpq/src/lib.rs @@ -127,7 +127,6 @@ pub mod builder; pub mod compare; pub mod compression; pub mod crypto; -pub mod database; pub mod error; pub mod header; pub mod io; @@ -172,7 +171,8 @@ pub use tables::{BetFileInfo, BetTable, BlockEntry, BlockTable, HashEntry, HashT // Re-export crypto for CLI usage pub use crypto::{ - decrypt_block, decrypt_dword, encrypt_block, hash_string, hash_type, jenkins_hash, + calculate_het_hashes, calculate_mpq_hashes, decrypt_block, decrypt_dword, encrypt_block, + hash_string, hash_type, jenkins_hash, }; // Re-export compression for testing diff --git a/warcraft-rs/Cargo.toml b/warcraft-rs/Cargo.toml index d5d3fa27..f95ebdb9 100644 --- a/warcraft-rs/Cargo.toml +++ b/warcraft-rs/Cargo.toml @@ -46,6 +46,12 @@ console = "0.15" chrono = "0.4" glob = "0.3" +# Async runtime +tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +# Database (only needed with mpq feature) +turso = { version = "0.4", default-features = false, optional = true } +directories = { version = "5.0", optional = true } + # Image processing (for BLP) image = { version = "0.25", optional = true } @@ -73,7 +79,7 @@ full = [ "parallel", "yaml", ] -mpq = [] +mpq = ["dep:turso", "dep:directories"] dbc = ["dep:wow-cdbc"] blp = ["dep:wow-blp", "dep:image"] m2 = ["dep:wow-m2"] diff --git a/warcraft-rs/src/commands/mpq.rs b/warcraft-rs/src/commands/mpq.rs index 0500fa84..8c8a37b1 100644 --- a/warcraft-rs/src/commands/mpq.rs +++ b/warcraft-rs/src/commands/mpq.rs @@ -387,7 +387,7 @@ pub enum ImportSourceArg { Directory, } -pub fn execute(command: MpqCommands) -> Result<()> { +pub async fn execute(command: MpqCommands) -> Result<()> { match command { MpqCommands::List { archive, @@ -396,7 +396,7 @@ pub fn execute(command: MpqCommands) -> Result<()> { use_db, record_to_db, show_patches, - } => list_archive(&archive, long, filter, use_db, record_to_db, show_patches), + } => list_archive(&archive, long, filter, use_db, record_to_db, show_patches).await, MpqCommands::Extract { archive, output, @@ -516,11 +516,11 @@ pub fn execute(command: MpqCommands) -> Result<()> { patches, detailed, } => visualize_patch_chain(&base, patches, detailed), - MpqCommands::Db(db_command) => execute_db_command(db_command), + MpqCommands::Db(db_command) => execute_db_command(db_command).await, } } -fn list_archive( +async fn list_archive( path: &str, long: bool, filter: Option, @@ -528,7 +528,7 @@ fn list_archive( record_to_db: bool, show_patches: bool, ) -> Result<()> { - use wow_mpq::database::Database; + use crate::database::Database; let spinner = create_spinner("Opening archive..."); let mut archive = Archive::open(path).context("Failed to open archive")?; @@ -536,14 +536,18 @@ fn list_archive( // Open database if needed let db = if use_db || record_to_db { - Some(Database::open_default().context("Failed to open database")?) + Some( + Database::open_default() + .await + .context("Failed to open database")?, + ) } else { None }; // Record filenames to database if requested if record_to_db && let Some(ref db) = db { - let count = archive.record_listfile_to_db(db)?; + let count = record_listfile_to_db(&mut archive, db).await?; if count > 0 { println!("Recorded {count} filenames to database"); } @@ -552,7 +556,7 @@ fn list_archive( // Get file list let entries = if use_db { if let Some(ref db) = db { - archive.list_with_db(db)? + list_with_db(&mut archive, db).await? } else { archive.list()? } @@ -614,6 +618,58 @@ fn list_archive( Ok(()) } +/// List files in an archive with database lookup for names +async fn list_with_db( + archive: &mut Archive, + db: &crate::database::Database, +) -> Result> { + use crate::database::HashLookup; + + let mut entries = archive.list_all_with_hashes()?; + + for entry in &mut entries { + if let Some((hash_a, hash_b)) = entry.hashes + && let Ok(Some(filename)) = db.lookup_filename(hash_a, hash_b).await + { + entry.name = filename; + } + } + + Ok(entries) +} + +/// Record all filenames from an archive's listfile to the database +async fn record_listfile_to_db( + archive: &mut Archive, + db: &crate::database::Database, +) -> Result { + use crate::database::HashLookup; + + if archive.find_file("(listfile)")?.is_some() + && let Ok(listfile_data) = archive.read_file("(listfile)") + && let Ok(filenames) = wow_mpq::special_files::parse_listfile(&listfile_data) + { + let source = format!("archive:{}", archive.path().display()); + let filenames_with_source: Vec<(&str, Option<&str>)> = filenames + .iter() + .map(|f| (f.as_str(), Some(source.as_str()))) + .collect(); + + match db.store_filenames(&filenames_with_source).await { + Ok((new_count, updated_count)) => { + log::info!("Recorded {new_count} new and {updated_count} updated filenames"); + return Ok(new_count + updated_count); + } + Err(e) => { + log::error!("Failed to store filenames in database: {e}"); + anyhow::bail!("Database error: {e}"); + } + } + } + + Ok(0) +} + struct ExtractOptions { archive_path: String, output_dir: String, @@ -2041,19 +2097,26 @@ fn show_entry_at_index(archive: &mut Archive, index: usize, _raw_dump: bool) -> } // Database command implementation -fn execute_db_command(command: DbCommands) -> Result<()> { +async fn execute_db_command(command: DbCommands) -> Result<()> { + use crate::database::{Database, HashLookup, ImportSource, Importer}; + use crate::database::{calculate_het_hashes, calculate_mpq_hashes}; use std::io::Write; - use wow_mpq::database::{Database, HashLookup, ImportSource, Importer}; - use wow_mpq::database::{calculate_het_hashes, calculate_mpq_hashes}; match command { DbCommands::Status { detailed } => { - let db = Database::open_default().context("Failed to open database")?; + let db = Database::open_default() + .await + .context("Failed to open database")?; let conn = db.connection(); // Get basic statistics - let filename_count: i64 = - conn.query_row("SELECT COUNT(*) FROM filenames", [], |row| row.get(0))?; + let filename_count: i64 = { + let mut rows = conn.query("SELECT COUNT(*) FROM filenames", ()).await?; + match rows.next().await? { + Some(row) => row.get(0)?, + None => 0, + } + }; println!("MPQ Hash Database Status"); println!("========================"); @@ -2064,16 +2127,17 @@ fn execute_db_command(command: DbCommands) -> Result<()> { println!("\nDetailed Statistics:"); // Count by source - let mut stmt = conn.prepare( - "SELECT source, COUNT(*) FROM filenames GROUP BY source ORDER BY COUNT(*) DESC", - )?; - let sources = stmt.query_map([], |row| { - Ok((row.get::<_, Option>(0)?, row.get::<_, i64>(1)?)) - })?; + let mut rows = conn + .query( + "SELECT source, COUNT(*) FROM filenames GROUP BY source ORDER BY COUNT(*) DESC", + (), + ) + .await?; println!("\nFilenames by source:"); - for source in sources { - let (src, count) = source?; + while let Some(row) = rows.next().await? { + let src: Option = row.get(0).ok(); + let count: i64 = row.get(1)?; println!( " {}: {}", src.unwrap_or_else(|| "unknown".to_string()), @@ -2082,16 +2146,17 @@ fn execute_db_command(command: DbCommands) -> Result<()> { } // Recent additions - let mut stmt = conn.prepare( - "SELECT filename, created_at FROM filenames ORDER BY created_at DESC LIMIT 10", - )?; - let recent = stmt.query_map([], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) - })?; + let mut rows = conn + .query( + "SELECT filename, created_at FROM filenames ORDER BY created_at DESC LIMIT 10", + (), + ) + .await?; println!("\nMost recent additions:"); - for entry in recent { - let (filename, created_at) = entry?; + while let Some(row) = rows.next().await? { + let filename: String = row.get(0)?; + let created_at: String = row.get(1)?; println!(" {filename} - {created_at}"); } } @@ -2104,7 +2169,9 @@ fn execute_db_command(command: DbCommands) -> Result<()> { source_type, show_progress, } => { - let db = Database::open_default().context("Failed to open database")?; + let db = Database::open_default() + .await + .context("Failed to open database")?; let importer = Importer::new(&db); let import_source = match source_type { @@ -2121,6 +2188,7 @@ fn execute_db_command(command: DbCommands) -> Result<()> { let stats = importer .import(Path::new(&path), import_source) + .await .context("Import failed")?; if let Some(s) = spinner { @@ -2142,13 +2210,15 @@ fn execute_db_command(command: DbCommands) -> Result<()> { archive, include_anonymous, } => { - let db = Database::open_default().context("Failed to open database")?; + let db = Database::open_default() + .await + .context("Failed to open database")?; let mut mpq = Archive::open(&archive).context("Failed to open archive")?; let spinner = create_spinner("Analyzing archive..."); // Record listfile entries - let count = mpq.record_listfile_to_db(&db)?; + let count = record_listfile_to_db(&mut mpq, &db).await?; spinner.finish_with_message(format!("Recorded {count} filenames from listfile")); @@ -2161,7 +2231,9 @@ fn execute_db_command(command: DbCommands) -> Result<()> { } DbCommands::Lookup { filename } => { - let db = Database::open_default().context("Failed to open database")?; + let db = Database::open_default() + .await + .context("Failed to open database")?; // Calculate and display hashes let (hash_a, hash_b, hash_offset) = calculate_mpq_hashes(&filename); @@ -2195,7 +2267,7 @@ fn execute_db_command(command: DbCommands) -> Result<()> { ); // Check if it exists in database - if db.filename_exists(&filename)? { + if db.filename_exists(&filename).await? { println!("\n✓ Filename exists in database"); } else { println!("\n✗ Filename not found in database"); @@ -2205,36 +2277,42 @@ fn execute_db_command(command: DbCommands) -> Result<()> { } DbCommands::Export { output, source } => { - let db = Database::open_default().context("Failed to open database")?; + let db = Database::open_default() + .await + .context("Failed to open database")?; let conn = db.connection(); - let mut query = "SELECT DISTINCT filename FROM filenames".to_string(); - let mut params: Vec = Vec::new(); + let mut file = fs::File::create(&output).context("Failed to create output file")?; + let mut count = 0; if let Some(src) = source { - query.push_str(" WHERE source = ?1"); - params.push(src); - } - - query.push_str(" ORDER BY filename"); - - let mut stmt = conn.prepare(&query)?; - let filenames: Vec = if params.is_empty() { - stmt.query_map([], |row| row.get::<_, String>(0))? - .collect::, _>>()? + let mut rows = conn + .query( + "SELECT DISTINCT filename FROM filenames WHERE source = ?1 ORDER BY filename", + turso::params![src], + ) + .await?; + + while let Some(row) = rows.next().await? { + let filename: String = row.get(0)?; + writeln!(file, "{filename}")?; + count += 1; + } } else { - stmt.query_map([¶ms[0]], |row| row.get::<_, String>(0))? - .collect::, _>>()? + let mut rows = conn + .query( + "SELECT DISTINCT filename FROM filenames ORDER BY filename", + (), + ) + .await?; + + while let Some(row) = rows.next().await? { + let filename: String = row.get(0)?; + writeln!(file, "{filename}")?; + count += 1; + } }; - let mut file = fs::File::create(&output).context("Failed to create output file")?; - let mut count = 0; - - for filename in filenames { - writeln!(file, "{filename}")?; - count += 1; - } - println!("Exported {count} filenames to {output}"); Ok(()) @@ -2245,58 +2323,60 @@ fn execute_db_command(command: DbCommands) -> Result<()> { long, limit, } => { - let db = Database::open_default().context("Failed to open database")?; + let db = Database::open_default() + .await + .context("Failed to open database")?; let conn = db.connection(); - let mut query = "SELECT filename, hash_a, hash_b, source FROM filenames".to_string(); - let mut params: Vec = Vec::new(); + let mut result_rows: Vec<(String, i64, i64, Option)> = Vec::new(); if let Some(pattern) = filter { - query.push_str(" WHERE filename LIKE ?1"); - params.push(pattern.replace('*', "%")); - } - - query.push_str(&format!(" ORDER BY filename LIMIT {limit}")); - - let mut stmt = conn.prepare(&query)?; - let rows: Vec<(String, u32, u32, Option)> = if params.is_empty() { - stmt.query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, u32>(1)?, - row.get::<_, u32>(2)?, - row.get::<_, Option>(3)?, - )) - })? - .collect::, _>>()? + let like_pattern = pattern.replace('*', "%"); + let query = format!( + "SELECT filename, hash_a, hash_b, source FROM filenames WHERE filename LIKE ?1 ORDER BY filename LIMIT {limit}" + ); + let mut rows = conn.query(&query, turso::params![like_pattern]).await?; + + while let Some(row) = rows.next().await? { + result_rows.push(( + row.get::(0)?, + row.get::(1)?, + row.get::(2)?, + row.get::>(3).ok().flatten(), + )); + } } else { - stmt.query_map([¶ms[0]], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, u32>(1)?, - row.get::<_, u32>(2)?, - row.get::<_, Option>(3)?, - )) - })? - .collect::, _>>()? - }; + let query = format!( + "SELECT filename, hash_a, hash_b, source FROM filenames ORDER BY filename LIMIT {limit}" + ); + let mut rows = conn.query(&query, ()).await?; + + while let Some(row) = rows.next().await? { + result_rows.push(( + row.get::(0)?, + row.get::(1)?, + row.get::(2)?, + row.get::>(3).ok().flatten(), + )); + } + } if long { let mut table = create_table(vec!["Filename", "Hash A", "Hash B", "Source"]); - for (filename, hash_a, hash_b, source) in rows { + for (filename, hash_a, hash_b, source) in result_rows { add_table_row( &mut table, vec![ filename, - format!("0x{:08X}", hash_a), - format!("0x{:08X}", hash_b), + format!("0x{hash_a:08X}"), + format!("0x{hash_b:08X}"), source.unwrap_or_else(|| "unknown".to_string()), ], ); } println!("{table}"); } else { - for (filename, _, _, _) in rows { + for (filename, _, _, _) in result_rows { println!("{filename}"); } } diff --git a/file-formats/archives/wow-mpq/src/database/connection.rs b/warcraft-rs/src/database/connection.rs similarity index 61% rename from file-formats/archives/wow-mpq/src/database/connection.rs rename to warcraft-rs/src/database/connection.rs index 67399aa6..db0e71f9 100644 --- a/file-formats/archives/wow-mpq/src/database/connection.rs +++ b/warcraft-rs/src/database/connection.rs @@ -1,7 +1,6 @@ -//! Database connection management +//! Database connection management using turso (async SQLite) use directories::ProjectDirs; -use rusqlite::{Connection, Result as SqlResult}; use std::path::{Path, PathBuf}; use thiserror::Error; @@ -10,9 +9,9 @@ use super::schema; /// Errors that can occur during database operations #[derive(Error, Debug)] pub enum DatabaseError { - /// SQLite database error - #[error("SQLite error: {0}")] - Sqlite(#[from] rusqlite::Error), + /// Database error from turso + #[error("Database error: {0}")] + Database(#[from] turso::Error), /// Filesystem I/O error #[error("I/O error: {0}")] @@ -25,31 +24,41 @@ pub enum DatabaseError { pub(super) type Result = std::result::Result; -/// Database connection wrapper +/// Database connection wrapper. +/// +/// Holds both the `turso::Database` (which must outlive the connection) and +/// a `turso::Connection` used for all queries. #[derive(Debug)] pub struct Database { - conn: Connection, + /// The turso database handle -- must outlive `conn`. + _db: turso::Database, + conn: turso::Connection, path: PathBuf, } impl Database { /// Open or create a database at the default location - pub fn open_default() -> Result { + pub async fn open_default() -> Result { let path = Self::default_path()?; - Self::open(&path) + Self::open(&path).await } /// Open or create a database at the specified path - pub fn open(path: &Path) -> Result { + pub async fn open(path: &Path) -> Result { // Ensure parent directory exists if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } - let conn = Connection::open(path)?; - schema::init_schema(&conn)?; + let db = turso::Builder::new_local(&path.to_string_lossy()) + .build() + .await?; + let conn = db.connect()?; + + schema::init_schema(&conn).await?; Ok(Self { + _db: db, conn, path: path.to_path_buf(), }) @@ -66,27 +75,12 @@ impl Database { } /// Get a reference to the underlying connection - pub fn connection(&self) -> &Connection { + pub fn connection(&self) -> &turso::Connection { &self.conn } - /// Get a mutable reference to the underlying connection - pub fn connection_mut(&mut self) -> &mut Connection { - &mut self.conn - } - /// Get the database file path pub fn path(&self) -> &Path { &self.path } - - /// Begin a transaction - pub fn transaction(&mut self) -> SqlResult> { - self.conn.transaction() - } - - /// Execute a batch of SQL statements - pub fn execute_batch(&self, sql: &str) -> SqlResult<()> { - self.conn.execute_batch(sql) - } } diff --git a/file-formats/archives/wow-mpq/src/database/import.rs b/warcraft-rs/src/database/import.rs similarity index 83% rename from file-formats/archives/wow-mpq/src/database/import.rs rename to warcraft-rs/src/database/import.rs index 39e01ecf..024b862c 100644 --- a/file-formats/archives/wow-mpq/src/database/import.rs +++ b/warcraft-rs/src/database/import.rs @@ -1,15 +1,14 @@ //! Import functionality for populating the database -use glob; use std::fs::File; use std::io::{BufRead, BufReader}; use std::path::Path; use thiserror::Error; +use super::lookup::HashLookup; use super::{Database, DatabaseError}; -use crate::Archive; -use crate::database::HashLookup; +use wow_mpq::Archive; #[derive(Error, Debug)] pub enum ImportError { @@ -20,7 +19,7 @@ pub enum ImportError { Io(#[from] std::io::Error), #[error("Archive error: {0}")] - Archive(#[from] crate::Error), + Archive(#[from] wow_mpq::Error), } pub(super) type ImportResult = std::result::Result; @@ -58,16 +57,20 @@ impl<'a> Importer<'a> { } /// Import filenames from a source - pub fn import(&self, path: &Path, source_type: ImportSource) -> ImportResult { + pub async fn import( + &self, + path: &Path, + source_type: ImportSource, + ) -> ImportResult { match source_type { - ImportSource::Listfile => self.import_listfile(path), - ImportSource::Archive => self.import_archive(path), - ImportSource::Directory => self.import_directory(path), + ImportSource::Listfile => self.import_listfile(path).await, + ImportSource::Archive => self.import_archive(path).await, + ImportSource::Directory => self.import_directory(path).await, } } /// Import from a listfile - fn import_listfile(&self, path: &Path) -> ImportResult { + async fn import_listfile(&self, path: &Path) -> ImportResult { let mut stats = ImportStats::default(); let file = File::open(path)?; let reader = BufReader::new(file); @@ -80,14 +83,14 @@ impl<'a> Importer<'a> { match line { Ok(filename) => { - let filename = filename.trim(); + let filename = filename.trim().to_string(); if !filename.is_empty() && !filename.starts_with('#') { - batch.push((filename.to_string(), Some(source.clone()))); + batch.push((filename, Some(source.clone()))); // Process in batches if batch.len() >= 1000 { - match self.process_batch(&mut batch, &mut stats) { - Ok(_) => {} + match self.process_batch(&mut batch, &mut stats).await { + Ok(()) => {} Err(e) => { log::error!("Error processing batch: {e}"); stats.errors += batch.len(); @@ -105,19 +108,19 @@ impl<'a> Importer<'a> { // Process remaining if !batch.is_empty() { - self.process_batch(&mut batch, &mut stats)?; + self.process_batch(&mut batch, &mut stats).await?; } Ok(stats) } /// Import from an MPQ archive - fn import_archive(&self, path: &Path) -> ImportResult { + async fn import_archive(&self, path: &Path) -> ImportResult { let mut stats = ImportStats::default(); let mut archive = Archive::open(path)?; let source = format!("archive:{}", path.display()); - // Get list of files from the archive + // Get list of files from the archive (sync I/O) let files = archive.list()?; stats.files_processed = files.len(); @@ -128,21 +131,21 @@ impl<'a> Importer<'a> { batch.push((file.name, Some(source.clone()))); if batch.len() >= 1000 { - self.process_batch(&mut batch, &mut stats)?; + self.process_batch(&mut batch, &mut stats).await?; } } } // Process remaining if !batch.is_empty() { - self.process_batch(&mut batch, &mut stats)?; + self.process_batch(&mut batch, &mut stats).await?; } Ok(stats) } /// Import from a directory (scan for WoW file patterns) - fn import_directory(&self, path: &Path) -> ImportResult { + async fn import_directory(&self, path: &Path) -> ImportResult { let mut stats = ImportStats::default(); let source = format!("directory:{}", path.display()); @@ -188,7 +191,7 @@ impl<'a> Importer<'a> { batch.push((filename.to_string(), Some(source.clone()))); if batch.len() >= 1000 { - self.process_batch(&mut batch, &mut stats)?; + self.process_batch(&mut batch, &mut stats).await?; } } } @@ -197,14 +200,14 @@ impl<'a> Importer<'a> { // Process remaining if !batch.is_empty() { - self.process_batch(&mut batch, &mut stats)?; + self.process_batch(&mut batch, &mut stats).await?; } Ok(stats) } /// Process a batch of filenames - fn process_batch( + async fn process_batch( &self, batch: &mut Vec<(String, Option)>, stats: &mut ImportStats, @@ -214,7 +217,7 @@ impl<'a> Importer<'a> { .map(|(f, s)| (f.as_str(), s.as_deref())) .collect(); - match self.db.store_filenames(&filenames) { + match self.db.store_filenames(&filenames).await { Ok((new_count, updated_count)) => { stats.new_entries += new_count; stats.updated_entries += updated_count; diff --git a/warcraft-rs/src/database/lookup.rs b/warcraft-rs/src/database/lookup.rs new file mode 100644 index 00000000..e0acec9a --- /dev/null +++ b/warcraft-rs/src/database/lookup.rs @@ -0,0 +1,290 @@ +//! Hash lookup functionality + +use wow_mpq::{calculate_het_hashes, calculate_mpq_hashes}; + +use super::Database; + +type Result = super::connection::Result; + +// These traits are only used within this crate, so auto-trait bounds on the +// returned futures are not a concern. + +/// Trait for traditional MPQ hash lookup operations +#[allow(async_fn_in_trait, dead_code)] +pub trait HashLookup { + /// Look up a filename by its traditional MPQ hash values + async fn lookup_filename(&self, hash_a: u32, hash_b: u32) -> Result>; + + /// Look up multiple filenames by their hash values + async fn lookup_filenames( + &self, + hashes: &[(u32, u32)], + ) -> Result)>>; + + /// Store a filename and calculate all its hashes + async fn store_filename(&self, filename: &str, source: Option<&str>) -> Result<()>; + + /// Store multiple filenames and return (new_entries, updated_entries) + async fn store_filenames(&self, filenames: &[(&str, Option<&str>)]) -> Result<(usize, usize)>; + + /// Check if a filename exists in the database + async fn filename_exists(&self, filename: &str) -> Result; + + /// Get hashes that have no known filename in the database + async fn get_unknown_hashes(&self, hashes: &[(u32, u32)]) -> Result>; +} + +/// Trait for HET hash lookup operations +#[allow(async_fn_in_trait, dead_code)] +pub trait HetHashLookup { + /// Look up a filename by its HET hash values + async fn lookup_filename_het( + &self, + file_hash: u64, + name_hash: u64, + hash_bits: u8, + ) -> Result>; + + /// Look up multiple filenames by their HET hash values + async fn lookup_filenames_het( + &self, + hashes: &[(u64, u64)], + hash_bits: u8, + ) -> Result)>>; +} + +impl HashLookup for Database { + async fn lookup_filename(&self, hash_a: u32, hash_b: u32) -> Result> { + let conn = self.connection(); + let mut rows = conn + .query( + "SELECT filename FROM filenames WHERE hash_a = ?1 AND hash_b = ?2 LIMIT 1", + turso::params![i64::from(hash_a), i64::from(hash_b)], + ) + .await?; + + match rows.next().await? { + Some(row) => Ok(Some(row.get::(0)?)), + None => Ok(None), + } + } + + async fn lookup_filenames( + &self, + hashes: &[(u32, u32)], + ) -> Result)>> { + if hashes.is_empty() { + return Ok(Vec::new()); + } + + let mut results = Vec::with_capacity(hashes.len()); + + for &(hash_a, hash_b) in hashes { + let filename = self.lookup_filename(hash_a, hash_b).await?; + results.push((hash_a, hash_b, filename)); + } + + Ok(results) + } + + async fn store_filename(&self, filename: &str, source: Option<&str>) -> Result<()> { + let (hash_a, hash_b, hash_offset) = calculate_mpq_hashes(filename); + + // Calculate HET hashes for common bit sizes + let het_40 = calculate_het_hashes(filename, 40); + let het_48 = calculate_het_hashes(filename, 48); + let het_56 = calculate_het_hashes(filename, 56); + let het_64 = calculate_het_hashes(filename, 64); + + self.connection() + .execute( + "INSERT OR REPLACE INTO filenames ( + filename, hash_a, hash_b, hash_offset, + het_hash_40_file, het_hash_40_name, + het_hash_48_file, het_hash_48_name, + het_hash_56_file, het_hash_56_name, + het_hash_64_file, het_hash_64_name, + source + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", + turso::params![ + filename, + i64::from(hash_a), + i64::from(hash_b), + i64::from(hash_offset), + het_40.0 as i64, + het_40.1 as i64, + het_48.0 as i64, + het_48.1 as i64, + het_56.0 as i64, + het_56.1 as i64, + het_64.0 as i64, + het_64.1 as i64, + source + ], + ) + .await?; + + Ok(()) + } + + async fn store_filenames(&self, filenames: &[(&str, Option<&str>)]) -> Result<(usize, usize)> { + if filenames.is_empty() { + return Ok((0, 0)); + } + + let conn = self.connection(); + let mut total_affected: usize = 0; + + // Wrap all inserts in a single transaction to avoid per-row fsync overhead. + conn.execute("BEGIN", ()).await?; + + let result = async { + // Prepare the statement once and reuse it for all rows. + let mut stmt = conn + .prepare_cached( + "INSERT OR REPLACE INTO filenames ( + filename, hash_a, hash_b, hash_offset, + het_hash_40_file, het_hash_40_name, + het_hash_48_file, het_hash_48_name, + het_hash_56_file, het_hash_56_name, + het_hash_64_file, het_hash_64_name, + source + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", + ) + .await?; + + for &(filename, source) in filenames { + let (hash_a, hash_b, hash_offset) = calculate_mpq_hashes(filename); + let het_40 = calculate_het_hashes(filename, 40); + let het_48 = calculate_het_hashes(filename, 48); + let het_56 = calculate_het_hashes(filename, 56); + let het_64 = calculate_het_hashes(filename, 64); + + let affected = stmt + .execute(turso::params![ + filename, + i64::from(hash_a), + i64::from(hash_b), + i64::from(hash_offset), + het_40.0 as i64, + het_40.1 as i64, + het_48.0 as i64, + het_48.1 as i64, + het_56.0 as i64, + het_56.1 as i64, + het_64.0 as i64, + het_64.1 as i64, + source + ]) + .await?; + + total_affected += affected as usize; + } + Ok::<_, super::connection::DatabaseError>(()) + } + .await; + + match result { + Ok(()) => { + conn.execute("COMMIT", ()).await?; + } + Err(e) => { + let _ = conn.execute("ROLLBACK", ()).await; + return Err(e); + } + } + + // INSERT OR REPLACE doesn't distinguish new vs updated at the SQL level, + // so we report all as new_entries for simplicity. + Ok((total_affected, 0)) + } + + async fn filename_exists(&self, filename: &str) -> Result { + let (hash_a, hash_b, _) = calculate_mpq_hashes(filename); + + let mut rows = self + .connection() + .query( + "SELECT COUNT(*) FROM filenames WHERE hash_a = ?1 AND hash_b = ?2", + turso::params![i64::from(hash_a), i64::from(hash_b)], + ) + .await?; + + match rows.next().await? { + Some(row) => { + let count: i64 = row.get(0)?; + Ok(count > 0) + } + None => Ok(false), + } + } + + async fn get_unknown_hashes(&self, hashes: &[(u32, u32)]) -> Result> { + let mut unknown = Vec::new(); + + for &(hash_a, hash_b) in hashes { + if self.lookup_filename(hash_a, hash_b).await?.is_none() { + unknown.push((hash_a, hash_b)); + } + } + + Ok(unknown) + } +} + +impl HetHashLookup for Database { + async fn lookup_filename_het( + &self, + file_hash: u64, + name_hash: u64, + hash_bits: u8, + ) -> Result> { + let (column_file, column_name) = match hash_bits { + 40 => ("het_hash_40_file", "het_hash_40_name"), + 48 => ("het_hash_48_file", "het_hash_48_name"), + 56 => ("het_hash_56_file", "het_hash_56_name"), + 64 => ("het_hash_64_file", "het_hash_64_name"), + _ => return Ok(None), + }; + + let query = format!( + "SELECT filename FROM filenames WHERE {column_file} = ?1 AND {column_name} = ?2 LIMIT 1" + ); + + let mut rows = self + .connection() + .query(&query, turso::params![file_hash as i64, name_hash as i64]) + .await?; + + match rows.next().await? { + Some(row) => Ok(Some(row.get::(0)?)), + None => Ok(None), + } + } + + async fn lookup_filenames_het( + &self, + hashes: &[(u64, u64)], + hash_bits: u8, + ) -> Result)>> { + if hashes.is_empty() { + return Ok(Vec::new()); + } + + let valid_bits = matches!(hash_bits, 40 | 48 | 56 | 64); + if !valid_bits { + return Ok(hashes.iter().map(|_| (0, 0, None)).collect()); + } + + let mut results = Vec::with_capacity(hashes.len()); + + for &(file_hash, name_hash) in hashes { + let filename = self + .lookup_filename_het(file_hash, name_hash, hash_bits) + .await?; + results.push((file_hash, name_hash, filename)); + } + + Ok(results) + } +} diff --git a/warcraft-rs/src/database/mod.rs b/warcraft-rs/src/database/mod.rs new file mode 100644 index 00000000..49853f53 --- /dev/null +++ b/warcraft-rs/src/database/mod.rs @@ -0,0 +1,21 @@ +//! Database functionality for storing and retrieving MPQ filename-to-hash mappings +//! +//! This module provides a persistent SQLite database (via turso) that records +//! relationships between filenames and their MPQ hash values, enabling file +//! resolution even when archives lack internal listfiles. + +mod connection; +mod import; +mod lookup; +mod models; +mod schema; + +pub use connection::{Database, DatabaseError}; +pub use import::{ImportSource, Importer}; +#[allow(unused_imports)] +pub use lookup::{HashLookup, HetHashLookup}; +#[allow(unused_imports)] +pub use models::{ArchiveRecord, FileRecord, HashType}; + +// Re-export hash computation functions from wow-mpq +pub use wow_mpq::{calculate_het_hashes, calculate_mpq_hashes}; diff --git a/file-formats/archives/wow-mpq/src/database/models.rs b/warcraft-rs/src/database/models.rs similarity index 75% rename from file-formats/archives/wow-mpq/src/database/models.rs rename to warcraft-rs/src/database/models.rs index 28ffa49e..594fbe6e 100644 --- a/file-formats/archives/wow-mpq/src/database/models.rs +++ b/warcraft-rs/src/database/models.rs @@ -3,6 +3,7 @@ use chrono::{DateTime, Utc}; /// Type of hash used in the archive +#[allow(dead_code)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HashType { /// Traditional MPQ hash table @@ -12,6 +13,7 @@ pub enum HashType { } /// Represents a filename and all its associated hashes +#[allow(dead_code)] #[derive(Debug, Clone)] pub struct FileRecord { /// Unique database identifier @@ -39,6 +41,7 @@ pub struct FileRecord { } /// Represents an analyzed MPQ archive +#[allow(dead_code)] #[derive(Debug, Clone)] pub struct ArchiveRecord { /// Unique database identifier @@ -54,26 +57,3 @@ pub struct ArchiveRecord { /// Number of files in the archive pub file_count: Option, } - -/// Represents a file found in an archive (unused, for future functionality) -#[allow(dead_code)] -#[derive(Debug, Clone)] -pub(super) struct ArchiveFileRecord { - pub archive_id: i64, - pub hash_a: u32, - pub hash_b: u32, - pub file_size: Option, - pub compressed_size: Option, - pub flags: Option, - pub filename_id: Option, -} - -/// Statistics about the database (unused, for future functionality) -#[allow(dead_code)] -#[derive(Debug)] -pub(super) struct DatabaseStats { - pub total_filenames: usize, - pub total_archives: usize, - pub total_archive_files: usize, - pub filenames_by_source: Vec<(String, usize)>, -} diff --git a/file-formats/archives/wow-mpq/src/database/schema.rs b/warcraft-rs/src/database/schema.rs similarity index 61% rename from file-formats/archives/wow-mpq/src/database/schema.rs rename to warcraft-rs/src/database/schema.rs index 1b1b6956..9c06af7c 100644 --- a/file-formats/archives/wow-mpq/src/database/schema.rs +++ b/warcraft-rs/src/database/schema.rs @@ -1,14 +1,21 @@ //! Database schema and migration management -use rusqlite::{Connection, Result}; +use turso::Connection; + +use super::connection::Result; /// Current schema version -pub(super) const SCHEMA_VERSION: i32 = 1; +pub(super) const SCHEMA_VERSION: i64 = 1; /// Initialize the database schema -pub(super) fn init_schema(conn: &Connection) -> Result<()> { +pub(super) async fn init_schema(conn: &Connection) -> Result<()> { + // Enable WAL mode for better write concurrency and throughput. + // PRAGMA returns a result row, so use query() rather than execute(). + let _ = conn.query("PRAGMA journal_mode = WAL", ()).await?; + // NORMAL sync is safe with WAL — only syncs at checkpoint, not every commit + let _ = conn.query("PRAGMA synchronous = NORMAL", ()).await?; // Enable foreign keys - conn.execute("PRAGMA foreign_keys = ON", [])?; + conn.execute("PRAGMA foreign_keys = ON", ()).await?; // Create version table conn.execute( @@ -16,26 +23,31 @@ pub(super) fn init_schema(conn: &Connection) -> Result<()> { version INTEGER PRIMARY KEY, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )", - [], - )?; + (), + ) + .await?; // Check current version - let current_version: Option = conn - .query_row("SELECT MAX(version) FROM schema_version", [], |row| { - row.get(0) - }) - .unwrap_or(None); + let current_version: Option = { + let mut rows = conn + .query("SELECT MAX(version) FROM schema_version", ()) + .await?; + match rows.next().await? { + Some(row) => row.get::>(0).ok().flatten(), + None => None, + } + }; // Apply migrations - if current_version.is_none() || current_version.unwrap() < 1 { - migrate_v1(conn)?; + if current_version.is_none() || current_version.is_some_and(|v| v < 1) { + migrate_v1(conn).await?; } Ok(()) } /// Migration to version 1: Initial schema -fn migrate_v1(conn: &Connection) -> Result<()> { +async fn migrate_v1(conn: &Connection) -> Result<()> { // Unified table for all filename-hash mappings conn.execute( "CREATE TABLE IF NOT EXISTS filenames ( @@ -46,19 +58,20 @@ fn migrate_v1(conn: &Connection) -> Result<()> { hash_b INTEGER NOT NULL, hash_offset INTEGER NOT NULL, -- HET hashes (stored for multiple bit sizes) - het_hash_40_file INTEGER, -- 40-bit HET file hash - het_hash_40_name INTEGER, -- 40-bit HET name hash - het_hash_48_file INTEGER, -- 48-bit HET file hash - het_hash_48_name INTEGER, -- 48-bit HET name hash - het_hash_56_file INTEGER, -- 56-bit HET file hash - het_hash_56_name INTEGER, -- 56-bit HET name hash - het_hash_64_file INTEGER, -- 64-bit HET file hash - het_hash_64_name INTEGER, -- 64-bit HET name hash + het_hash_40_file INTEGER, + het_hash_40_name INTEGER, + het_hash_48_file INTEGER, + het_hash_48_name INTEGER, + het_hash_56_file INTEGER, + het_hash_56_name INTEGER, + het_hash_64_file INTEGER, + het_hash_64_name INTEGER, source TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )", - [], - )?; + (), + ) + .await?; // Archive analysis results conn.execute( @@ -70,8 +83,9 @@ fn migrate_v1(conn: &Connection) -> Result<()> { mpq_version INTEGER, file_count INTEGER )", - [], - )?; + (), + ) + .await?; // Files found in archives conn.execute( @@ -85,51 +99,60 @@ fn migrate_v1(conn: &Connection) -> Result<()> { filename_id INTEGER REFERENCES filenames(id), PRIMARY KEY (archive_id, hash_a, hash_b) )", - [], - )?; + (), + ) + .await?; // Create indices for fast lookups conn.execute( "CREATE INDEX IF NOT EXISTS idx_filename_hashes ON filenames(hash_a, hash_b)", - [], - )?; + (), + ) + .await?; conn.execute( "CREATE INDEX IF NOT EXISTS idx_archive_files_hashes ON archive_files(hash_a, hash_b)", - [], - )?; + (), + ) + .await?; conn.execute( "CREATE INDEX IF NOT EXISTS idx_archive_analysis_path ON archive_analysis(archive_path)", - [], - )?; + (), + ) + .await?; // Indices for HET table lookups at different bit sizes conn.execute( "CREATE INDEX IF NOT EXISTS idx_het_40 ON filenames(het_hash_40_file, het_hash_40_name)", - [], - )?; + (), + ) + .await?; conn.execute( "CREATE INDEX IF NOT EXISTS idx_het_48 ON filenames(het_hash_48_file, het_hash_48_name)", - [], - )?; + (), + ) + .await?; conn.execute( "CREATE INDEX IF NOT EXISTS idx_het_56 ON filenames(het_hash_56_file, het_hash_56_name)", - [], - )?; + (), + ) + .await?; conn.execute( "CREATE INDEX IF NOT EXISTS idx_het_64 ON filenames(het_hash_64_file, het_hash_64_name)", - [], - )?; + (), + ) + .await?; // Record migration conn.execute( "INSERT INTO schema_version (version) VALUES (?1)", - [SCHEMA_VERSION], - )?; + turso::params![SCHEMA_VERSION], + ) + .await?; Ok(()) } diff --git a/warcraft-rs/src/lib.rs b/warcraft-rs/src/lib.rs index ff2cac3e..ef4988e3 100644 --- a/warcraft-rs/src/lib.rs +++ b/warcraft-rs/src/lib.rs @@ -4,4 +4,6 @@ pub mod cli; pub mod commands; +#[cfg(feature = "mpq")] +pub mod database; pub mod utils; diff --git a/warcraft-rs/src/main.rs b/warcraft-rs/src/main.rs index 7ff053f5..b6f3cde1 100644 --- a/warcraft-rs/src/main.rs +++ b/warcraft-rs/src/main.rs @@ -2,6 +2,8 @@ mod cli; mod commands; +#[cfg(feature = "mpq")] +mod database; mod utils; use anyhow::Result; @@ -12,7 +14,8 @@ use std::io; use crate::cli::{Cli, Commands}; -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { // Initialize logger env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("warn")).init(); @@ -33,7 +36,7 @@ fn main() -> Result<()> { // Execute command match cli.command { #[cfg(feature = "mpq")] - Commands::Mpq { command } => commands::mpq::execute(command), + Commands::Mpq { command } => commands::mpq::execute(command).await, #[cfg(feature = "dbc")] Commands::Dbc { command } => commands::dbc::execute(command),