diff --git a/coprocessor/fhevm-engine/Cargo.lock b/coprocessor/fhevm-engine/Cargo.lock index 7bbf02f59f..7a97db3b45 100644 --- a/coprocessor/fhevm-engine/Cargo.lock +++ b/coprocessor/fhevm-engine/Cargo.lock @@ -266,9 +266,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alloy" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f63701831729cb154cf0b6945256af46c426074646c98b9d123148ba1d8bde" +checksum = "67031be093311a96afdd146fb5de209ceaf0f347f2284ed71902368cd15a77ff" dependencies = [ "alloy-consensus", "alloy-contract", @@ -304,9 +304,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a3bd0305a44fb457cae77de1e82856eadd42ea3cdf0dae29df32eb3b592979" +checksum = "0cd9d29a6a0bb8d4832ff7685dcbb430011b832f2ccec1af9571a0e75c1f7e9c" dependencies = [ "alloy-eips", "alloy-primitives 1.3.1", @@ -323,15 +323,16 @@ dependencies = [ "rand 0.8.5", "secp256k1", "serde", + "serde_json", "serde_with", "thiserror 2.0.16", ] [[package]] name = "alloy-consensus-any" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a842b4023f571835e62ac39fb8d523d19fcdbacfa70bf796ff96e7e19586f50" +checksum = "ce038cb325f9a85a10fb026fb1b70cb8c62a004d85d22f8516e5d173e3eec612" dependencies = [ "alloy-consensus", "alloy-eips", @@ -343,9 +344,9 @@ dependencies = [ [[package]] name = "alloy-contract" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "591104333286b52b03ec4e8162983e31122b318d21ae2b0900d1e8b51727ad40" +checksum = "a376305e5c3b3285e84a553fa3f9aee4f5f0e1b0aad4944191b843cd8228788d" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -431,9 +432,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cd749c57f38f8cbf433e651179fc5a676255e6b95044f467d49255d2b81725a" +checksum = "4bfec530782b30151e2564edf3c900f1fa6852128b7a993e458e8e3815d8b915" dependencies = [ "alloy-eip2124", "alloy-eip2930", @@ -453,9 +454,9 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d32cbf6c26d7d87e8a4e5925bbce41456e0bbeed95601add3443af277cd604e" +checksum = "956e6a23eb880dd93123e8ebea028584325b9af22f991eec2c499c54c277c073" dependencies = [ "alloy-eips", "alloy-primitives 1.3.1", @@ -504,9 +505,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f614019a029c8fec14ae661aa7d4302e6e66bdbfb869dab40e78dcfba935fc97" +checksum = "be436893c0d1f7a57d1d8f1b6b9af9db04174468410b7e6e1d1893e78110a3bc" dependencies = [ "alloy-primitives 1.3.1", "alloy-sol-types", @@ -519,9 +520,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8b6d58e98803017bbfea01dde96c4d270a29e7aed3beb65c8d28b5ab464e0e" +checksum = "f18959e1a1b40e05578e7a705f65ff4e6b354e38335da4b33ccbee876bde7c26" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -545,9 +546,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db489617bffe14847bf89f175b1c183e5dd7563ef84713936e2c34255cfbd845" +checksum = "1da0037ac546c0cae2eb776bed53687b7bbf776f4e7aa2fea0b8b89e734c319b" dependencies = [ "alloy-consensus", "alloy-eips", @@ -558,9 +559,9 @@ dependencies = [ [[package]] name = "alloy-node-bindings" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50536212c8b686907f66dbac78799b8d39e18558648963594544ac1a05f4306d" +checksum = "fd89c9e72e62d95b51be0b92468282526b37d3d1015f5e31069a35c2e020872f" dependencies = [ "alloy-genesis", "alloy-hardforks", @@ -635,9 +636,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed90278374435e076a04dbddbb6d714bdd518eb274a64dbd70f65701429dd747" +checksum = "4ca97e31bc05bd6d4780254fbb60b16d33b3548d1c657a879fffb0e7ebb642e9" dependencies = [ "alloy-chains", "alloy-consensus", @@ -678,9 +679,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f539a4caaa5496ad54af38f5615adb54cc7b3ec1a42e530e706291cce074f4a" +checksum = "f7bb37096e97de25133cf904e08df2aa72168af64f429e3c43a112649e131930" dependencies = [ "alloy-json-rpc", "alloy-primitives 1.3.1", @@ -722,9 +723,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33732242ca63f107f5f8284190244038905fb233280f4b7c41f641d4f584d40d" +checksum = "dbeeeffa0bb7e95cb79f2b4b46b591763afeccfa9a797183c1b192377ffb6fac" dependencies = [ "alloy-json-rpc", "alloy-primitives 1.3.1", @@ -747,9 +748,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2683049c5f3037d64722902e2c1081f3d45de68696aca0511bbea834905746" +checksum = "a21fe4c370b9e733d884ffd953eb6d654d053b1b22e26ffd591ef597a9e2bc49" dependencies = [ "alloy-primitives 1.3.1", "alloy-rpc-types-anvil", @@ -760,9 +761,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b757081f2a68e683de3731108494fa058036d5651bf10141ec2430bc1315c362" +checksum = "eb44412ed075c19d37698f33213b83f0bf8ccc2d4e928527f2622555a31723de" dependencies = [ "alloy-primitives 1.3.1", "alloy-rpc-types-eth", @@ -772,9 +773,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-any" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f27c0c41a16cd0af4f5dbf791f7be2a60502ca8b0e840e0ad29803fac2d587" +checksum = "65423baf6af0ff356e254d7824b3824aa34d8ca9bd857a4e298f74795cc4b69d" dependencies = [ "alloy-consensus-any", "alloy-rpc-types-eth", @@ -783,9 +784,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f5812f81c3131abc2cd8953dc03c41999e180cff7252abbccaba68676e15027" +checksum = "848f8ea4063bed834443081d77f840f31075f68d0d49723027f5a209615150bf" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -804,9 +805,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dfe41a47805a34b848c83448946ca96f3d36842e8c074bcf8fa0870e337d12" +checksum = "19c3835bdc128f2f3418f5d6c76aec63a245d72973e0eaacc9720aa0787225c5" dependencies = [ "alloy-primitives 1.3.1", "serde", @@ -815,9 +816,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79237b4c1b0934d5869deea4a54e6f0a7425a8cd943a739d6293afdf893d847" +checksum = "42084a7b455ef0b94ed201b7494392a759c3e20faac2d00ded5d5762fcf71dee" dependencies = [ "alloy-primitives 1.3.1", "async-trait", @@ -830,9 +831,9 @@ dependencies = [ [[package]] name = "alloy-signer-aws" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5cbc38711b7e5ee7b53a7489e8e718a54c6b138d382cfb86e293a2ad5ac894d" +checksum = "d190ee456bba27fc4c3bf7aa65001dd3c71cd431591f98ca3b07960d530d2336" dependencies = [ "alloy-consensus", "alloy-network", @@ -849,9 +850,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6e90a3858da59d1941f496c17db8d505f643260f7e97cdcdd33823ddca48fc1" +checksum = "6312ccc048a4a88aed7311fc448a2e23da55c60c2b3b6dcdb794f759d02e49d7" dependencies = [ "alloy-consensus", "alloy-network", @@ -948,9 +949,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb43750e137fe3a69a325cd89a8f8e2bbf4f83e70c0f60fbe49f22511ca075e8" +checksum = "68f77fa71f6dad3aa9b97ab6f6e90f257089fb9eaa959892d153a1011618e2d6" dependencies = [ "alloy-json-rpc", "alloy-primitives 1.3.1", @@ -972,9 +973,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9b42c7d8b666eed975739201f407afc3320d3cd2e4d807639c2918fc736ea67" +checksum = "0ab1a5d0f5dd5e07187a4170bdcb7ceaff18b1133cd6b8585bc316ab442cd78a" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -987,9 +988,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83db1cc29cce5f692844d6cf1b6b270ae308219c5d90a7246a74f3479b9201c2" +checksum = "6a21442472bad4494cfb1f11d975ae83059882a11cdda6a3aa8c0d2eb444beb6" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -1021,9 +1022,9 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "1.0.32" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e434e0917dce890f755ea774f59d6f12557bc8c7dd9fa06456af80cfe0f0181e" +checksum = "cc79013f9ac3a8ddeb60234d43da09e6d6abfc1c9dd29d3fe97adfbece3f4a08" dependencies = [ "alloy-primitives 1.3.1", "darling 0.21.3", @@ -1377,6 +1378,12 @@ dependencies = [ "rayon", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.7.6" @@ -2198,6 +2205,19 @@ dependencies = [ "wyz", ] +[[package]] +name = "blake3" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -2600,6 +2620,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.7.1" @@ -6294,9 +6320,14 @@ dependencies = [ "anyhow", "daggy", "fhevm-engine-common", + "hex", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "rayon", "tfhe", "tokio", + "tracing", ] [[package]] @@ -6479,9 +6510,9 @@ checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" [[package]] name = "serde" -version = "1.0.225" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6c24dee235d0da097043389623fb913daddf92c76e9f5a1db88607a0bcbd1d" +checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" dependencies = [ "serde_core", "serde_derive", @@ -6489,18 +6520,18 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.225" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "659356f9a0cb1e529b24c01e43ad2bdf520ec4ceaf83047b83ddcc2251f96383" +checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.225" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea936adf78b1f766949a4977b91d2f5595825bd6ec079aa9543ad2685fc4516" +checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" dependencies = [ "proc-macro2", "quote", @@ -6760,7 +6791,7 @@ dependencies = [ "sqlx", "test-harness", "tfhe", - "tfhe-versionable 0.5.0", + "tfhe-versionable", "thiserror 2.0.16", "tokio", "tokio-util", @@ -7214,7 +7245,7 @@ dependencies = [ "test-harness", "testcontainers", "tfhe", - "tfhe-versionable 0.5.0", + "tfhe-versionable", "tfhe-worker", "thiserror 2.0.16", "tokio", @@ -7495,12 +7526,13 @@ dependencies = [ [[package]] name = "tfhe" -version = "1.3.3" +version = "1.4.0-alpha.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81d2bfa12b8ed34e6c70d04489c9201daefde2030bc48f437eeba2ee14bcb46a" +checksum = "217d37960449b4c3c9a6c9cf60982bd1f7eaaa9f6a5e64aa4abfc2bfe91fad60" dependencies = [ "aligned-vec", "bincode", + "blake3", "bytemuck", "dyn-stack", "itertools 0.14.0", @@ -7515,26 +7547,29 @@ dependencies = [ "tfhe-cuda-backend", "tfhe-fft", "tfhe-ntt", - "tfhe-versionable 0.6.1", + "tfhe-versionable", "tfhe-zk-pok", ] [[package]] name = "tfhe-csprng" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06450766ae375cd305281c5d691a41d8877e40d3cc7510a33244775d62f60ad4" +checksum = "58dbda4dff1373da441b7d1f131873c0229a09294327288acdbade48b38d128d" dependencies = [ "aes", + "getrandom 0.2.16", "libc", "rayon", + "serde", + "tfhe-versionable", ] [[package]] name = "tfhe-cuda-backend" -version = "0.11.0" +version = "0.12.0-alpha.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcea29526dc65bee515c160a54665a595fa56ec36846b003d941d196165ec29" +checksum = "faa52cf7fb9f827718c2f4535df9edf625be2830dc8ccaa8d2948e901c2a5c9d" dependencies = [ "bindgen 0.71.1", "cmake", @@ -7558,9 +7593,9 @@ dependencies = [ [[package]] name = "tfhe-ntt" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c64925e25d6ba666f75332ce44fbd2e80be64dbd657e3c2652a6a07f2868d17" +checksum = "27e21dbb266f4472337dec6a608ffca2cf911d680b52873db8027c572f22d0fc" dependencies = [ "aligned-vec", "bytemuck", @@ -7569,44 +7604,21 @@ dependencies = [ [[package]] name = "tfhe-versionable" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af3a2785eb45d8bbb1cf26eb19277a7398b2fc818ad5136b57f5ee280bf0c430" -dependencies = [ - "aligned-vec", - "num-complex", - "serde", - "tfhe-versionable-derive 0.5.0", -] - -[[package]] -name = "tfhe-versionable" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d41b22104d3cfb329a012fdbd9764decc31efd1d71289c74df913be1f7051dac" +checksum = "f59040e53a3581e1270a18eec2f1798e7bfd1c724b0f5d9db3f2a6f6cff89195" dependencies = [ "aligned-vec", "num-complex", "serde", - "tfhe-versionable-derive 0.6.1", + "tfhe-versionable-derive", ] [[package]] name = "tfhe-versionable-derive" -version = "0.5.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f221336486fd08a09dcfd035845f18ae4b742bbf6b6c21dc2121f6b3322705f7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "tfhe-versionable-derive" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5a8431efbbcda5c501d6573928ebbc9c99f4d0ffd34b228395cf948e33f443a" +checksum = "b9a5d55a09b8aff152dee2fa3d28af8999c3af3f69a8bce728855de21638727e" dependencies = [ "proc-macro2", "quote", @@ -7661,9 +7673,9 @@ dependencies = [ [[package]] name = "tfhe-zk-pok" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e73edceedf380f2c28a42f5f6e25ca5cce7721fcf18cc17a3e248ea490ec03ec" +checksum = "8f7b4fa4d6b7d0bd2b5d37e237e2628dc464fc09d40ad4b95fcd3c5d2461cf98" dependencies = [ "ark-bls12-381", "ark-ec", @@ -7674,7 +7686,7 @@ dependencies = [ "rayon", "serde", "sha3", - "tfhe-versionable 0.6.1", + "tfhe-versionable", "zeroize", ] diff --git a/coprocessor/fhevm-engine/Cargo.toml b/coprocessor/fhevm-engine/Cargo.toml index d837868c6b..a563cd6b55 100644 --- a/coprocessor/fhevm-engine/Cargo.toml +++ b/coprocessor/fhevm-engine/Cargo.toml @@ -50,11 +50,11 @@ opentelemetry-semantic-conventions = "0.29.0" prometheus = "0.14.0" prost = "0.13.5" rand = "0.9.1" -rayon = "1.10.0" +rayon = "1.11.0" reqwest = "0.12.20" rustls = { version = "0.23", features = ["aws-lc-rs"] } semver = "1.0.26" -serde = "1.0.219" +serde = "1.0.225" serde_json = "1.0.140" serial_test = "3.2.0" sha3 = "0.10.8" @@ -69,14 +69,14 @@ sqlx = { version = "0.8.6", default-features = false, features = [ ] } testcontainers = "0.24.0" thiserror = "2.0.12" -tfhe = { version = "=1.3", features = [ +tfhe = { version = "1.4.0-alpha.3", features = [ "boolean", "shortint", "integer", "zk-pok", "experimental-force_fft_algo_dif4", ] } -tfhe-versionable = "=0.5.0" +tfhe-versionable = "=0.6.2" tfhe-zk-pok = "0.7.2" tokio = { version = "1.45.0", features = ["full"] } tokio-util = "0.7.15" diff --git a/coprocessor/fhevm-engine/db-migration/migrations/20250920080000_computations_scheduling.sql b/coprocessor/fhevm-engine/db-migration/migrations/20250920080000_computations_scheduling.sql new file mode 100644 index 0000000000..db7ab4505e --- /dev/null +++ b/coprocessor/fhevm-engine/db-migration/migrations/20250920080000_computations_scheduling.sql @@ -0,0 +1,28 @@ +ALTER TABLE computations + ADD COLUMN IF NOT EXISTS is_allowed BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS schedule_order TIMESTAMP NOT NULL DEFAULT NOW(), + ADD COLUMN IF NOT EXISTS uncomputable_counter SMALLINT NOT NULL DEFAULT 1; + +-- We update is_allowed flag of all computations that are not yet +-- computed and producing an allowed handle +UPDATE computations + SET is_allowed = TRUE + WHERE (output_handle, tenant_id) IN ( + SELECT handle, tenant_id FROM allowed_handles + WHERE is_computed = FALSE + ); + +CREATE INDEX IF NOT EXISTS idx_computations_is_allowed + ON computations USING BTREE (is_allowed) + WHERE is_completed = false; +CREATE INDEX IF NOT EXISTS idx_computations_schedule_order + ON computations USING BTREE (schedule_order) + WHERE is_completed = false; +CREATE INDEX IF NOT EXISTS idx_computations_pk + ON computations USING BTREE (tenant_id, output_handle, transaction_id); + +DROP INDEX IF EXISTS idx_allowed_handles_schedule_order; +ALTER TABLE allowed_handles DROP COLUMN IF EXISTS schedule_order; +ALTER TABLE allowed_handles DROP COLUMN IF EXISTS uncomputable_counter; +ALTER TABLE allowed_handles DROP COLUMN IF EXISTS is_computed; + diff --git a/coprocessor/fhevm-engine/fhevm-engine-common/src/keys.rs b/coprocessor/fhevm-engine/fhevm-engine-common/src/keys.rs index eeb866d966..11f1772430 100644 --- a/coprocessor/fhevm-engine/fhevm-engine-common/src/keys.rs +++ b/coprocessor/fhevm-engine/fhevm-engine-common/src/keys.rs @@ -2,18 +2,15 @@ use std::{fs::read, sync::Arc}; #[cfg(feature = "gpu")] use tfhe::core_crypto::gpu::get_number_of_gpus; +#[cfg(feature = "gpu")] +use tfhe::shortint::parameters::v1_4::meta::gpu::V1_4_META_PARAM_GPU_2_2_MULTI_BIT_GROUP_4_KS_PBS_TUNIFORM_2M128 as gpu_meta_parameters; +use tfhe::shortint::AtomicPatternParameters; use tfhe::{ set_server_key, shortint::parameters::{ - v1_0::{ - compact_public_key_only::p_fail_2_minus_128::ks_pbs::V1_0_PARAM_PKE_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128, - key_switching::p_fail_2_minus_128::ks_pbs::V1_0_PARAM_KEYSWITCH_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128, - list_compression::V1_0_COMP_PARAM_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128, - }, - v1_3::{self, V1_3_NOISE_SQUASHING_COMP_PARAM_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128}, - CompactPublicKeyEncryptionParameters, CompressionParameters, - NoiseSquashingCompressionParameters, NoiseSquashingParameters, - ShortintKeySwitchingParameters, + meta::DedicatedCompactPublicKeyParameters, + v1_4::meta::cpu::V1_4_META_PARAM_CPU_2_2_KS_PBS_PKE_TO_SMALL_ZKV2_TUNIFORM_2M128 as cpu_meta_parameters, + CompressionParameters, MetaNoiseSquashingParameters, ShortintKeySwitchingParameters, }, zk::CompactPkeCrs, ClientKey, CompactPublicKey, CompressedServerKey, Config, ConfigBuilder, ServerKey, @@ -21,29 +18,37 @@ use tfhe::{ use crate::utils::{safe_deserialize_key, safe_serialize_key}; -pub const TFHE_COMPRESSION_PARAMS: CompressionParameters = - V1_0_COMP_PARAM_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128; -pub const TFHE_COMPACT_PK_ENCRYPTION_PARAMS: CompactPublicKeyEncryptionParameters = - V1_0_PARAM_PKE_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128; -pub const TFHE_KS_PARAMS: ShortintKeySwitchingParameters = - V1_0_PARAM_KEYSWITCH_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128; -pub const TFHE_NOISE_SQUASHING_PARAMS: NoiseSquashingParameters = - v1_3::V1_3_NOISE_SQUASHING_PARAM_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128; -pub const TFHE_NOISE_SQUASHING_PARAMS_COMPRESSED: NoiseSquashingCompressionParameters = - V1_3_NOISE_SQUASHING_COMP_PARAM_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128; - #[cfg(not(feature = "gpu"))] -pub const TFHE_PARAMS: tfhe::shortint::ClassicPBSParameters = - tfhe::shortint::parameters::PARAM_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128; +pub const TFHE_PARAMS: AtomicPatternParameters = cpu_meta_parameters.compute_parameters; +#[cfg(not(feature = "gpu"))] +pub const TFHE_COMPRESSION_PARAMS: CompressionParameters = cpu_meta_parameters + .compression_parameters + .expect("Missing compression parameters"); + +pub const TFHE_COMPACT_PK_PARAMS: DedicatedCompactPublicKeyParameters = cpu_meta_parameters + .dedicated_compact_public_key_parameters + .expect("Missing compact public key parameters"); +pub const TFHE_NOISE_SQUASHING_PARAMS: MetaNoiseSquashingParameters = cpu_meta_parameters + .noise_squashing_parameters + .expect("Missing noise squashing parameters"); +pub const TFHE_PKS_RERANDOMIZATION_PARAMS: ShortintKeySwitchingParameters = cpu_meta_parameters + .re_randomization_parameters + .expect("Missing rerandomisation parameters"); + #[cfg(feature = "gpu")] -pub const TFHE_PARAMS: tfhe::shortint::parameters::MultiBitPBSParameters = - tfhe::shortint::parameters::PARAM_GPU_MULTI_BIT_GROUP_4_MESSAGE_2_CARRY_2_KS_PBS; +pub const TFHE_PARAMS: AtomicPatternParameters = gpu_meta_parameters.compute_parameters; +#[cfg(feature = "gpu")] +pub const TFHE_COMPRESSION_PARAMS: CompressionParameters = gpu_meta_parameters + .compression_parameters + .expect("Missing compression parameters"); pub const MAX_BITS_TO_PROVE: usize = 2048; #[derive(Clone)] pub struct FhevmKeys { pub server_key: ServerKey, + #[cfg(not(feature = "gpu"))] + pub server_key_without_ns: ServerKey, pub client_key: Option, pub compact_public_key: CompactPublicKey, pub public_params: Arc, @@ -56,6 +61,8 @@ pub struct FhevmKeys { pub struct SerializedFhevmKeys { #[cfg(not(feature = "gpu"))] pub server_key: Vec, + #[cfg(not(feature = "gpu"))] + pub server_key_without_ns: Vec, pub client_key: Option>, pub compact_public_key: Vec, pub public_params: Vec, @@ -77,8 +84,34 @@ impl FhevmKeys { let compact_public_key = CompactPublicKey::new(&client_key); let crs = CompactPkeCrs::from_config(config, MAX_BITS_TO_PROVE).expect("CRS creation"); let compressed_server_key = CompressedServerKey::new(&client_key); + let server_key = compressed_server_key.clone().decompress(); + #[cfg(not(feature = "gpu"))] + let ( + sks, + kskm, + compression_key, + decompression_key, + _noise_squashing_key, + _noise_squashing_compression_key, + re_randomization_keyswitching_key, + tag, + ) = server_key.clone().into_raw_parts(); + #[cfg(not(feature = "gpu"))] + let server_key_without_ns = ServerKey::from_raw_parts( + sks, + kskm, + compression_key, + decompression_key, + None, // noise squashing key excluded + None, // noise squashing compression key excluded + re_randomization_keyswitching_key, + tag, + ); + FhevmKeys { - server_key: compressed_server_key.clone().decompress(), + server_key, + #[cfg(not(feature = "gpu"))] + server_key_without_ns, client_key: Some(client_key), compact_public_key, public_params: Arc::new(crs.clone()), @@ -89,7 +122,7 @@ impl FhevmKeys { gpu_server_key: vec![compressed_server_key.decompress_to_gpu()], #[cfg(feature = "gpu")] #[cfg(not(feature = "latency"))] - gpu_server_key: (0..get_number_of_gpus() as u32) + gpu_server_key: (0..get_number_of_gpus()) .map(|i| compressed_server_key.decompress_to_specific_gpu(tfhe::GpuIndex::new(i))) .collect::>(), } @@ -97,13 +130,18 @@ impl FhevmKeys { pub fn new_config() -> Config { ConfigBuilder::with_custom_parameters(TFHE_PARAMS) - .enable_noise_squashing(TFHE_NOISE_SQUASHING_PARAMS) - .enable_noise_squashing_compression(TFHE_NOISE_SQUASHING_PARAMS_COMPRESSED) + .enable_noise_squashing(TFHE_NOISE_SQUASHING_PARAMS.parameters) + .enable_noise_squashing_compression( + TFHE_NOISE_SQUASHING_PARAMS + .compression_parameters + .expect("Missing noise squahing compression parameters."), + ) .enable_compression(TFHE_COMPRESSION_PARAMS) .use_dedicated_compact_public_key_parameters(( - TFHE_COMPACT_PK_ENCRYPTION_PARAMS, - TFHE_KS_PARAMS, + TFHE_COMPACT_PK_PARAMS.pke_params, + TFHE_COMPACT_PK_PARAMS.ksk_params, )) + .enable_ciphertext_re_randomization(TFHE_PKS_RERANDOMIZATION_PARAMS) .build() } @@ -128,6 +166,8 @@ impl SerializedFhevmKeys { const PKS: &'static str = "../fhevm-keys/pks"; #[cfg(not(feature = "gpu"))] const PUBLIC_PARAMS: &'static str = "../fhevm-keys/pp"; + #[cfg(not(feature = "gpu"))] + const FULL_SKS: &'static str = "../fhevm-keys/sns_pk"; #[cfg(feature = "gpu")] const GPU_CSKS: &'static str = "../fhevm-keys/gpu-csks"; @@ -145,7 +185,10 @@ impl SerializedFhevmKeys { #[cfg(not(feature = "gpu"))] { println!("Creating file {}", Self::SKS); - std::fs::write(Self::SKS, self.server_key).expect("write sks"); + std::fs::write(Self::SKS, self.server_key_without_ns).expect("write sks"); + + println!("Creating file {}", Self::FULL_SKS); + std::fs::write(Self::FULL_SKS, self.server_key).expect("write sns_pk"); if self.client_key.is_some() { println!("Creating file {}", Self::CKS); @@ -179,12 +222,14 @@ impl SerializedFhevmKeys { pub fn load_from_disk(keys_directory: &str) -> Self { let keys_dir = std::path::Path::new(&keys_directory); - let (sks, cks, pks, pp) = if !cfg!(feature = "gpu") { - ("sks", "cks", "pks", "pp") + let (sns_pk, sks, cks, pks, pp) = if !cfg!(feature = "gpu") { + ("sns_pk", "sks", "cks", "pks", "pp") } else { - ("gpu-csks", "gpu-cks", "gpu-pks", "gpu-pp") + ("_unused_", "gpu-csks", "gpu-cks", "gpu-pks", "gpu-pp") }; - let server_key = read(keys_dir.join(sks)).expect("read server key"); + let server_key = read(keys_dir.join(sns_pk)).expect("read full server key (sns_pk)"); + #[cfg(not(feature = "gpu"))] + let server_key_without_ns = read(keys_dir.join(sks)).expect("read server key"); let client_key = read(keys_dir.join(cks)).ok(); let compact_public_key = read(keys_dir.join(pks)).expect("read compact public key"); let public_params = read(keys_dir.join(pp)).expect("read public params"); @@ -194,6 +239,8 @@ impl SerializedFhevmKeys { public_params, #[cfg(not(feature = "gpu"))] server_key, + #[cfg(not(feature = "gpu"))] + server_key_without_ns, #[cfg(feature = "gpu")] compressed_server_key: server_key, } @@ -208,6 +255,8 @@ impl From for SerializedFhevmKeys { public_params: safe_serialize_key(f.public_params.as_ref()), #[cfg(not(feature = "gpu"))] server_key: safe_serialize_key(&f.server_key), + #[cfg(not(feature = "gpu"))] + server_key_without_ns: safe_serialize_key(&f.server_key_without_ns), #[cfg(feature = "gpu")] compressed_server_key: safe_serialize_key(&f.compressed_server_key), } @@ -232,7 +281,11 @@ impl From for FhevmKeys { safe_deserialize_key(&f.public_params).expect("deserialize public params"), ), #[cfg(not(feature = "gpu"))] - server_key: safe_deserialize_key(&f.server_key).expect("deserialize server key"), + server_key: safe_deserialize_key(&f.server_key) + .expect("deserialize full server key (sns_pk)"), + #[cfg(not(feature = "gpu"))] + server_key_without_ns: safe_deserialize_key(&f.server_key_without_ns) + .expect("deserialize server key"), #[cfg(feature = "gpu")] compressed_server_key: compressed_server_key.clone(), #[cfg(feature = "gpu")] @@ -240,7 +293,7 @@ impl From for FhevmKeys { gpu_server_key: vec![compressed_server_key.decompress_to_gpu()], #[cfg(feature = "gpu")] #[cfg(not(feature = "latency"))] - gpu_server_key: (0..get_number_of_gpus() as u32) + gpu_server_key: (0..get_number_of_gpus()) .map(|i| compressed_server_key.decompress_to_specific_gpu(tfhe::GpuIndex::new(i))) .collect::>(), #[cfg(feature = "gpu")] diff --git a/coprocessor/fhevm-engine/fhevm-engine-common/src/types.rs b/coprocessor/fhevm-engine/fhevm-engine-common/src/types.rs index 6922c9b2a7..d8080cc8c7 100644 --- a/coprocessor/fhevm-engine/fhevm-engine-common/src/types.rs +++ b/coprocessor/fhevm-engine/fhevm-engine-common/src/types.rs @@ -5,11 +5,14 @@ use alloy_provider::fillers::{ use anyhow::Result; use bigdecimal::num_bigint::BigInt; use tfhe::integer::bigint::StaticUnsignedBigInt; -use tfhe::integer::ciphertext::BaseRadixCiphertext; +use tfhe::integer::ciphertext::{BaseRadixCiphertext, ReRandomizationSeed}; use tfhe::integer::U256; -use tfhe::prelude::{CiphertextList, FheDecrypt}; +use tfhe::prelude::{CiphertextList, FheDecrypt, ReRandomize}; use tfhe::shortint::Ciphertext; -use tfhe::{CompressedCiphertextList, CompressedCiphertextListBuilder}; +use tfhe::{ + CompactPublicKey, CompressedCiphertextList, CompressedCiphertextListBuilder, + ReRandomizationContext, +}; use crate::utils::{safe_deserialize, safe_serialize}; @@ -19,6 +22,7 @@ pub enum FhevmError { UnknownFheType(i32), DeserializationError(Box), CiphertextExpansionError(tfhe::Error), + ReRandomisationError(tfhe::Error), CiphertextExpansionUnsupportedCiphertextKind(tfhe::FheTypes), FheOperationOnlyOneOperandCanBeScalar { fhe_operation: i32, @@ -146,6 +150,9 @@ impl std::fmt::Display for FhevmError { Self::CiphertextExpansionError(e) => { write!(f, "error expanding compact ciphertext list: {:?}", e) } + Self::ReRandomisationError(e) => { + write!(f, "error re-randomising ciphertext: {:?}", e) + } Self::CiphertextExpansionUnsupportedCiphertextKind(e) => { write!( f, @@ -626,6 +633,170 @@ impl SupportedFheCiphertexts { | SupportedFheCiphertexts::Scalar(_) => false, } } + + pub fn add_to_re_randomization_context(&self, context: &mut ReRandomizationContext) { + match self { + SupportedFheCiphertexts::FheBool(ct) => { + context.add_ciphertext(ct); + } + SupportedFheCiphertexts::FheUint4(ct) => { + context.add_ciphertext(ct); + } + SupportedFheCiphertexts::FheUint8(ct) => { + context.add_ciphertext(ct); + } + SupportedFheCiphertexts::FheUint16(ct) => { + context.add_ciphertext(ct); + } + SupportedFheCiphertexts::FheUint32(ct) => { + context.add_ciphertext(ct); + } + SupportedFheCiphertexts::FheUint64(ct) => { + context.add_ciphertext(ct); + } + SupportedFheCiphertexts::FheUint128(ct) => { + context.add_ciphertext(ct); + } + SupportedFheCiphertexts::FheUint160(ct) => { + context.add_ciphertext(ct); + } + SupportedFheCiphertexts::FheUint256(ct) => { + context.add_ciphertext(ct); + } + SupportedFheCiphertexts::FheBytes64(ct) => { + context.add_ciphertext(ct); + } + SupportedFheCiphertexts::FheBytes128(ct) => { + context.add_ciphertext(ct); + } + SupportedFheCiphertexts::FheBytes256(ct) => { + context.add_ciphertext(ct); + } + SupportedFheCiphertexts::Scalar(_) => (), + } + } + + pub fn add_re_randomization_metadata(&mut self, hash_data: &[u8]) { + match self { + SupportedFheCiphertexts::FheBool(ct) => { + ct.re_randomization_metadata_mut().set_data(hash_data); + } + SupportedFheCiphertexts::FheUint4(ct) => { + ct.re_randomization_metadata_mut().set_data(hash_data); + } + SupportedFheCiphertexts::FheUint8(ct) => { + ct.re_randomization_metadata_mut().set_data(hash_data); + } + SupportedFheCiphertexts::FheUint16(ct) => { + ct.re_randomization_metadata_mut().set_data(hash_data); + } + SupportedFheCiphertexts::FheUint32(ct) => { + ct.re_randomization_metadata_mut().set_data(hash_data); + } + SupportedFheCiphertexts::FheUint64(ct) => { + ct.re_randomization_metadata_mut().set_data(hash_data); + } + SupportedFheCiphertexts::FheUint128(ct) => { + ct.re_randomization_metadata_mut().set_data(hash_data); + } + SupportedFheCiphertexts::FheUint160(ct) => { + ct.re_randomization_metadata_mut().set_data(hash_data); + } + SupportedFheCiphertexts::FheUint256(ct) => { + ct.re_randomization_metadata_mut().set_data(hash_data); + } + SupportedFheCiphertexts::FheBytes64(ct) => { + ct.re_randomization_metadata_mut().set_data(hash_data); + } + SupportedFheCiphertexts::FheBytes128(ct) => { + ct.re_randomization_metadata_mut().set_data(hash_data); + } + SupportedFheCiphertexts::FheBytes256(ct) => { + ct.re_randomization_metadata_mut().set_data(hash_data); + } + SupportedFheCiphertexts::Scalar(_) => (), + } + } + + pub fn add_to_rerandomisation_context(&self, context: &mut ReRandomizationContext) { + match self { + SupportedFheCiphertexts::FheBool(c) => context.add_ciphertext(c), + SupportedFheCiphertexts::FheUint4(c) => context.add_ciphertext(c), + SupportedFheCiphertexts::FheUint8(c) => context.add_ciphertext(c), + SupportedFheCiphertexts::FheUint16(c) => context.add_ciphertext(c), + SupportedFheCiphertexts::FheUint32(c) => context.add_ciphertext(c), + SupportedFheCiphertexts::FheUint64(c) => context.add_ciphertext(c), + SupportedFheCiphertexts::FheUint128(c) => context.add_ciphertext(c), + SupportedFheCiphertexts::FheUint160(c) => context.add_ciphertext(c), + SupportedFheCiphertexts::FheUint256(c) => context.add_ciphertext(c), + SupportedFheCiphertexts::FheBytes64(c) => context.add_ciphertext(c), + SupportedFheCiphertexts::FheBytes128(c) => context.add_ciphertext(c), + SupportedFheCiphertexts::FheBytes256(c) => context.add_ciphertext(c), + SupportedFheCiphertexts::Scalar(_) => { + // Do nothing + } + }; + } + pub fn re_randomise( + &mut self, + cpk: &CompactPublicKey, + seed: ReRandomizationSeed, + ) -> Result<(), FhevmError> { + match self { + SupportedFheCiphertexts::FheBool(c) => { + c.re_randomize(cpk, seed) + .map_err(FhevmError::ReRandomisationError)?; + } + SupportedFheCiphertexts::FheUint4(c) => { + c.re_randomize(cpk, seed) + .map_err(FhevmError::ReRandomisationError)?; + } + SupportedFheCiphertexts::FheUint8(c) => { + c.re_randomize(cpk, seed) + .map_err(FhevmError::ReRandomisationError)?; + } + SupportedFheCiphertexts::FheUint16(c) => { + c.re_randomize(cpk, seed) + .map_err(FhevmError::ReRandomisationError)?; + } + SupportedFheCiphertexts::FheUint32(c) => { + c.re_randomize(cpk, seed) + .map_err(FhevmError::ReRandomisationError)?; + } + SupportedFheCiphertexts::FheUint64(c) => { + c.re_randomize(cpk, seed) + .map_err(FhevmError::ReRandomisationError)?; + } + SupportedFheCiphertexts::FheUint128(c) => { + c.re_randomize(cpk, seed) + .map_err(FhevmError::ReRandomisationError)?; + } + SupportedFheCiphertexts::FheUint160(c) => { + c.re_randomize(cpk, seed) + .map_err(FhevmError::ReRandomisationError)?; + } + SupportedFheCiphertexts::FheUint256(c) => { + c.re_randomize(cpk, seed) + .map_err(FhevmError::ReRandomisationError)?; + } + SupportedFheCiphertexts::FheBytes64(c) => { + c.re_randomize(cpk, seed) + .map_err(FhevmError::ReRandomisationError)?; + } + SupportedFheCiphertexts::FheBytes128(c) => { + c.re_randomize(cpk, seed) + .map_err(FhevmError::ReRandomisationError)?; + } + SupportedFheCiphertexts::FheBytes256(c) => { + c.re_randomize(cpk, seed) + .map_err(FhevmError::ReRandomisationError)?; + } + SupportedFheCiphertexts::Scalar(_s) => { + // Do nothing + } + } + Ok(()) + } } impl SupportedFheOperations { diff --git a/coprocessor/fhevm-engine/fhevm-keys/cks b/coprocessor/fhevm-engine/fhevm-keys/cks index e03d942b3c..aa556f56cc 100644 --- a/coprocessor/fhevm-engine/fhevm-keys/cks +++ b/coprocessor/fhevm-engine/fhevm-keys/cks @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:03d5c7919dab456544b8e8f42123c4fe2f2437d3ceca3fb329665b71ad8eee45 -size 49006 +oid sha256:ec017cc84e340df46b7c674ed897284b91cff6ef84eec652dd3911763a7dbbac +size 213173 diff --git a/coprocessor/fhevm-engine/fhevm-keys/gpu-cks b/coprocessor/fhevm-engine/fhevm-keys/gpu-cks index b8f38fae2e..4d1e8ef36f 100644 --- a/coprocessor/fhevm-engine/fhevm-keys/gpu-cks +++ b/coprocessor/fhevm-engine/fhevm-keys/gpu-cks @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4c4ac87a56c067eb0129ebe463910ed4874a9f760d6cea4bad24bf7979d87cd4 -size 48654 +oid sha256:c3ded4b50b9e8c15369b0258146ba808761f81f361cbca92d75d3c0cd7667b4e +size 213194 diff --git a/coprocessor/fhevm-engine/fhevm-keys/gpu-csks b/coprocessor/fhevm-engine/fhevm-keys/gpu-csks index 13dd677303..985b378fb6 100644 --- a/coprocessor/fhevm-engine/fhevm-keys/gpu-csks +++ b/coprocessor/fhevm-engine/fhevm-keys/gpu-csks @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a32c673203bcbf14bad4c8a255116d4b1e4c0e5467bd9dabad179b326f82bcb1 -size 123093790 +oid sha256:46de963b70db1ce9745533de024b42d8373e3b285c8b9f950a3bbece20eb2ae0 +size 505316550 diff --git a/coprocessor/fhevm-engine/fhevm-keys/gpu-pks b/coprocessor/fhevm-engine/fhevm-keys/gpu-pks index 669f62343b..5c3123d09b 100644 --- a/coprocessor/fhevm-engine/fhevm-keys/gpu-pks +++ b/coprocessor/fhevm-engine/fhevm-keys/gpu-pks @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:68263ce89b13d85e571c53a6af388f98a7b8fd98e03dc5a00d9d88033daf89af +oid sha256:1919770a7b40bc1a210dc0443a35b67c58b3236cb51c7e604e640eb44ce0dabf size 33018 diff --git a/coprocessor/fhevm-engine/fhevm-keys/gpu-pp b/coprocessor/fhevm-engine/fhevm-keys/gpu-pp index 8ac04cb48c..f89eab7573 100644 --- a/coprocessor/fhevm-engine/fhevm-keys/gpu-pp +++ b/coprocessor/fhevm-engine/fhevm-keys/gpu-pp @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:aa76f098f0ae30be65523696271bc5935336c1d80e9633b0e98e61dbfc406d04 -size 4573811 +oid sha256:655f906cea53ca862d5d6e008e6044cc69349c109512714061516b21fbfb6f13 +size 4571368 diff --git a/coprocessor/fhevm-engine/fhevm-keys/pks b/coprocessor/fhevm-engine/fhevm-keys/pks index 494bb26053..d072fbdf49 100644 --- a/coprocessor/fhevm-engine/fhevm-keys/pks +++ b/coprocessor/fhevm-engine/fhevm-keys/pks @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d74037feb1394aec0eb724afc4a690e16a0203eb1dc868de08179f2e4828f335 +oid sha256:0ae739cc6de1f11271cb2d3c8006a41ef25f26da7aac5514d73319f5cf2597f3 size 33018 diff --git a/coprocessor/fhevm-engine/fhevm-keys/pp b/coprocessor/fhevm-engine/fhevm-keys/pp index cbf75f2d3e..6dfeac5937 100644 --- a/coprocessor/fhevm-engine/fhevm-keys/pp +++ b/coprocessor/fhevm-engine/fhevm-keys/pp @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e4e3f0deef6492bde2fac3f2c698112ace878d4a3656b272197ed332391c48fa -size 4573811 +oid sha256:0386e37bc427c73083081d4d1082128f9dec7cc36aeb43d082fdadd8dbc0b3f3 +size 4571368 diff --git a/coprocessor/fhevm-engine/fhevm-keys/sks b/coprocessor/fhevm-engine/fhevm-keys/sks index 79e4adc58d..573790f688 100644 --- a/coprocessor/fhevm-engine/fhevm-keys/sks +++ b/coprocessor/fhevm-engine/fhevm-keys/sks @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:812b602c11693dda4bfa68db78269849972b6f1920a725a98fc78cf619a6ab5f -size 321356585 +oid sha256:cb2130c20d9f28707230410e4ab258ad7fbe7d0f1dedb0f97933cb2d7900d7fc +size 344274375 diff --git a/coprocessor/fhevm-engine/fhevm-keys/sns_pk b/coprocessor/fhevm-engine/fhevm-keys/sns_pk index 84dce00bb7..60df8dc468 100644 --- a/coprocessor/fhevm-engine/fhevm-keys/sns_pk +++ b/coprocessor/fhevm-engine/fhevm-keys/sns_pk @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:668921c86a66413333924752f8be03ebe58a09ff1ce531e8c02bda1f2d609cfc -size 1144197531 +oid sha256:a12d504899839d9558d745d28f7e3b2f4b47b2fc25b54c337a5814a759e97243 +size 1626224351 diff --git a/coprocessor/fhevm-engine/fhevm-keys/sns_sk b/coprocessor/fhevm-engine/fhevm-keys/sns_sk deleted file mode 100644 index cd2028eadf..0000000000 --- a/coprocessor/fhevm-engine/fhevm-keys/sns_sk +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cb53bc8bd6aaa8781fd28cede2a749dc2d883bcbcfdae1de0011c19c3a445feb -size 65709 diff --git a/coprocessor/fhevm-engine/gw-listener/src/sks_key.rs b/coprocessor/fhevm-engine/gw-listener/src/sks_key.rs index 3e954b8889..66907e6676 100644 --- a/coprocessor/fhevm-engine/gw-listener/src/sks_key.rs +++ b/coprocessor/fhevm-engine/gw-listener/src/sks_key.rs @@ -10,20 +10,35 @@ pub fn extract_server_key_without_ns(sns_key: &[u8]) -> anyhow::Result> let server_key: ServerKey = safe_deserialize_sns_key(sns_key)?; - let (sks, kskm, compression_key, decompression_key, _, noise_squashing_key, tag) = - server_key.into_raw_parts(); + let ( + sks, + kskm, + compression_key, + decompression_key, + noise_squashing_key, + noise_squashing_compression_key, + re_randomization_keyswitching_key, + tag, + ) = server_key.into_raw_parts(); if noise_squashing_key.is_none() { anyhow::bail!("Server key does not have noise squashing"); } + if noise_squashing_compression_key.is_none() { + anyhow::bail!("Server key does not have noise squashing compresion"); + } + if re_randomization_keyswitching_key.is_none() { + anyhow::bail!("Server key does not have rerandomisation"); + } Ok(safe_serialize_key(&ServerKey::from_raw_parts( sks, kskm, compression_key, decompression_key, - None, // noise squashing key excluded - None, // noise squashing compression key excluded + None, // noise squashing key excluded + None, // noise squashing compression key excluded + re_randomization_keyswitching_key, // rerandomisation keyswitching key excluded tag, ))) } diff --git a/coprocessor/fhevm-engine/host-listener/.sqlx/query-e4260b9ba59d978d5504b787a0002c6eec8c6b95b2cb8b5da83f8885fb273c39.json b/coprocessor/fhevm-engine/host-listener/.sqlx/query-0fe024db8ce4cb065633e9f02bd96e67b62460c4d725030ce6fb3b488d0541a7.json similarity index 61% rename from coprocessor/fhevm-engine/host-listener/.sqlx/query-e4260b9ba59d978d5504b787a0002c6eec8c6b95b2cb8b5da83f8885fb273c39.json rename to coprocessor/fhevm-engine/host-listener/.sqlx/query-0fe024db8ce4cb065633e9f02bd96e67b62460c4d725030ce6fb3b488d0541a7.json index f3ff350e33..9afd9248b4 100644 --- a/coprocessor/fhevm-engine/host-listener/.sqlx/query-e4260b9ba59d978d5504b787a0002c6eec8c6b95b2cb8b5da83f8885fb273c39.json +++ b/coprocessor/fhevm-engine/host-listener/.sqlx/query-0fe024db8ce4cb065633e9f02bd96e67b62460c4d725030ce6fb3b488d0541a7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO computations (\n tenant_id,\n output_handle,\n dependencies,\n fhe_operation,\n is_scalar,\n dependence_chain_id,\n transaction_id\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ON CONFLICT (tenant_id, output_handle, transaction_id) DO NOTHING\n ", + "query": "\n INSERT INTO computations (\n tenant_id,\n output_handle,\n dependencies,\n fhe_operation,\n is_scalar,\n dependence_chain_id,\n transaction_id,\n is_allowed\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (tenant_id, output_handle, transaction_id) DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -11,10 +11,11 @@ "Int2", "Bool", "Bytea", - "Bytea" + "Bytea", + "Bool" ] }, "nullable": [] }, - "hash": "e4260b9ba59d978d5504b787a0002c6eec8c6b95b2cb8b5da83f8885fb273c39" + "hash": "0fe024db8ce4cb065633e9f02bd96e67b62460c4d725030ce6fb3b488d0541a7" } diff --git a/coprocessor/fhevm-engine/host-listener/src/cmd/mod.rs b/coprocessor/fhevm-engine/host-listener/src/cmd/mod.rs index ea86603d6e..35f9f43a98 100644 --- a/coprocessor/fhevm-engine/host-listener/src/cmd/mod.rs +++ b/coprocessor/fhevm-engine/host-listener/src/cmd/mod.rs @@ -8,7 +8,7 @@ use anyhow::{anyhow, Result}; use futures_util::stream::StreamExt; use sqlx::types::Uuid; -use std::collections::VecDeque; +use std::collections::{HashSet, VecDeque}; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; @@ -22,11 +22,13 @@ use rustls; use tokio_util::sync::CancellationToken; use fhevm_engine_common::healthz_server::HttpServer as HealthHttpServer; -use fhevm_engine_common::types::BlockchainProvider; +use fhevm_engine_common::types::{BlockchainProvider, Handle}; use fhevm_engine_common::utils::HeartBeat; use crate::contracts::{AclContract, TfheContract}; -use crate::database::tfhe_event_propagate::{ChainId, Database, LogTfhe}; +use crate::database::tfhe_event_propagate::{ + acl_result_handles, tfhe_result_handle, ChainId, Database, LogTfhe, +}; use crate::health_check::HealthCheck; pub mod block_history; @@ -819,35 +821,35 @@ async fn db_insert_block_no_retry( tfhe_contract_address: &Option
, ) -> std::result::Result<(), sqlx::Error> { let mut tx = db.new_transaction().await?; + let mut is_allowed = HashSet::::new(); + let mut tfhe_event_log = vec![]; for log in &block_logs.logs { - info!( - block = ?log.block_number, - tx = ?log.transaction_hash, - log_index = ?log.log_index, - "Log", - ); let current_address = Some(log.inner.address); + let is_acl_address = ¤t_address == acl_contract_address; + if acl_contract_address.is_none() || is_acl_address { + if let Ok(event) = + AclContract::AclContractEvents::decode_log(&log.inner) + { + info!(acl_event = ?event, "ACL event"); + let handles = acl_result_handles(&event); + for handle in handles { + is_allowed.insert(handle.to_vec()); + } + db.handle_acl_event(&mut tx, &event).await?; + continue; + } + } let is_tfhe_address = ¤t_address == tfhe_contract_address; if tfhe_contract_address.is_none() || is_tfhe_address { if let Ok(event) = TfheContract::TfheContractEvents::decode_log(&log.inner) { - info!(tfhe_event = ?event, "TFHE event"); let log = LogTfhe { event, transaction_hash: log.transaction_hash, + is_allowed: false, // updated in the next loop }; - db.insert_tfhe_event(&mut tx, &log).await?; - continue; - } - } - let is_acl_address = ¤t_address == acl_contract_address; - if acl_contract_address.is_none() || is_acl_address { - if let Ok(event) = - AclContract::AclContractEvents::decode_log(&log.inner) - { - info!(acl_event = ?event, "ACL event"); - db.handle_acl_event(&mut tx, &event).await?; + tfhe_event_log.push(log); continue; } } @@ -856,10 +858,25 @@ async fn db_insert_block_no_retry( event_address = ?log.inner.address, acl_contract_address = ?acl_contract_address, tfhe_contract_address = ?tfhe_contract_address, + log = ?log, "Cannot decode event", ); } } + for tfhe_log in tfhe_event_log { + info!(tfhe_log = ?tfhe_log, "TFHE event"); + let is_allowed = + if let Some(result_handle) = tfhe_result_handle(&tfhe_log.event) { + is_allowed.contains(&result_handle.to_vec()) + } else { + false + }; + let tfhe_log = LogTfhe { + is_allowed, + ..tfhe_log + }; + db.insert_tfhe_event(&mut tx, &tfhe_log).await?; + } db.mark_block_as_valid(&mut tx, &block_logs.summary).await?; tx.commit().await } diff --git a/coprocessor/fhevm-engine/host-listener/src/database/tfhe_event_propagate.rs b/coprocessor/fhevm-engine/host-listener/src/database/tfhe_event_propagate.rs index f5725ade24..62376d4fdc 100644 --- a/coprocessor/fhevm-engine/host-listener/src/database/tfhe_event_propagate.rs +++ b/coprocessor/fhevm-engine/host-listener/src/database/tfhe_event_propagate.rs @@ -77,9 +77,11 @@ pub struct Database { pub tick: HeartBeat, } +#[derive(Debug)] pub struct LogTfhe { pub event: Log, pub transaction_hash: Option, + pub is_allowed: bool, } pub type Transaction<'l> = sqlx::Transaction<'l, Postgres>; @@ -277,9 +279,10 @@ impl Database { fhe_operation, is_scalar, dependence_chain_id, - transaction_id + transaction_id, + is_allowed ) - VALUES ($1, $2, $3, $4, $5, $6, $7) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (tenant_id, output_handle, transaction_id) DO NOTHING "#, tenant_id as i32, @@ -288,7 +291,8 @@ impl Database { fhe_operation as i16, is_scalar, bucket.to_vec(), - log.transaction_hash.map(|txh| txh.to_vec()) + log.transaction_hash.map(|txh| txh.to_vec()), + log.is_allowed, ); query.execute(tx.deref_mut()).await.map(|_| ()) } @@ -674,3 +678,62 @@ pub fn event_name(op: &TfheContractEvents) -> &'static str { E::VerifyCiphertext(_) => "VerifyCiphertext", } } + +pub fn tfhe_result_handle(op: &TfheContractEvents) -> Option { + use TfheContract as C; + use TfheContractEvents as E; + match op { + E::Cast(C::Cast { result, .. }) + | E::FheAdd(C::FheAdd { result, .. }) + | E::FheBitAnd(C::FheBitAnd { result, .. }) + | E::FheBitOr(C::FheBitOr { result, .. }) + | E::FheBitXor(C::FheBitXor { result, .. }) + | E::FheDiv(C::FheDiv { result, .. }) + | E::FheMax(C::FheMax { result, .. }) + | E::FheMin(C::FheMin { result, .. }) + | E::FheMul(C::FheMul { result, .. }) + | E::FheRem(C::FheRem { result, .. }) + | E::FheRotl(C::FheRotl { result, .. }) + | E::FheRotr(C::FheRotr { result, .. }) + | E::FheShl(C::FheShl { result, .. }) + | E::FheShr(C::FheShr { result, .. }) + | E::FheSub(C::FheSub { result, .. }) + | E::FheIfThenElse(C::FheIfThenElse { result, .. }) + | E::FheEq(C::FheEq { result, .. }) + | E::FheGe(C::FheGe { result, .. }) + | E::FheGt(C::FheGt { result, .. }) + | E::FheLe(C::FheLe { result, .. }) + | E::FheLt(C::FheLt { result, .. }) + | E::FheNe(C::FheNe { result, .. }) + | E::FheNeg(C::FheNeg { result, .. }) + | E::FheNot(C::FheNot { result, .. }) + | E::FheRand(C::FheRand { result, .. }) + | E::FheRandBounded(C::FheRandBounded { result, .. }) + | E::TrivialEncrypt(C::TrivialEncrypt { result, .. }) => Some(*result), + + E::Initialized(_) + | E::OwnershipTransferStarted(_) + | E::OwnershipTransferred(_) + | E::Upgraded(_) + | E::VerifyCiphertext(_) => None, + } +} + +pub fn acl_result_handles(event: &Log) -> Vec { + let data = &event.data; + match data { + AclContractEvents::Allowed(allowed) => vec![allowed.handle], + AclContractEvents::AllowedForDecryption(allowed_for_decryption) => { + allowed_for_decryption.handlesList.clone() + } + AclContractEvents::Initialized(_) + | AclContractEvents::NewDelegation(_) + | AclContractEvents::OwnershipTransferStarted(_) + | AclContractEvents::OwnershipTransferred(_) + | AclContractEvents::RevokedDelegation(_) + | AclContractEvents::Upgraded(_) + | AclContractEvents::Paused(_) + | AclContractEvents::Unpaused(_) + | AclContractEvents::UpdatePauser(_) => vec![], + } +} diff --git a/coprocessor/fhevm-engine/scheduler/Cargo.toml b/coprocessor/fhevm-engine/scheduler/Cargo.toml index 6e11b9191c..309a40114a 100644 --- a/coprocessor/fhevm-engine/scheduler/Cargo.toml +++ b/coprocessor/fhevm-engine/scheduler/Cargo.toml @@ -8,9 +8,14 @@ license.workspace = true # workspace dependencies anyhow = { workspace = true } daggy = { workspace = true } +hex = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry-otlp = { workspace = true } +opentelemetry_sdk = { workspace = true } rayon = { workspace = true } tfhe = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } # local dependencies fhevm-engine-common = { path = "../fhevm-engine-common" } @@ -18,4 +23,3 @@ fhevm-engine-common = { path = "../fhevm-engine-common" } [features] nightly-avx512 = ["tfhe/nightly-avx512"] gpu = ["tfhe/gpu"] - diff --git a/coprocessor/fhevm-engine/scheduler/src/dfg.rs b/coprocessor/fhevm-engine/scheduler/src/dfg.rs index 095a9505bb..30dcf87ff9 100644 --- a/coprocessor/fhevm-engine/scheduler/src/dfg.rs +++ b/coprocessor/fhevm-engine/scheduler/src/dfg.rs @@ -1,51 +1,313 @@ pub mod scheduler; pub mod types; +use std::collections::HashMap; + use crate::dfg::types::*; use anyhow::Result; use daggy::petgraph::{ - visit::{EdgeRef, IntoEdgesDirected}, + visit::{EdgeRef, IntoEdgesDirected, IntoNodeReferences}, Direction, }; use daggy::{petgraph::graph::node_index, Dag, NodeIndex}; -use fhevm_engine_common::types::Handle; +use fhevm_engine_common::types::{Handle, SupportedFheCiphertexts, SupportedFheOperations}; +#[derive(Debug)] +pub struct DFGOp { + pub output_handle: Handle, + pub fhe_op: SupportedFheOperations, + pub inputs: Vec, + pub is_allowed: bool, +} +pub type TxEdge = (); +#[derive(Default)] +pub struct TxNode { + // Inner dataflow graph + pub graph: DFGraph, + // Allowed handles or verified input handles, with a map of + // internal DFG node indexes to input positions in the + // corresponding FHE op + pub inputs: HashMap>, + // Only allowed handles can be results (used beyond the + // transaction) + pub results: Vec, + pub transaction_id: Handle, + pub is_uncomputable: bool, + pub intermediate_handles: Vec, +} +impl TxNode { + pub fn build(&mut self, mut operations: Vec, transaction_id: &Handle) -> Result<()> { + self.transaction_id = transaction_id.clone(); + self.is_uncomputable = false; + // Gather all handles produced within the transaction + let mut produced_handles: HashMap = HashMap::new(); + for (index, op) in operations.iter().enumerate() { + produced_handles.insert(op.output_handle.clone(), index); + } + let mut dependence_pairs = vec![]; + for (index, op) in operations.iter_mut().enumerate() { + for (pos, i) in op.inputs.iter().enumerate() { + match i { + DFGTaskInput::Dependence(dh) => { + // Check which dependences are satisfied internally, + // all missing ones are exposed as required inputs at + // transaction level. + let producer = produced_handles.get(dh); + if let Some(producer) = producer { + dependence_pairs.push((*producer, index, pos)); + } else { + self.inputs.entry(dh.clone()).or_insert(None); + } + } + DFGTaskInput::Value(_) | DFGTaskInput::Compressed(_) => {} + } + } + if op.is_allowed { + self.results.push(op.output_handle.clone()); + } else { + self.intermediate_handles.push(op.output_handle.clone()); + } + assert!( + index + == self + .graph + .add_node( + op.output_handle.clone(), + (op.fhe_op as i16).into(), + std::mem::take(&mut op.inputs), + op.is_allowed, + ) + .index() + ); + } + for (source, destination, pos) in dependence_pairs { + // This returns an error in case of circular + // dependences. This should not be possible. + self.graph.add_dependence(source, destination, pos)?; + } + Ok(()) + } + pub fn add_input(&mut self, handle: &[u8], cct: DFGTxInput) { + self.inputs + .entry(handle.to_vec()) + .and_modify(|v| *v = Some(cct)); + } +} +impl std::fmt::Debug for TxNode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let _ = writeln!(f, "Transaction: [{:?}]", self.transaction_id); + let _ = writeln!( + f, + "{:?}", + daggy::petgraph::dot::Dot::with_config(self.graph.graph.graph(), &[]) + ); + let _ = writeln!(f, "Inputs :"); + for i in self.inputs.iter() { + let _ = writeln!(f, "\t {:?}", i); + } + let _ = writeln!(f, "Results :"); + for r in self.results.iter() { + let _ = writeln!(f, "\t {:?}", r); + } + writeln!(f) + } +} + +#[derive(Default)] +pub struct DFTxGraph { + pub graph: Dag, + pub needed_map: HashMap>, + pub allowed_map: HashMap, + pub results: Vec, +} +impl DFTxGraph { + pub fn build(&mut self, nodes: &mut Vec) -> Result<()> { + while let Some(tx) = nodes.pop() { + self.graph.add_node(tx); + } + // Gather handles produced within the graph + for (producer, tx) in self.graph.node_references() { + for r in tx.results.iter() { + self.allowed_map.insert(r.clone(), producer); + } + } + // Identify all dependence pairs (producer, consumer) + let mut dependence_pairs = vec![]; + for (consumer, tx) in self.graph.node_references() { + for i in tx.inputs.keys() { + if let Some(producer) = self.allowed_map.get(i) { + dependence_pairs.push((producer, consumer)); + } else { + self.needed_map + .entry(i.clone()) + .and_modify(|uses| uses.push(consumer)) + .or_insert(vec![consumer]); + } + } + } + // Add transaction dependence edges + for (producer, consumer) in dependence_pairs { + // Error only occurs in case of cyclic dependence which + // shoud not be possible between transactions. In that + // case, the whole cycle should be put in an error state. + self.graph + .add_edge(*producer, consumer, ()) + .map_err(|_| SchedulerError::CyclicDependence)?; + } + Ok(()) + } + + pub fn add_input(&mut self, handle: &[u8], input: &DFGTxInput) -> Result<()> { + if let Some(nodes) = self.needed_map.get(handle) { + for n in nodes.iter() { + let node = self + .graph + .node_weight_mut(*n) + .ok_or(SchedulerError::DataflowGraphError)?; + node.add_input(handle, input.clone()); + } + } + Ok(()) + } + pub fn add_output( + &mut self, + handle: &[u8], + result: Result<(SupportedFheCiphertexts, i16, Vec)>, + edges: &Dag<(), TxEdge>, + ) -> Result<()> { + if let Some(producer) = self.allowed_map.get(handle).cloned() { + if let Ok(ref result) = result { + // Traverse immediate dependents and add this result as an input + for edge in edges.edges_directed(producer, Direction::Outgoing) { + let dependent_tx_index = edge.target(); + let dependent_tx = self + .graph + .node_weight_mut(dependent_tx_index) + .ok_or(SchedulerError::DataflowGraphError)?; + dependent_tx + .inputs + .entry(handle.to_vec()) + .and_modify(|v| *v = Some(DFGTxInput::Value(result.0.clone()))); + } + } else { + // If this result was an error, mark this transaction + // and all its dependents as uncomputable, we will + // skip them during scheduling + self.set_uncomputable(producer, edges)?; + } + // Finally add the output (either error or compressed + // ciphertext) to the graph's outputs + let producer_tx = self + .graph + .node_weight_mut(producer) + .ok_or(SchedulerError::DataflowGraphError)?; + self.results.push(DFGTxResult { + transaction_id: producer_tx.transaction_id.clone(), + handle: handle.to_vec(), + compressed_ct: result.map(|rok| (rok.1, rok.2)), + }); + } + Ok(()) + } + // Set a node as uncomputable and recursively traverse graph to + // set its dependents as uncomputable as well + fn set_uncomputable( + &mut self, + tx_node_index: NodeIndex, + edges: &Dag<(), TxEdge>, + ) -> Result<()> { + let tx_node = self + .graph + .node_weight_mut(tx_node_index) + .ok_or(SchedulerError::DataflowGraphError)?; + tx_node.is_uncomputable = true; + for edge in edges.edges_directed(tx_node_index, Direction::Outgoing) { + let dependent_tx_index = edge.target(); + self.set_uncomputable(dependent_tx_index, edges)?; + } + Ok(()) + } + pub fn get_results(&mut self) -> Vec { + std::mem::take(&mut self.results) + } + pub fn get_intermediate_handles(&mut self) -> Vec<(Handle, Handle)> { + let mut res = vec![]; + for tx in self.graph.node_weights_mut() { + if !tx.is_uncomputable { + res.append( + &mut (std::mem::take(&mut tx.intermediate_handles)) + .into_iter() + .map(|h| (h, tx.transaction_id.clone())) + .collect::>(), + ); + } + } + res + } +} +impl std::fmt::Debug for DFTxGraph { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let _ = writeln!(f, "Transaction Graph:",); + let _ = writeln!( + f, + "{:?}", + daggy::petgraph::dot::Dot::with_config(self.graph.graph(), &[]) + ); + let _ = writeln!(f, "Needed Inputs :"); + for i in self.needed_map.iter() { + let _ = writeln!(f, "\t {:?}", i); + } + let _ = writeln!(f, "Results :"); + for r in self.results.iter() { + let _ = writeln!(f, "\t {:?}", r); + } + writeln!(f) + } +} + +pub struct DFGResult { + pub handle: Handle, + pub result: Result)>>, + pub work_index: usize, +} +pub type OpEdge = u8; pub struct OpNode { opcode: i32, - result: DFGTaskResult, result_handle: Handle, inputs: Vec, #[cfg(feature = "gpu")] locality: i32, is_allowed: bool, - is_needed: bool, - work_index: usize, } -pub type OpEdge = u8; - -pub struct DFGResult { - pub handle: Handle, - pub result: Result)>>, - pub work_index: usize, -} - impl std::fmt::Debug for OpNode { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OpNode") .field("OP", &self.opcode) - .field( - "Result handle", - &format_args!("{:02X?}", &self.result_handle), - ) + .field("Result handle", &format_args!("{:?}", &self.result_handle)) .finish() } } +impl OpNode { + fn check_ready_inputs(&mut self, ct_map: &mut HashMap>) -> bool { + for i in self.inputs.iter_mut() { + if !matches!(i, DFGTaskInput::Value(_)) { + let DFGTaskInput::Dependence(d) = i else { + return false; + }; + let Some(Some(DFGTxInput::Value(val))) = ct_map.get(d) else { + return false; + }; + *i = DFGTaskInput::Value(val.clone()); + } + } + true + } +} #[derive(Default, Debug)] pub struct DFGraph { pub graph: Dag, } - impl DFGraph { pub fn add_node( &mut self, @@ -53,19 +315,15 @@ impl DFGraph { opcode: i32, inputs: Vec, is_allowed: bool, - work_index: usize, - ) -> Result { - Ok(self.graph.add_node(OpNode { + ) -> NodeIndex { + self.graph.add_node(OpNode { opcode, - result: None, result_handle: rh, inputs, #[cfg(feature = "gpu")] locality: -1, is_allowed, - is_needed: is_allowed, - work_index, - })) + }) } pub fn add_dependence( &mut self, @@ -73,8 +331,6 @@ impl DFGraph { destination: usize, consumer_input: usize, ) -> Result<()> { - let consumer_index = node_index(destination); - self.graph[consumer_index].inputs[consumer_input] = DFGTaskInput::Dependence(Some(source)); let _edge = self .graph .add_edge( @@ -85,91 +341,4 @@ impl DFGraph { .map_err(|_| SchedulerError::CyclicDependence)?; Ok(()) } - - pub fn get_results(&mut self) -> Vec { - let mut res = Vec::with_capacity(self.graph.node_count()); - for index in 0..self.graph.node_count() { - let node = self.graph.node_weight_mut(NodeIndex::new(index)).unwrap(); - if let Some(ct) = std::mem::take(&mut node.result) { - if let Ok(ct) = ct { - if node.is_allowed { - res.push(DFGResult { - handle: node.result_handle.clone(), - result: Ok(ct.1), - work_index: node.work_index, - }); - } else { - res.push(DFGResult { - handle: node.result_handle.clone(), - result: Ok(None), - work_index: node.work_index, - }); - } - } else { - res.push(DFGResult { - handle: node.result_handle.clone(), - result: Err(ct.err().unwrap()), - work_index: node.work_index, - }); - } - } else { - res.push(DFGResult { - handle: node.result_handle.clone(), - result: Err(SchedulerError::DataflowGraphError.into()), - work_index: node.work_index, - }); - } - } - res - } - - fn is_needed(&self, index: usize) -> bool { - let node_index = NodeIndex::new(index); - let node = self.graph.node_weight(node_index).unwrap(); - if node.is_allowed || node.is_needed { - true - } else { - for edge in self.graph.edges_directed(node_index, Direction::Outgoing) { - // If any outgoing dependence is needed, so is this node - if self.is_needed(edge.target().index()) { - return true; - } - } - false - } - } - - pub fn finalize(&mut self) { - // Traverse in reverse order and mark nodes as needed as the - // graph order is roughly computable, so allowed nodes should - // generally be later in the graph. - for index in (0..self.graph.node_count()).rev() { - if self.is_needed(index) { - let node = self.graph.node_weight_mut(NodeIndex::new(index)).unwrap(); - node.is_needed = true; - } - } - // Prune graph of all unneeded nodes and edges - let mut unneeded_nodes = Vec::new(); - for index in 0..self.graph.node_count() { - let node_index = NodeIndex::new(index); - let Some(node) = self.graph.node_weight(node_index) else { - continue; - }; - if !node.is_needed { - unneeded_nodes.push(index); - } - } - unneeded_nodes.sort(); - // Remove unneeded nodes and their edges - for index in unneeded_nodes.iter().rev() { - let node_index = NodeIndex::new(*index); - let Some(node) = self.graph.node_weight(node_index) else { - continue; - }; - if !node.is_needed { - self.graph.remove_node(node_index); - } - } - } } diff --git a/coprocessor/fhevm-engine/scheduler/src/dfg/scheduler.rs b/coprocessor/fhevm-engine/scheduler/src/dfg/scheduler.rs index 913399f092..94c25c1949 100644 --- a/coprocessor/fhevm-engine/scheduler/src/dfg/scheduler.rs +++ b/coprocessor/fhevm-engine/scheduler/src/dfg/scheduler.rs @@ -1,28 +1,34 @@ -use crate::dfg::{types::*, OpEdge, OpNode}; +use crate::dfg::{types::*, TxEdge}; use anyhow::Result; use daggy::{ petgraph::{ csr::IndexType, graph::node_index, visit::{ - EdgeRef, IntoEdgeReferences, IntoEdgesDirected, IntoNeighbors, VisitMap, Visitable, + EdgeRef, IntoEdgeReferences, IntoEdgesDirected, IntoNeighbors, IntoNodeIdentifiers, + VisitMap, Visitable, }, - Direction, - Direction::Incoming, + Direction::{self, Incoming}, }, Dag, NodeIndex, }; use fhevm_engine_common::common::FheOperation; use fhevm_engine_common::tfhe_ops::perform_fhe_operation; -use fhevm_engine_common::types::SupportedFheCiphertexts; - +use fhevm_engine_common::types::{Handle, SupportedFheCiphertexts}; use fhevm_engine_common::utils::HeartBeat; -use rayon::prelude::*; -use std::{ - collections::HashMap, - sync::{atomic::AtomicUsize, mpsc::channel}, +use opentelemetry::{ + trace::{Span, Tracer}, + KeyValue, }; +use std::{collections::HashMap, sync::atomic::AtomicUsize}; +use tfhe::ReRandomizationContext; use tokio::task::JoinSet; +use tracing::{error, info, warn}; + +use super::{DFGraph, DFTxGraph, OpNode}; + +const TRANSACTION_RERANDOMISATION_DOMAIN_SEPARATOR: [u8; 8] = *b"TFHE_Rrd"; +const COMPACT_PUBLIC_ENCRYPTION_DOMAIN_SEPARATOR: [u8; 8] = *b"TFHE_Enc"; struct ExecNode { df_nodes: Vec, @@ -50,482 +56,179 @@ impl std::fmt::Debug for ExecNode { } } +enum DeviceSelection { + #[allow(dead_code)] + Index(usize), + RoundRobin, + #[allow(dead_code)] + NA, +} + pub struct Scheduler<'a> { - graph: &'a mut Dag, - edges: Dag<(), OpEdge>, + graph: &'a mut DFTxGraph, + edges: Dag<(), TxEdge>, sks: tfhe::ServerKey, + cpk: tfhe::CompactPublicKey, #[cfg(feature = "gpu")] csks: Vec, activity_heartbeat: HeartBeat, } impl<'a> Scheduler<'a> { - fn is_ready(node: &OpNode) -> bool { - let mut ready = true; - for i in node.inputs.iter() { - if let DFGTaskInput::Dependence(_) = i { - ready = false; - } - } - ready - } fn is_ready_task(&self, node: &ExecNode) -> bool { node.dependence_counter .load(std::sync::atomic::Ordering::SeqCst) == 0 } pub fn new( - graph: &'a mut Dag, + graph: &'a mut DFTxGraph, sks: tfhe::ServerKey, + cpk: tfhe::CompactPublicKey, #[cfg(feature = "gpu")] csks: Vec, activity_heartbeat: HeartBeat, ) -> Self { - let edges = graph.map(|_, _| (), |_, edge| *edge); + let edges = graph.graph.map(|_, _| (), |_, edge| *edge); Self { graph, edges, sks: sks.clone(), + cpk: cpk.clone(), #[cfg(feature = "gpu")] csks: csks.clone(), activity_heartbeat, } } - pub async fn schedule(&mut self) -> Result<()> { + pub async fn schedule(&mut self, loop_ctx: &'a opentelemetry::Context) -> Result<()> { let schedule_type = std::env::var("FHEVM_DF_SCHEDULE"); match schedule_type { Ok(val) => match val.as_str() { "MAX_PARALLELISM" => { - self.schedule_coarse_grain(PartitionStrategy::MaxParallelism) + self.schedule_coarse_grain(PartitionStrategy::MaxParallelism, loop_ctx) .await } "MAX_LOCALITY" => { - self.schedule_coarse_grain(PartitionStrategy::MaxLocality) + self.schedule_coarse_grain(PartitionStrategy::MaxLocality, loop_ctx) + .await + } + unhandled => { + error!(target: "scheduler", { strategy = ?unhandled }, + "Scheduling strategy does not exist"); + info!(target: "scheduler", { }, + "Reverting to default (generally best performance) strategy MAX_PARALLELISM"); + self.schedule_coarse_grain(PartitionStrategy::MaxParallelism, loop_ctx) .await } - "LOOP" => self.schedule_component_loop().await, - "FINE_GRAIN" => self.schedule_fine_grain().await, - unhandled => panic!("Scheduling strategy {:?} does not exist", unhandled), }, // Use overall best strategy as default #[cfg(not(feature = "gpu"))] _ => { - self.schedule_coarse_grain(PartitionStrategy::MaxParallelism) + self.schedule_coarse_grain(PartitionStrategy::MaxParallelism, loop_ctx) .await } #[cfg(feature = "gpu")] _ => { - self.schedule_coarse_grain(PartitionStrategy::MaxParallelism) + self.schedule_coarse_grain(PartitionStrategy::MaxParallelism, loop_ctx) .await } } } #[cfg(not(feature = "gpu"))] - async fn schedule_fine_grain(&mut self) -> Result<()> { - let mut set: JoinSet<(usize, TaskResult)> = JoinSet::new(); - let sks = self.sks.clone(); - tfhe::set_server_key(sks.clone()); - // Prime the scheduler with all nodes without dependences - for idx in 0..self.graph.node_count() { - let sks = sks.clone(); - let index = NodeIndex::new(idx); - let node = self - .graph - .node_weight_mut(index) - .ok_or(SchedulerError::DataflowGraphError)?; - if Self::is_ready(node) { - let opcode = node.opcode; - let is_allowed = node.is_allowed; - let inputs: Vec = node - .inputs - .iter() - .map(|i| match i { - DFGTaskInput::Value(i) => Ok(i.clone()), - DFGTaskInput::Compressed((t, c)) => { - SupportedFheCiphertexts::decompress_no_memcheck(*t, c) - } - _ => Err(SchedulerError::UnsatisfiedDependence.into()), - }) - .collect::>>()?; - set.spawn_blocking(move || { - tfhe::set_server_key(sks.clone()); - run_computation(opcode, inputs, idx, is_allowed, 0) - }); - } - } - // Get results from computations and update dependences of remaining computations - while let Some(result) = set.join_next().await { - self.activity_heartbeat.update(); - let result = result?; - let index = result.0; - let node_index = NodeIndex::new(index); - if let Ok(output) = &result.1 { - // Satisfy deps from the executed task - for edge in self.edges.edges_directed(node_index, Direction::Outgoing) { - let sks = sks.clone(); - let child_index = edge.target(); - let child_node = self - .graph - .node_weight_mut(child_index) - .ok_or(SchedulerError::DataflowGraphError)?; - child_node.inputs[*edge.weight() as usize] = - DFGTaskInput::Value(output.0.clone()); - if Self::is_ready(child_node) { - let opcode = child_node.opcode; - let is_allowed = child_node.is_allowed; - let inputs: Vec = child_node - .inputs - .iter() - .map(|i| match i { - DFGTaskInput::Value(i) => Ok(i.clone()), - DFGTaskInput::Compressed((t, c)) => { - SupportedFheCiphertexts::decompress_no_memcheck(*t, c) - } - _ => Err(SchedulerError::UnsatisfiedDependence.into()), - }) - .collect::>>()?; - set.spawn_blocking(move || { - tfhe::set_server_key(sks.clone()); - run_computation(opcode, inputs, child_index.index(), is_allowed, 0) - }); - } - } - } - let node_index = NodeIndex::new(result.0); - self.graph[node_index].result = Some(result.1); - } - Ok(()) - } - - #[cfg(not(feature = "gpu"))] - async fn schedule_coarse_grain(&mut self, strategy: PartitionStrategy) -> Result<()> { - let sks = self.sks.clone(); - tfhe::set_server_key(sks.clone()); - let mut set: JoinSet<(Vec<(usize, TaskResult)>, NodeIndex)> = JoinSet::new(); - let mut execution_graph: Dag = Dag::default(); - let _ = match strategy { - PartitionStrategy::MaxLocality => { - partition_components(self.graph, &mut execution_graph) - } - PartitionStrategy::MaxParallelism => { - partition_preserving_parallelism(self.graph, &mut execution_graph) - } - }; - let task_dependences = execution_graph.map(|_, _| (), |_, edge| *edge); - - // Prime the scheduler with all nodes without dependences - for idx in 0..execution_graph.node_count() { - let sks = sks.clone(); - let index = NodeIndex::new(idx); - let node = execution_graph - .node_weight_mut(index) - .ok_or(SchedulerError::DataflowGraphError)?; - if self.is_ready_task(node) { - let mut args = Vec::with_capacity(node.df_nodes.len()); - for nidx in node.df_nodes.iter() { - let n = self - .graph - .node_weight_mut(*nidx) - .ok_or(SchedulerError::DataflowGraphError)?; - let opcode = n.opcode; - let is_allowed = n.is_allowed; - args.push((opcode, std::mem::take(&mut n.inputs), *nidx, is_allowed)); - } - set.spawn_blocking(move || { - tfhe::set_server_key(sks.clone()); - execute_partition(args, index, 0) - }); - } - } - // Get results from computations and update dependences of remaining computations - while let Some(result) = set.join_next().await { - self.activity_heartbeat.update(); - let mut result = result?; - let task_index = result.1; - while let Some((node_index, node_result)) = result.0.pop() { - let node_index = NodeIndex::new(node_index); - // If this node result is an error, we can't satisfy - // any dependences with it, so skip - all dependences - // on this will remain unsatisfied and result in - // further errors. - if let Ok(ref node_result) = node_result { - // Satisfy deps from the executed computation in the DFG - for edge in self.edges.edges_directed(node_index, Direction::Outgoing) { - let child_index = edge.target(); - let child_node = self - .graph - .node_weight_mut(child_index) - .ok_or(SchedulerError::DataflowGraphError)?; - if !child_node.inputs.is_empty() { - // Here cannot be an error - child_node.inputs[*edge.weight() as usize] = - DFGTaskInput::Value(node_result.0.clone()); - } - } - } - self.graph[node_index].result = Some(node_result); - } - for edge in task_dependences.edges_directed(task_index, Direction::Outgoing) { - let sks = sks.clone(); - let dependent_task_index = edge.target(); - let dependent_task = execution_graph - .node_weight_mut(dependent_task_index) - .ok_or(SchedulerError::DataflowGraphError)?; - dependent_task - .dependence_counter - .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); - if self.is_ready_task(dependent_task) { - let mut args = Vec::with_capacity(dependent_task.df_nodes.len()); - for nidx in dependent_task.df_nodes.iter() { - let n = self - .graph - .node_weight_mut(*nidx) - .ok_or(SchedulerError::DataflowGraphError)?; - let opcode = n.opcode; - let is_allowed = n.is_allowed; - args.push((opcode, std::mem::take(&mut n.inputs), *nidx, is_allowed)); - } - set.spawn_blocking(move || { - tfhe::set_server_key(sks.clone()); - execute_partition(args, dependent_task_index, 0) - }); - } - } - } - Ok(()) - } - - #[cfg(not(feature = "gpu"))] - async fn schedule_component_loop(&mut self) -> Result<()> { - let mut execution_graph: Dag = Dag::default(); - let _ = partition_components(self.graph, &mut execution_graph); - let mut comps = vec![]; - let sks = self.sks.clone(); - tfhe::set_server_key(sks.clone()); - rayon::broadcast(|_| { - tfhe::set_server_key(sks.clone()); - }); - - // Prime the scheduler with all nodes without dependences - for idx in 0..execution_graph.node_count() { - let index = NodeIndex::new(idx); - let node = execution_graph - .node_weight_mut(index) - .ok_or(SchedulerError::DataflowGraphError)?; - if self.is_ready_task(node) { - let mut args = Vec::with_capacity(node.df_nodes.len()); - for nidx in node.df_nodes.iter() { - let n = self - .graph - .node_weight_mut(*nidx) - .ok_or(SchedulerError::DataflowGraphError)?; - let opcode = n.opcode; - let is_allowed = n.is_allowed; - args.push((opcode, std::mem::take(&mut n.inputs), *nidx, is_allowed)); - } - comps.push((std::mem::take(&mut args), index)); - } - } - - let (src, dest) = channel(); - tokio::task::spawn_blocking(move || { - tfhe::set_server_key(sks.clone()); - comps.par_iter().for_each_with(src, |src, (args, index)| { - src.send(execute_partition(args.to_vec(), *index, 0)) - .unwrap(); - }); - }) - .await?; - let mut results = vec![]; - for v in dest.iter() { - self.activity_heartbeat.update(); - results.push(v); - } - for mut result in results { - while let Some(o) = result.0.pop() { - let index = o.0; - let node_index = NodeIndex::new(index); - self.graph[node_index].result = Some(o.1); - } - } - Ok(()) + fn get_keys( + &self, + _target: DeviceSelection, + ) -> Result<(tfhe::ServerKey, tfhe::CompactPublicKey)> { + Ok((self.sks.clone(), self.cpk.clone())) } - #[cfg(feature = "gpu")] - async fn schedule_fine_grain(&mut self) -> Result<()> { - let now = std::time::SystemTime::now(); - let mut set: JoinSet<(usize, TaskResult)> = JoinSet::new(); - let keys = self.csks.clone(); - let mut rr = 0; - // Prime the scheduler with all nodes without dependences - for idx in 0..self.graph.node_count() { - let index = NodeIndex::new(idx); - let node = self - .graph - .node_weight_mut(index) - .ok_or(SchedulerError::DataflowGraphError)?; - if Self::is_ready(node) { - let gpu_index = rr % keys.len(); - let key = keys[gpu_index].clone(); - node.locality = (gpu_index) as i32; - rr += 1; - tfhe::set_server_key(key.clone()); - let opcode = node.opcode; - let is_allowed = node.is_allowed; - let inputs: Vec = node - .inputs - .iter() - .map(|i| match i { - DFGTaskInput::Value(i) => Ok(i.clone()), - DFGTaskInput::Compressed((t, c)) => { - SupportedFheCiphertexts::decompress(*t, c, gpu_index) - } - _ => Err(SchedulerError::UnsatisfiedDependence.into()), - }) - .collect::>>()?; - set.spawn_blocking(move || { - tfhe::set_server_key(key); - run_computation(opcode, inputs, idx, is_allowed, gpu_index) - }); - } - } - // Get results from computations and update dependences of remaining computations - while let Some(result) = set.join_next().await { - self.activity_heartbeat.update(); - let result = result?; - let index = result.0; - let node_index = NodeIndex::new(index); - let loc = self.graph[node_index].locality; - if let Ok(output) = &result.1 { - // Satisfy deps from the executed task - for edge in self.edges.edges_directed(node_index, Direction::Outgoing) { - let child_index = edge.target(); - let child_node = self - .graph - .node_weight_mut(child_index) - .ok_or(SchedulerError::DataflowGraphError)?; - child_node.locality = loc; - child_node.inputs[*edge.weight() as usize] = - DFGTaskInput::Value(output.0.clone()); - if Self::is_ready(child_node) { - let loc = if child_node.locality == -1 { - let loc = rr % keys.len(); - rr += 1; - loc - } else { - child_node.locality as usize - }; - let key = keys[loc].clone(); - tfhe::set_server_key(key.clone()); - let opcode = child_node.opcode; - let is_allowed = child_node.is_allowed; - let inputs: Vec = child_node - .inputs - .iter() - .map(|i| match i { - DFGTaskInput::Value(i) => Ok(i.clone()), - DFGTaskInput::Compressed((t, c)) => { - SupportedFheCiphertexts::decompress(*t, c, loc) - } - _ => Err(SchedulerError::UnsatisfiedDependence.into()), - }) - .collect::>>()?; - set.spawn_blocking(move || { - tfhe::set_server_key(key); - run_computation(opcode, inputs, child_index.index(), is_allowed, loc) - }); - } + fn get_keys( + &self, + target: DeviceSelection, + ) -> Result<(tfhe::CudaServerKey, tfhe::CompactPublicKey)> { + match target { + DeviceSelection::Index(i) => { + if i > self.csks.len() { + error!(target: "scheduler", {index = ?i }, + "Wrong device index"); + // Instead of giving up, we'll use device 0 (which + // should always be safe to use) and keep making + // progress even if suboptimally + Ok((self.csks[0].clone(), self.cpk.clone())) + } else { + Ok((self.csks[i].clone(), self.cpk.clone())) } } - let node_index = NodeIndex::new(result.0); - self.graph[node_index].result = Some(result.1); + DeviceSelection::RoundRobin => { + static LAST: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); + let i = LAST.load(std::sync::atomic::Ordering::Acquire); + LAST.store( + (i + 1) % self.csks.len(), + std::sync::atomic::Ordering::Release, + ); + Ok((self.csks[i].clone(), self.cpk.clone())) + } + DeviceSelection::NA => Ok((self.csks[0].clone(), self.cpk.clone())), } - println!( - "Scheduler time for block of {}: {}", - self.graph.node_count(), - now.elapsed().unwrap().as_millis() - ); - Ok(()) } - #[cfg(feature = "gpu")] - async fn schedule_coarse_grain(&mut self, strategy: PartitionStrategy) -> Result<()> { - let keys = self.csks.clone(); - tfhe::set_server_key(keys[0].clone()); - let mut set: JoinSet<(Vec<(usize, TaskResult)>, NodeIndex)> = JoinSet::new(); + async fn schedule_coarse_grain( + &mut self, + strategy: PartitionStrategy, + loop_ctx: &'a opentelemetry::Context, + ) -> Result<()> { let mut execution_graph: Dag = Dag::default(); - let _ = match strategy { + match strategy { PartitionStrategy::MaxLocality => { - partition_components(self.graph, &mut execution_graph) + partition_components(&self.graph.graph, &mut execution_graph)? } PartitionStrategy::MaxParallelism => { - partition_preserving_parallelism(self.graph, &mut execution_graph) + partition_preserving_parallelism(&self.graph.graph, &mut execution_graph)? } }; let task_dependences = execution_graph.map(|_, _| (), |_, edge| *edge); - let now = std::time::SystemTime::now(); // Prime the scheduler with all nodes without dependences - let mut rr = 0; + let mut set: JoinSet<(HashMap, NodeIndex)> = JoinSet::new(); for idx in 0..execution_graph.node_count() { - let loc = rr % keys.len(); - let key = keys[loc].clone(); - rr += 1; let index = NodeIndex::new(idx); let node = execution_graph .node_weight_mut(index) .ok_or(SchedulerError::DataflowGraphError)?; - node.locality = loc as i32; if self.is_ready_task(node) { let mut args = Vec::with_capacity(node.df_nodes.len()); for nidx in node.df_nodes.iter() { - let n = self + let tx = self + .graph .graph .node_weight_mut(*nidx) .ok_or(SchedulerError::DataflowGraphError)?; - let opcode = n.opcode; - let is_allowed = n.is_allowed; - args.push((opcode, std::mem::take(&mut n.inputs), *nidx, is_allowed)); + args.push(( + std::mem::take(&mut tx.graph), + std::mem::take(&mut tx.inputs), + tx.transaction_id.clone(), + )); } - set.spawn_blocking(move || { - tfhe::set_server_key(key); - execute_partition(args, index, loc) - }); + let (sks, cpk) = self.get_keys(DeviceSelection::RoundRobin)?; + let loop_ctx = loop_ctx.clone(); + set.spawn( + async move { execute_partition(args, index, 0, sks, cpk, &loop_ctx).await }, + ); } } - // Get results from computations and update dependences of remaining computations while let Some(result) = set.join_next().await { self.activity_heartbeat.update(); - let mut result = result?; + // The result contains all outputs (allowed handles) + // computed within the finished partition. Now check the + // outputs and update the trnsaction inputs of downstream + // transactions + let result = result?; let task_index = result.1; - while let Some((node_index, node_result)) = result.0.pop() { - let node_index = NodeIndex::new(node_index); - let loc: usize = if self.graph[node_index].locality < 0 { - 0 - } else { - self.graph[node_index].locality as usize - }; - // If this node result is an error, we can't satisfy - // any dependences with it, so skip - all dependences - // on this will remain unsatisfied and result in - // further errors. - if let Ok(ref node_result) = node_result { - // Satisfy deps from the executed computation in the DFG - for edge in self.edges.edges_directed(node_index, Direction::Outgoing) { - let child_index = edge.target(); - let child_node = self - .graph - .node_weight_mut(child_index) - .ok_or(SchedulerError::DataflowGraphError)?; - if !child_node.inputs.is_empty() { - tfhe::set_server_key(keys[loc].clone()); - // Here cannot be an error - child_node.inputs[*edge.weight() as usize] = - DFGTaskInput::Value(node_result.0.clone()); - } - } - } - self.graph[node_index].result = Some(node_result); + for (handle, node_result) in result.0.into_iter() { + // Add computed allowed handles to the graph. These + // can be used as inputs and forwarded to subsequent, + // dependent transactions + self.graph.add_output(&handle, node_result, &self.edges)?; } for edge in task_dependences.edges_directed(task_index, Direction::Outgoing) { let dependent_task_index = edge.target(); @@ -536,117 +239,38 @@ impl<'a> Scheduler<'a> { .dependence_counter .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); if self.is_ready_task(dependent_task) { - let loc = rr % keys.len(); - let key = keys[loc].clone(); - dependent_task.locality = loc as i32; - rr += 1; let mut args = Vec::with_capacity(dependent_task.df_nodes.len()); for nidx in dependent_task.df_nodes.iter() { - let n = self + let tx = self + .graph .graph .node_weight_mut(*nidx) .ok_or(SchedulerError::DataflowGraphError)?; - let opcode = n.opcode; - let is_allowed = n.is_allowed; - args.push((opcode, std::mem::take(&mut n.inputs), *nidx, is_allowed)); + // Skip transactions that cannot complete + // because of missing dependences. + if tx.is_uncomputable { + continue; + } + args.push(( + std::mem::take(&mut tx.graph), + std::mem::take(&mut tx.inputs), + tx.transaction_id.clone(), + )); } - set.spawn_blocking(move || { - tfhe::set_server_key(key); - execute_partition(args, dependent_task_index, loc) + let (sks, cpk) = self.get_keys(DeviceSelection::RoundRobin)?; + let loop_ctx = loop_ctx.clone(); + set.spawn(async move { + execute_partition(args, dependent_task_index, 0, sks, cpk, &loop_ctx).await }); } } } - println!( - "Scheduler time for block of {}: {}", - self.graph.node_count(), - now.elapsed().unwrap().as_millis() - ); - Ok(()) - } - - #[cfg(feature = "gpu")] - async fn schedule_component_loop(&mut self) -> Result<()> { - let mut execution_graph: Dag = Dag::default(); - let _ = partition_components(self.graph, &mut execution_graph); - let mut comps = vec![]; - - let now = std::time::SystemTime::now(); - // Prime the scheduler with all nodes without dependences - for idx in 0..execution_graph.node_count() { - let index = NodeIndex::new(idx); - let node = execution_graph - .node_weight_mut(index) - .ok_or(SchedulerError::DataflowGraphError)?; - if self.is_ready_task(node) { - let mut args = Vec::with_capacity(node.df_nodes.len()); - for nidx in node.df_nodes.iter() { - let n = self - .graph - .node_weight_mut(*nidx) - .ok_or(SchedulerError::DataflowGraphError)?; - let opcode = n.opcode; - let is_allowed = n.is_allowed; - args.push((opcode, std::mem::take(&mut n.inputs), *nidx, is_allowed)); - } - comps.push((std::mem::take(&mut args), index)); - } - } - if comps.is_empty() { - return Ok(()); - } - - let keys = self.csks.clone(); - let (src, dest) = channel(); - tokio::task::spawn_blocking(move || { - let num_streams_per_gpu = 8; // TODO: add config variable for this - let chunk_size = comps.len() / keys.len() + (comps.len() % keys.len() != 0) as usize; - - comps - .par_chunks(chunk_size) // Split into as many chunks as GPUs available - .enumerate() // Get the index for GPU - .for_each_with(src, |src, (i, comps_i)| { - // Process chunks within each GPU - let stream_chunk_size = comps_i.len() / num_streams_per_gpu - + (comps_i.len() % num_streams_per_gpu != 0) as usize; - let src = src.clone(); - comps_i - .par_chunks(stream_chunk_size) // Further split chunks into as many chunks as we allow streams per GPU - .for_each_with(src, |src, chunk| { - // Set the server key for the current GPU - tfhe::set_server_key(keys[i].clone()); - // Sequential iteration over the chunks of data for each stream - chunk.iter().for_each(|(args, index)| { - src.send(execute_partition(args.to_vec(), *index, i)) - .unwrap(); - }); - }); - }); - }) - .await?; - let mut results = vec![]; - for v in dest.iter() { - self.activity_heartbeat.update(); - results.push(v); - } - for mut result in results { - while let Some(o) = result.0.pop() { - let index = o.0; - let node_index = NodeIndex::new(index); - self.graph[node_index].result = Some(o.1); - } - } - println!( - "Scheduler time for block of {}: {}", - self.graph.node_count(), - now.elapsed().unwrap().as_millis() - ); Ok(()) } } -fn add_execution_depedences( - graph: &Dag, +fn add_execution_depedences( + graph: &Dag, execution_graph: &mut Dag, node_map: HashMap, ) -> Result<()> { @@ -676,8 +300,8 @@ fn add_execution_depedences( Ok(()) } -fn partition_preserving_parallelism( - graph: &Dag, +fn partition_preserving_parallelism( + graph: &Dag, execution_graph: &mut Dag, ) -> Result<()> { // First sort the DAG in a schedulable order @@ -720,8 +344,8 @@ fn partition_preserving_parallelism( add_execution_depedences(graph, execution_graph, node_map) } -fn partition_components( - graph: &Dag, +fn partition_components( + graph: &Dag, execution_graph: &mut Dag, ) -> Result<()> { // First sort the DAG in a schedulable order @@ -746,7 +370,7 @@ fn partition_components( } } } - // Apply topsort to component nodes + // Apply toposort to component nodes df_nodes.sort_by_key(|x| tsmap.get(x).unwrap()); execution_graph .add_node(ExecNode { @@ -764,58 +388,269 @@ fn partition_components( Ok(()) } -type TaskResult = Result<(SupportedFheCiphertexts, Option<(i16, Vec)>)>; +fn re_randomise_transaction_inputs( + inputs: &mut HashMap>, + transaction_id: &Handle, + gpu_idx: usize, + cpk: tfhe::CompactPublicKey, +) -> Result<()> { + let mut re_rand_context = ReRandomizationContext::new( + TRANSACTION_RERANDOMISATION_DOMAIN_SEPARATOR, + [transaction_id.as_slice()], + COMPACT_PUBLIC_ENCRYPTION_DOMAIN_SEPARATOR, + ); + for txinput in inputs.values_mut() { + match txinput { + Some(DFGTxInput::Value(val)) => { + val.add_to_re_randomization_context(&mut re_rand_context); + } + Some(DFGTxInput::Compressed((t, c))) => { + let decomp = SupportedFheCiphertexts::decompress(*t, c, gpu_idx)?; + decomp.add_to_rerandomisation_context(&mut re_rand_context); + *txinput = Some(DFGTxInput::Value(decomp)); + } + None => { + error!(target: "scheduler", { transaction_id = ?hex::encode(transaction_id) }, + "Missing transaction input while trying to re-randomise"); + return Err(SchedulerError::MissingInputs.into()); + } + } + } + let mut seed_gen = re_rand_context.finalize(); + for txinput in inputs.values_mut() { + match txinput { + Some(DFGTxInput::Value(ref mut val)) => { + val.re_randomise(&cpk, seed_gen.next_seed()?)?; + } + Some(DFGTxInput::Compressed(_)) => { + error!(target: "scheduler", { transaction_id = ?hex::encode(transaction_id) }, + "Failed to re-randomise inputs for transaction"); + return Err(SchedulerError::ReRandomisationError.into()); + } + None => { + error!(target: "scheduler", { transaction_id = ?hex::encode(transaction_id) }, + "Failed to re-randomise inputs for transaction"); + return Err(SchedulerError::ReRandomisationError.into()); + } + } + } + Ok(()) +} +fn decompress_transaction_inputs( + inputs: &mut HashMap>, + transaction_id: &Handle, + gpu_idx: usize, + _cpk: tfhe::CompactPublicKey, +) -> Result<()> { + // TODO: implement re-randomisation on GPU. For now just decompress inputs + for txinput in inputs.values_mut() { + match txinput { + Some(DFGTxInput::Value(_)) => {} + Some(DFGTxInput::Compressed((t, c))) => { + let decomp = SupportedFheCiphertexts::decompress(*t, c, gpu_idx)?; + *txinput = Some(DFGTxInput::Value(decomp)); + } + None => { + error!(target: "scheduler", { transaction_id = ?hex::encode(transaction_id) }, + "Missing transaction input while trying to decompress"); + return Err(SchedulerError::MissingInputs.into()); + } + } + } + Ok(()) +} -fn execute_partition( - computations: Vec<(i32, Vec, NodeIndex, bool)>, +type TaskResult = Result<(SupportedFheCiphertexts, i16, Vec)>; +async fn execute_partition( + transactions: Vec<(DFGraph, HashMap>, Handle)>, task_id: NodeIndex, gpu_idx: usize, -) -> (Vec<(usize, TaskResult)>, NodeIndex) { - let mut res: HashMap = HashMap::with_capacity(computations.len()); - 'comps: for (opcode, inputs, nidx, is_allowed) in computations { - let mut cts = Vec::with_capacity(inputs.len()); - for i in inputs.iter() { - match i { - DFGTaskInput::Dependence(d) => { - if let Some(d) = d { - if let Some(Ok(ct)) = res.get(d) { - cts.push(ct.0.clone()); - } else { + #[cfg(not(feature = "gpu"))] sks: tfhe::ServerKey, + #[cfg(feature = "gpu")] sks: tfhe::CudaServerKey, + cpk: tfhe::CompactPublicKey, + loop_ctx: &opentelemetry::Context, +) -> (HashMap, NodeIndex) { + let mut res: HashMap = HashMap::with_capacity(transactions.len()); + let tracer = opentelemetry::global::tracer("tfhe_worker"); + // Traverse transactions within the partition. The transactions + // are topologically sorted so the order is executable + 'tx: for (ref mut dfg, ref mut tx_inputs, tid) in transactions { + tfhe::set_server_key(sks.clone()); + // Update the transaction inputs based on allowed handles so + // far. If any input is still missing, and we cannot fill it + // (e.g., error in the producer transaction) we cannot execute + // this transaction and possibly more downstream. + for (h, i) in tx_inputs.iter_mut() { + if i.is_none() { + let Some(Ok(ct)) = res.get(h) else { + warn!(target: "scheduler", {transaction_id = ?tid }, + "Missing input to compute transaction - skipping"); + for nidx in dfg.graph.node_identifiers() { + let Some(node) = dfg.graph.node_weight_mut(nidx) else { + error!(target: "scheduler", {index = ?nidx.index() }, "Wrong dataflow graph index"); + continue; + }; + if node.is_allowed { res.insert( - nidx.index(), - Err(SchedulerError::UnsatisfiedDependence.into()), + node.result_handle.clone(), + Err(SchedulerError::MissingInputs.into()), ); - continue 'comps; } } + continue 'tx; + }; + *i = Some(DFGTxInput::Value(ct.0.clone())); + } + } + let mut s = tracer.start_with_context("rerandomise_inputs", loop_ctx); + if !cfg!(feature = "gpu") { + // Re-randomise inputs of the transaction - this also + // decompresses ciphertexts + if let Err(e) = re_randomise_transaction_inputs(tx_inputs, &tid, gpu_idx, cpk.clone()) { + error!(target: "scheduler", {transaction_id = ?tid, error = ?e }, + "Error while re-randomising inputs"); + for nidx in dfg.graph.node_identifiers() { + let Some(node) = dfg.graph.node_weight_mut(nidx) else { + error!(target: "scheduler", {index = ?nidx.index() }, "Wrong dataflow graph index"); + continue; + }; + if node.is_allowed { + res.insert( + node.result_handle.clone(), + Err(SchedulerError::ReRandomisationError.into()), + ); + } } - DFGTaskInput::Value(v) => { - cts.push(v.clone()); + continue 'tx; + } + } else { + // If re-randomisation is not available (e.g., on GPU), + // only decompress ciphertexts + if let Err(e) = decompress_transaction_inputs(tx_inputs, &tid, gpu_idx, cpk.clone()) { + error!(target: "scheduler", {transaction_id = ?tid, error = ?e }, + "Error while decompressing inputs"); + for nidx in dfg.graph.node_identifiers() { + let Some(node) = dfg.graph.node_weight_mut(nidx) else { + error!(target: "scheduler", {index = ?nidx.index() }, "Wrong dataflow graph index"); + continue; + }; + if node.is_allowed { + res.insert( + node.result_handle.clone(), + Err(SchedulerError::ReRandomisationError.into()), + ); + } } - DFGTaskInput::Compressed((t, c)) => { - let decomp = SupportedFheCiphertexts::decompress(*t, c, gpu_idx); - if let Ok(decomp) = decomp { - cts.push(decomp); - } else { - res.insert(nidx.index(), Err(decomp.err().unwrap())); - continue 'comps; + continue 'tx; + } + } + s.end(); + + // Prime the scheduler with ready ops from the transaction's subgraph + let mut s = tracer.start_with_context("execute_transaction", loop_ctx); + s.set_attribute(KeyValue::new( + "transaction_hash", + format!("0x{}", hex::encode(&tid)), + )); + let mut set: JoinSet<(usize, OpResult)> = JoinSet::new(); + for nidx in dfg.graph.node_identifiers() { + let Some(node) = dfg.graph.node_weight_mut(nidx) else { + error!(target: "scheduler", {index = ?nidx.index() }, "Wrong dataflow graph index"); + continue; + }; + try_schedule_node( + node, + nidx.index(), + &mut set, + tx_inputs, + gpu_idx, + sks.clone(), + ); + } + let edges = dfg.graph.map(|_, _| (), |_, edge| *edge); + while let Some(result) = set.join_next().await { + tfhe::set_server_key(sks.clone()); + if let Ok(result) = result { + let nidx = NodeIndex::new(result.0); + if result.1.is_ok() { + for edge in edges.edges_directed(nidx, Direction::Outgoing) { + let child_index = edge.target(); + let Some(child_node) = dfg.graph.node_weight_mut(child_index) else { + error!(target: "scheduler", {index = ?child_index.index() }, "Wrong dataflow graph index"); + continue; + }; + // Update input of consumers + if let Ok(ref res) = result.1 { + child_node.inputs[*edge.weight() as usize] = + DFGTaskInput::Value(res.0.clone()); + } + try_schedule_node( + child_node, + child_index.index(), + &mut set, + tx_inputs, + gpu_idx, + sks.clone(), + ); } } + // Update partition's outputs (allowed handles only) + let node = dfg.graph.node_weight_mut(nidx).unwrap(); + if node.is_allowed { + res.insert( + node.result_handle.clone(), + result + .1 + .map(|v| (v.0, v.1.as_ref().unwrap().0, v.1.unwrap().1)), + ); + } } } - let (node_index, result) = run_computation(opcode, cts, nidx.index(), is_allowed, gpu_idx); - res.insert(node_index, result); + s.end(); + } + (res, task_id) +} + +fn try_schedule_node( + node: &mut OpNode, + node_index: usize, + set: &mut JoinSet<(usize, OpResult)>, + tx_inputs: &mut HashMap>, + gpu_idx: usize, + #[cfg(not(feature = "gpu"))] sks: tfhe::ServerKey, + #[cfg(feature = "gpu")] sks: tfhe::CudaServerKey, +) { + if !node.check_ready_inputs(tx_inputs) { + return; + } + let mut cts = Vec::with_capacity(node.inputs.len()); + for i in std::mem::take(&mut node.inputs) { + if let DFGTaskInput::Value(i) = i { + cts.push(i); + } else { + // That should not be possible as we called the checker. + error!(target: "scheduler", { handle = ?node.result_handle }, "Computation missing inputs"); + return; + } } - (Vec::from_iter(res), task_id) + + let opcode = node.opcode; + let is_allowed = node.is_allowed; + let sks = sks.clone(); + set.spawn_blocking(move || { + tfhe::set_server_key(sks.clone()); + run_computation(opcode, cts, node_index, is_allowed, gpu_idx) + }); } +type OpResult = Result<(SupportedFheCiphertexts, Option<(i16, Vec)>)>; fn run_computation( operation: i32, inputs: Vec, graph_node_index: usize, is_allowed: bool, gpu_idx: usize, -) -> (usize, TaskResult) { +) -> (usize, OpResult) { let op = FheOperation::try_from(operation); match op { Ok(FheOperation::FheGetCiphertext) => { @@ -836,9 +671,6 @@ fn run_computation( } Err(e) => (graph_node_index, Err(e.into())), }, - _ => ( - graph_node_index, - Err(SchedulerError::UnknownOperation(operation).into()), - ), + Err(e) => (graph_node_index, Err(e.into())), } } diff --git a/coprocessor/fhevm-engine/scheduler/src/dfg/types.rs b/coprocessor/fhevm-engine/scheduler/src/dfg/types.rs index d4a224bd15..29a8229e21 100644 --- a/coprocessor/fhevm-engine/scheduler/src/dfg/types.rs +++ b/coprocessor/fhevm-engine/scheduler/src/dfg/types.rs @@ -1,22 +1,62 @@ use anyhow::Result; -use fhevm_engine_common::types::SupportedFheCiphertexts; +use fhevm_engine_common::types::{Handle, SupportedFheCiphertexts}; -pub type DFGTaskResult = Option)>)>>; +pub struct DFGTxResult { + pub handle: Handle, + pub transaction_id: Handle, + pub compressed_ct: Result<(i16, Vec)>, +} +impl std::fmt::Debug for DFGTxResult { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let _ = writeln!( + f, + "Result: [{:?}] - tid [{:?}]", + self.handle, self.transaction_id + ); + if self.compressed_ct.is_err() { + let _ = write!(f, "\t ERROR"); + } else { + let _ = write!(f, "\t OK"); + } + writeln!(f) + } +} +#[derive(Clone)] +pub enum DFGTxInput { + Value(SupportedFheCiphertexts), + Compressed((i16, Vec)), +} +impl std::fmt::Debug for DFGTxInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Value(_) => write!(f, "DecCT"), + Self::Compressed(_) => write!(f, "ComCT"), + } + } +} #[derive(Clone)] pub enum DFGTaskInput { Value(SupportedFheCiphertexts), Compressed((i16, Vec)), - Dependence(Option), + Dependence(Handle), +} +impl std::fmt::Debug for DFGTaskInput { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Value(_) => write!(f, "DecCT"), + Self::Compressed(_) => write!(f, "ComCT"), + Self::Dependence(_) => write!(f, "DepHL"), + } + } } #[derive(Debug, Copy, Clone)] pub enum SchedulerError { - UnsatisfiedDependence, CyclicDependence, DataflowGraphError, - UnknownOperation(i32), - InvalidInputs, + MissingInputs, + ReRandomisationError, SchedulerError, } @@ -25,23 +65,17 @@ impl std::error::Error for SchedulerError {} impl std::fmt::Display for SchedulerError { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - Self::UnsatisfiedDependence => { - write!( - f, - "Unsatisfied depence in dataflow graph at scheduling time" - ) - } Self::CyclicDependence => { write!(f, "Depence cycle in dataflow graph") } Self::DataflowGraphError => { write!(f, "Inconsistent dataflow graph error") } - Self::UnknownOperation(op) => { - write!(f, "Unknown operation with code: {op}") + Self::MissingInputs => { + write!(f, "Missing inputs") } - Self::InvalidInputs => { - write!(f, "Invalid inputs to FHE operation") + Self::ReRandomisationError => { + write!(f, "Re-randomisation error") } Self::SchedulerError => { write!(f, "Generic scheduler error") diff --git a/coprocessor/fhevm-engine/sns-worker/src/executor.rs b/coprocessor/fhevm-engine/sns-worker/src/executor.rs index eb28274906..521cbb283f 100644 --- a/coprocessor/fhevm-engine/sns-worker/src/executor.rs +++ b/coprocessor/fhevm-engine/sns-worker/src/executor.rs @@ -630,20 +630,6 @@ async fn update_computations_status( ) .execute(db_txn.as_mut()) .await?; - // We need to update the allowed_handles table as well for - // the case where an input handle (that is not the output - // of a FHE computation) is allowed. This means that the - // TFHE worker never sees this handle and therefore cannot - // update its computed status. - sqlx::query!( - " - UPDATE allowed_handles - SET is_computed = TRUE - WHERE handle = $1;", - task.handle - ) - .execute(db_txn.as_mut()) - .await?; } else { error!( handle = ?task.handle, "Large ciphertext not computed for task"); } diff --git a/coprocessor/fhevm-engine/stress-test-generator/src/dex.rs b/coprocessor/fhevm-engine/stress-test-generator/src/dex.rs index 0b956e2d49..90e72a075d 100644 --- a/coprocessor/fhevm-engine/stress-test-generator/src/dex.rs +++ b/coprocessor/fhevm-engine/stress-test-generator/src/dex.rs @@ -60,7 +60,7 @@ async fn dex_swap_request_update_dex_balance( result: sent_amount, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; Ok((sent_amount, new_current_balance)) } @@ -97,7 +97,7 @@ async fn dex_swap_request_finalize( result: pending_in, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, true).await?; let pending_total_token_in = next_random_handle(DEF_TYPE); let event = tfhe_event(TfheContractEvents::FheAdd(TfheContract::FheAdd { caller, @@ -106,7 +106,7 @@ async fn dex_swap_request_finalize( result: pending_total_token_in, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, true).await?; Ok((pending_in, pending_total_token_in)) } @@ -298,7 +298,7 @@ async fn dex_swap_claim_prepare( toType: crate::utils::FheType::FheUint128 as u8, result: big_pending_1_in, })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; let total_dex_token_0_out_te = generate_trivial_encrypt( contract_address, user_address, @@ -306,6 +306,7 @@ async fn dex_swap_claim_prepare( listener_event_to_db, Some(crate::utils::FheType::FheUint128), Some(total_dex_token_0_out.into()), + false, ) .await?; let big_amount_0_out_mul = next_random_handle(crate::utils::FheType::FheUint128); @@ -316,7 +317,7 @@ async fn dex_swap_claim_prepare( result: big_amount_0_out_mul, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; let total_dex_token_1_in_te = generate_trivial_encrypt( contract_address, user_address, @@ -324,6 +325,7 @@ async fn dex_swap_claim_prepare( listener_event_to_db, Some(crate::utils::FheType::FheUint128), Some(total_dex_token_1_in.into()), + false, ) .await?; let big_amount_0_out_div = next_random_handle(crate::utils::FheType::FheUint128); @@ -334,7 +336,7 @@ async fn dex_swap_claim_prepare( result: big_amount_0_out_div, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; amount_0_out = next_random_handle(crate::utils::FheType::FheUint64); let event = tfhe_event(TfheContractEvents::Cast(TfheContract::Cast { caller, @@ -342,7 +344,7 @@ async fn dex_swap_claim_prepare( toType: crate::utils::FheType::FheUint64 as u8, result: amount_0_out, })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; } if total_dex_token_0_in != 0 { let big_pending_0_in = next_random_handle(crate::utils::FheType::FheUint128); @@ -352,7 +354,7 @@ async fn dex_swap_claim_prepare( toType: crate::utils::FheType::FheUint128 as u8, result: big_pending_0_in, })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; let total_dex_token_1_out_te = generate_trivial_encrypt( contract_address, user_address, @@ -360,6 +362,7 @@ async fn dex_swap_claim_prepare( listener_event_to_db, Some(crate::utils::FheType::FheUint128), Some(total_dex_token_1_out.into()), + false, ) .await?; let big_amount_1_out_mul = next_random_handle(crate::utils::FheType::FheUint128); @@ -370,7 +373,7 @@ async fn dex_swap_claim_prepare( result: big_amount_1_out_mul, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; let total_dex_token_0_in_te = generate_trivial_encrypt( contract_address, user_address, @@ -378,6 +381,7 @@ async fn dex_swap_claim_prepare( listener_event_to_db, Some(crate::utils::FheType::FheUint128), Some(total_dex_token_0_in.into()), + false, ) .await?; let big_amount_1_out_div = next_random_handle(crate::utils::FheType::FheUint128); @@ -388,7 +392,7 @@ async fn dex_swap_claim_prepare( result: big_amount_1_out_div, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; amount_1_out = next_random_handle(crate::utils::FheType::FheUint64); let event = tfhe_event(TfheContractEvents::Cast(TfheContract::Cast { caller, @@ -396,7 +400,7 @@ async fn dex_swap_claim_prepare( toType: crate::utils::FheType::FheUint64 as u8, result: amount_1_out, })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; } Ok((amount_0_out, amount_1_out)) } diff --git a/coprocessor/fhevm-engine/stress-test-generator/src/erc20.rs b/coprocessor/fhevm-engine/stress-test-generator/src/erc20.rs index 023b8a627d..58ba7ef952 100644 --- a/coprocessor/fhevm-engine/stress-test-generator/src/erc20.rs +++ b/coprocessor/fhevm-engine/stress-test-generator/src/erc20.rs @@ -54,7 +54,7 @@ pub async fn erc20_transaction( result: has_enough_funds, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; let new_source = next_random_handle(DEF_TYPE); let new_destination = next_random_handle(DEF_TYPE); match variant { @@ -67,8 +67,7 @@ pub async fn erc20_transaction( result: new_destination_target, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; - + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; let event = tfhe_event(TfheContractEvents::FheIfThenElse( TfheContract::FheIfThenElse { caller, @@ -78,6 +77,7 @@ pub async fn erc20_transaction( result: new_destination, }, )); + insert_tfhe_event(listener_event_to_db, transaction_id, event, true).await?; allow_handle( &new_destination.to_vec(), AllowEvents::AllowedForDecryption, @@ -85,8 +85,6 @@ pub async fn erc20_transaction( pool, ) .await?; - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; - let new_source_target = next_random_handle(DEF_TYPE); let event = tfhe_event(TfheContractEvents::FheSub(TfheContract::FheSub { caller, @@ -95,7 +93,7 @@ pub async fn erc20_transaction( result: new_source_target, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; let event = tfhe_event(TfheContractEvents::FheIfThenElse( TfheContract::FheIfThenElse { caller, @@ -105,6 +103,7 @@ pub async fn erc20_transaction( result: new_source, }, )); + insert_tfhe_event(listener_event_to_db, transaction_id, event, true).await?; allow_handle( &new_source.to_vec(), AllowEvents::AllowedForDecryption, @@ -112,7 +111,6 @@ pub async fn erc20_transaction( pool, ) .await?; - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; } ERCTransferVariant::NoCMUX => { let cast_has_enough_funds = next_random_handle(DEF_TYPE); @@ -122,8 +120,7 @@ pub async fn erc20_transaction( toType: 5u8, result: cast_has_enough_funds, })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; - + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; let select_amount = next_random_handle(DEF_TYPE); let event = tfhe_event(TfheContractEvents::FheMul(TfheContract::FheMul { caller, @@ -132,8 +129,7 @@ pub async fn erc20_transaction( result: select_amount, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; - + insert_tfhe_event(listener_event_to_db, transaction_id, event, false).await?; let event = tfhe_event(TfheContractEvents::FheAdd(TfheContract::FheAdd { caller, lhs: destination, @@ -141,8 +137,7 @@ pub async fn erc20_transaction( result: new_destination, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; - + insert_tfhe_event(listener_event_to_db, transaction_id, event, true).await?; allow_handle( &new_destination.to_vec(), AllowEvents::AllowedForDecryption, @@ -157,8 +152,7 @@ pub async fn erc20_transaction( result: new_source, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; - + insert_tfhe_event(listener_event_to_db, transaction_id, event, true).await?; allow_handle( &new_source.to_vec(), AllowEvents::AllowedForDecryption, diff --git a/coprocessor/fhevm-engine/stress-test-generator/src/synthetics.rs b/coprocessor/fhevm-engine/stress-test-generator/src/synthetics.rs index a2c36994cf..d597fca606 100644 --- a/coprocessor/fhevm-engine/stress-test-generator/src/synthetics.rs +++ b/coprocessor/fhevm-engine/stress-test-generator/src/synthetics.rs @@ -26,7 +26,7 @@ pub async fn add_chain_transaction( ) -> Result<(Handle, Handle), Box> { let caller = user_address.parse().unwrap(); let transaction_id = transaction_id.unwrap_or_else(|| next_random_handle(DEF_TYPE)); - let counter = + let mut counter = generate_random_handle_amount_if_none(ctx, counter, contract_address, user_address).await?; let amount = match amount { @@ -39,12 +39,13 @@ pub async fn add_chain_transaction( listener_event_to_db, Some(DEF_TYPE), None, + false, ) .await? } }; - for _ in 0..length { + for i in 0..length { let new_counter = next_random_handle(FheType::FheUint64); let event = tfhe_event(TfheContractEvents::FheAdd(TfheContract::FheAdd { caller, @@ -53,7 +54,8 @@ pub async fn add_chain_transaction( result: new_counter, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, i == length - 1).await?; + counter = new_counter; } allow_handle( &counter.to_vec(), @@ -92,12 +94,13 @@ pub async fn mul_chain_transaction( listener_event_to_db, Some(DEF_TYPE), None, + false, ) .await? } }; - for _ in 0..length { + for i in 0..length { let new_counter = next_random_handle(FheType::FheUint64); let event = tfhe_event(TfheContractEvents::FheMul(TfheContract::FheMul { caller, @@ -106,7 +109,7 @@ pub async fn mul_chain_transaction( result: new_counter, scalarByte: ScalarByte::from(false as u8), })); - insert_tfhe_event(listener_event_to_db, transaction_id, event).await?; + insert_tfhe_event(listener_event_to_db, transaction_id, event, i == length - 1).await?; counter = new_counter; } allow_handle( @@ -145,6 +148,7 @@ pub async fn generate_pub_decrypt_handles_types( listener_event_to_db, Some(type_num.into()), Some(type_num.into()), + true, ) .await?; allow_handle( @@ -185,6 +189,7 @@ pub async fn generate_user_decrypt_handles_types( listener_event_to_db, Some(type_num.into()), Some(type_num.into()), + true, ) .await?; allow_handle( diff --git a/coprocessor/fhevm-engine/stress-test-generator/src/utils.rs b/coprocessor/fhevm-engine/stress-test-generator/src/utils.rs index 05bbf964a3..4ce0a801c4 100644 --- a/coprocessor/fhevm-engine/stress-test-generator/src/utils.rs +++ b/coprocessor/fhevm-engine/stress-test-generator/src/utils.rs @@ -208,6 +208,7 @@ pub async fn generate_trivial_encrypt( listener_event_to_db: &mut ListenerDatabase, ct_type: Option, ct_value: Option, + is_allowed: bool, ) -> Result> { let caller = user_address.parse().unwrap(); let ct_type = ct_type.unwrap_or(DEF_TYPE); @@ -223,6 +224,7 @@ pub async fn generate_trivial_encrypt( }, )), transaction_hash: Some(transaction_hash), + is_allowed, }; let mut tx = listener_event_to_db.new_transaction().await?; listener_event_to_db @@ -368,12 +370,14 @@ pub async fn insert_tfhe_event( listener_event_to_db: &ListenerDatabase, transaction_hash: TransactionHash, event: Log, + is_allowed: bool, ) -> Result<(), Box> { let started_at = tokio::time::Instant::now(); let mut tx = listener_event_to_db.new_transaction().await?; let log = LogTfhe { event, transaction_hash: Some(transaction_hash), + is_allowed, }; listener_event_to_db .insert_tfhe_event(&mut tx, &log) diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-0b85af1e88f24290121400feb960ef80ce040e2b877b259da17188668e6c404a.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-0b85af1e88f24290121400feb960ef80ce040e2b877b259da17188668e6c404a.json new file mode 100644 index 0000000000..fbbf9d8f5e --- /dev/null +++ b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-0b85af1e88f24290121400feb960ef80ce040e2b877b259da17188668e6c404a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT MAX(block_number) FROM host_chain_blocks_valid WHERE chain_id = $1;\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0b85af1e88f24290121400feb960ef80ce040e2b877b259da17188668e6c404a" +} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-e4260b9ba59d978d5504b787a0002c6eec8c6b95b2cb8b5da83f8885fb273c39.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-0fe024db8ce4cb065633e9f02bd96e67b62460c4d725030ce6fb3b488d0541a7.json similarity index 61% rename from coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-e4260b9ba59d978d5504b787a0002c6eec8c6b95b2cb8b5da83f8885fb273c39.json rename to coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-0fe024db8ce4cb065633e9f02bd96e67b62460c4d725030ce6fb3b488d0541a7.json index f3ff350e33..9afd9248b4 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-e4260b9ba59d978d5504b787a0002c6eec8c6b95b2cb8b5da83f8885fb273c39.json +++ b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-0fe024db8ce4cb065633e9f02bd96e67b62460c4d725030ce6fb3b488d0541a7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO computations (\n tenant_id,\n output_handle,\n dependencies,\n fhe_operation,\n is_scalar,\n dependence_chain_id,\n transaction_id\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ON CONFLICT (tenant_id, output_handle, transaction_id) DO NOTHING\n ", + "query": "\n INSERT INTO computations (\n tenant_id,\n output_handle,\n dependencies,\n fhe_operation,\n is_scalar,\n dependence_chain_id,\n transaction_id,\n is_allowed\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (tenant_id, output_handle, transaction_id) DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -11,10 +11,11 @@ "Int2", "Bool", "Bytea", - "Bytea" + "Bytea", + "Bool" ] }, "nullable": [] }, - "hash": "e4260b9ba59d978d5504b787a0002c6eec8c6b95b2cb8b5da83f8885fb273c39" + "hash": "0fe024db8ce4cb065633e9f02bd96e67b62460c4d725030ce6fb3b488d0541a7" } diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-1dde98bc8c1076c5708f985512b88082da81a35fd1411d6f9871a5414075a666.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-1dde98bc8c1076c5708f985512b88082da81a35fd1411d6f9871a5414075a666.json new file mode 100644 index 0000000000..0374d7023c --- /dev/null +++ b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-1dde98bc8c1076c5708f985512b88082da81a35fd1411d6f9871a5414075a666.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE computations\n SET schedule_order = CURRENT_TIMESTAMP + INTERVAL '1 second' * uncomputable_counter,\n uncomputable_counter = LEAST(uncomputable_counter * 2, 32000)::SMALLINT\n WHERE tenant_id = $1\n AND (output_handle, transaction_id) IN (\n SELECT * FROM unnest($2::BYTEA[], $3::BYTEA[])\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "ByteaArray", + "ByteaArray" + ] + }, + "nullable": [] + }, + "hash": "1dde98bc8c1076c5708f985512b88082da81a35fd1411d6f9871a5414075a666" +} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-416ef65e70058585ce4cec14ef80330cd688076d02e375486d7ab07fab628280.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-416ef65e70058585ce4cec14ef80330cd688076d02e375486d7ab07fab628280.json deleted file mode 100644 index 23e15bca47..0000000000 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-416ef65e70058585ce4cec14ef80330cd688076d02e375486d7ab07fab628280.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO blocks_valid (chain_id, block_hash, block_number, listener_tfhe)\n VALUES ($1, $2, $3, true)\n ON CONFLICT (chain_id, block_hash) DO UPDATE SET listener_tfhe = true;\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "Bytea", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "416ef65e70058585ce4cec14ef80330cd688076d02e375486d7ab07fab628280" -} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-5061d9ee69dd4a22aa118bd4d1ca26b4a02f9f690def1c3f08c34bf67ec95f8c.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-5061d9ee69dd4a22aa118bd4d1ca26b4a02f9f690def1c3f08c34bf67ec95f8c.json deleted file mode 100644 index 1f53dfcee9..0000000000 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-5061d9ee69dd4a22aa118bd4d1ca26b4a02f9f690def1c3f08c34bf67ec95f8c.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE allowed_handles\n SET schedule_order = CURRENT_TIMESTAMP + INTERVAL '1 second' * uncomputable_counter,\n uncomputable_counter = LEAST(uncomputable_counter * 2, 32000)::SMALLINT\n WHERE tenant_id = $1\n AND handle = ANY($2::BYTEA[])\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "ByteaArray" - ] - }, - "nullable": [] - }, - "hash": "5061d9ee69dd4a22aa118bd4d1ca26b4a02f9f690def1c3f08c34bf67ec95f8c" -} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-78a2670fabfffa3f6fd187cd0288b86b13b401468cabe15089d75a08c5d919c0.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-54ecfc9b50024b3f14bce1ee1f91b721b843292e0c314c058fb1adaaea118b96.json similarity index 62% rename from coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-78a2670fabfffa3f6fd187cd0288b86b13b401468cabe15089d75a08c5d919c0.json rename to coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-54ecfc9b50024b3f14bce1ee1f91b721b843292e0c314c058fb1adaaea118b96.json index ee77874e78..65e29dfdba 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-78a2670fabfffa3f6fd187cd0288b86b13b401468cabe15089d75a08c5d919c0.json +++ b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-54ecfc9b50024b3f14bce1ee1f91b721b843292e0c314c058fb1adaaea118b96.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO pbs_computations(tenant_id, handle) VALUES($1, $2) \n ON CONFLICT DO NOTHING;", + "query": "INSERT INTO pbs_computations(tenant_id, handle) VALUES($1, $2)\n ON CONFLICT DO NOTHING;", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "78a2670fabfffa3f6fd187cd0288b86b13b401468cabe15089d75a08c5d919c0" + "hash": "54ecfc9b50024b3f14bce1ee1f91b721b843292e0c314c058fb1adaaea118b96" } diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-641036eba016313ea7cf191d71f2b69c1def70ea46139dd02fb510581b6322c2.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-641036eba016313ea7cf191d71f2b69c1def70ea46139dd02fb510581b6322c2.json new file mode 100644 index 0000000000..5c3a86a7b7 --- /dev/null +++ b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-641036eba016313ea7cf191d71f2b69c1def70ea46139dd02fb510581b6322c2.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO host_chain_blocks_valid (chain_id, block_hash, block_number)\n VALUES ($1, $2, $3)\n ON CONFLICT (chain_id, block_hash) DO NOTHING;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Bytea", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "641036eba016313ea7cf191d71f2b69c1def70ea46139dd02fb510581b6322c2" +} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-648e618238efe4b918570e78a235947075cc241f3af841e4d34fba3587268570.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-648e618238efe4b918570e78a235947075cc241f3af841e4d34fba3587268570.json deleted file mode 100644 index 9b3b3bd246..0000000000 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-648e618238efe4b918570e78a235947075cc241f3af841e4d34fba3587268570.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE allowed_handles\n SET is_computed = TRUE\n WHERE tenant_id = $1\n AND handle = ANY($2::BYTEA[])\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "ByteaArray" - ] - }, - "nullable": [] - }, - "hash": "648e618238efe4b918570e78a235947075cc241f3af841e4d34fba3587268570" -} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-69eea32069c04ab351856d18f4b53d059e01468778eea9588aff3f8aee6171a7.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-69eea32069c04ab351856d18f4b53d059e01468778eea9588aff3f8aee6171a7.json deleted file mode 100644 index c5e79af515..0000000000 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-69eea32069c04ab351856d18f4b53d059e01468778eea9588aff3f8aee6171a7.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT block_number FROM blocks_valid WHERE chain_id = $1 ORDER BY block_number DESC LIMIT 1;\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "block_number", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int4" - ] - }, - "nullable": [ - false - ] - }, - "hash": "69eea32069c04ab351856d18f4b53d059e01468778eea9588aff3f8aee6171a7" -} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-6613f90d85256adebd7f00e73b1e8d6b4db9d9309cae6810b4830188ea97088e.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-6f7b577ff92824540d1baee24d9a4303e567bf70930a2869b934153b98837c38.json similarity index 78% rename from coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-6613f90d85256adebd7f00e73b1e8d6b4db9d9309cae6810b4830188ea97088e.json rename to coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-6f7b577ff92824540d1baee24d9a4303e567bf70930a2869b934153b98837c38.json index db25ab32e2..58970a2004 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-6613f90d85256adebd7f00e73b1e8d6b4db9d9309cae6810b4830188ea97088e.json +++ b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-6f7b577ff92824540d1baee24d9a4303e567bf70930a2869b934153b98837c38.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT tenant_id, handle, ciphertext, ciphertext_type\n FROM ciphertexts\n WHERE tenant_id = ANY($1::INT[])\n AND handle = ANY($2::BYTEA[])\n ", + "query": "\n SELECT tenant_id, handle, ciphertext, ciphertext_type\n FROM ciphertexts\n WHERE tenant_id = $1\n AND handle = ANY($2::BYTEA[])\n ", "describe": { "columns": [ { @@ -26,7 +26,7 @@ ], "parameters": { "Left": [ - "Int4Array", + "Int4", "ByteaArray" ] }, @@ -37,5 +37,5 @@ false ] }, - "hash": "6613f90d85256adebd7f00e73b1e8d6b4db9d9309cae6810b4830188ea97088e" + "hash": "6f7b577ff92824540d1baee24d9a4303e567bf70930a2869b934153b98837c38" } diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-3427af810410f385b774ff7e2f382c08bf38acc91d45a6efae187870b602d619.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-7d2b6f2d8d54967820ba63bb3e30a1cf15755280e85b0de61e234f192c55f736.json similarity index 58% rename from coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-3427af810410f385b774ff7e2f382c08bf38acc91d45a6efae187870b602d619.json rename to coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-7d2b6f2d8d54967820ba63bb3e30a1cf15755280e85b0de61e234f192c55f736.json index 70f16f677e..47210ba3a2 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-3427af810410f385b774ff7e2f382c08bf38acc91d45a6efae187870b602d619.json +++ b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-7d2b6f2d8d54967820ba63bb3e30a1cf15755280e85b0de61e234f192c55f736.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO computations(\n tenant_id,\n output_handle,\n dependencies,\n fhe_operation,\n is_completed,\n is_scalar,\n dependence_chain_id,\n transaction_id\n )\n VALUES($1, $2, $3, $4, false, $5, $6, $7)\n ON CONFLICT (tenant_id, output_handle, transaction_id) DO NOTHING\n ", + "query": "\n INSERT INTO computations(\n tenant_id,\n output_handle,\n dependencies,\n fhe_operation,\n is_completed,\n is_scalar,\n dependence_chain_id,\n transaction_id,\n is_allowed\n )\n VALUES($1, $2, $3, $4, false, $5, $6, $7, $8)\n ON CONFLICT (tenant_id, output_handle, transaction_id) DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -11,10 +11,11 @@ "Int2", "Bool", "Bytea", - "Bytea" + "Bytea", + "Bool" ] }, "nullable": [] }, - "hash": "3427af810410f385b774ff7e2f382c08bf38acc91d45a6efae187870b602d619" + "hash": "7d2b6f2d8d54967820ba63bb3e30a1cf15755280e85b0de61e234f192c55f736" } diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-82afc3c943c43f1f76e150c5d3354bef4482c3da82403ad6c03add0840e71966.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-82afc3c943c43f1f76e150c5d3354bef4482c3da82403ad6c03add0840e71966.json deleted file mode 100644 index d1176fb971..0000000000 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-82afc3c943c43f1f76e150c5d3354bef4482c3da82403ad6c03add0840e71966.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\nWITH selected_computations AS (\n -- Get all computations from such transactions\n (\n SELECT \n c.tenant_id, \n c.output_handle,\n c.transaction_id,\n ah.handle, \n ah.is_computed\n FROM computations c\n LEFT JOIN allowed_handles ah\n ON c.output_handle = ah.handle\n AND c.tenant_id = ah.tenant_id\n WHERE c.transaction_id IN (\n -- Select transaction IDs with uncomputed handles\n -- out of the dependence buckets\n SELECT DISTINCT transaction_id \n FROM computations\n WHERE is_error = FALSE\n AND (tenant_id, output_handle) IN (\n SELECT tenant_id, handle\n FROM allowed_handles\n WHERE is_computed = FALSE\n ORDER BY schedule_order\n LIMIT $1\n )\n LIMIT $2\n )\n )\n)\n-- Acquire all computations from this transaction set\nSELECT \n c.tenant_id, \n c.output_handle, \n c.dependencies, \n c.fhe_operation, \n c.is_scalar,\n sc.handle IS NOT NULL AS is_allowed, \n c.dependence_chain_id,\n COALESCE(sc.is_computed) AS is_computed,\n c.transaction_id\nFROM computations c\nJOIN selected_computations sc\n ON c.tenant_id = sc.tenant_id\n AND c.output_handle = sc.output_handle\n AND c.transaction_id = sc.transaction_id\nFOR UPDATE SKIP LOCKED ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tenant_id", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "output_handle", - "type_info": "Bytea" - }, - { - "ordinal": 2, - "name": "dependencies", - "type_info": "ByteaArray" - }, - { - "ordinal": 3, - "name": "fhe_operation", - "type_info": "Int2" - }, - { - "ordinal": 4, - "name": "is_scalar", - "type_info": "Bool" - }, - { - "ordinal": 5, - "name": "is_allowed", - "type_info": "Bool" - }, - { - "ordinal": 6, - "name": "dependence_chain_id", - "type_info": "Bytea" - }, - { - "ordinal": 7, - "name": "is_computed", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "transaction_id", - "type_info": "Bytea" - } - ], - "parameters": { - "Left": [ - "Int8", - "Int8" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - null, - true, - null, - false - ] - }, - "hash": "82afc3c943c43f1f76e150c5d3354bef4482c3da82403ad6c03add0840e71966" -} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-86869a71d7bd80262a82765e77b819bab29e95c1d60ff31525172fb859c61cee.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-86869a71d7bd80262a82765e77b819bab29e95c1d60ff31525172fb859c61cee.json new file mode 100644 index 0000000000..09259407cd --- /dev/null +++ b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-86869a71d7bd80262a82765e77b819bab29e95c1d60ff31525172fb859c61cee.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "\nWITH selected_computations AS (\n (\n SELECT DISTINCT\n c.transaction_id\n FROM (\n SELECT transaction_id\n FROM computations \n WHERE is_completed = FALSE\n AND is_error = FALSE\n AND is_allowed = TRUE\n ORDER BY schedule_order\n LIMIT $1\n ) as c\n )\n)\n-- Acquire all computations from this transaction set\nSELECT\n c.tenant_id, \n c.output_handle, \n c.dependencies, \n c.fhe_operation, \n c.is_scalar,\n c.is_allowed, \n c.dependence_chain_id,\n c.transaction_id\nFROM computations c\nJOIN selected_computations sc\n ON c.transaction_id = sc.transaction_id\nFOR UPDATE SKIP LOCKED ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tenant_id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "output_handle", + "type_info": "Bytea" + }, + { + "ordinal": 2, + "name": "dependencies", + "type_info": "ByteaArray" + }, + { + "ordinal": 3, + "name": "fhe_operation", + "type_info": "Int2" + }, + { + "ordinal": 4, + "name": "is_scalar", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "is_allowed", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "dependence_chain_id", + "type_info": "Bytea" + }, + { + "ordinal": 7, + "name": "transaction_id", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + false + ] + }, + "hash": "86869a71d7bd80262a82765e77b819bab29e95c1d60ff31525172fb859c61cee" +} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-8767303b387a1e36636aac24abb209ae4ad9a106cc132b78048faf1e16b0b913.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-8767303b387a1e36636aac24abb209ae4ad9a106cc132b78048faf1e16b0b913.json new file mode 100644 index 0000000000..634e01fda5 --- /dev/null +++ b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-8767303b387a1e36636aac24abb209ae4ad9a106cc132b78048faf1e16b0b913.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE computations\n SET is_error = true, error_message = $1\n WHERE tenant_id = $2\n AND output_handle = $3\n AND transaction_id = $4\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int4", + "Bytea", + "Bytea" + ] + }, + "nullable": [] + }, + "hash": "8767303b387a1e36636aac24abb209ae4ad9a106cc132b78048faf1e16b0b913" +} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-af96d042a4d211bb5ea04ce2b6e7deac02e28246b9adc05b4df7772b20d4c0ee.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-af96d042a4d211bb5ea04ce2b6e7deac02e28246b9adc05b4df7772b20d4c0ee.json deleted file mode 100644 index dadb62981d..0000000000 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-af96d042a4d211bb5ea04ce2b6e7deac02e28246b9adc05b4df7772b20d4c0ee.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO allowed_handles(tenant_id, handle, account_address, event_type)\n SELECT * FROM UNNEST($1::INTEGER[], $2::BYTEA[], $3::TEXT[], $4::SMALLINT[])\n ON CONFLICT DO NOTHING;", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4Array", - "ByteaArray", - "TextArray", - "Int2Array" - ] - }, - "nullable": [] - }, - "hash": "af96d042a4d211bb5ea04ce2b6e7deac02e28246b9adc05b4df7772b20d4c0ee" -} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-e24d50902fd179dee66a7375ad1cdda51748bd9a0af8ec42a403d13bd132888c.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-bbfc9833ed9cb4c8e98834dc715f70f2eab863af8b79a6c0faf9c249ac6ecaf0.json similarity index 75% rename from coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-e24d50902fd179dee66a7375ad1cdda51748bd9a0af8ec42a403d13bd132888c.json rename to coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-bbfc9833ed9cb4c8e98834dc715f70f2eab863af8b79a6c0faf9c249ac6ecaf0.json index a162acc380..303c80a663 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-e24d50902fd179dee66a7375ad1cdda51748bd9a0af8ec42a403d13bd132888c.json +++ b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-bbfc9833ed9cb4c8e98834dc715f70f2eab863af8b79a6c0faf9c249ac6ecaf0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO allowed_handles(tenant_id, handle, account_address, event_type) VALUES($1, $2, $3, $4)\n ON CONFLICT DO NOTHING;", + "query": "INSERT INTO allowed_handles(tenant_id, handle, account_address, event_type) VALUES($1, $2, $3, $4)\n ON CONFLICT DO NOTHING;", "describe": { "columns": [], "parameters": { @@ -13,5 +13,5 @@ }, "nullable": [] }, - "hash": "e24d50902fd179dee66a7375ad1cdda51748bd9a0af8ec42a403d13bd132888c" + "hash": "bbfc9833ed9cb4c8e98834dc715f70f2eab863af8b79a6c0faf9c249ac6ecaf0" } diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-c0e1b25662f8d9b3ec8813c028e0de5388e5094d8c316b29e6762bc2783939d2.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-c0e1b25662f8d9b3ec8813c028e0de5388e5094d8c316b29e6762bc2783939d2.json deleted file mode 100644 index 394956b8b7..0000000000 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-c0e1b25662f8d9b3ec8813c028e0de5388e5094d8c316b29e6762bc2783939d2.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE computations\n SET is_completed = true, completed_at = CURRENT_TIMESTAMP\n WHERE tenant_id = $1\n AND (output_handle, transaction_id) IN (\n SELECT * FROM unnest($2::BYTEA[], $3::BYTEA[])\n )\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "ByteaArray", - "ByteaArray" - ] - }, - "nullable": [] - }, - "hash": "c0e1b25662f8d9b3ec8813c028e0de5388e5094d8c316b29e6762bc2783939d2" -} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-d12da013cb1d11f23595b7d4f6e4d79f886ec342b77ab4166b6d1a8febd62f1a.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-d12da013cb1d11f23595b7d4f6e4d79f886ec342b77ab4166b6d1a8febd62f1a.json deleted file mode 100644 index 126397d041..0000000000 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-d12da013cb1d11f23595b7d4f6e4d79f886ec342b77ab4166b6d1a8febd62f1a.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE computations\n SET is_error = true, error_message = $1\n WHERE tenant_id = $2\n AND output_handle = $3\n AND transaction_id = $4\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int4", - "Bytea", - "Bytea" - ] - }, - "nullable": [] - }, - "hash": "d12da013cb1d11f23595b7d4f6e4d79f886ec342b77ab4166b6d1a8febd62f1a" -} diff --git a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-7553e242070661c2ef0eaffccda9476eb243ff0be8c4c133301ddd6e51796bdf.json b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-d94483044765504ae794c16487fd225297876c170ba807360ae413fb9f837e5d.json similarity index 58% rename from coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-7553e242070661c2ef0eaffccda9476eb243ff0be8c4c133301ddd6e51796bdf.json rename to coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-d94483044765504ae794c16487fd225297876c170ba807360ae413fb9f837e5d.json index 470d7be682..e39bfe317a 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-7553e242070661c2ef0eaffccda9476eb243ff0be8c4c133301ddd6e51796bdf.json +++ b/coprocessor/fhevm-engine/tfhe-worker/.sqlx/query-d94483044765504ae794c16487fd225297876c170ba807360ae413fb9f837e5d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT count(1) FROM allowed_handles WHERE is_computed = FALSE", + "query": "SELECT count(1) FROM computations WHERE is_allowed = TRUE AND is_completed = FALSE", "describe": { "columns": [ { @@ -16,5 +16,5 @@ null ] }, - "hash": "7553e242070661c2ef0eaffccda9476eb243ff0be8c4c133301ddd6e51796bdf" + "hash": "d94483044765504ae794c16487fd225297876c170ba807360ae413fb9f837e5d" } diff --git a/coprocessor/fhevm-engine/tfhe-worker/benches/dex.rs b/coprocessor/fhevm-engine/tfhe-worker/benches/dex.rs index ae982e231b..079218eb1f 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/benches/dex.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/benches/dex.rs @@ -1,9 +1,8 @@ #[path = "./utils.rs"] mod utils; use crate::utils::{ - allow_handle, default_api_key, default_tenant_id, query_tenant_keys, random_handle, - setup_test_app, wait_until_all_allowed_handles_computed, write_to_json, EnvConfig, - OperatorType, + default_api_key, default_tenant_id, query_tenant_keys, random_handle, setup_test_app, + wait_until_all_allowed_handles_computed, write_to_json, EnvConfig, OperatorType, }; use criterion::{ async_executor::FuturesExecutor, measurement::WallTime, Bencher, Criterion, Throughput, @@ -323,12 +322,14 @@ async fn swap_request_whitepaper( transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle_0.clone(), inputs: vec![from_balance_0.clone(), amount_0.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: new_to_amount_target_handle_0.clone(), inputs: vec![current_dex_balance_0.clone(), amount_0.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -343,6 +344,7 @@ async fn swap_request_whitepaper( }, current_dex_balance_0.clone(), ], + is_allowed: false, }); // async_computations.push(AsyncComputation { // operation: FheOperation::FheSub.into(), @@ -373,12 +375,14 @@ async fn swap_request_whitepaper( transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle_1.clone(), inputs: vec![from_balance_1.clone(), amount_1.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: new_to_amount_target_handle_1.clone(), inputs: vec![current_dex_balance_1.clone(), amount_1.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -393,6 +397,7 @@ async fn swap_request_whitepaper( }, current_dex_balance_1.clone(), ], + is_allowed: false, }); // async_computations.push(AsyncComputation { // operation: FheOperation::FheSub.into(), @@ -429,12 +434,14 @@ async fn swap_request_whitepaper( transaction_id: transaction_id.clone(), output_handle: sent_0_handle.clone(), inputs: vec![new_current_balance_0.clone(), current_dex_balance_0.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), transaction_id: transaction_id.clone(), output_handle: sent_1_handle.clone(), inputs: vec![new_current_balance_1.clone(), current_dex_balance_1.clone()], + is_allowed: false, }); let sent_0 = AsyncComputationInput { input: Some(Input::InputHandle(sent_0_handle.clone())), @@ -447,30 +454,29 @@ async fn swap_request_whitepaper( transaction_id: transaction_id.clone(), output_handle: pending_0_in_handle.clone(), inputs: vec![to_balance_0.clone(), sent_0.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: pending_1_in_handle.clone(), inputs: vec![to_balance_1.clone(), sent_1.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: pending_total_token_0_in.clone(), inputs: vec![total_dex_token_0_in.clone(), sent_0.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: pending_total_token_1_in.clone(), inputs: vec![total_dex_token_1_in.clone(), sent_1.clone()], + is_allowed: true, }); - - allow_handle(&pending_0_in_handle, &pool).await?; - allow_handle(&pending_1_in_handle, &pool).await?; - allow_handle(&pending_total_token_0_in, &pool).await?; - allow_handle(&pending_total_token_1_in, &pool).await?; } let mut compute_request = tonic::Request::new(AsyncComputeRequest { @@ -643,6 +649,7 @@ async fn swap_request_no_cmux( transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle_0.clone(), inputs: vec![from_balance_0.clone(), amount_0.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -656,6 +663,7 @@ async fn swap_request_no_cmux( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheMul.into(), @@ -667,6 +675,7 @@ async fn swap_request_no_cmux( input: Some(Input::InputHandle(cast_has_enough_funds_handle_0.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -678,6 +687,7 @@ async fn swap_request_no_cmux( input: Some(Input::InputHandle(select_amount_handle_0.clone())), }, ], + is_allowed: false, }); // async_computations.push(AsyncComputation { // operation: FheOperation::FheSub.into(), @@ -701,6 +711,7 @@ async fn swap_request_no_cmux( transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle_1.clone(), inputs: vec![from_balance_1.clone(), amount_1.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -714,6 +725,7 @@ async fn swap_request_no_cmux( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheMul.into(), @@ -725,6 +737,7 @@ async fn swap_request_no_cmux( input: Some(Input::InputHandle(cast_has_enough_funds_handle_1.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -736,6 +749,7 @@ async fn swap_request_no_cmux( input: Some(Input::InputHandle(select_amount_handle_1.clone())), }, ], + is_allowed: false, }); // async_computations.push(AsyncComputation { // operation: FheOperation::FheSub.into(), @@ -765,12 +779,14 @@ async fn swap_request_no_cmux( transaction_id: transaction_id.clone(), output_handle: sent_0_handle.clone(), inputs: vec![new_current_balance_0.clone(), current_dex_balance_0.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), transaction_id: transaction_id.clone(), output_handle: sent_1_handle.clone(), inputs: vec![new_current_balance_1.clone(), current_dex_balance_1.clone()], + is_allowed: false, }); let sent_0 = AsyncComputationInput { input: Some(Input::InputHandle(sent_0_handle.clone())), @@ -783,30 +799,29 @@ async fn swap_request_no_cmux( transaction_id: transaction_id.clone(), output_handle: pending_0_in_handle.clone(), inputs: vec![to_balance_0.clone(), sent_0.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: pending_1_in_handle.clone(), inputs: vec![to_balance_1.clone(), sent_1.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: pending_total_token_0_in.clone(), inputs: vec![total_dex_token_0_in.clone(), sent_0.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: pending_total_token_1_in.clone(), inputs: vec![total_dex_token_1_in.clone(), sent_1.clone()], + is_allowed: true, }); - - allow_handle(&pending_0_in_handle, &pool).await?; - allow_handle(&pending_1_in_handle, &pool).await?; - allow_handle(&pending_total_token_0_in, &pool).await?; - allow_handle(&pending_total_token_1_in, &pool).await?; } let mut compute_request = tonic::Request::new(AsyncComputeRequest { @@ -971,6 +986,7 @@ async fn swap_claim_whitepaper( input: Some(Input::Scalar(vec![6u8])), }, ], + is_allowed: false, }); let mul_temp = next_handle(); async_computations.push(AsyncComputation { @@ -987,6 +1003,7 @@ async fn swap_claim_whitepaper( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheDiv.into(), @@ -1002,6 +1019,7 @@ async fn swap_claim_whitepaper( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -1015,6 +1033,7 @@ async fn swap_claim_whitepaper( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); // Transfer let has_enough_funds_handle_0 = next_handle(); @@ -1032,6 +1051,7 @@ async fn swap_claim_whitepaper( input: Some(Input::InputHandle(amount_0_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -1043,6 +1063,7 @@ async fn swap_claim_whitepaper( input: Some(Input::InputHandle(amount_0_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -1057,6 +1078,7 @@ async fn swap_claim_whitepaper( }, old_balance_0.clone(), ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), @@ -1068,6 +1090,7 @@ async fn swap_claim_whitepaper( input: Some(Input::InputHandle(amount_0_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -1082,9 +1105,8 @@ async fn swap_claim_whitepaper( }, current_dex_balance_0.clone(), ], + is_allowed: true, }); - allow_handle(&new_from_amount_handle_0, &pool).await?; - allow_handle(&new_to_amount_handle_0, &pool).await?; } if total_dex_token_0_in != 0 { async_computations.push(AsyncComputation { @@ -1097,6 +1119,7 @@ async fn swap_claim_whitepaper( input: Some(Input::Scalar(vec![6u8])), }, ], + is_allowed: false, }); let mul_temp = next_handle(); async_computations.push(AsyncComputation { @@ -1113,6 +1136,7 @@ async fn swap_claim_whitepaper( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheDiv.into(), @@ -1128,6 +1152,7 @@ async fn swap_claim_whitepaper( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -1141,6 +1166,7 @@ async fn swap_claim_whitepaper( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); // Transfer let has_enough_funds_handle_1 = next_handle(); @@ -1158,6 +1184,7 @@ async fn swap_claim_whitepaper( input: Some(Input::InputHandle(amount_1_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -1169,6 +1196,7 @@ async fn swap_claim_whitepaper( input: Some(Input::InputHandle(amount_1_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -1183,6 +1211,7 @@ async fn swap_claim_whitepaper( }, old_balance_1.clone(), ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), @@ -1194,6 +1223,7 @@ async fn swap_claim_whitepaper( input: Some(Input::InputHandle(amount_1_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -1208,9 +1238,8 @@ async fn swap_claim_whitepaper( }, current_dex_balance_1.clone(), ], + is_allowed: true, }); - allow_handle(&new_from_amount_handle_1, &pool).await?; - allow_handle(&new_to_amount_handle_1, &pool).await?; } } @@ -1376,6 +1405,7 @@ async fn swap_claim_no_cmux( input: Some(Input::Scalar(vec![6u8])), }, ], + is_allowed: false, }); let mul_temp = next_handle(); async_computations.push(AsyncComputation { @@ -1392,6 +1422,7 @@ async fn swap_claim_no_cmux( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheDiv.into(), @@ -1407,6 +1438,7 @@ async fn swap_claim_no_cmux( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -1420,6 +1452,7 @@ async fn swap_claim_no_cmux( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); // Transfer @@ -1438,6 +1471,7 @@ async fn swap_claim_no_cmux( input: Some(Input::InputHandle(amount_0_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -1451,6 +1485,7 @@ async fn swap_claim_no_cmux( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheMul.into(), @@ -1464,6 +1499,7 @@ async fn swap_claim_no_cmux( input: Some(Input::InputHandle(cast_has_enough_funds_handle_0.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -1475,6 +1511,7 @@ async fn swap_claim_no_cmux( input: Some(Input::InputHandle(select_amount_handle_0.clone())), }, ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), @@ -1486,9 +1523,8 @@ async fn swap_claim_no_cmux( input: Some(Input::InputHandle(select_amount_handle_0.clone())), }, ], + is_allowed: true, }); - allow_handle(&new_from_amount_handle_0, &pool).await?; - allow_handle(&new_to_amount_handle_0, &pool).await?; } if total_dex_token_0_in != 0 { @@ -1502,6 +1538,7 @@ async fn swap_claim_no_cmux( input: Some(Input::Scalar(vec![6u8])), }, ], + is_allowed: false, }); let mul_temp = next_handle(); async_computations.push(AsyncComputation { @@ -1518,6 +1555,7 @@ async fn swap_claim_no_cmux( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheDiv.into(), @@ -1533,6 +1571,7 @@ async fn swap_claim_no_cmux( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -1546,6 +1585,7 @@ async fn swap_claim_no_cmux( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); // Transfer let has_enough_funds_handle_1 = next_handle(); @@ -1563,6 +1603,7 @@ async fn swap_claim_no_cmux( input: Some(Input::InputHandle(amount_1_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -1576,6 +1617,7 @@ async fn swap_claim_no_cmux( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheMul.into(), @@ -1589,6 +1631,7 @@ async fn swap_claim_no_cmux( input: Some(Input::InputHandle(cast_has_enough_funds_handle_1.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -1600,6 +1643,7 @@ async fn swap_claim_no_cmux( input: Some(Input::InputHandle(select_amount_handle_1.clone())), }, ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), @@ -1611,9 +1655,8 @@ async fn swap_claim_no_cmux( input: Some(Input::InputHandle(select_amount_handle_1.clone())), }, ], + is_allowed: true, }); - allow_handle(&new_from_amount_handle_1, &pool).await?; - allow_handle(&new_to_amount_handle_1, &pool).await?; } } @@ -1790,12 +1833,14 @@ async fn swap_request_whitepaper_dep( transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle_0.clone(), inputs: vec![from_balance_0.clone(), amount_0.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: new_to_amount_target_handle_0.clone(), inputs: vec![current_dex_balance_0.clone(), amount_0.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -1810,6 +1855,7 @@ async fn swap_request_whitepaper_dep( }, current_dex_balance_0.clone(), ], + is_allowed: false, }); // async_computations.push(AsyncComputation { // operation: FheOperation::FheSub.into(), @@ -1840,12 +1886,14 @@ async fn swap_request_whitepaper_dep( transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle_1.clone(), inputs: vec![from_balance_1.clone(), amount_1.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: new_to_amount_target_handle_1.clone(), inputs: vec![current_dex_balance_1.clone(), amount_1.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -1860,6 +1908,7 @@ async fn swap_request_whitepaper_dep( }, current_dex_balance_1.clone(), ], + is_allowed: false, }); // async_computations.push(AsyncComputation { // operation: FheOperation::FheSub.into(), @@ -1896,12 +1945,14 @@ async fn swap_request_whitepaper_dep( transaction_id: transaction_id.clone(), output_handle: sent_0_handle.clone(), inputs: vec![new_current_balance_0.clone(), current_dex_balance_0.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), transaction_id: transaction_id.clone(), output_handle: sent_1_handle.clone(), inputs: vec![new_current_balance_1.clone(), current_dex_balance_1.clone()], + is_allowed: false, }); let sent_0 = AsyncComputationInput { input: Some(Input::InputHandle(sent_0_handle.clone())), @@ -1914,33 +1965,32 @@ async fn swap_request_whitepaper_dep( transaction_id: transaction_id.clone(), output_handle: pending_0_in_handle.clone(), inputs: vec![to_balance_0.clone(), sent_0.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: pending_1_in_handle.clone(), inputs: vec![to_balance_1.clone(), sent_1.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: pending_total_token_0_in.clone(), inputs: vec![total_dex_token_0_in.clone(), sent_0.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: pending_total_token_1_in.clone(), inputs: vec![total_dex_token_1_in.clone(), sent_1.clone()], + is_allowed: true, }); // Update DEX balance handles current_dex_balance_0 = new_current_balance_0.clone(); current_dex_balance_1 = new_current_balance_1.clone(); - - allow_handle(&pending_0_in_handle, &pool).await?; - allow_handle(&pending_1_in_handle, &pool).await?; - allow_handle(&pending_total_token_0_in, &pool).await?; - allow_handle(&pending_total_token_1_in, &pool).await?; } let mut compute_request = tonic::Request::new(AsyncComputeRequest { @@ -2115,6 +2165,7 @@ async fn swap_request_no_cmux_dep( transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle_0.clone(), inputs: vec![from_balance_0.clone(), amount_0.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -2128,6 +2179,7 @@ async fn swap_request_no_cmux_dep( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheMul.into(), @@ -2139,6 +2191,7 @@ async fn swap_request_no_cmux_dep( input: Some(Input::InputHandle(cast_has_enough_funds_handle_0.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -2150,6 +2203,7 @@ async fn swap_request_no_cmux_dep( input: Some(Input::InputHandle(select_amount_handle_0.clone())), }, ], + is_allowed: false, }); // async_computations.push(AsyncComputation { // operation: FheOperation::FheSub.into(), @@ -2173,6 +2227,7 @@ async fn swap_request_no_cmux_dep( transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle_1.clone(), inputs: vec![from_balance_1.clone(), amount_1.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -2186,6 +2241,7 @@ async fn swap_request_no_cmux_dep( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheMul.into(), @@ -2197,6 +2253,7 @@ async fn swap_request_no_cmux_dep( input: Some(Input::InputHandle(cast_has_enough_funds_handle_1.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -2208,6 +2265,7 @@ async fn swap_request_no_cmux_dep( input: Some(Input::InputHandle(select_amount_handle_1.clone())), }, ], + is_allowed: false, }); // async_computations.push(AsyncComputation { // operation: FheOperation::FheSub.into(), @@ -2237,12 +2295,14 @@ async fn swap_request_no_cmux_dep( transaction_id: transaction_id.clone(), output_handle: sent_0_handle.clone(), inputs: vec![new_current_balance_0.clone(), current_dex_balance_0.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), transaction_id: transaction_id.clone(), output_handle: sent_1_handle.clone(), inputs: vec![new_current_balance_1.clone(), current_dex_balance_1.clone()], + is_allowed: false, }); let sent_0 = AsyncComputationInput { input: Some(Input::InputHandle(sent_0_handle.clone())), @@ -2255,33 +2315,32 @@ async fn swap_request_no_cmux_dep( transaction_id: transaction_id.clone(), output_handle: pending_0_in_handle.clone(), inputs: vec![to_balance_0.clone(), sent_0.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: pending_1_in_handle.clone(), inputs: vec![to_balance_1.clone(), sent_1.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: pending_total_token_0_in.clone(), inputs: vec![total_dex_token_0_in.clone(), sent_0.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: pending_total_token_1_in.clone(), inputs: vec![total_dex_token_1_in.clone(), sent_1.clone()], + is_allowed: true, }); // Update DEX balance handles current_dex_balance_0 = new_current_balance_0.clone(); current_dex_balance_1 = new_current_balance_1.clone(); - - allow_handle(&pending_0_in_handle, &pool).await?; - allow_handle(&pending_1_in_handle, &pool).await?; - allow_handle(&pending_total_token_0_in, &pool).await?; - allow_handle(&pending_total_token_1_in, &pool).await?; } let mut compute_request = tonic::Request::new(AsyncComputeRequest { @@ -2448,6 +2507,7 @@ async fn swap_claim_whitepaper_dep( input: Some(Input::Scalar(vec![6u8])), }, ], + is_allowed: false, }); let mul_temp = next_handle(); async_computations.push(AsyncComputation { @@ -2464,6 +2524,7 @@ async fn swap_claim_whitepaper_dep( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheDiv.into(), @@ -2479,6 +2540,7 @@ async fn swap_claim_whitepaper_dep( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -2492,6 +2554,7 @@ async fn swap_claim_whitepaper_dep( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); // Transfer let has_enough_funds_handle_0 = next_handle(); @@ -2509,6 +2572,7 @@ async fn swap_claim_whitepaper_dep( input: Some(Input::InputHandle(amount_0_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -2520,6 +2584,7 @@ async fn swap_claim_whitepaper_dep( input: Some(Input::InputHandle(amount_0_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -2534,6 +2599,7 @@ async fn swap_claim_whitepaper_dep( }, old_balance_0.clone(), ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), @@ -2545,6 +2611,7 @@ async fn swap_claim_whitepaper_dep( input: Some(Input::InputHandle(amount_0_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -2559,13 +2626,12 @@ async fn swap_claim_whitepaper_dep( }, current_dex_balance_0.clone(), ], + is_allowed: true, }); // Update DEX balance handles current_dex_balance_0 = AsyncComputationInput { input: Some(Input::InputHandle(new_from_amount_handle_0.clone())), }; - allow_handle(&new_from_amount_handle_0, &pool).await?; - allow_handle(&new_to_amount_handle_0, &pool).await?; } if total_dex_token_0_in != 0 { async_computations.push(AsyncComputation { @@ -2578,6 +2644,7 @@ async fn swap_claim_whitepaper_dep( input: Some(Input::Scalar(vec![6u8])), }, ], + is_allowed: false, }); let mul_temp = next_handle(); async_computations.push(AsyncComputation { @@ -2594,6 +2661,7 @@ async fn swap_claim_whitepaper_dep( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheDiv.into(), @@ -2609,6 +2677,7 @@ async fn swap_claim_whitepaper_dep( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -2622,6 +2691,7 @@ async fn swap_claim_whitepaper_dep( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); // Transfer let has_enough_funds_handle_1 = next_handle(); @@ -2639,6 +2709,7 @@ async fn swap_claim_whitepaper_dep( input: Some(Input::InputHandle(amount_1_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -2650,6 +2721,7 @@ async fn swap_claim_whitepaper_dep( input: Some(Input::InputHandle(amount_1_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -2664,6 +2736,7 @@ async fn swap_claim_whitepaper_dep( }, old_balance_1.clone(), ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), @@ -2675,6 +2748,7 @@ async fn swap_claim_whitepaper_dep( input: Some(Input::InputHandle(amount_1_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -2689,13 +2763,12 @@ async fn swap_claim_whitepaper_dep( }, current_dex_balance_1.clone(), ], + is_allowed: true, }); // Update DEX balance handles current_dex_balance_1 = AsyncComputationInput { input: Some(Input::InputHandle(new_from_amount_handle_1.clone())), }; - allow_handle(&new_from_amount_handle_1, &pool).await?; - allow_handle(&new_to_amount_handle_1, &pool).await?; } } @@ -2863,6 +2936,7 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::Scalar(vec![6u8])), }, ], + is_allowed: false, }); let mul_temp = next_handle(); async_computations.push(AsyncComputation { @@ -2879,6 +2953,7 @@ async fn swap_claim_no_cmux_dep( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheDiv.into(), @@ -2894,6 +2969,7 @@ async fn swap_claim_no_cmux_dep( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -2907,6 +2983,7 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); // Transfer @@ -2925,6 +3002,7 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::InputHandle(amount_0_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -2938,6 +3016,7 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheMul.into(), @@ -2951,6 +3030,7 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::InputHandle(cast_has_enough_funds_handle_0.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -2962,6 +3042,7 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::InputHandle(select_amount_handle_0.clone())), }, ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), @@ -2973,13 +3054,12 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::InputHandle(select_amount_handle_0.clone())), }, ], + is_allowed: true, }); // Update DEX balance handles current_dex_balance_0 = AsyncComputationInput { input: Some(Input::InputHandle(new_from_amount_handle_0.clone())), }; - allow_handle(&new_from_amount_handle_0, &pool).await?; - allow_handle(&new_to_amount_handle_0, &pool).await?; } if total_dex_token_0_in != 0 { @@ -2993,6 +3073,7 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::Scalar(vec![6u8])), }, ], + is_allowed: false, }); let mul_temp = next_handle(); async_computations.push(AsyncComputation { @@ -3009,6 +3090,7 @@ async fn swap_claim_no_cmux_dep( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheDiv.into(), @@ -3024,6 +3106,7 @@ async fn swap_claim_no_cmux_dep( )), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -3037,6 +3120,7 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); // Transfer let has_enough_funds_handle_1 = next_handle(); @@ -3054,6 +3138,7 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::InputHandle(amount_1_out.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -3067,6 +3152,7 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheMul.into(), @@ -3080,6 +3166,7 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::InputHandle(cast_has_enough_funds_handle_1.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -3091,6 +3178,7 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::InputHandle(select_amount_handle_1.clone())), }, ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), @@ -3102,13 +3190,12 @@ async fn swap_claim_no_cmux_dep( input: Some(Input::InputHandle(select_amount_handle_1.clone())), }, ], + is_allowed: true, }); // Update DEX balance handles current_dex_balance_1 = AsyncComputationInput { input: Some(Input::InputHandle(new_from_amount_handle_1.clone())), }; - allow_handle(&new_from_amount_handle_1, &pool).await?; - allow_handle(&new_to_amount_handle_1, &pool).await?; } } diff --git a/coprocessor/fhevm-engine/tfhe-worker/benches/erc20.rs b/coprocessor/fhevm-engine/tfhe-worker/benches/erc20.rs index 652c4cf614..5bcbd07a6d 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/benches/erc20.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/benches/erc20.rs @@ -1,9 +1,8 @@ #[path = "./utils.rs"] mod utils; use crate::utils::{ - allow_handle, default_api_key, default_tenant_id, query_tenant_keys, random_handle, - setup_test_app, wait_until_all_allowed_handles_computed, write_to_json, EnvConfig, - OperatorType, + default_api_key, default_tenant_id, query_tenant_keys, random_handle, setup_test_app, + wait_until_all_allowed_handles_computed, write_to_json, EnvConfig, OperatorType, }; use criterion::{ async_executor::FuturesExecutor, measurement::WallTime, Bencher, Criterion, Throughput, @@ -215,12 +214,14 @@ async fn schedule_erc20_whitepaper( transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle.clone(), inputs: vec![bals.clone(), trxa.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: new_to_amount_target_handle.clone(), inputs: vec![bald.clone(), trxa.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -235,12 +236,14 @@ async fn schedule_erc20_whitepaper( }, bald.clone(), ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), transaction_id: transaction_id.clone(), output_handle: new_from_amount_target_handle.clone(), inputs: vec![bals.clone(), trxa.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -255,10 +258,8 @@ async fn schedule_erc20_whitepaper( }, bals.clone(), ], + is_allowed: true, }); - - allow_handle(&new_to_amount_handle, &pool).await?; - allow_handle(&new_from_amount_handle, &pool).await?; } let mut compute_request = tonic::Request::new(AsyncComputeRequest { @@ -401,6 +402,7 @@ async fn schedule_erc20_no_cmux( transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle.clone(), inputs: vec![bals.clone(), trxa.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -414,6 +416,7 @@ async fn schedule_erc20_no_cmux( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheMul.into(), @@ -425,6 +428,7 @@ async fn schedule_erc20_no_cmux( input: Some(Input::InputHandle(cast_has_enough_funds_handle.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -436,6 +440,7 @@ async fn schedule_erc20_no_cmux( input: Some(Input::InputHandle(select_amount_handle.clone())), }, ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), @@ -447,10 +452,8 @@ async fn schedule_erc20_no_cmux( input: Some(Input::InputHandle(select_amount_handle.clone())), }, ], + is_allowed: true, }); - - allow_handle(&new_to_amount_handle, &pool).await?; - allow_handle(&new_from_amount_handle, &pool).await?; } let mut compute_request = tonic::Request::new(AsyncComputeRequest { @@ -615,12 +618,14 @@ async fn schedule_dependent_erc20_whitepaper( transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle.clone(), inputs: vec![bals.clone(), trxa.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: new_to_amount_target_handle.clone(), inputs: vec![bald.clone(), trxa.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -635,12 +640,14 @@ async fn schedule_dependent_erc20_whitepaper( }, bald.clone(), ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), transaction_id: transaction_id.clone(), output_handle: new_from_amount_target_handle.clone(), inputs: vec![bals.clone(), trxa.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -655,11 +662,9 @@ async fn schedule_dependent_erc20_whitepaper( }, bals.clone(), ], + is_allowed: true, }); - allow_handle(&new_to_amount_handle, &pool).await?; - allow_handle(&new_from_amount_handle, &pool).await?; - bald = AsyncComputationInput { input: Some(Input::InputHandle(new_to_amount_handle.clone())), }; @@ -828,6 +833,7 @@ async fn schedule_dependent_erc20_no_cmux( transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle.clone(), inputs: vec![bals.clone(), trxa.clone()], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -841,6 +847,7 @@ async fn schedule_dependent_erc20_no_cmux( input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheMul.into(), @@ -852,6 +859,7 @@ async fn schedule_dependent_erc20_no_cmux( input: Some(Input::InputHandle(cast_has_enough_funds_handle.clone())), }, ], + is_allowed: false, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -863,6 +871,7 @@ async fn schedule_dependent_erc20_no_cmux( input: Some(Input::InputHandle(select_amount_handle.clone())), }, ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), @@ -874,11 +883,9 @@ async fn schedule_dependent_erc20_no_cmux( input: Some(Input::InputHandle(select_amount_handle.clone())), }, ], + is_allowed: true, }); - allow_handle(&new_to_amount_handle, &pool).await?; - allow_handle(&new_from_amount_handle, &pool).await?; - bald = AsyncComputationInput { input: Some(Input::InputHandle(new_to_amount_handle.clone())), }; diff --git a/coprocessor/fhevm-engine/tfhe-worker/benches/synthetics.rs b/coprocessor/fhevm-engine/tfhe-worker/benches/synthetics.rs index f33cd9ff47..173bf863ec 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/benches/synthetics.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/benches/synthetics.rs @@ -1,8 +1,8 @@ #[path = "./utils.rs"] mod utils; use crate::utils::{ - allow_handle, default_api_key, default_tenant_id, query_tenant_keys, random_handle, - setup_test_app, wait_until_all_allowed_handles_computed, write_to_json, OperatorType, + default_api_key, default_tenant_id, query_tenant_keys, random_handle, setup_test_app, + wait_until_all_allowed_handles_computed, write_to_json, OperatorType, }; use criterion::{ async_executor::FuturesExecutor, measurement::WallTime, Bencher, Criterion, Throughput, @@ -160,7 +160,7 @@ async fn counter_increment( }; let transaction_id = next_handle(); - for _ in 0..=(num_samples - 1) as u32 { + for i in 0..num_samples { let new_counter = next_handle(); output_handles.push(new_counter.clone()); @@ -175,6 +175,7 @@ async fn counter_increment( input: Some(Input::Scalar(vec![7u8])), }, ], + is_allowed: i == (num_samples - 1), }); counter = AsyncComputationInput { @@ -182,8 +183,6 @@ async fn counter_increment( }; } - allow_handle(output_handles.last().unwrap(), &pool).await?; - let mut compute_request = tonic::Request::new(AsyncComputeRequest { computations: async_computations, }); @@ -318,6 +317,7 @@ async fn tree_reduction( transaction_id: transaction_id.clone(), output_handle: output_handle.clone(), inputs: vec![level_inputs[2 * i].clone(), level_inputs[2 * i + 1].clone()], + is_allowed: true, }); } num_comps_at_level /= 2; @@ -327,7 +327,6 @@ async fn tree_reduction( level_inputs = std::mem::take(&mut level_outputs); } output_handles.push(output_handle.clone()); - allow_handle(&output_handle, &pool).await?; let mut compute_request = tonic::Request::new(AsyncComputeRequest { computations: async_computations, diff --git a/coprocessor/fhevm-engine/tfhe-worker/benches/utils.rs b/coprocessor/fhevm-engine/tfhe-worker/benches/utils.rs index c5d2ed40bc..f2f833b039 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/benches/utils.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/benches/utils.rs @@ -1,7 +1,6 @@ -use fhevm_engine_common::types::AllowEvents; use fhevm_engine_common::utils::safe_deserialize_key; use rand::Rng; -use sqlx::{query, Postgres}; +use sqlx::query; use std::sync::atomic::{AtomicU16, Ordering}; use std::sync::Arc; use testcontainers::{core::WaitFor, runners::AsyncRunner, GenericImage, ImageExt}; @@ -94,9 +93,9 @@ async fn start_coprocessor(rx: Receiver, app_port: u16, db_url: &str) { work_items_batch_size: ecfg.batch_size, dependence_chains_per_batch: 2000, tenant_key_cache_size: 4, - coprocessor_fhe_threads: 128, + coprocessor_fhe_threads: 64, maximum_handles_per_input: 255, - tokio_threads: 16, + tokio_threads: 32, pg_pool_max_connections: 2, server_addr: format!("127.0.0.1:{app_port}"), metrics_addr: "".to_string(), @@ -167,48 +166,6 @@ async fn setup_test_app_custom_docker() -> Result, - pool: &sqlx::Pool, -) -> Result<(), Box> { - let tenant_id = default_tenant_id(); - let account_address = String::new(); - let event_type = AllowEvents::AllowedForDecryption; - let _query = - sqlx::query!( - "INSERT INTO allowed_handles(tenant_id, handle, account_address, event_type) VALUES($1, $2, $3, $4) - ON CONFLICT DO NOTHING;", - tenant_id, - handle, - account_address, - event_type as i16, - ).execute(pool).await?; - Ok(()) -} - -#[allow(dead_code)] -pub async fn allow_handles( - handles: &Vec>, - pool: &sqlx::Pool, -) -> Result<(), Box> { - let tenant_id = vec![default_tenant_id(); handles.len()]; - let account_address = vec![String::new(); handles.len()]; - let event_type = vec![AllowEvents::AllowedForDecryption as i16; handles.len()]; - let _query = sqlx::query!( - "INSERT INTO allowed_handles(tenant_id, handle, account_address, event_type) - SELECT * FROM UNNEST($1::INTEGER[], $2::BYTEA[], $3::TEXT[], $4::SMALLINT[]) - ON CONFLICT DO NOTHING;", - &tenant_id, - handles, - &account_address, - &event_type, - ) - .execute(pool) - .await?; - Ok(()) -} - #[allow(dead_code)] pub async fn wait_until_all_allowed_handles_computed( db_url: String, @@ -220,9 +177,11 @@ pub async fn wait_until_all_allowed_handles_computed( loop { tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - let count = sqlx::query!("SELECT count(1) FROM allowed_handles WHERE is_computed = FALSE") - .fetch_one(&pool) - .await?; + let count = sqlx::query!( + "SELECT count(1) FROM computations WHERE is_allowed = TRUE AND is_completed = FALSE" + ) + .fetch_one(&pool) + .await?; let current_count = count.count.unwrap(); if current_count == 0 { break; diff --git a/coprocessor/fhevm-engine/tfhe-worker/src/bin/cli.rs b/coprocessor/fhevm-engine/tfhe-worker/src/bin/cli.rs index bb7c6e9d66..2df73cc83d 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/src/bin/cli.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/src/bin/cli.rs @@ -147,7 +147,8 @@ fn smoke_test(tenant_api_key: String, coprocessor_url: String) { AsyncComputationInput { input: Some(tfhe_worker::server::tfhe_worker::async_computation_input::Input::InputHandle(handle_b.clone())), }, - ] + ], + is_allowed: true, }, ] } diff --git a/coprocessor/fhevm-engine/tfhe-worker/src/bin/utils.rs b/coprocessor/fhevm-engine/tfhe-worker/src/bin/utils.rs index 9710782b58..c3726b5e6b 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/src/bin/utils.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/src/bin/utils.rs @@ -33,8 +33,16 @@ pub fn extract_server_key_without_ns(src_path: String, dest_path: &String) -> bo let server_key: ServerKey = safe_deserialize_sns_key(&read(src_path).expect("read server key")) .expect("deserialize server key"); - let (sks, kskm, compression_key, decompression_key, _, noise_squashing_key, tag) = - server_key.into_raw_parts(); + let ( + sks, + kskm, + compression_key, + decompression_key, + noise_squashing_key, + _noise_squashing_compression_key, + re_randomization_keyswitching_key, + tag, + ) = server_key.into_raw_parts(); if noise_squashing_key.is_none() { error!("Server key does not have noise squashing"); return false; @@ -49,6 +57,7 @@ pub fn extract_server_key_without_ns(src_path: String, dest_path: &String) -> bo decompression_key, None, // noise squashing key excluded None, // noise squashing compression key excluded + re_randomization_keyswitching_key, tag, )); diff --git a/coprocessor/fhevm-engine/tfhe-worker/src/server.rs b/coprocessor/fhevm-engine/tfhe-worker/src/server.rs index 00bcb84a91..cd8dc8f6d5 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/src/server.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/src/server.rs @@ -695,9 +695,10 @@ impl CoprocessorService { is_completed, is_scalar, dependence_chain_id, - transaction_id + transaction_id, + is_allowed ) - VALUES($1, $2, $3, $4, false, $5, $6, $7) + VALUES($1, $2, $3, $4, false, $5, $6, $7, $8) ON CONFLICT (tenant_id, output_handle, transaction_id) DO NOTHING ", tenant_id, @@ -706,7 +707,8 @@ impl CoprocessorService { fhe_operation, are_comps_scalar[idx], computation_buckets[idx], - comp.transaction_id + comp.transaction_id, + comp.is_allowed ) .execute(trx.as_mut()) .await diff --git a/coprocessor/fhevm-engine/tfhe-worker/src/tests/errors.rs b/coprocessor/fhevm-engine/tfhe-worker/src/tests/errors.rs index 68b1be3015..3212dc5284 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/src/tests/errors.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/src/tests/errors.rs @@ -342,6 +342,7 @@ async fn test_coprocessor_computation_errors() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { input: Some(Input::Scalar(vec![0x00, 0x10])), }, ], + is_allowed: true, }, AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -97,6 +97,7 @@ async fn test_smoke() -> Result<(), Box> { input: Some(Input::InputHandle(h2.to_vec())), }, ], + is_allowed: true, }, ], }); @@ -108,8 +109,6 @@ async fn test_smoke() -> Result<(), Box> { println!("compute request: {:?}", resp); } - allow_handle(&h3.to_vec(), &pool).await?; - allow_handle(&h4.to_vec(), &pool).await?; println!("sleeping for computation to complete..."); wait_until_all_allowed_handles_computed(&app).await?; diff --git a/coprocessor/fhevm-engine/tfhe-worker/src/tests/operators.rs b/coprocessor/fhevm-engine/tfhe-worker/src/tests/operators.rs index b3b8960ef0..17210408f6 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/src/tests/operators.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/src/tests/operators.rs @@ -4,7 +4,7 @@ use crate::server::tfhe_worker::{ AsyncComputation, AsyncComputeRequest, TrivialEncryptBatch, TrivialEncryptRequestSingle, }; use crate::tests::utils::{ - allow_handle, decrypt_ciphertexts, random_handle, wait_until_all_allowed_handles_computed, + decrypt_ciphertexts, random_handle, wait_until_all_allowed_handles_computed, }; use crate::{ server::tfhe_worker::{async_computation_input::Input, AsyncComputationInput}, @@ -152,8 +152,8 @@ async fn test_fhe_binary_operands() -> Result<(), Box> { transaction_id: transaction_id.clone(), output_handle: output_handle.clone(), inputs, + is_allowed: true, }); - allow_handle(&output_handle, &pool).await?; } println!("Encrypting inputs..."); @@ -263,8 +263,8 @@ async fn test_fhe_unary_operands() -> Result<(), Box> { inputs: vec![AsyncComputationInput { input: Some(Input::InputHandle(input_handle)), }], + is_allowed: true, }); - allow_handle(&output_handle, &pool).await?; } println!("Encrypting inputs..."); @@ -400,8 +400,8 @@ async fn test_fhe_casts() -> Result<(), Box> { input: Some(Input::Scalar(vec![*type_to as u8])), }, ], + is_allowed: true, }); - allow_handle(&output_handle, &pool).await?; } } @@ -519,8 +519,8 @@ async fn test_op_trivial_encrypt() -> Result<(), Box> { input: Some(Input::Scalar(vec![case.inp_type as u8])), }, ], + is_allowed: true, }); - allow_handle(&output_handle, &pool).await?; } println!("Scheduling computations..."); @@ -673,8 +673,8 @@ async fn test_fhe_if_then_else() -> Result<(), Box> { input: Some(Input::InputHandle(right_handle.clone())), }, ], + is_allowed: true, }); - allow_handle(&output_handle, &pool).await?; } } diff --git a/coprocessor/fhevm-engine/tfhe-worker/src/tests/operators_from_events.rs b/coprocessor/fhevm-engine/tfhe-worker/src/tests/operators_from_events.rs index e50817d22a..8c7b798d6f 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/src/tests/operators_from_events.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/src/tests/operators_from_events.rs @@ -35,10 +35,12 @@ async fn insert_tfhe_event( db: &ListenerDatabase, tx: &mut Transaction<'_>, log: alloy::rpc::types::Log, + is_allowed: bool, ) -> Result<(), sqlx::Error> { let event = LogTfhe { event: log.inner, transaction_hash: log.transaction_hash, + is_allowed, }; db.insert_tfhe_event(tx, &event).await } @@ -313,7 +315,7 @@ async fn test_fhe_binary_operands_events() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box removed: false, }; let mut tx = listener_event_to_db.new_transaction().await?; + insert_tfhe_event(&listener_event_to_db, &mut tx, log, true).await?; allow_handle(&listener_event_to_db, &mut tx, false_handle.as_ref()).await?; - insert_tfhe_event(&listener_event_to_db, &mut tx, log).await?; let log = alloy::rpc::types::Log { inner: tfhe_event(TfheContractEvents::TrivialEncrypt( @@ -556,8 +558,8 @@ async fn test_fhe_if_then_else_events() -> Result<(), Box log_index: None, removed: false, }; + insert_tfhe_event(&listener_event_to_db, &mut tx, log, true).await?; allow_handle(&listener_event_to_db, &mut tx, true_handle.as_ref()).await?; - insert_tfhe_event(&listener_event_to_db, &mut tx, log).await?; tx.commit().await?; for input_types in supported_types() { @@ -590,7 +592,7 @@ async fn test_fhe_if_then_else_events() -> Result<(), Box removed: false, }; let mut tx = listener_event_to_db.new_transaction().await?; - insert_tfhe_event(&listener_event_to_db, &mut tx, log).await?; + insert_tfhe_event(&listener_event_to_db, &mut tx, log, false).await?; let log = alloy::rpc::types::Log { inner: tfhe_event(TfheContractEvents::TrivialEncrypt( @@ -609,7 +611,7 @@ async fn test_fhe_if_then_else_events() -> Result<(), Box log_index: None, removed: false, }; - insert_tfhe_event(&listener_event_to_db, &mut tx, log).await?; + insert_tfhe_event(&listener_event_to_db, &mut tx, log, false).await?; let output_handle = next_handle(); let (expected_result, input_handle) = if test_value { @@ -641,8 +643,8 @@ async fn test_fhe_if_then_else_events() -> Result<(), Box log_index: None, removed: false, }; + insert_tfhe_event(&listener_event_to_db, &mut tx, log, true).await?; allow_handle(&listener_event_to_db, &mut tx, output_handle.as_ref()).await?; - insert_tfhe_event(&listener_event_to_db, &mut tx, log).await?; tx.commit().await?; wait_until_all_allowed_handles_computed(&app).await?; let decrypt_request = vec![output_handle.to_vec()]; @@ -718,7 +720,7 @@ async fn test_fhe_cast_events() -> Result<(), Box> { }; let mut tx = listener_event_to_db.new_transaction().await?; - insert_tfhe_event(&listener_event_to_db, &mut tx, log).await?; + insert_tfhe_event(&listener_event_to_db, &mut tx, log, false).await?; let log = alloy::rpc::types::Log { inner: tfhe_event(TfheContractEvents::Cast(TfheContract::Cast { @@ -735,7 +737,7 @@ async fn test_fhe_cast_events() -> Result<(), Box> { log_index: None, removed: false, }; - insert_tfhe_event(&listener_event_to_db, &mut tx, log).await?; + insert_tfhe_event(&listener_event_to_db, &mut tx, log, true).await?; allow_handle(&listener_event_to_db, &mut tx, output_handle.as_ref()).await?; tx.commit().await?; @@ -803,7 +805,7 @@ async fn test_fhe_rand_events() -> Result<(), Box> { }; let mut tx = listener_event_to_db.new_transaction().await?; - insert_tfhe_event(&listener_event_to_db, &mut tx, log).await?; + insert_tfhe_event(&listener_event_to_db, &mut tx, log, true).await?; allow_handle(&listener_event_to_db, &mut tx, output1_handle.as_ref()).await?; let log = alloy::rpc::types::Log { @@ -823,7 +825,7 @@ async fn test_fhe_rand_events() -> Result<(), Box> { log_index: None, removed: false, }; - insert_tfhe_event(&listener_event_to_db, &mut tx, log).await?; + insert_tfhe_event(&listener_event_to_db, &mut tx, log, true).await?; allow_handle(&listener_event_to_db, &mut tx, output2_handle.as_ref()).await?; let log = alloy::rpc::types::Log { @@ -847,7 +849,7 @@ async fn test_fhe_rand_events() -> Result<(), Box> { log_index: None, removed: false, }; - insert_tfhe_event(&listener_event_to_db, &mut tx, log).await?; + insert_tfhe_event(&listener_event_to_db, &mut tx, log, true).await?; allow_handle(&listener_event_to_db, &mut tx, output3_handle.as_ref()).await?; tx.commit().await?; diff --git a/coprocessor/fhevm-engine/tfhe-worker/src/tests/random.rs b/coprocessor/fhevm-engine/tfhe-worker/src/tests/random.rs index 4cf046f8f6..127c287f5a 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/src/tests/random.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/src/tests/random.rs @@ -12,7 +12,7 @@ use crate::{ }, }, tests::utils::{ - allow_handle, decrypt_ciphertexts, default_api_key, random_handle, setup_test_app, + decrypt_ciphertexts, default_api_key, random_handle, setup_test_app, wait_until_all_allowed_handles_computed, DecryptionResult, }, }; @@ -71,8 +71,8 @@ async fn test_fhe_random_basic() -> Result<(), Box> { input: Some(Input::Scalar(vec![*the_type as u8])), }, ], + is_allowed: true, }); - allow_handle(&output_handle, &pool).await?; } for the_type in random_test_types { @@ -92,8 +92,8 @@ async fn test_fhe_random_basic() -> Result<(), Box> { input: Some(Input::Scalar(vec![*the_type as u8])), }, ], + is_allowed: true, }); - allow_handle(&output_handle, &pool).await?; } let deterministic_seed = 124u8; @@ -114,8 +114,8 @@ async fn test_fhe_random_basic() -> Result<(), Box> { input: Some(Input::Scalar(vec![*the_type as u8])), }, ], + is_allowed: true, }); - allow_handle(&output_handle, &pool).await?; } println!("Scheduling computations..."); let mut compute_request = tonic::Request::new(AsyncComputeRequest { @@ -134,55 +134,55 @@ async fn test_fhe_random_basic() -> Result<(), Box> { let resp = decrypt_ciphertexts(&pool, 1, decrypt_request).await?; #[cfg(not(feature = "gpu"))] let expected: Vec = vec![ - DecryptionResult { value: "false".to_string(), output_type: 0 }, - DecryptionResult { value: "10".to_string(), output_type: 1 }, - DecryptionResult { value: "74".to_string(), output_type: 2 }, - DecryptionResult { value: "20042".to_string(), output_type: 3 }, - DecryptionResult { value: "2317110858".to_string(), output_type: 4 }, - DecryptionResult { value: "3781517732040429130".to_string(), output_type: 5 }, - DecryptionResult { value: "173067494276272686594848549919321247306".to_string(), output_type: 6 }, - DecryptionResult { value: "1094515384870572566830339459490292190735847149130".to_string(), output_type: 7 }, - DecryptionResult { value: "708862794572017520787824189501889310665101791528313059693660933975816556106".to_string(), output_type: 8 }, - DecryptionResult { value: "12730235082833568380334594754488662004233832431992980449804758406069160864275028020962155601245523679819346268699252919365791199191241634613862853049601610".to_string(), output_type: 9 }, - DecryptionResult { value: "108080741562249556931009330483536710085758122079454929899567131003800646884284657303009577197577435564912115080988607041335379236995946564760399287610685200564494202954787898550157622716115053586508022488999714877512770859433667427323545700100098005683083248085496846062049536709006011184102093099103655054922".to_string(), output_type: 10 }, - DecryptionResult { value: "20861508589877342907727084703622550598935352998746120842950395081753632435267183114988504993162213367166862517403829273790814314107740319854089900370739997411799274100417555415192972122100470155923419604034291807093541986108659977790911517837161376588397655175961399498725202642377614760931166783035115963547101181507813456254749873668327253075828261674997156280447164928833111149986197323155994656673039741155124831297907623530345138812274608251091216970172997278005915715893955210806849703477333174085073707097135005438554387123608323887293974874745441977610516712982266319433597579122608490187852748993192739491402".to_string(), output_type: 11 } + DecryptionResult { value: "true".to_string(), output_type: 0 }, + DecryptionResult { value: "14".to_string(), output_type: 1 }, + DecryptionResult { value: "110".to_string(), output_type: 2 }, + DecryptionResult { value: "25710".to_string(), output_type: 3 }, + DecryptionResult { value: "2301977710".to_string(), output_type: 4 }, + DecryptionResult { value: "2124469003621852270".to_string(), output_type: 5 }, + DecryptionResult { value: "302505226437502196976291892336864355438".to_string(), output_type: 6 }, + DecryptionResult { value: "1108505784518728418941925401722025312826563060846".to_string(), output_type: 7 }, + DecryptionResult { value: "91126377170185961821859299570816872415960892403427050990577017965002762839150".to_string(), output_type: 8 }, + DecryptionResult { value: "6922152174502812819541117929565886419979032194201713031364562045239089177214539727726262702229414643692655368798839136842332367875784413253251929757541486".to_string(), output_type: 9 }, + DecryptionResult { value: "46292958359651000446442970822030237984729180059428482069495508220766054410039851432242501028059680884559363235277734516881603292424012563704460966342066139304397727662470435440766471281500712626984856035127831101691856273260533177950768622436605340597015447884662833094737801854406574285721065553810196620398".to_string(), output_type: 10 }, + DecryptionResult { value: "19366715041382087641703612484552249244187072336000885470385288586981801677187904586405444522508800282573665369274273890968159817127001139444178452126787109107071987648588764258452153875757367092164435262611470070330101502732418033994048490250401360902192046837680752648261413452195148137545219876661724890351331347604933664429281088816477375296876876531304998388499127947392826174401162163758970998813610929416858500220952663636485501844293391382813958363730841469302791359890855173063921329851153104221774931052580016325789314293273437735628912181501829333776496467493732834524085385578650231652302155872382861272174".to_string(), output_type: 11 } ]; #[cfg(feature = "gpu")] let expected: Vec = vec![ DecryptionResult { - value: "false".to_string(), + value: "true".to_string(), output_type: 0, }, DecryptionResult { - value: "5".to_string(), + value: "10".to_string(), output_type: 1, }, DecryptionResult { - value: "149".to_string(), + value: "42".to_string(), output_type: 2, }, DecryptionResult { - value: "19349".to_string(), + value: "13098".to_string(), output_type: 3, }, DecryptionResult { - value: "417483669".to_string(), + value: "24916778".to_string(), output_type: 4, }, DecryptionResult { - value: "18215029179758562197".to_string(), + value: "13546293436763353898".to_string(), output_type: 5, }, DecryptionResult { - value: "285171577856650533455529174907356334997".to_string(), + value: "244133020262364485587543763849414718250".to_string(), output_type: 6, }, DecryptionResult { - value: "152905830138175504431271466872053891096341793685".to_string(), + value: "861771350222443153516072989587279784988534125354".to_string(), output_type: 7, }, DecryptionResult { - value: "75047619794910340784669152516253626220419211295364484836071532923723869997973" + value: "2972409412398590319234166868940218748864587370013358645802893111142926529322" .to_string(), output_type: 8, }, @@ -199,7 +199,7 @@ async fn test_fhe_random_basic() -> Result<(), Box> { ); let resp_repeated = decrypt_ciphertexts(&pool, 1, other_seed_output_handles).await?; - assert_ne!(resp, resp_repeated, "seed has change, so must the values"); + assert_ne!(resp, resp_repeated, "seed has changed, so must the values"); Ok(()) } @@ -248,30 +248,30 @@ async fn test_fhe_random_bounded() -> Result<(), Box> { ]; #[cfg(not(feature = "gpu"))] let results = [ - "false", + "true", "2", - "10", - "3658", - "169627210", - "3781517732040429130", - "2926310815803454863161246203437141578", - "363764566205121107728497043132150680907880877642", - "708862794572017520787824189501889310665101791528313059693660933975816556106", - "2674379135376620555654076005834277408624308066548685416512087323277837841719867788360749877620346109051822375059388381225475787082281707154037616295038538", - "18196084819133761544544070944085473404859273132339601262852090424934308981534175736655338536373667554352058141052910362505984352588738253513975572290948138375610256242355155912006512915492006526781481012957211993093695518262435986586589144829684387101407992743203696942075913739766153031684414934291542986314", - "4703005554221839257369646359287574618713301663888378826885222367991304865833237668389904287400756635322503536942880264043754534532494859306545824177515855851483835416767057369317873246905644102525390784842258022955145876787350099709992470667923291353106832249943246977281414696607081856627390583473150770786144014812979285044407386275044968328400173657333995251408262099167598053631967166080865360240951182792153029438676694851545963236123785298889368163556353662392352873538545105944271152613867512350234435806806656471031388489431824568186212291550723309838715645265651517110938339820132416091074822187662941376074", + "46", + "9326", + "154494062", + "2124469003621852270", + "47293451246798349378760936763038196846", + "12379556520551230289161777184813048084613653614", + "4282310242198815254181060814300941526008403904196627960983829959067915609198", + "218248209531514269754105430462963356239349283905516342502781323378207162177766239325325553145962929847639439705596111415455426469811128280035105254499438", + "1350629988093102753210341052304619644279755585870817751137987931332885458664610649065381697457796879279334765309886177466905850220408408081249108682197608209955754306254064121690916381189189097121585297106579659482318602674917457582290344801398531306177820213516258534750990369786645209512226471404140586094", + "3208212005726583991346174140217273263965021001143143454320115873219474107753959139806843816747343550729306388813324881221100037551755678896634375933562967546756548964938266212577055000562541038766406443419436286191705393411108155913129443081163275666901223911662600126817625506424615233241443677099759697590374180910099493218938601423195090549448788513641837359460225117727313078046932006683841702381522371053886698361721734957686326268142568430612109557114197853689228517535445068201342778987687442486935659762251667358266315659096938416521149598307110666004695399777118032201426146276174157555524229066853063156846", ]; #[cfg(feature = "gpu")] let results = [ - "false", - "1", - "85", - "2965", - "417483669", - "4379971124476398485", - "29959802665946685857998219333530176405", - "152905830138175504431271466872053891096341793685", - "17151575176252243072883660011909672293784218962544202816342740919767305178005", + "true", + "2", + "42", + "13098", + "24916778", + "4322921399908578090", + "73991836801895253855856460133530612522", + "131020531556991694414230573229138275160567853866", + "2972409412398590319234166868940218748864587370013358645802893111142926529322", ]; for (idx, the_type) in random_test_supported_types().iter().enumerate() { @@ -294,8 +294,8 @@ async fn test_fhe_random_bounded() -> Result<(), Box> { input: Some(Input::Scalar(vec![*the_type as u8])), }, ], + is_allowed: true, }); - allow_handle(&output_handle, &pool).await?; } println!("Scheduling computations..."); diff --git a/coprocessor/fhevm-engine/tfhe-worker/src/tests/scheduling_bench.rs b/coprocessor/fhevm-engine/tfhe-worker/src/tests/scheduling_bench.rs index 6e01c412d9..1062dd124e 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/src/tests/scheduling_bench.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/src/tests/scheduling_bench.rs @@ -6,8 +6,8 @@ use crate::server::tfhe_worker::{ InputToUpload, InputUploadBatch, }; use crate::tests::utils::{ - allow_handle, decrypt_ciphertexts, default_api_key, default_tenant_id, random_handle, - setup_test_app, wait_until_all_allowed_handles_computed, + decrypt_ciphertexts, default_api_key, default_tenant_id, random_handle, setup_test_app, + wait_until_all_allowed_handles_computed, }; use fhevm_engine_common::utils::safe_serialize; use std::str::FromStr; @@ -42,7 +42,7 @@ async fn schedule_erc20_whitepaper() -> Result<(), Box> { let mut output_handles = vec![]; let mut async_computations = vec![]; - let mut num_samples: usize = 2; + let mut num_samples: usize = 7; let samples = std::env::var("FHEVM_TEST_NUM_SAMPLES"); if let Ok(samples) = samples { num_samples = samples.parse::().unwrap(); @@ -115,12 +115,14 @@ async fn schedule_erc20_whitepaper() -> Result<(), Box> { transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle.clone(), inputs: vec![bals.clone(), trxa.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), transaction_id: transaction_id.clone(), output_handle: new_to_amount_target_handle.clone(), inputs: vec![bald.clone(), trxa.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -135,12 +137,14 @@ async fn schedule_erc20_whitepaper() -> Result<(), Box> { }, bald.clone(), ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), transaction_id: transaction_id.clone(), output_handle: new_from_amount_target_handle.clone(), inputs: vec![bals.clone(), trxa.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheIfThenElse.into(), @@ -155,6 +159,7 @@ async fn schedule_erc20_whitepaper() -> Result<(), Box> { }, bals.clone(), ], + is_allowed: true, }); } @@ -168,10 +173,6 @@ async fn schedule_erc20_whitepaper() -> Result<(), Box> { ); let _resp = client.async_compute(compute_request).await?; - for h in output_handles.iter() { - allow_handle(h, &pool).await?; - } - println!("Computations scheduled, waiting upon completion..."); wait_until_all_allowed_handles_computed(&app).await?; @@ -215,7 +216,7 @@ async fn schedule_erc20_no_cmux() -> Result<(), Box> { let mut output_handles = vec![]; let mut async_computations = vec![]; - let mut num_samples: usize = 2; + let mut num_samples: usize = 7; let samples = std::env::var("FHEVM_TEST_NUM_SAMPLES"); if let Ok(samples) = samples { num_samples = samples.parse::().unwrap(); @@ -288,6 +289,7 @@ async fn schedule_erc20_no_cmux() -> Result<(), Box> { transaction_id: transaction_id.clone(), output_handle: has_enough_funds_handle.clone(), inputs: vec![bals.clone(), trxa.clone()], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheCast.into(), @@ -301,6 +303,7 @@ async fn schedule_erc20_no_cmux() -> Result<(), Box> { input: Some(Input::Scalar(vec![5u8])), }, ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheMul.into(), @@ -312,6 +315,7 @@ async fn schedule_erc20_no_cmux() -> Result<(), Box> { input: Some(Input::InputHandle(cast_has_enough_funds_handle.clone())), }, ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheAdd.into(), @@ -323,6 +327,7 @@ async fn schedule_erc20_no_cmux() -> Result<(), Box> { input: Some(Input::InputHandle(select_amount_handle.clone())), }, ], + is_allowed: true, }); async_computations.push(AsyncComputation { operation: FheOperation::FheSub.into(), @@ -334,6 +339,7 @@ async fn schedule_erc20_no_cmux() -> Result<(), Box> { input: Some(Input::InputHandle(select_amount_handle.clone())), }, ], + is_allowed: true, }); } @@ -347,10 +353,6 @@ async fn schedule_erc20_no_cmux() -> Result<(), Box> { ); let _resp = client.async_compute(compute_request).await?; - for h in output_handles.iter() { - allow_handle(h, &pool).await?; - } - println!("Computations scheduled, waiting upon completion..."); wait_until_all_allowed_handles_computed(&app).await?; @@ -394,7 +396,7 @@ async fn schedule_dependent_erc20_no_cmux() -> Result<(), Box().unwrap(); @@ -490,6 +492,7 @@ async fn schedule_dependent_erc20_no_cmux() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { let mut output_handles = vec![]; let mut async_computations = vec![]; - let mut num_samples: usize = 2; + let mut num_samples: usize = 7; let samples = std::env::var("FHEVM_TEST_NUM_SAMPLES"); if let Ok(samples) = samples { num_samples = samples.parse::().unwrap(); @@ -659,6 +662,7 @@ async fn counter_increment() -> Result<(), Box> { input: Some(Input::Scalar(vec![7u8])), }, ], + is_allowed: true, }); counter = AsyncComputationInput { @@ -676,9 +680,6 @@ async fn counter_increment() -> Result<(), Box> { ); let _resp = client.async_compute(compute_request).await?; - for h in output_handles.iter() { - allow_handle(h, &pool).await?; - } println!("Computations scheduled, waiting upon completion..."); wait_until_all_allowed_handles_computed(&app).await?; @@ -783,6 +784,7 @@ async fn tree_reduction() -> Result<(), Box> { transaction_id: transaction_id.clone(), output_handle: output_handle.clone(), inputs: vec![level_inputs[2 * i].clone(), level_inputs[2 * i + 1].clone()], + is_allowed: true, }); } num_comps_at_level /= 2; @@ -803,10 +805,6 @@ async fn tree_reduction() -> Result<(), Box> { ); let _resp = client.async_compute(compute_request).await?; - for h in output_handles.iter() { - allow_handle(h, &pool).await?; - } - println!("Computations scheduled, waiting upon completion..."); wait_until_all_allowed_handles_computed(&app).await?; diff --git a/coprocessor/fhevm-engine/tfhe-worker/src/tests/utils.rs b/coprocessor/fhevm-engine/tfhe-worker/src/tests/utils.rs index 6731c7beee..4ba79ff66c 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/src/tests/utils.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/src/tests/utils.rs @@ -1,10 +1,8 @@ use crate::daemon_cli::Args; use fhevm_engine_common::tfhe_ops::current_ciphertext_version; -use fhevm_engine_common::types::AllowEvents; use fhevm_engine_common::types::SupportedFheCiphertexts; use fhevm_engine_common::utils::{safe_deserialize, safe_deserialize_key}; use rand::Rng; -use sqlx::Postgres; use std::collections::BTreeMap; use std::sync::atomic::{AtomicU16, Ordering}; use testcontainers::{core::WaitFor, runners::AsyncRunner, GenericImage, ImageExt}; @@ -194,9 +192,11 @@ pub async fn wait_until_all_allowed_handles_computed( loop { tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; - let count = sqlx::query!("SELECT count(1) FROM allowed_handles WHERE is_computed = FALSE") - .fetch_one(&pool) - .await?; + let count = sqlx::query!( + "SELECT count(1) FROM computations WHERE is_allowed = TRUE AND is_completed = FALSE" + ) + .fetch_one(&pool) + .await?; let current_count = count.count.unwrap(); if current_count == 0 { println!("All computations completed"); @@ -340,22 +340,3 @@ pub async fn decrypt_ciphertexts( let values = values.into_iter().map(|i| i.1).collect::>(); Ok(values) } - -pub async fn allow_handle( - handle: &Vec, - pool: &sqlx::Pool, -) -> Result<(), Box> { - let tenant_id = default_tenant_id(); - let account_address = String::new(); - let event_type = AllowEvents::AllowedForDecryption; - let _query = - sqlx::query!( - "INSERT INTO allowed_handles(tenant_id, handle, account_address, event_type) VALUES($1, $2, $3, $4) - ON CONFLICT DO NOTHING;", - tenant_id, - handle, - account_address, - event_type as i16, - ).execute(pool).await?; - Ok(()) -} diff --git a/coprocessor/fhevm-engine/tfhe-worker/src/tfhe_worker.rs b/coprocessor/fhevm-engine/tfhe-worker/src/tfhe_worker.rs index 0e2160339d..60da9e9ee9 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/src/tfhe_worker.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/src/tfhe_worker.rs @@ -8,8 +8,10 @@ use lazy_static::lazy_static; use opentelemetry::trace::{Span, TraceContextExt, Tracer}; use opentelemetry::KeyValue; use prometheus::{register_int_counter, IntCounter}; -use scheduler::dfg::types::SchedulerError; -use scheduler::dfg::{scheduler::Scheduler, types::DFGTaskInput, DFGraph}; +use scheduler::dfg::types::{DFGTxInput, SchedulerError}; +use scheduler::dfg::{scheduler::Scheduler, types::DFGTaskInput}; +use scheduler::dfg::{DFGOp, DFTxGraph, TxNode}; +use sqlx::Postgres; use sqlx::{postgres::PgListener, query, Acquire}; use std::{ collections::{BTreeSet, HashMap}, @@ -73,24 +75,20 @@ async fn tfhe_worker_cycle( health_check: crate::health_check::HealthCheck, ) -> Result<(), Box> { let tracer = opentelemetry::global::tracer("tfhe_worker"); - let tenant_key_cache: std::sync::Arc>> = std::sync::Arc::new(tokio::sync::RwLock::new(lru::LruCache::new( NonZeroUsize::new(args.tenant_key_cache_size as usize).unwrap(), ))); - let db_url = crate::utils::db_url(args); let pool = sqlx::postgres::PgPoolOptions::new() .max_connections(args.pg_pool_max_connections) .connect(&db_url) .await?; + let mut listener = PgListener::connect_with(&pool).await?; + listener.listen("work_available").await?; #[cfg(feature = "bench")] populate_cache_with_tenant_keys(vec![1i32], &pool, &tenant_key_cache).await?; - - let mut listener = PgListener::connect_with(&pool).await.unwrap(); - listener.listen("event_allowed_handle").await?; - let mut immedially_poll_more_work = false; loop { // only if previous iteration had no work done do the wait @@ -98,7 +96,7 @@ async fn tfhe_worker_cycle( tokio::select! { _ = listener.try_recv() => { WORK_ITEMS_NOTIFICATIONS_COUNTER.inc(); - info!(target: "tfhe_worker", "Received event_allowed_handle notification from postgres"); + info!(target: "tfhe_worker", "Received work_available notification from postgres"); }, _ = tokio::time::sleep(tokio::time::Duration::from_millis(args.worker_polling_interval_ms)) => { WORK_ITEMS_POLL_COUNTER.inc(); @@ -107,6 +105,8 @@ async fn tfhe_worker_cycle( }; } immedially_poll_more_work = false; + #[cfg(feature = "bench")] + let now = std::time::SystemTime::now(); let loop_span = tracer.start("worker_iteration"); let loop_ctx = opentelemetry::Context::current_with_span(loop_span); let mut s = tracer.start_with_context("acquire_connection", &loop_ctx); @@ -115,236 +115,261 @@ async fn tfhe_worker_cycle( let mut s = tracer.start_with_context("begin_transaction", &loop_ctx); let mut trx = conn.begin().await?; s.end(); - // This query locks our work items so other worker doesn't select them. - let mut s = tracer.start_with_context("query_work_items", &loop_ctx); + + // Query for transactions to execute, and if relevant the associated keys + let mut transactions = + query_for_work(args, &health_check, &mut trx, &tracer, &loop_ctx).await?; + if transactions.is_empty() { + continue; + } else { + // We've fetched work, so we'll poll again without waiting + // for a notification after this cycle. + immedially_poll_more_work = true; + } + query_tenants_and_keys( + &transactions, + &tenant_key_cache, + &mut trx, + &tracer, + &loop_ctx, + ) + .await?; + + // Execute transactions segregated by tenant + for (tenant_id, ref mut tenant_txs) in transactions.iter_mut() { + let mut tx_graph = build_transaction_graph_and_execute( + tenant_id, + tenant_txs, + &tenant_key_cache, + &health_check, + &mut trx, + &tracer, + &loop_ctx, + ) + .await?; + upload_transaction_graph_results( + tenant_id, + &mut tx_graph, + &mut trx, + &tracer, + &loop_ctx, + ) + .await?; + } + s.end(); + trx.commit().await?; + let _guard = loop_ctx.attach(); #[cfg(feature = "bench")] - let now = std::time::SystemTime::now(); - let the_work = query!( - " + { + let prev_cycle_time = TIMING.load(std::sync::atomic::Ordering::SeqCst); + TIMING.store( + now.elapsed().unwrap().as_micros() as u64 + prev_cycle_time, + std::sync::atomic::Ordering::SeqCst, + ); + } + } +} + +async fn query_tenants_and_keys<'a>( + transactions: &[(i32, Vec)], + tenant_key_cache: &std::sync::Arc>>, + trx: &mut sqlx::Transaction<'a, Postgres>, + tracer: &opentelemetry::global::BoxedTracer, + loop_ctx: &opentelemetry::Context, +) -> Result<(), Box> { + let mut s = tracer.start_with_context("populate_key_cache", loop_ctx); + let mut tenants_to_query: BTreeSet = BTreeSet::new(); + let mut keys_to_query: BTreeSet = BTreeSet::new(); + let key_cache = tenant_key_cache.read().await; + for (tenant_id, _) in transactions.iter() { + let _ = tenants_to_query.insert(*tenant_id); + if !key_cache.contains(tenant_id) { + let _ = keys_to_query.insert(*tenant_id); + } + } + drop(key_cache); + let tenants_to_query = tenants_to_query.into_iter().collect::>(); + let keys_to_query = keys_to_query.into_iter().collect::>(); + s.set_attribute(KeyValue::new("keys_to_query", keys_to_query.len() as i64)); + s.set_attribute(KeyValue::new( + "tenants_to_query", + tenants_to_query.len() as i64, + )); + populate_cache_with_tenant_keys(keys_to_query, trx.as_mut(), tenant_key_cache).await?; + s.end(); + Ok(()) +} + +async fn query_ciphertexts<'a>( + cts_to_query: &[Vec], + tenant_id: i32, + trx: &mut sqlx::Transaction<'a, Postgres>, + tracer: &opentelemetry::global::BoxedTracer, + loop_ctx: &opentelemetry::Context, +) -> Result, (i16, Vec)>, Box> { + let mut s = tracer.start_with_context("query_ciphertext_batch", loop_ctx); + s.set_attribute(KeyValue::new("cts_to_query", cts_to_query.len() as i64)); + // TODO: select all the ciphertexts where they're contained in the tuples + let ciphertexts_rows = query!( + " + SELECT tenant_id, handle, ciphertext, ciphertext_type + FROM ciphertexts + WHERE tenant_id = $1 + AND handle = ANY($2::BYTEA[]) + ", + &tenant_id, + &cts_to_query + ) + .fetch_all(trx.as_mut()) + .await + .map_err(|err| { + error!(target: "tfhe_worker", { error = %err }, "error while querying ciphertexts"); + err + })?; + + s.end(); + // index ciphertexts in hashmap + let mut ciphertext_map: HashMap, (i16, Vec)> = + HashMap::with_capacity(ciphertexts_rows.len()); + for row in &ciphertexts_rows { + let _ = ciphertext_map.insert( + row.handle.clone(), + (row.ciphertext_type, row.ciphertext.clone()), + ); + } + Ok(ciphertext_map) +} + +// Update uncomputable ops schedule orders +async fn update_uncomputable_handles<'a>( + uncomputable: Vec<(Handle, Handle)>, + tenant_id: i32, + trx: &mut sqlx::Transaction<'a, Postgres>, + tracer: &opentelemetry::global::BoxedTracer, + loop_ctx: &opentelemetry::Context, +) -> Result<(), Box> { + let mut s = tracer.start_with_context("update_unschedulable_computations", loop_ctx); + let (handles, transactions): (Vec<_>, Vec<_>) = uncomputable.into_iter().unzip(); + + s.set_attribute(KeyValue::new("tenant_id", tenant_id as i64)); + s.set_attributes( + handles + .iter() + .map(|h| KeyValue::new("handle", format!("0x{}", hex::encode(h)))), + ); + s.set_attributes( + transactions + .iter() + .map(|tid| KeyValue::new("transaction_id", format!("0x{}", hex::encode(tid)))), + ); + let _ = query!( + " + UPDATE computations + SET schedule_order = CURRENT_TIMESTAMP + INTERVAL '1 second' * uncomputable_counter, + uncomputable_counter = LEAST(uncomputable_counter * 2, 32000)::SMALLINT + WHERE tenant_id = $1 + AND (output_handle, transaction_id) IN ( + SELECT * FROM unnest($2::BYTEA[], $3::BYTEA[]) + ) + ", + tenant_id, + &handles, + &transactions + ) + .execute(trx.as_mut()) + .await.map_err(|err| { + error!(target: "tfhe_worker", { tenant_id = tenant_id, error = %err }, "error while marking computations as unschedulable"); + err + })?; + s.end(); + Ok(()) +} + +async fn query_for_work<'a>( + args: &crate::daemon_cli::Args, + health_check: &crate::health_check::HealthCheck, + trx: &mut sqlx::Transaction<'a, Postgres>, + tracer: &opentelemetry::global::BoxedTracer, + loop_ctx: &opentelemetry::Context, +) -> Result)>, Box> { + // This query locks our work items so other worker doesn't select them. + let mut s = tracer.start_with_context("query_work_items", loop_ctx); + let the_work = query!( + " WITH selected_computations AS ( - -- Get all computations from such transactions ( - SELECT - c.tenant_id, - c.output_handle, - c.transaction_id, - ah.handle, - ah.is_computed - FROM computations c - LEFT JOIN allowed_handles ah - ON c.output_handle = ah.handle - AND c.tenant_id = ah.tenant_id - WHERE c.transaction_id IN ( - -- Select transaction IDs with uncomputed handles - -- out of the dependence buckets - SELECT DISTINCT transaction_id - FROM computations - WHERE is_error = FALSE - AND (tenant_id, output_handle) IN ( - SELECT tenant_id, handle - FROM allowed_handles - WHERE is_computed = FALSE - ORDER BY schedule_order - LIMIT $1 - ) - LIMIT $2 - ) + SELECT DISTINCT + c.transaction_id + FROM ( + SELECT transaction_id + FROM computations + WHERE is_completed = FALSE + AND is_error = FALSE + AND is_allowed = TRUE + ORDER BY schedule_order + LIMIT $1 + ) as c ) ) -- Acquire all computations from this transaction set -SELECT +SELECT c.tenant_id, c.output_handle, c.dependencies, c.fhe_operation, c.is_scalar, - sc.handle IS NOT NULL AS is_allowed, + c.is_allowed, c.dependence_chain_id, - COALESCE(sc.is_computed) AS is_computed, c.transaction_id FROM computations c JOIN selected_computations sc - ON c.tenant_id = sc.tenant_id - AND c.output_handle = sc.output_handle - AND c.transaction_id = sc.transaction_id + ON c.transaction_id = sc.transaction_id FOR UPDATE SKIP LOCKED ", - args.work_items_batch_size as i32, - args.dependence_chains_per_batch as i32, - ) - .fetch_all(trx.as_mut()) - .await - .map_err(|err| { - error!(target: "tfhe_worker", { error = %err }, "error while querying work items"); - err - })?; - - s.set_attribute(KeyValue::new("count", the_work.len() as i64)); - s.end(); - health_check.update_db_access(); - if the_work.is_empty() { - health_check.update_activity(); - continue; - } - WORK_ITEMS_FOUND_COUNTER.inc_by(the_work.len() as u64); - info!(target: "tfhe_worker", { count = the_work.len() }, "Processing work items"); - - // Make sure we process each tenant independently to avoid - // setting different keys from different tenants in the worker - // threads - let mut work_by_tenant = the_work.into_iter().into_group_map_by(|k| k.tenant_id); - - let mut s = tracer.start_with_context("populate_key_cache", &loop_ctx); - let mut cts_to_query: BTreeSet<&[u8]> = BTreeSet::new(); - let mut tenants_to_query: BTreeSet = BTreeSet::new(); - let mut keys_to_query: BTreeSet = BTreeSet::new(); - let key_cache = tenant_key_cache.read().await; - // Clear unneeded work items - for (_, work) in work_by_tenant.iter_mut() { - let mut work_to_remove: BTreeSet = BTreeSet::new(); - let mut transactions: HashMap<&Handle, (bool, Vec)> = HashMap::new(); - for (idx, w) in work.iter_mut().enumerate() { - transactions - .entry(&w.transaction_id) - .and_modify(|(_needed, ops)| ops.push(idx)) - .or_insert((false, vec![idx])); - // If this handle is already marked as computed in - // allowed_handles, we don't need to re-compute it - // (nor any other intermediate handles it depends on, - // which will be marked as unneeded during compute - // graph finalization). Remove it from the work set. - if w.is_computed.unwrap_or(false) { - work_to_remove.insert(idx); - } else if w.is_allowed.unwrap_or(false) { - // If for a given transaction we never set the - // needed flag, we'll remove all of its operations - // from the work set. This happens if there is no - // computable allowed handle present, in which - // case no useful work can be done. - transactions - .entry(&w.transaction_id) - .and_modify(|(needed, _ops)| *needed = true) - .or_insert((true, vec![idx])); - } - } - for (_, (needed, ops)) in transactions { - if !needed { - ops.iter().for_each(|idx| { - work_to_remove.insert(*idx); - }); - } - } - for idx in work_to_remove.iter().rev() { - work.remove(*idx); - } - } - for (tenant_id, work) in work_by_tenant.iter_mut() { - let _ = tenants_to_query.insert(*tenant_id); - if !key_cache.contains(tenant_id) { - let _ = keys_to_query.insert(*tenant_id); - } - for w in work.iter_mut() { - for dh in &w.dependencies { - let _ = cts_to_query.insert(dh); - } - } - } - drop(key_cache); - let cts_to_query = cts_to_query - .into_iter() - .map(|i| i.to_vec()) - .collect::>(); - let tenants_to_query = tenants_to_query.into_iter().collect::>(); - let keys_to_query = keys_to_query.into_iter().collect::>(); - s.set_attribute(KeyValue::new("keys_to_query", keys_to_query.len() as i64)); - s.set_attribute(KeyValue::new( - "tenants_to_query", - tenants_to_query.len() as i64, - )); - populate_cache_with_tenant_keys(keys_to_query, trx.as_mut(), &tenant_key_cache).await?; - s.end(); - let mut s = tracer.start_with_context("query_ciphertext_batch", &loop_ctx); - s.set_attribute(KeyValue::new("cts_to_query", cts_to_query.len() as i64)); - // TODO: select all the ciphertexts where they're contained in the tuples - let ciphertexts_rows = query!( - " - SELECT tenant_id, handle, ciphertext, ciphertext_type - FROM ciphertexts - WHERE tenant_id = ANY($1::INT[]) - AND handle = ANY($2::BYTEA[]) - ", - &tenants_to_query, - &cts_to_query - ) - .fetch_all(trx.as_mut()) - .await - .map_err(|err| { - error!(target: "tfhe_worker", { error = %err }, "error while querying ciphertexts"); - err - })?; - - s.end(); - // index ciphertexts in hashmap - let mut ciphertext_map: HashMap<(i32, &[u8]), _> = - HashMap::with_capacity(ciphertexts_rows.len()); - for row in &ciphertexts_rows { - let _ = ciphertext_map.insert((row.tenant_id, &row.handle), row); - } - - // Process tenants in sequence to avoid switching keys during execution - for (tenant_id, work) in work_by_tenant.iter() { - let mut s_schedule = tracer.start_with_context("schedule_fhe_work", &loop_ctx); - s_schedule.set_attribute(KeyValue::new("work_items", work.len() as i64)); - s_schedule.set_attribute(KeyValue::new("tenant_id", *tenant_id as i64)); - // We need to ensure that no handles are missing from - // either DB inputs or values produced within this batch - // before this batch is scheduled. - let mut produced_handles: HashMap<&Handle, ()> = HashMap::new(); - for w in work.iter() { - produced_handles.insert(&w.output_handle, ()); - } - let mut produced_handles_count = produced_handles.len(); - loop { - 'work_items: for w in work.iter() { - let fhe_op: SupportedFheOperations = w - .fhe_operation - .try_into() - .expect("only valid fhe ops must have been put in db"); - for (idx, dh) in w.dependencies.iter().enumerate() { - let is_operand_scalar = - w.is_scalar && idx == 1 || fhe_op.does_have_more_than_one_scalar(); - if !is_operand_scalar - && !ciphertext_map.contains_key(&(w.tenant_id, dh)) - && !produced_handles.contains_key(dh) - { - // As this operation is not computable, remove - // the output handle from those produced in - // this batch. - produced_handles.remove(&w.output_handle); - continue 'work_items; - } - } - } - // Test if we've reached the fixpoint - if produced_handles_count == produced_handles.len() { - break; - } - produced_handles_count = produced_handles.len(); - } - - // Now build the DF graph for the computations that can - // proceed and record those that can't - let mut graph = DFGraph::default(); - let mut uncomputable: HashMap = HashMap::new(); - let mut producer_indexes: HashMap<&Handle, usize> = HashMap::new(); - let mut consumer_indexes: HashMap = HashMap::new(); - 'work_items: for (widx, w) in work.iter().enumerate() { - let mut s = tracer.start_with_context("tfhe_computation", &loop_ctx); + args.work_items_batch_size as i32, + ) + .fetch_all(trx.as_mut()) + .await + .map_err(|err| { + error!(target: "tfhe_worker", { error = %err }, "error while querying work items"); + err + })?; + s.set_attribute(KeyValue::new("count", the_work.len() as i64)); + s.end(); + health_check.update_db_access(); + if the_work.is_empty() { + health_check.update_activity(); + return Ok(vec![]); + } + WORK_ITEMS_FOUND_COUNTER.inc_by(the_work.len() as u64); + info!(target: "tfhe_worker", { count = the_work.len() }, "Processing work items"); + // Make sure we process each tenant independently to avoid + // setting different keys from different tenants in the worker + // threads + let mut s_prep = tracer.start_with_context("prepare_dataflow_graphs", loop_ctx); + s_prep.set_attribute(KeyValue::new("work_items", the_work.len() as i64)); + // Partition work by tenant + let work_by_tenant = the_work.into_iter().into_group_map_by(|k| k.tenant_id); + // Partition the work by transaction + let mut work_by_tenant_by_transaction: HashMap>> = HashMap::new(); + for (tenant_id, work) in work_by_tenant.into_iter() { + work_by_tenant_by_transaction.insert( + tenant_id, + work.into_iter() + .into_group_map_by(|k| k.transaction_id.clone()), + ); + } + // Traverse transactions and build transaction nodes + let mut transactions: Vec<(i32, Vec)> = vec![]; + for (tenant_id, work_by_transaction) in work_by_tenant_by_transaction.iter() { + let mut tenant_transactions: Vec = vec![]; + for (transaction_id, txwork) in work_by_transaction.iter() { + let mut ops = vec![]; + for w in txwork { let fhe_op: SupportedFheOperations = w .fhe_operation .try_into() .expect("only valid fhe ops must have been put in db"); - let mut input_ciphertexts: Vec = - Vec::with_capacity(w.dependencies.len()); + let mut inputs: Vec = Vec::with_capacity(w.dependencies.len()); let mut this_comp_inputs: Vec> = Vec::with_capacity(w.dependencies.len()); let mut is_scalar_op_vec: Vec = Vec::with_capacity(w.dependencies.len()); for (idx, dh) in w.dependencies.iter().enumerate() { @@ -353,272 +378,200 @@ FOR UPDATE SKIP LOCKED ", is_scalar_op_vec.push(is_operand_scalar); this_comp_inputs.push(dh.clone()); if is_operand_scalar { - input_ciphertexts.push(DFGTaskInput::Value( - SupportedFheCiphertexts::Scalar(dh.clone()), - )); - } else if let Some(ct_map_val) = ciphertext_map.get(&(w.tenant_id, dh)) { - input_ciphertexts.push(DFGTaskInput::Compressed(( - ct_map_val.ciphertext_type, - ct_map_val.ciphertext.clone().to_vec(), + inputs.push(DFGTaskInput::Value(SupportedFheCiphertexts::Scalar( + dh.clone(), ))); - } else if produced_handles.contains_key(dh) { - input_ciphertexts.push(DFGTaskInput::Dependence(None)); } else { - // If this cannot be computed, we need to - // exclude it from the DF graph. - uncomputable.insert(widx, w.output_handle.clone()); - continue 'work_items; + inputs.push(DFGTaskInput::Dependence(dh.clone())); } } - check_fhe_operand_types( w.fhe_operation.into(), &this_comp_inputs, &is_scalar_op_vec, ) .map_err(CoprocessorError::FhevmError)?; + ops.push(DFGOp { + output_handle: w.output_handle.clone(), + fhe_op, + inputs, + is_allowed: w.is_allowed, + }); + } + let mut txn = TxNode::default(); + txn.build(ops, transaction_id)?; + tenant_transactions.push(txn); + } + transactions.push((*tenant_id, tenant_transactions)); + } + s_prep.end(); + Ok(transactions) +} - let n = graph.add_node( - w.output_handle.clone(), - w.fhe_operation.into(), - input_ciphertexts.clone(), - w.is_allowed.unwrap_or(true), - widx, - )?; - producer_indexes.insert(&w.output_handle, n.index()); - consumer_indexes.insert(widx, n.index()); +async fn build_transaction_graph_and_execute<'a>( + tenant_id: &i32, + tenant_txs: &mut Vec, + tenant_key_cache: &std::sync::Arc>>, + health_check: &crate::health_check::HealthCheck, + trx: &mut sqlx::Transaction<'a, Postgres>, + tracer: &opentelemetry::global::BoxedTracer, + loop_ctx: &opentelemetry::Context, +) -> Result> { + let mut tx_graph = DFTxGraph::default(); + tx_graph.build(tenant_txs)?; + let cts_to_query = tx_graph.needed_map.keys().cloned().collect::>(); + let ciphertext_map = + query_ciphertexts(&cts_to_query, *tenant_id, trx, tracer, loop_ctx).await?; + for (handle, (ct_type, mut ct)) in ciphertext_map.into_iter() { + tx_graph.add_input( + &handle, + &DFGTxInput::Compressed((ct_type, std::mem::take(&mut ct))), + )?; + } + // Execute the DFG with the current tenant's keys + let mut s_compute = tracer.start_with_context("compute_fhe_ops", loop_ctx); + { + let mut rk = tenant_key_cache.write().await; + let keys = rk.get(tenant_id).expect("Can't get tenant key from cache"); + // Schedule computations in parallel as dependences allow + tfhe::set_server_key(keys.sks.clone()); + let mut sched = Scheduler::new( + &mut tx_graph, + keys.sks.clone(), + keys.pks.clone(), + #[cfg(feature = "gpu")] + keys.gpu_sks.clone(), + health_check.activity_heartbeat.clone(), + ); + sched.schedule(loop_ctx).await?; + } + s_compute.end(); + Ok(tx_graph) +} - s.set_attribute(KeyValue::new("fhe_operation", w.fhe_operation as i64)); - s.set_attribute(KeyValue::new( - "handle", - format!("0x{}", hex::encode(&w.output_handle)), +async fn upload_transaction_graph_results<'a>( + tenant_id: &i32, + tx_graph: &mut DFTxGraph, + trx: &mut sqlx::Transaction<'a, Postgres>, + tracer: &opentelemetry::global::BoxedTracer, + loop_ctx: &opentelemetry::Context, +) -> Result<(), Box> { + // Get computation results + let graph_results = tx_graph.get_results(); + + // Traverse computations that have been scheduled and + // upload their results/errors + let mut handles_to_update = tx_graph.get_intermediate_handles(); + let mut cts_to_insert = vec![]; + let mut uncomputable = vec![]; + for result in graph_results.into_iter() { + match result.compressed_ct { + Ok((db_type, db_bytes)) => { + cts_to_insert.push(( + *tenant_id, + ( + result.handle.clone(), + (db_bytes, (current_ciphertext_version(), db_type)), + ), )); - let input_types = input_ciphertexts - .iter() - .map(|i| match i { - DFGTaskInput::Value(i) => i.type_num().to_string(), - DFGTaskInput::Compressed(_) => "Compressed value".to_string(), - DFGTaskInput::Dependence(_) => "Temporary value".to_string(), - }) - .collect::>() - .join(","); - s.set_attribute(KeyValue::new("input_types", input_types)); - s.end(); + handles_to_update.push((result.handle.clone(), result.transaction_id.clone())); + WORK_ITEMS_PROCESSED_COUNTER.inc(); } - s.end(); - // Traverse computations and add dependences/edges as required - for (index, w) in work.iter().enumerate() { - if uncomputable.contains_key(&index) { - continue; - } - for (input_idx, input) in w.dependencies.iter().enumerate() { - let fhe_op: SupportedFheOperations = w - .fhe_operation - .try_into() - .expect("only valid fhe ops must have been put in db"); - let is_operand_scalar = - w.is_scalar && input_idx == 1 || fhe_op.does_have_more_than_one_scalar(); - if !is_operand_scalar && !ciphertext_map.contains_key(&(w.tenant_id, input)) { - if let Some(producer_index) = producer_indexes.get(input) { - let consumer_index = consumer_indexes.get(&index).unwrap(); - graph.add_dependence(*producer_index, *consumer_index, input_idx)?; - } + Err(mut err) => { + let cerr: Box = + if err.downcast_ref::().is_some() { + let mut swap_val = FhevmError::BadInputs; + std::mem::swap( + &mut *err.downcast_mut::().unwrap(), + &mut swap_val, + ); + CoprocessorError::FhevmError(swap_val).into() + } else { + CoprocessorError::SchedulerError( + *err.downcast_ref::() + .unwrap_or(&SchedulerError::SchedulerError), + ) + .into() + }; + // Downgrade SchedulerError to warning when the + // error is not about the operations themselves. + // Do not set the error flag in the DB in such cases. + if let Some(err) = cerr.downcast_ref::() { + if matches!( + err, + CoprocessorError::SchedulerError(SchedulerError::DataflowGraphError) + ) || matches!( + err, + CoprocessorError::SchedulerError(SchedulerError::SchedulerError) + ) { + warn!(target: "tfhe_worker", + { tenant_id = tenant_id, error = cerr, + output_handle = format!("0x{}", hex::encode(&result.handle)) }, + "scheduler encountered an error while processing work item" + ); + continue; + } + if matches!( + err, + CoprocessorError::SchedulerError(SchedulerError::MissingInputs) + ) { + uncomputable.push((result.handle.clone(), result.transaction_id.clone())); } } - } - graph.finalize(); - s_schedule.end(); - - // Execute the DFG with the current tenant's keys - let mut s_outer = tracer.start_with_context("wait_and_update_fhe_work", &loop_ctx); - { - let mut rk = tenant_key_cache.write().await; - let keys = rk.get(tenant_id).expect("Can't get tenant key from cache"); - - // Schedule computations in parallel as dependences allow - tfhe::set_server_key(keys.sks.clone()); - let mut sched = Scheduler::new( - &mut graph.graph, - keys.sks.clone(), - #[cfg(feature = "gpu")] - keys.gpu_sks.clone(), - health_check.activity_heartbeat.clone(), - ); - sched.schedule().await?; - } - // Extract the results from the graph - let mut graph_results = graph.get_results(); - - // Update uncomputable ops schedule orders - { - let mut s = - tracer.start_with_context("update_unschedulable_computations", &loop_ctx); + WORKER_ERRORS_COUNTER.inc(); + error!(target: "tfhe_worker", + { tenant_id = tenant_id, error = cerr, + output_handle = format!("0x{}", hex::encode(&result.handle)) }, + "error while processing work item" + ); + let mut s = tracer.start_with_context("set_computation_error_in_db", loop_ctx); s.set_attribute(KeyValue::new("tenant_id", *tenant_id as i64)); - s.set_attributes( - uncomputable - .values() - .map(|h| KeyValue::new("handle", format!("0x{}", hex::encode(h)))), - ); + s.set_attribute(KeyValue::new( + "handle", + format!("0x{}", hex::encode(&result.handle)), + )); + let err_string = err.to_string(); + s.set_status(opentelemetry::trace::Status::Error { + description: err_string.clone().into(), + }); + let _ = query!( " - UPDATE allowed_handles - SET schedule_order = CURRENT_TIMESTAMP + INTERVAL '1 second' * uncomputable_counter, - uncomputable_counter = LEAST(uncomputable_counter * 2, 32000)::SMALLINT - WHERE tenant_id = $1 - AND handle = ANY($2::BYTEA[]) - ", + UPDATE computations + SET is_error = true, error_message = $1 + WHERE tenant_id = $2 + AND output_handle = $3 + AND transaction_id = $4 + ", + err_string, *tenant_id, - &uncomputable.into_values().collect::>() + result.handle, + result.transaction_id ) .execute(trx.as_mut()) - .await.map_err(|err| { - error!(target: "tfhe_worker", { tenant_id = *tenant_id, error = %err }, "error while marking computations as unschedulable"); - err - })?; + .await?; s.end(); } - // Traverse computations that have been scheduled and - // upload their results/errors - let mut handles_to_update = vec![]; - let mut intermediate_handles_to_update = vec![]; - let mut cts_to_insert = vec![]; - for result in graph_results.iter_mut() { - let idx = result.work_index; - let result = &mut result.result; - - #[allow(clippy::type_complexity)] - let finished_work_unit: Result< - _, - ( - Box, - i32, - Vec, - Vec, - ), - > = result - .as_mut() - .map(|rok| { - if let Some((ct_type, ref mut ct_bytes)) = rok { - (&work[idx], Some((ct_type, std::mem::take(ct_bytes)))) - } else { - (&work[idx], None) - } - }) - .map_err(|rerr| { - if rerr.downcast_ref::().is_some() { - let mut swap_val = FhevmError::BadInputs; - std::mem::swap( - &mut *rerr.downcast_mut::().unwrap(), - &mut swap_val, - ); - ( - CoprocessorError::FhevmError(swap_val).into(), - work[idx].tenant_id, - work[idx].output_handle.clone(), - work[idx].transaction_id.clone(), - ) - } else { - ( - CoprocessorError::SchedulerError( - *rerr - .downcast_ref::() - .unwrap_or(&SchedulerError::SchedulerError), - ) - .into(), - work[idx].tenant_id, - work[idx].output_handle.clone(), - work[idx].transaction_id.clone(), - ) - } - }); - match finished_work_unit { - Ok((w, Some((db_type, db_bytes)))) => { - cts_to_insert.push(( - w.tenant_id, - ( - w.output_handle.clone(), - (db_bytes, (current_ciphertext_version(), *db_type)), - ), - )); - handles_to_update.push((w.output_handle.clone(), w.transaction_id.clone())); - // As we've completed useful computation on an - // allowed handle, we poll for work again - // without waiting for a notification. - immedially_poll_more_work = true; - WORK_ITEMS_PROCESSED_COUNTER.inc(); - } - Ok((w, None)) => { - // Non allowed handles are still marked as - // complete but we don't upload the CT - intermediate_handles_to_update - .push((w.output_handle.clone(), w.transaction_id.clone())); - WORK_ITEMS_PROCESSED_COUNTER.inc(); - } - Err((err, tenant_id, output_handle, transaction_id)) => { - // Downgrade SchedulerError to warning as the - // error is not about the operations themselves. - // Do not set the error flag in the DB. - if let Some(cerr) = err.downcast_ref::() { - if matches!(cerr, CoprocessorError::SchedulerError(_)) { - warn!(target: "tfhe_worker", - { tenant_id = tenant_id, error = err, - output_handle = format!("0x{}", hex::encode(&output_handle)) }, - "scheduler encountered an error while processing work item" - ); - continue; - } - } - WORKER_ERRORS_COUNTER.inc(); - error!(target: "tfhe_worker", - { tenant_id = tenant_id, error = err, - output_handle = format!("0x{}", hex::encode(&output_handle)) }, - "error while processing work item" - ); - let mut s = - tracer.start_with_context("set_computation_error_in_db", &loop_ctx); - s.set_attribute(KeyValue::new("tenant_id", tenant_id as i64)); - s.set_attribute(KeyValue::new( - "handle", - format!("0x{}", hex::encode(&output_handle)), - )); - let err_string = err.to_string(); - s.set_status(opentelemetry::trace::Status::Error { - description: err_string.clone().into(), - }); - - let _ = query!( - " - UPDATE computations - SET is_error = true, error_message = $1 - WHERE tenant_id = $2 - AND output_handle = $3 - AND transaction_id = $4 - ", - err_string, - tenant_id, - output_handle, - transaction_id - ) - .execute(trx.as_mut()) - .await?; - s.end(); - } - } - } - let mut s = tracer.start_with_context("insert_ct_into_db", &loop_ctx); - s.set_attribute(KeyValue::new("tenant_id", *tenant_id as i64)); - s.set_attributes(cts_to_insert.iter().map(|(_, (h, (_, (_, _))))| { - KeyValue::new("handle", format!("0x{}", hex::encode(h))) - })); - s.set_attributes(cts_to_insert.iter().map(|(_, (_, (_, (_, db_type))))| { - KeyValue::new("ciphertext_type", *db_type as i64) - })); - #[allow(clippy::type_complexity)] - let (tenant_ids, (handles, (ciphertexts, (ciphertext_versions, ciphertext_types)))): ( - Vec<_>, - (Vec<_>, (Vec<_>, (Vec<_>, Vec<_>))), - ) = cts_to_insert.into_iter().unzip(); - let _ = query!( + } + } + let mut s = tracer.start_with_context("insert_ct_into_db", loop_ctx); + s.set_attribute(KeyValue::new("tenant_id", *tenant_id as i64)); + s.set_attributes( + cts_to_insert + .iter() + .map(|(_, (h, (_, (_, _))))| KeyValue::new("handle", format!("0x{}", hex::encode(h)))), + ); + s.set_attributes( + cts_to_insert + .iter() + .map(|(_, (_, (_, (_, db_type))))| KeyValue::new("ciphertext_type", *db_type as i64)), + ); + #[allow(clippy::type_complexity)] + let (tenant_ids, (handles, (ciphertexts, (ciphertext_versions, ciphertext_types)))): ( + Vec<_>, + (Vec<_>, (Vec<_>, (Vec<_>, Vec<_>))), + ) = cts_to_insert.into_iter().unzip(); + let _ = query!( " INSERT INTO ciphertexts(tenant_id, handle, ciphertext, ciphertext_version, ciphertext_type) SELECT * FROM UNNEST($1::INTEGER[], $2::BYTEA[], $3::BYTEA[], $4::SMALLINT[], $5::SMALLINT[]) @@ -630,25 +583,24 @@ FOR UPDATE SKIP LOCKED ", error!(target: "tfhe_worker", { tenant_id = *tenant_id, error = %err }, "error while inserting new ciphertexts"); err })?; - // Notify all workers that new ciphertext is inserted - // For now, it's only the SnS workers that are listening for these events - let _ = sqlx::query!("SELECT pg_notify($1, '')", EVENT_CIPHERTEXT_COMPUTED) - .execute(trx.as_mut()) - .await?; - s.end(); + // Notify all workers that new ciphertext is inserted + // For now, it's only the SnS workers that are listening for these events + let _ = sqlx::query!("SELECT pg_notify($1, '')", EVENT_CIPHERTEXT_COMPUTED) + .execute(trx.as_mut()) + .await?; + s.end(); - let mut s = tracer.start_with_context("update_computation", &loop_ctx); - s.set_attribute(KeyValue::new("tenant_id", *tenant_id as i64)); - s.set_attributes( - handles_to_update - .iter() - .map(|(h, _)| KeyValue::new("handle", format!("0x{}", hex::encode(h)))), - ); + let mut s = tracer.start_with_context("update_computation", loop_ctx); + s.set_attribute(KeyValue::new("tenant_id", *tenant_id as i64)); + s.set_attributes( + handles_to_update + .iter() + .map(|(h, _)| KeyValue::new("handle", format!("0x{}", hex::encode(h)))), + ); - let (handles_vec, txn_ids_vec): (Vec<_>, Vec<_>) = - handles_to_update.iter().cloned().unzip(); + let (handles_vec, txn_ids_vec): (Vec<_>, Vec<_>) = handles_to_update.into_iter().unzip(); - let _ = query!( + let _ = query!( " UPDATE computations SET is_completed = true, completed_at = CURRENT_TIMESTAMP @@ -667,83 +619,8 @@ FOR UPDATE SKIP LOCKED ", err })?; - s.end(); - let mut s = tracer.start_with_context("update_allowed_handles_is_computed", &loop_ctx); - s.set_attribute(KeyValue::new("tenant_id", *tenant_id as i64)); - s.set_attributes( - handles_to_update - .iter() - .map(|(h, _)| KeyValue::new("handle", format!("0x{}", hex::encode(h)))), - ); - let _ = query!( - " - UPDATE allowed_handles - SET is_computed = TRUE - WHERE tenant_id = $1 - AND handle = ANY($2::BYTEA[]) - ", - *tenant_id, - &handles_to_update - .iter() - .map(|(handle, _)| handle.clone()) - .collect::>() - ) - .execute(trx.as_mut()) - .await.map_err(|err| { - error!(target: "tfhe_worker", { tenant_id = *tenant_id, error = %err }, "error while marking allowed handles as computed"); - err - })?; + s.end(); - s.end(); - let mut s = tracer.start_with_context("update_intermediate_computation", &loop_ctx); - s.set_attribute(KeyValue::new("tenant_id", *tenant_id as i64)); - s.set_attributes( - intermediate_handles_to_update - .iter() - .map(|(h, _)| KeyValue::new("handle", format!("0x{}", hex::encode(h)))), - ); - - let _ = query!( - " - UPDATE computations - SET is_completed = true, completed_at = CURRENT_TIMESTAMP - WHERE tenant_id = $1 - AND (output_handle, transaction_id) IN ( - SELECT * FROM unnest($2::BYTEA[], $3::BYTEA[]) - ) - ", - *tenant_id, - &intermediate_handles_to_update - .iter() - .map(|(handle, _)| handle.clone()) - .collect::>(), - &intermediate_handles_to_update - .iter() - .map(|(_, txn_id)| txn_id.clone()) - .collect::>() - ) - .execute(trx.as_mut()) - .await.map_err(|err| { - error!(target: "tfhe_worker", { tenant_id = *tenant_id, error = %err }, "error while updating intermediate computations as completed"); - err - })?; - s.end(); - - s_outer.end(); - } - s.end(); - - trx.commit().await?; - - let _guard = loop_ctx.attach(); - - #[cfg(feature = "bench")] - { - let prev_cycle_time = TIMING.load(std::sync::atomic::Ordering::SeqCst); - TIMING.store( - now.elapsed().unwrap().as_micros() as u64 + prev_cycle_time, - std::sync::atomic::Ordering::SeqCst, - ); - } - } + update_uncomputable_handles(uncomputable, *tenant_id, trx, tracer, loop_ctx).await?; + Ok(()) } diff --git a/coprocessor/fhevm-engine/tfhe-worker/src/utils.rs b/coprocessor/fhevm-engine/tfhe-worker/src/utils.rs index b1ff432138..6c91bdb761 100644 --- a/coprocessor/fhevm-engine/tfhe-worker/src/utils.rs +++ b/coprocessor/fhevm-engine/tfhe-worker/src/utils.rs @@ -189,6 +189,7 @@ fn test_invalid_handle_too_short() { input: Some(Input::InputHandle(vec![2])), }, ], + is_allowed: true, }]; match sort_computations_by_dependencies(&comp) { @@ -215,6 +216,7 @@ fn test_invalid_handle_too_long() { input: Some(Input::InputHandle(vec![2])), }, ], + is_allowed: true, }]; match sort_computations_by_dependencies(&comp) { @@ -240,6 +242,7 @@ fn test_simple_circular_dependency_detection() { input: Some(Input::InputHandle(vec![2])), }, ], + is_allowed: true, }, AsyncComputation { operation: 1, @@ -253,6 +256,7 @@ fn test_simple_circular_dependency_detection() { input: Some(Input::InputHandle(vec![2])), }, ], + is_allowed: true, }, ]; @@ -286,6 +290,7 @@ fn test_multi_level_circular_dependency_detection() { input: Some(Input::InputHandle(vec![3])), }, ], + is_allowed: true, }, AsyncComputation { operation: 1, @@ -299,6 +304,7 @@ fn test_multi_level_circular_dependency_detection() { input: Some(Input::InputHandle(vec![4])), }, ], + is_allowed: true, }, AsyncComputation { operation: 1, @@ -312,6 +318,7 @@ fn test_multi_level_circular_dependency_detection() { input: Some(Input::InputHandle(vec![2])), }, ], + is_allowed: true, }, ]; diff --git a/coprocessor/fhevm-engine/zkproof-worker/src/tests/mod.rs b/coprocessor/fhevm-engine/zkproof-worker/src/tests/mod.rs index ba4ca3c769..5fdbd6d50f 100644 --- a/coprocessor/fhevm-engine/zkproof-worker/src/tests/mod.rs +++ b/coprocessor/fhevm-engine/zkproof-worker/src/tests/mod.rs @@ -82,7 +82,7 @@ async fn test_max_input_index() { ) .await .expect("valid db insert"), - 50 + 5000 ) .await .expect("non-expired db query")); @@ -99,7 +99,7 @@ async fn test_max_input_index() { ) .await .expect("valid db insert"), - 500 + 5000 ) .await .expect("non-expired db query")); diff --git a/coprocessor/fhevm-engine/zkproof-worker/src/verifier.rs b/coprocessor/fhevm-engine/zkproof-worker/src/verifier.rs index 8b264d13b6..0aca5bcbdf 100644 --- a/coprocessor/fhevm-engine/zkproof-worker/src/verifier.rs +++ b/coprocessor/fhevm-engine/zkproof-worker/src/verifier.rs @@ -35,6 +35,9 @@ use tracing::{debug, error, info}; const MAX_CACHED_TENANT_KEYS: usize = 100; const EVENT_CIPHERTEXT_COMPUTED: &str = "event_ciphertext_computed"; +const RAW_CT_HASH_DOMAIN_SEPARATOR: [u8; 8] = *b"ZK-w_rct"; +const HANDLE_HASH_DOMAIN_SEPARATOR: [u8; 8] = *b"ZK-w_hdl"; + pub(crate) struct Ciphertext { handle: Vec, compressed: Vec, @@ -341,7 +344,7 @@ pub(crate) fn verify_proof( set_server_key(keys.server_key.clone()); let mut s = t.child_span("verify_and_expand"); - let cts = match try_verify_and_expand_ciphertext_list(request_id, raw_ct, keys, aux_data) { + let mut cts = match try_verify_and_expand_ciphertext_list(request_id, raw_ct, keys, aux_data) { Ok(cts) => { telemetry::attribute(&mut s, "count", cts.len().to_string()); telemetry::end_span(s); @@ -357,11 +360,12 @@ pub(crate) fn verify_proof( let _s = t.child_span("create_ciphertext"); let mut h = Keccak256::new(); + h.update(RAW_CT_HASH_DOMAIN_SEPARATOR); h.update(raw_ct); let blob_hash = h.finalize().to_vec(); let cts = cts - .iter() + .iter_mut() .enumerate() .map(|(idx, ct)| create_ciphertext(request_id, &blob_hash, idx, ct, aux_data)) .collect::, ExecutionError>>()?; @@ -412,15 +416,18 @@ fn create_ciphertext( request_id: i64, blob_hash: &[u8], ct_idx: usize, - the_ct: &SupportedFheCiphertexts, + the_ct: &mut SupportedFheCiphertexts, aux_data: &auxiliary::ZkData, ) -> Result { - let (serialized_type, compressed) = the_ct.compress(); + if ct_idx > MAX_INPUT_INDEX as usize { + return Err(ExecutionError::TooManyInputs(ct_idx)); + } + let chain_id_bytes: [u8; 32] = alloy_primitives::U256::from(aux_data.chain_id) .to_owned() .to_be_bytes(); - let mut handle_hash = Keccak256::new(); + handle_hash.update(HANDLE_HASH_DOMAIN_SEPARATOR); handle_hash.update(blob_hash); handle_hash.update([ct_idx as u8]); handle_hash.update( @@ -430,12 +437,13 @@ fn create_ciphertext( ); handle_hash.update(chain_id_bytes); let mut handle = handle_hash.finalize().to_vec(); - assert_eq!(handle.len(), 32); - if ct_idx > MAX_INPUT_INDEX as usize { - return Err(ExecutionError::TooManyInputs(ct_idx)); - } + // Add the full 256bit hash as re-randomization metadata, NOT the + // truncated hash of the handle + the_ct.add_re_randomization_metadata(&handle); + let (serialized_type, compressed) = the_ct.compress(); + // idx cast to u8 must succeed because we don't allow // more handles than u8 size handle[21] = ct_idx as u8; diff --git a/coprocessor/proto/coprocessor.proto b/coprocessor/proto/coprocessor.proto index 718666f6fb..700bb455d3 100644 --- a/coprocessor/proto/coprocessor.proto +++ b/coprocessor/proto/coprocessor.proto @@ -51,6 +51,7 @@ message AsyncComputation { bytes transaction_id = 2; bytes output_handle = 3; repeated AsyncComputationInput inputs = 4; + bool is_allowed = 5; } message AsyncComputationInput { diff --git a/package-lock.json b/package-lock.json index 6e106534f0..f1c662bec5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,9 @@ "test-suite/e2e", "test-suite/benchmarks" ], + "dependencies": { + "relayer-sdk": "^1.0.1" + }, "devDependencies": { "@trivago/prettier-plugin-sort-imports": "^5.2.2", "prettier": "^3.5.3", @@ -21992,6 +21995,125 @@ "node": ">=10" } }, + "node_modules/relayer-sdk": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/relayer-sdk/-/relayer-sdk-1.0.1.tgz", + "integrity": "sha512-A/KpkwnuWGO42IDpc//v4tWnQZPPJASjTuVHUApOxfHUpHiXwGhYAAeJP+72n7/cHykunsz43Tr2pqdW7UxJvg==", + "license": "ISC", + "dependencies": { + "isomorphic-unfetch": "^4.0.2" + } + }, + "node_modules/relayer-sdk/node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/relayer-sdk/node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/relayer-sdk/node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/relayer-sdk/node_modules/isomorphic-unfetch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/isomorphic-unfetch/-/isomorphic-unfetch-4.0.2.tgz", + "integrity": "sha512-1Yd+CF/7al18/N2BDbsLBcp6RO3tucSW+jcLq24dqdX5MNbCNTw1z4BsGsp4zNmjr/Izm2cs/cEqZPp4kvWSCA==", + "license": "MIT", + "dependencies": { + "node-fetch": "^3.2.0", + "unfetch": "^5.0.0" + } + }, + "node_modules/relayer-sdk/node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/relayer-sdk/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/relayer-sdk/node_modules/unfetch": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unfetch/-/unfetch-5.0.0.tgz", + "integrity": "sha512-3xM2c89siXg0nHvlmYsQ2zkLASvVMBisZm5lF3gFDqfF2xonNStDJyMpvaOBe0a1Edxmqrf2E0HBdmy9QyZaeg==", + "license": "MIT", + "workspaces": [ + "./packages/isomorphic-unfetch" + ] + }, + "node_modules/relayer-sdk/node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "test-suite/benchmarks": { "name": "@fhevm/e2e-benchmarks", "dependencies": { @@ -30131,7 +30253,7 @@ "dependencies": { "@fhevm/solidity": "*", "@openzeppelin/contracts": "^5.3.0", - "@zama-fhe/relayer-sdk": "^0.3.0-0", + "@zama-fhe/relayer-sdk": "^0.3.0-2", "bigint-buffer": "^1.1.5", "dotenv": "^16.0.3", "encrypted-types": "^0.0.4" @@ -31749,19 +31871,19 @@ } }, "test-suite/e2e/node_modules/@zama-fhe/relayer-sdk": { - "version": "0.3.0-0", - "resolved": "https://registry.npmjs.org/@zama-fhe/relayer-sdk/-/relayer-sdk-0.3.0-0.tgz", - "integrity": "sha512-TLPc5kEbIo9psx3kt4SGZQ/JXTVk28Tbw2szMNBRFQ/2/YJEWUcZcJvvCrklMnKHsA7zF3JLiMMg48Jrp25loA==", + "version": "0.3.0-2", + "resolved": "https://registry.npmjs.org/@zama-fhe/relayer-sdk/-/relayer-sdk-0.3.0-2.tgz", + "integrity": "sha512-cUs8UcERlUivUh+ZRNYJAJEdaaQgavXzhpRmRDRPw6kIxRayzCHAtZkq9ys1RFXKcEJjNBDfcWTKQk3YA7jtdA==", "license": "BSD-3-Clause-Clear", "dependencies": { "commander": "^14.0.0", "ethers": "^6.15.0", "fetch-retry": "^6.0.0", "keccak": "^3.0.4", - "node-tfhe": "1.3.0", - "node-tkms": "^0.11.0", - "tfhe": "1.3.0", - "tkms": "^0.11.0", + "node-tfhe": "1.4.0-alpha.3", + "node-tkms": "^0.12.0", + "tfhe": "1.4.0-alpha.3", + "tkms": "^0.12.0", "wasm-feature-detect": "^1.8.0" }, "bin": { @@ -34878,13 +35000,15 @@ } }, "test-suite/e2e/node_modules/node-tfhe": { - "version": "1.3.0", + "version": "1.4.0-alpha.3", + "resolved": "https://registry.npmjs.org/node-tfhe/-/node-tfhe-1.4.0-alpha.3.tgz", + "integrity": "sha512-oTcWL0OFA6t6BhScmDiGQ3VA8tU8T3EXCzIzpNxQxcuJDgQtiUF5CV6dgJLOrpWck4KCp1Bo/xLhv07uwn3q6Q==", "license": "BSD-3-Clause-Clear" }, "test-suite/e2e/node_modules/node-tkms": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/node-tkms/-/node-tkms-0.11.1.tgz", - "integrity": "sha512-AWciFzfvjEYECHiAJXv1KLz6K28fX/0DDlaktAbslF2XpaIGsc9sCKjYPJHubrJfNrtUWUI5qfqhJOP3BD/mcw==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/node-tkms/-/node-tkms-0.12.0.tgz", + "integrity": "sha512-XIVFDOYDQXchW3fDPFKbekg/azQacn55FW2P2lFz0Q3kerRo6u6hJKsZuBSQQ0iz8iBZF197TFqEkxMicFff0A==", "license": "BSD-3-Clause-Clear" }, "test-suite/e2e/node_modules/nofilter": { @@ -36272,7 +36396,9 @@ } }, "test-suite/e2e/node_modules/tfhe": { - "version": "1.3.0", + "version": "1.4.0-alpha.3", + "resolved": "https://registry.npmjs.org/tfhe/-/tfhe-1.4.0-alpha.3.tgz", + "integrity": "sha512-xdla7hi2WzLFIdAx2/ihRZ/bKlKcgDDabTJGtoqp1E5oqhLM1PzTXsJE0p7tW8+ebrvxiMGfbgMAWnU3f2ZAIQ==", "license": "BSD-3-Clause-Clear" }, "test-suite/e2e/node_modules/then-request": { @@ -36369,9 +36495,9 @@ } }, "test-suite/e2e/node_modules/tkms": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/tkms/-/tkms-0.11.1.tgz", - "integrity": "sha512-FNpnwZKsUUMs0q4aAwZatpw7fz1UBG9cdh3LZYgWYN3rvouS+v4zysB642dG8J35KgNF6WCFAzTyRKagdL8x7g==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/tkms/-/tkms-0.12.0.tgz", + "integrity": "sha512-3JEpetW+hvE+PaYGcI4vPzPihWgEnRl31NGjGCyKJt6duhJyo9EbXE2FUz5UGhow582br9asNb6M/AhJkYOA/A==", "license": "BSD-3-Clause-Clear" }, "test-suite/e2e/node_modules/tmp": { diff --git a/package.json b/package.json index 046a1b1c71..1026981bb7 100644 --- a/package.json +++ b/package.json @@ -10,5 +10,8 @@ "@trivago/prettier-plugin-sort-imports": "^5.2.2", "prettier": "^3.5.3", "prettier-plugin-solidity": "^2.0.0" + }, + "dependencies": { + "relayer-sdk": "^1.0.1" } -} \ No newline at end of file +} diff --git a/test-suite/e2e/package.json b/test-suite/e2e/package.json index 107a543418..854b6e3479 100644 --- a/test-suite/e2e/package.json +++ b/test-suite/e2e/package.json @@ -15,7 +15,7 @@ "dependencies": { "@fhevm/solidity": "*", "@openzeppelin/contracts": "^5.3.0", - "@zama-fhe/relayer-sdk": "^0.3.0-0", + "@zama-fhe/relayer-sdk": "^0.3.0-2", "bigint-buffer": "^1.1.5", "dotenv": "^16.0.3", "encrypted-types": "^0.0.4" diff --git a/test-suite/fhevm/config/core-client/config.toml b/test-suite/fhevm/config/core-client/config.toml index 501c01dbbd..f0ffc44047 100644 --- a/test-suite/fhevm/config/core-client/config.toml +++ b/test-suite/fhevm/config/core-client/config.toml @@ -1,14 +1,15 @@ -s3_endpoint = "http://minio:9000/kms-public" -object_folder = ["PUB"] -core_addresses = ["kms-core:50051"] num_majority = 1 num_reconstruct = 1 -# Decryption mode used in user decryption reconstruction. -# Options are NoiseFloodSmall and BitDecSmall. -# Must match the deployed core/service setting! -# Only relevant for threshold. decryption_mode = "NoiseFloodSmall" # fhe_params = "Test" # Uncomment for testing purposes -fhe_params = "Default" \ No newline at end of file +fhe_params = "Default" + +kms_type = "centralized" + +[[cores]] +party_id = 1 +address = "kms-core:50051" +s3_endpoint = "http://minio:9000/kms-public" +object_folder = "PUB"