diff --git a/.config/nextest.toml b/.config/nextest.toml index 75597f9..513c7c7 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,3 +1,9 @@ [profile.default] failure-output = "immediate-final" fail-fast = false +default-filter = 'not test(merkle::smt::full::concurrent)' + +[profile.smt-concurrent] +failure-output = "immediate-final" +fail-fast = false +default-filter = 'test(merkle::smt::full::concurrent)' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2194ef6..386ef3f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,3 +26,20 @@ jobs: run: | rustup update --no-self-update ${{matrix.toolchain}} make test-${{matrix.args}} + + test-smt-concurrent: + name: test-smt-concurrent ${{ matrix.toolchain }} + runs-on: ubuntu-latest + if: ${{ github.event_name == 'pull_request' && (github.base_ref == 'main' || github.base_ref == 'next') }} + strategy: + fail-fast: false + matrix: + toolchain: [stable, nightly] + timeout-minutes: 30 + steps: + - uses: actions/checkout@main + - uses: taiki-e/install-action@nextest + - name: Perform concurrent SMT tests + run: | + rustup update --no-self-update ${{matrix.toolchain}} + make test-smt-concurrent \ No newline at end of file diff --git a/.gitignore b/.gitignore index c7b497e..3668393 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ cmake-build-* # VS Code .vscode/ + +# Proptest +tests.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index de54686..fe493fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ - [BREAKING] Updated Winterfell dependency to v0.12 (#374). - Added debug-only duplicate column check in `build_subtree` (#378). - Filter out empty values in concurrent version of `Smt::with_entries` to fix a panic (#383). +- Added property-based testing (proptest) and fuzzing for `Smt::with_entries` and `Smt::compute_mutations` (#385). +- Sort keys in a leaf in the concurrent implementation of `Smt::with_entries`, ensuring consistency with the sequential version (#385). +- Skip unchanged leaves in the concurrent implementation of `Smt::compute_mutations` (#385). - Add range checks to `ntru_gen` for Falcon DSA (#391). ## 0.13.3 (2025-02-18) diff --git a/Cargo.toml b/Cargo.toml index d161859..7e1125f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,3 +91,7 @@ seq-macro = { version = "0.3" } [build-dependencies] cc = { version = "1.2", optional = true, features = ["parallel"] } glob = "0.3" + +[lints.rust] +# Suppress warnings about `cfg(fuzzing)`, which is automatically set when using `cargo-fuzz`. +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] } diff --git a/Makefile b/Makefile index 6ab285d..af430a9 100644 --- a/Makefile +++ b/Makefile @@ -54,9 +54,12 @@ test-smt-hashmaps: ## Run tests with `smt_hashmaps` feature enabled test-no-std: ## Run tests with `no-default-features` (std) $(DEBUG_OVERFLOW_INFO) cargo nextest run --profile default --release --no-default-features +.PHONY: test-smt-concurrent +test-smt-concurrent: ## Run only concurrent SMT tests + $(DEBUG_OVERFLOW_INFO) cargo nextest run --profile smt-concurrent --release --all-features .PHONY: test -test: test-default test-smt-hashmaps test-no-std ## Run all tests +test: test-default test-smt-hashmaps test-no-std ## Run all tests except concurrent SMT tests # --- checking ------------------------------------------------------------------------------------ @@ -91,3 +94,9 @@ bench: ## Run crypto benchmarks .PHONY: bench-smt-concurrent bench-smt-concurrent: ## Run SMT benchmarks with concurrent feature cargo run --release --features concurrent,executable -- --size 1000000 + +# --- fuzzing -------------------------------------------------------------------------------- + +.PHONY: fuzz-smt +fuzz-smt: ## Run fuzzing for SMT + cargo +nightly fuzz run smt --release -- -max_len=10485760 diff --git a/README.md b/README.md index 1addcb8..cd59c3d 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,22 @@ We do that by enabling some special [flags](https://doc.rust-lang.org/cargo/refe RUSTFLAGS="-C debug-assertions -C overflow-checks -C debuginfo=2" cargo test --release ``` +## Fuzzing + +The `fuzz-smt` fuzz target is designed to test the `Smt` implementation. It generates random SMT entries and updates, and then compares the results of the sequential and parallel implementations. + +Before running the fuzz tests, ensure you have `cargo-fuzz` installed: + +```shell +cargo install cargo-fuzz +``` + +To run the fuzz target, use: + +```shell +make fuzz-smt +``` + ## License This project is [MIT licensed](./LICENSE). diff --git a/fuzz/.gitignore b/fuzz/.gitignore new file mode 100644 index 0000000..1a45eee --- /dev/null +++ b/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock new file mode 100644 index 0000000..201f30a --- /dev/null +++ b/fuzz/Cargo.lock @@ -0,0 +1,555 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "blake3" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1230237285e3e10cde447185e8975408ae24deaa67205ce684805c25bc0c7937" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cc" +version = "1.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", + "rayon", + "serde", +] + +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "libc" +version = "0.2.170" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "miden-crypto" +version = "0.14.0" +dependencies = [ + "blake3", + "cc", + "glob", + "hashbrown", + "num", + "num-complex", + "rand", + "rand_core", + "rayon", + "sha3", + "thiserror", + "winter-crypto", + "winter-math", + "winter-utils", +] + +[[package]] +name = "miden-crypto-fuzz" +version = "0.0.0" +dependencies = [ + "libfuzzer-sys", + "miden-crypto", + "rand", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "serde" +version = "1.0.218" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.218" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "unicode-ident" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "winter-crypto" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32247cde9f43e5bbd05362caa7274608790ea69b14f7c81cd509aae7127c5ff2" +dependencies = [ + "blake3", + "sha3", + "winter-math", + "winter-utils", +] + +[[package]] +name = "winter-math" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "326dfe4bfa4072b7c909133a88f8807820d3e49e5dfd246f67981771f74a0ed3" +dependencies = [ + "winter-utils", +] + +[[package]] +name = "winter-utils" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d47518e6931955dcac73a584cacb04550b82ab2f45c72880cbbbdbe13adb63c" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml new file mode 100644 index 0000000..9029557 --- /dev/null +++ b/fuzz/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "miden-crypto-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" +miden-crypto = { path = "..", features = ["concurrent"] } +rand = { version = "0.8", default-features = false } + +[[bin]] +name = "smt" +path = "fuzz_targets/smt.rs" +test = false +doc = false +bench = false diff --git a/fuzz/fuzz_targets/smt.rs b/fuzz/fuzz_targets/smt.rs new file mode 100644 index 0000000..8304292 --- /dev/null +++ b/fuzz/fuzz_targets/smt.rs @@ -0,0 +1,80 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use miden_crypto::{merkle::Smt, hash::rpo::RpoDigest, Word, Felt, ONE}; +use rand::Rng; // Needed for randomizing the split percentage + +struct FuzzInput { + entries: Vec<(RpoDigest, Word)>, + updates: Vec<(RpoDigest, Word)>, +} + +impl FuzzInput { + fn from_bytes(data: &[u8]) -> Self { + let mut rng = rand::thread_rng(); + let split_percentage = rng.gen_range(20..80); // Randomly choose between 20% and 80% + + let split_index = (data.len() * split_percentage) / 100; + let (construction_data, update_data) = data.split_at(split_index); + + let entries = Self::parse_entries(construction_data); + let updates = Self::parse_entries(update_data); + + Self { entries, updates } + } + + fn parse_entries(data: &[u8]) -> Vec<(RpoDigest, Word)> { + let mut entries = Vec::new(); + let num_entries = data.len() / 40; // Each entry is 40 bytes + + for chunk in data.chunks_exact(40).take(num_entries) { + let key = RpoDigest::new([ + Felt::new(u64::from_le_bytes(chunk[0..8].try_into().unwrap())), + Felt::new(u64::from_le_bytes(chunk[8..16].try_into().unwrap())), + Felt::new(u64::from_le_bytes(chunk[16..24].try_into().unwrap())), + Felt::new(u64::from_le_bytes(chunk[24..32].try_into().unwrap())), + ]); + let value = [ + ONE, + ONE, + ONE, + Felt::new(u64::from_le_bytes(chunk[32..40].try_into().unwrap())), + ]; + entries.push((key, value)); + } + + entries + } +} + +fuzz_target!(|data: &[u8]| { + let fuzz_input = FuzzInput::from_bytes(data); + run_fuzz_smt(fuzz_input); +}); + +fn run_fuzz_smt(fuzz_input: FuzzInput) { + let sequential_result = Smt::fuzz_with_entries_sequential(fuzz_input.entries.clone()); + let parallel_result = Smt::with_entries(fuzz_input.entries); + + match (sequential_result, parallel_result) { + (Ok(sequential_smt), Ok(parallel_smt)) => { + assert_eq!(sequential_smt.root(), parallel_smt.root(), "Mismatch in SMT roots!"); + + let sequential_mutations = sequential_smt.fuzz_compute_mutations_sequential(fuzz_input.updates.clone()); + let parallel_mutations = parallel_smt.compute_mutations(fuzz_input.updates); + + assert_eq!(sequential_mutations.root(), parallel_mutations.root(), "Mismatch in mutation results!"); + assert_eq!(sequential_mutations.node_mutations(), parallel_mutations.node_mutations(), "Node mutations mismatch!"); + assert_eq!(sequential_mutations.new_pairs(), parallel_mutations.new_pairs(), "New pairs mismatch!"); + } + (Err(e1), Err(e2)) => { + assert_eq!( + format!("{:?}", e1), + format!("{:?}", e2), + "Different errors returned" + ); + } + (Ok(_), Err(e)) => panic!("Sequential succeeded but parallel failed with: {:?}", e), + (Err(e), Ok(_)) => panic!("Parallel succeeded but sequential failed with: {:?}", e), + } +} diff --git a/src/merkle/smt/full/concurrent/mod.rs b/src/merkle/smt/full/concurrent/mod.rs index 1785642..14730e6 100644 --- a/src/merkle/smt/full/concurrent/mod.rs +++ b/src/merkle/smt/full/concurrent/mod.rs @@ -4,7 +4,7 @@ use core::mem; use num::Integer; use super::{ - EmptySubtreeRoots, InnerNode, InnerNodes, LeafIndex, Leaves, MerkleError, MutationSet, + leaf, EmptySubtreeRoots, InnerNode, InnerNodes, LeafIndex, Leaves, MerkleError, MutationSet, NodeIndex, RpoDigest, Smt, SmtLeaf, SparseMerkleTree, Word, SMT_DEPTH, }; use crate::merkle::smt::{NodeMutation, NodeMutations, UnorderedMap}; @@ -89,6 +89,17 @@ impl Smt { // Convert sorted pairs into mutated leaves and capture any new pairs let (mut subtree_leaves, new_pairs) = self.sorted_pairs_to_mutated_subtree_leaves(sorted_kv_pairs); + + // If no mutations, return an empty mutation set + if subtree_leaves.is_empty() { + return MutationSet { + old_root: self.root(), + new_root: self.root(), + node_mutations: NodeMutations::default(), + new_pairs, + }; + } + let mut node_mutations = NodeMutations::default(); // Process each depth level in reverse, stepping by the subtree depth @@ -111,13 +122,22 @@ impl Smt { debug_assert!(!subtree_leaves.is_empty()); } - // Finalize the mutation set with updated roots and mutations - MutationSet { + let new_root = subtree_leaves[0][0].hash; + + // Create mutation set + let mutation_set = MutationSet { old_root: self.root(), - new_root: subtree_leaves[0][0].hash, + new_root, node_mutations, new_pairs, - } + }; + + // There should be mutations and new pairs at this point + debug_assert!( + !mutation_set.node_mutations().is_empty() && !mutation_set.new_pairs().is_empty() + ); + + mutation_set } // SUBTREE MUTATION @@ -244,7 +264,9 @@ impl Smt { /// With debug assertions on, this function panics if it detects that `pairs` is not correctly /// sorted. Without debug assertions, the returned computations will be incorrect. fn sorted_pairs_to_leaves(pairs: Vec<(RpoDigest, Word)>) -> PairComputations { - Self::process_sorted_pairs_to_leaves(pairs, Self::pairs_to_leaf) + Self::process_sorted_pairs_to_leaves(pairs, |leaf_pairs| { + Some(Self::pairs_to_leaf(leaf_pairs)) + }) } /// Constructs a single leaf from an arbitrary amount of key-value pairs. @@ -253,6 +275,7 @@ impl Smt { assert!(!pairs.is_empty()); if pairs.len() > 1 { + pairs.sort_by(|(key_1, _), (key_2, _)| leaf::cmp_keys(*key_1, *key_2)); SmtLeaf::new_multiple(pairs).unwrap() } else { let (key, value) = pairs.pop().unwrap(); @@ -278,22 +301,32 @@ impl Smt { let accumulator = Self::process_sorted_pairs_to_leaves(pairs, |leaf_pairs| { let mut leaf = self.get_leaf(&leaf_pairs[0].0); + let mut leaf_changed = false; for (key, value) in leaf_pairs { // Check if the value has changed - let old_value = - new_pairs.get(&key).cloned().unwrap_or_else(|| self.get_value(&key)); + let old_value = new_pairs.get(&key).cloned().unwrap_or_else(|| { + // Safe to unwrap: `leaf_pairs` contains keys all belonging to this leaf. + // `SmtLeaf::get_value()` only returns `None` if the key does not belong to the + // leaf, which cannot happen due to the sorting/grouping + // logic in `process_sorted_pairs_to_leaves()`. + leaf.get_value(&key).unwrap() + }); - // Skip if the value hasn't changed - if value == old_value { - continue; + if value != old_value { + // Update the leaf and track the new key-value pair + leaf = self.construct_prospective_leaf(leaf, &key, &value); + new_pairs.insert(key, value); + leaf_changed = true; } - - // Otherwise, update the leaf and track the new key-value pair - leaf = self.construct_prospective_leaf(leaf, &key, &value); - new_pairs.insert(key, value); } - leaf + if leaf_changed { + // Only return the leaf if it actually changed + Some(leaf) + } else { + // Return None if leaf hasn't changed + None + } }); (accumulator.leaves, new_pairs) } @@ -326,7 +359,7 @@ impl Smt { mut process_leaf: F, ) -> PairComputations where - F: FnMut(Vec<(RpoDigest, Word)>) -> SmtLeaf, + F: FnMut(Vec<(RpoDigest, Word)>) -> Option, { use rayon::prelude::*; debug_assert!(pairs.is_sorted_by_key(|(key, _)| Self::key_to_leaf_index(key).value())); @@ -359,9 +392,9 @@ impl Smt { // Otherwise, the next pair is a different column, or there is no next pair. Either way // it's time to swap out our buffer. let leaf_pairs = mem::take(&mut current_leaf_buffer); - let leaf = process_leaf(leaf_pairs); - - accumulator.nodes.insert(col, leaf); + if let Some(leaf) = process_leaf(leaf_pairs) { + accumulator.nodes.insert(col, leaf); + } debug_assert!(current_leaf_buffer.is_empty()); } diff --git a/src/merkle/smt/full/concurrent/tests.rs b/src/merkle/smt/full/concurrent/tests.rs index 8407c35..8e42052 100644 --- a/src/merkle/smt/full/concurrent/tests.rs +++ b/src/merkle/smt/full/concurrent/tests.rs @@ -3,6 +3,7 @@ use alloc::{ vec::Vec, }; +use proptest::prelude::*; use rand::{prelude::IteratorRandom, thread_rng, Rng}; use super::{ @@ -10,7 +11,7 @@ use super::{ Smt, SmtLeaf, SparseMerkleTree, SubtreeLeaf, SubtreeLeavesIter, UnorderedMap, COLS_PER_SUBTREE, SMT_DEPTH, SUBTREE_DEPTH, }; -use crate::{merkle::smt::Felt, Word, EMPTY_WORD, ONE}; +use crate::{merkle::smt::Felt, Word, EMPTY_WORD, ONE, ZERO}; fn smtleaf_to_subtree_leaf(leaf: &SmtLeaf) -> SubtreeLeaf { SubtreeLeaf { @@ -465,3 +466,135 @@ fn test_compute_mutations_parallel() { assert_eq!(mutations.node_mutations(), control.node_mutations()); assert_eq!(mutations.new_pairs(), control.new_pairs()); } + +#[test] +fn test_smt_construction_with_entries_unsorted() { + let entries = [ + (RpoDigest::new([ONE, ONE, Felt::new(2_u64), ONE]), [ONE; 4]), + (RpoDigest::new([ONE; 4]), [ONE; 4]), + ]; + let control = Smt::with_entries_sequential(entries).unwrap(); + let smt = Smt::with_entries(entries).unwrap(); + assert_eq!(smt.root(), control.root()); + assert_eq!(smt, control); +} + +fn arb_felt() -> impl Strategy { + prop_oneof![any::().prop_map(Felt::new), Just(ZERO), Just(ONE),] +} + +/// Generate entries that are guaranteed to be in different subtrees +fn generate_cross_subtree_entries() -> impl Strategy> { + let subtree_offsets = prop::collection::vec(0..(COLS_PER_SUBTREE * 4), 1..100); + + subtree_offsets.prop_map(|offsets| { + offsets + .into_iter() + .map(|base_col| { + let key = RpoDigest::new([ONE, ONE, ONE, Felt::new(base_col)]); + let value = [ONE, ONE, ONE, Felt::new(base_col)]; + (key, value) + }) + .collect() + }) +} + +fn arb_entries() -> impl Strategy> { + // Combine random entries with guaranteed cross-subtree entries + prop_oneof![ + // Original random entry generation + prop::collection::vec( + prop_oneof![ + // Random values case + ( + prop::array::uniform4(arb_felt()).prop_map(RpoDigest::new), + prop::array::uniform4(arb_felt()) + ), + // Edge case values + ( + Just(RpoDigest::new([ONE, ONE, ONE, Felt::new(0)])), + Just([ONE, ONE, ONE, Felt::new(u64::MAX)]) + ) + ], + 1..1000, + ), + // Guaranteed cross-subtree entries + generate_cross_subtree_entries(), + // Mix of both (combine random and cross-subtree entries) + ( + generate_cross_subtree_entries(), + prop::collection::vec( + ( + prop::array::uniform4(arb_felt()).prop_map(RpoDigest::new), + prop::array::uniform4(arb_felt()) + ), + 1..1000, + ) + ) + .prop_map(|(mut cross_subtree, mut random)| { + cross_subtree.append(&mut random); + cross_subtree + }) + ] + .prop_map(|entries| { + // Ensure uniqueness of entries as `Smt::with_entries` returns an error if multiple values + // exist for the same key. + let mut used_indices = BTreeSet::new(); + let mut used_keys = BTreeSet::new(); + let mut result = Vec::new(); + + for (key, value) in entries { + let leaf_index = LeafIndex::::from(key).value(); + if used_indices.insert(leaf_index) && used_keys.insert(key) { + result.push((key, value)); + } + } + result + }) +} + +proptest! { + #[test] + fn test_with_entries_consistency(entries in arb_entries()) { + let sequential = Smt::with_entries_sequential(entries.clone()).unwrap(); + let concurrent = Smt::with_entries(entries.clone()).unwrap(); + prop_assert_eq!(concurrent, sequential); + } + + #[test] + fn test_compute_mutations_consistency( + initial_entries in arb_entries(), + update_entries in arb_entries().prop_filter( + "Update must not be empty and must differ from initial entries", + |updates| !updates.is_empty() + ) + ) { + let tree = Smt::with_entries_sequential(initial_entries.clone()).unwrap(); + + let has_real_changes = update_entries.iter().any(|(key, value)| { + match initial_entries.iter().find(|(init_key, _)| init_key == key) { + Some((_, init_value)) => init_value != value, + None => true, + } + }); + + let sequential = tree.compute_mutations_sequential(update_entries.clone()); + let concurrent = tree.compute_mutations(update_entries.clone()); + + // If there are real changes, the root should change + if has_real_changes { + let sequential_changed = sequential.old_root != sequential.new_root; + let concurrent_changed = concurrent.old_root != concurrent.new_root; + + prop_assert!( + sequential_changed || concurrent_changed, + "Root should have changed" + ); + } + + prop_assert_eq!(sequential.old_root, concurrent.old_root); + prop_assert_eq!(sequential.new_root, concurrent.new_root); + prop_assert_eq!(sequential.node_mutations(), concurrent.node_mutations()); + prop_assert_eq!(sequential.new_pairs.len(), concurrent.new_pairs.len()); + } +} diff --git a/src/merkle/smt/full/mod.rs b/src/merkle/smt/full/mod.rs index 11c28f4..2e751fb 100644 --- a/src/merkle/smt/full/mod.rs +++ b/src/merkle/smt/full/mod.rs @@ -103,7 +103,7 @@ impl Smt { /// /// # Errors /// Returns an error if the provided entries contain multiple values for the same key. - #[cfg(any(not(feature = "concurrent"), test))] + #[cfg(any(not(feature = "concurrent"), fuzzing, test))] fn with_entries_sequential( entries: impl IntoIterator, ) -> Result { @@ -501,6 +501,25 @@ impl Deserializable for Smt { } } +// FUZZING +// ================================================================================================ + +#[cfg(fuzzing)] +impl Smt { + pub fn fuzz_with_entries_sequential( + entries: impl IntoIterator, + ) -> Result { + Self::with_entries_sequential(entries) + } + + pub fn fuzz_compute_mutations_sequential( + &self, + kv_pairs: impl IntoIterator, + ) -> MutationSet { + >::compute_mutations(self, kv_pairs) + } +} + // TESTS // ================================================================================================ diff --git a/src/merkle/smt/full/tests.rs b/src/merkle/smt/full/tests.rs index 787c01a..4863205 100644 --- a/src/merkle/smt/full/tests.rs +++ b/src/merkle/smt/full/tests.rs @@ -485,6 +485,20 @@ fn test_prospective_insertion() { assert_eq!(smt.root(), root_3); } +#[test] +fn test_mutations_no_mutations() { + let key = RpoDigest::from([ONE, ONE, ONE, ONE]); + let value = [ONE; WORD_SIZE]; + let entries = [(key, value)]; + + let tree = Smt::with_entries(entries).unwrap(); + let mutations = tree.compute_mutations(entries); + + assert_eq!(mutations.root(), mutations.old_root(), "Root should not change"); + assert!(mutations.node_mutations().is_empty(), "Node mutations should be empty"); + assert!(mutations.new_pairs().is_empty(), "There should be no new pairs"); +} + #[test] fn test_mutations_revert() { let mut smt = Smt::default();