diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000000..6220db2a67 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,37 @@ +name: Benchmark + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + codspeed: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: Install cargo-codspeed + uses: taiki-e/install-action@v2 + with: + tool: cargo-codspeed + - name: Build the benchmark target(s) + run: cargo codspeed build --profile profiling --workspace + - name: Run the benchmarks + uses: CodSpeedHQ/action@v3 + with: + run: cargo codspeed run --workspace + token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/.github/workflows/cachegrind.yml b/.github/workflows/cachegrind.yml deleted file mode 100644 index 25050ae047..0000000000 --- a/.github/workflows/cachegrind.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Valgrind Cachegrind - -on: - pull_request: - branches: [main] - -jobs: - valgrind: - runs-on: ubuntu-latest - permissions: write-all - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup | Rust - uses: dtolnay/rust-toolchain@stable - - - name: Install Valgrind - run: | - sudo apt-get update - sudo apt-get install -y valgrind - - - name: Run Valgrind - run: | - cargo b -r -p revme - valgrind --tool=cachegrind target/release/revme bench snailtracer 2>&1 | tee cachegrind_results.txt - - - name: Valgrind results - id: valgrind_results - run: | - contents=$(printf "%s" "$(head -c 64000 cachegrind_results.txt)") - # dump line by line into a file - printf "Valgrind Results:\n\n\`\`\`\n%s\n\`\`\`" "$contents" > results.md - - - name: Comment on PR - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Try to edit the last comment - if gh pr comment ${{ github.event.pull_request.number }} --edit-last --body-file results.md; then - echo "Successfully edited last comment." - else - echo "Failed to edit last comment. Trying to add a new comment instead!" - # If editing last comment fails, try to add a new comment - if ! gh pr comment ${{ github.event.pull_request.number }} --body-file results.md; then - echo "Comment failed to be made, printing out results here:" - cat results.md - fi - fi diff --git a/Cargo.lock b/Cargo.lock index ebecebcbe1..a8a72552de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addchain" @@ -959,12 +959,44 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +[[package]] +name = "codspeed" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "450a0e9df9df1c154156f4344f99d8f6f6e69d0fc4de96ef6e2e68b2ec3bce97" +dependencies = [ + "colored", + "libc", + "serde_json", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb1a6cb9c20e177fde58cdef97c1c7c9264eb1424fe45c4fccedc2fb078a569" +dependencies = [ + "codspeed", + "colored", + "criterion", +] + [[package]] name = "colorchoice" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + [[package]] name = "console" version = "0.15.8" @@ -2346,9 +2378,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.1" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] @@ -2392,9 +2424,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.64" +version = "0.10.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" dependencies = [ "bitflags", "cfg-if", @@ -2424,9 +2456,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.102" +version = "0.9.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" dependencies = [ "cc", "libc", @@ -2843,9 +2875,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ "bitflags", ] @@ -2935,7 +2967,6 @@ dependencies = [ "alloy-provider", "alloy-sol-types", "anyhow", - "criterion", "ethers-contract", "indicatif", "reqwest", @@ -3006,7 +3037,6 @@ dependencies = [ "alloy-transport", "anyhow", "auto_impl", - "criterion", "indicatif", "revm-bytecode", "revm-context-interface", @@ -3026,7 +3056,6 @@ dependencies = [ "alloy-sol-types", "anyhow", "auto_impl", - "criterion", "indicatif", "revm-primitives", "revm-state", @@ -3093,7 +3122,6 @@ version = "1.0.0" dependencies = [ "alloy-sol-types", "anyhow", - "criterion", "indicatif", "once_cell", "revm", @@ -3112,7 +3140,7 @@ dependencies = [ "blst", "c-kzg", "cfg-if", - "criterion", + "codspeed-criterion-compat", "eyre", "k256", "kzg-rs", @@ -3179,6 +3207,7 @@ dependencies = [ "alloy-sol-macro", "alloy-sol-types", "clap", + "codspeed-criterion-compat", "hash-db", "hashbrown 0.14.5", "indicatif", diff --git a/Cargo.toml b/Cargo.toml index fba5545a7f..91d9fcc32b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,10 +55,11 @@ context-interface = { path = "crates/context/interface", package = "revm-context handler = { path = "crates/handler", package = "revm-handler", version = "1.0.0", default-features = false } handler-interface = { path = "crates/handler/interface", package = "revm-handler-interface", version = "1.0.0", default-features = false } -# mics +# misc cfg-if = { version = "1.0", default-features = false } auto_impl = { version = "1.2.0" } derive-where = { version = "1.2.7", default-features = false } +criterion = { package = "codspeed-criterion-compat", version = "2.7" } [workspace.package] license = "MIT" @@ -75,9 +76,23 @@ all-features = true rustdoc-args = ["--cfg", "docsrs"] [profile.release] -lto = true -codegen-units = 1 -debug = true +codegen-units = 16 +debug = "line-tables-only" +lto = "thin" +opt-level = 3 +panic = "abort" +strip = true + +# Use the `--profile profiling` flag to show symbols in release mode. +# e.g. `cargo build --profile profiling` +[profile.profiling] +debug = 2 +inherits = "release" +strip = false + +# Make sure debug symbols are in the bench profile +[profile.bench] +inherits = "profiling" [profile.ethtests] inherits = "test" diff --git a/bins/revme/Cargo.toml b/bins/revme/Cargo.toml index 89c4f99707..704334ae4f 100644 --- a/bins/revme/Cargo.toml +++ b/bins/revme/Cargo.toml @@ -36,3 +36,10 @@ thiserror = "1.0" triehash = "0.8" walkdir = "2.5" k256 = { version = "0.13.3", features = ["ecdsa"] } + +[dev-dependencies] +criterion.workspace = true + +[[bench]] +name = "evm" +harness = false diff --git a/bins/revme/benches/evm.rs b/bins/revme/benches/evm.rs new file mode 100644 index 0000000000..a4fc9bfb62 --- /dev/null +++ b/bins/revme/benches/evm.rs @@ -0,0 +1,17 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use revme::cmd::{ + bench::{self, BenchName}, + MainCmd, +}; + +fn evm(c: &mut Criterion) { + for &bench_name in BenchName::ALL { + let cmd = MainCmd::Bench(bench::Cmd { name: bench_name }); + c.bench_function(bench_name.as_str(), |b| { + b.iter(|| cmd.run().unwrap()); + }); + } +} + +criterion_group!(benches, evm); +criterion_main!(benches); diff --git a/bins/revme/src/cmd/bench.rs b/bins/revme/src/cmd/bench.rs index aab98b9c0e..ce313b6887 100644 --- a/bins/revme/src/cmd/bench.rs +++ b/bins/revme/src/cmd/bench.rs @@ -13,11 +13,30 @@ pub enum BenchName { Transfer, } +impl BenchName { + pub const ALL: &[BenchName] = &[ + BenchName::Analysis, + // TODO: Fails with OutOfGas(InvalidOperand) + // BenchName::Burntpix, + BenchName::Snailtracer, + BenchName::Transfer, + ]; + + pub fn as_str(self) -> &'static str { + match self { + BenchName::Analysis => "analysis", + BenchName::Burntpix => "burntpix", + BenchName::Snailtracer => "snailtracer", + BenchName::Transfer => "transfer", + } + } +} + /// `bytecode` subcommand #[derive(Parser, Debug)] pub struct Cmd { #[arg(value_enum)] - name: BenchName, + pub name: BenchName, } impl Cmd { diff --git a/bins/revme/src/cmd/bench/transfer.rs b/bins/revme/src/cmd/bench/transfer.rs index f36889b0bc..f53d27e8df 100644 --- a/bins/revme/src/cmd/bench/transfer.rs +++ b/bins/revme/src/cmd/bench/transfer.rs @@ -5,7 +5,6 @@ use revm::{ primitives::{TxKind, U256}, Context, MainEvm, }; -use std::time::Duration; pub fn run() { let context = Context::builder() @@ -24,10 +23,5 @@ pub fn run() { }); let mut evm = MainEvm::new(context, EthHandler::default()); - // Microbenchmark - let bench_options = microbench::Options::default().time(Duration::from_secs(3)); - - microbench::bench(&bench_options, "Simple value transfer", || { - let _ = evm.transact().unwrap(); - }); + let _ = evm.transact().unwrap(); } diff --git a/crates/database/Cargo.toml b/crates/database/Cargo.toml index 329db4acc7..306da97840 100644 --- a/crates/database/Cargo.toml +++ b/crates/database/Cargo.toml @@ -49,7 +49,6 @@ alloy-transport = { version = "0.6", optional = true, default-features = false } [dev-dependencies] serde_json = { version = "1.0", default-features = false, features = ["alloc"] } anyhow = "1.0.83" -criterion = "0.5" indicatif = "0.17" rstest = "0.22.0" alloy-sol-types = "0.8" diff --git a/crates/database/interface/Cargo.toml b/crates/database/interface/Cargo.toml index e3d3e51533..3d7325f3d2 100644 --- a/crates/database/interface/Cargo.toml +++ b/crates/database/interface/Cargo.toml @@ -41,7 +41,6 @@ tokio = { version = "1.40", optional = true } [dev-dependencies] anyhow = "1.0.83" -criterion = "0.5" indicatif = "0.17" rstest = "0.22.0" alloy-sol-types = "0.8" diff --git a/crates/optimism/Cargo.toml b/crates/optimism/Cargo.toml index 1896d73440..6c9db89c8c 100644 --- a/crates/optimism/Cargo.toml +++ b/crates/optimism/Cargo.toml @@ -40,7 +40,6 @@ serde = { version = "1.0", default-features = false, features = [ [dev-dependencies] database.workspace = true anyhow = "1.0.89" -criterion = "0.5" indicatif = "0.17" rstest = "0.23.0" alloy-sol-types = "0.8" diff --git a/crates/precompile/Cargo.toml b/crates/precompile/Cargo.toml index e2445b2a03..968cfd13a4 100644 --- a/crates/precompile/Cargo.toml +++ b/crates/precompile/Cargo.toml @@ -69,7 +69,7 @@ p256 = { version = "0.13.2", optional = true, default-features = false, features cfg-if = { version = "1.0", default-features = false } [dev-dependencies] -criterion = "0.5" +criterion.workspace = true rand = { version = "0.8", features = ["std"] } eyre = "0.6.12" rstest = "0.22.0" diff --git a/crates/precompile/benches/bench.rs b/crates/precompile/benches/bench.rs index 02e3a58574..1a14d283e0 100644 --- a/crates/precompile/benches/bench.rs +++ b/crates/precompile/benches/bench.rs @@ -1,4 +1,4 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use criterion::{criterion_group, criterion_main, Criterion}; use primitives::{hex, keccak256, U256}; use revm_precompile::{ bn128::{ @@ -76,9 +76,7 @@ pub fn benchmark_crypto_precompiles(c: &mut Criterion) { let message = Message::from_digest_slice(&hash[..]).unwrap(); let s = SECP256K1.sign_ecdsa_recoverable(&message, &secret_key); let (rec_id, data) = s.serialize_compact(); - let mut rec_id = rec_id.to_i32() as u8; - assert_eq!(rec_id, 0); - rec_id += 27; + let rec_id = rec_id.to_i32() as u8 + 27; let mut message_and_signature = [0u8; 128]; message_and_signature[0..32].copy_from_slice(&hash[..]); @@ -111,17 +109,11 @@ pub fn benchmark_crypto_precompiles(c: &mut Criterion) { println!("gas used by kzg precompile: {:?}", output.gas_used); group.bench_function(group_name("ecrecover precompile"), |b| { - b.iter(|| { - ec_recover_run(&message_and_signature, u64::MAX).unwrap(); - black_box(()) - }) + b.iter(|| ec_recover_run(&message_and_signature, u64::MAX).unwrap()) }); group.bench_function(group_name("bn128 add precompile"), |b| { - b.iter(|| { - run_add(&ecadd_input, ISTANBUL_ADD_GAS_COST, 150).unwrap(); - black_box(()) - }) + b.iter(|| run_add(&ecadd_input, ISTANBUL_ADD_GAS_COST, 150).unwrap()) }); group.bench_function(group_name("ecpairing precompile"), |b| { @@ -132,16 +124,12 @@ pub fn benchmark_crypto_precompiles(c: &mut Criterion) { ISTANBUL_PAIR_BASE, u64::MAX, ) - .unwrap(); - black_box(()) + .unwrap() }) }); group.bench_function(group_name("kzg precompile"), |b| { - b.iter(|| { - run(&kzg_input, gas).unwrap(); - black_box(()) - }) + b.iter(|| run(&kzg_input, gas).unwrap()) }); } diff --git a/crates/revm/Cargo.toml b/crates/revm/Cargo.toml index b5139ff74b..7dbcb50c2b 100644 --- a/crates/revm/Cargo.toml +++ b/crates/revm/Cargo.toml @@ -49,7 +49,6 @@ alloy-sol-types = { version = "0.8.2", default-features = false, features = [ ] } ethers-contract = { version = "2.0.14", default-features = false } anyhow = "1.0.89" -criterion = "0.5" indicatif = "0.17" reqwest = { version = "0.12" } rstest = "0.22.0"