feat: integrate codspeed

.github/workflows/bench.yml

@@ -0,0 +1,37 @@
+name: Benchmark
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  codspeed:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+      - name: Install cargo-codspeed
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-codspeed
+      - name: Build the benchmark target(s)
+        run: cargo codspeed build --profile profiling --workspace
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@v3
+        with:
+          run: cargo codspeed run --workspace
+          token: ${{ secrets.CODSPEED_TOKEN }}

Cargo.toml

@@ -55,10 +55,11 @@ context-interface = { path = "crates/context/interface", package = "revm-context
 handler = { path = "crates/handler", package = "revm-handler", version = "1.0.0", default-features = false }
 handler-interface = { path = "crates/handler/interface", package = "revm-handler-interface", version = "1.0.0", default-features = false }
 
-# mics
+# misc
 cfg-if = { version = "1.0", default-features = false }
 auto_impl = { version = "1.2.0" }
 derive-where = { version = "1.2.7", default-features = false }
+criterion = { package = "codspeed-criterion-compat", version = "2.7" }
 
 [workspace.package]
 license = "MIT"
@@ -75,9 +76,23 @@ all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
 [profile.release]
-lto = true
-codegen-units = 1
-debug = true
+codegen-units = 16
+debug = "line-tables-only"
+lto = "thin"
+opt-level = 3
+panic = "abort"
+strip = true
+
+# Use the `--profile profiling` flag to show symbols in release mode.
+# e.g. `cargo build --profile profiling`
+[profile.profiling]
+debug = 2
+inherits = "release"
+strip = false
+
+# Make sure debug symbols are in the bench profile
+[profile.bench]
+inherits = "profiling"
 
 [profile.ethtests]
 inherits = "test"

bins/revme/Cargo.toml

@@ -36,3 +36,10 @@ thiserror = "1.0"
 triehash = "0.8"
 walkdir = "2.5"
 k256 = { version = "0.13.3", features = ["ecdsa"] }
+
+[dev-dependencies]
+criterion.workspace = true
+
+[[bench]]
+name = "evm"
+harness = false

bins/revme/benches/evm.rs

@@ -0,0 +1,17 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use revme::cmd::{
+    bench::{self, BenchName},
+    MainCmd,
+};
+
+fn evm(c: &mut Criterion) {
+    for &bench_name in BenchName::ALL {
+        let cmd = MainCmd::Bench(bench::Cmd { name: bench_name });
+        c.bench_function(bench_name.as_str(), |b| {
+            b.iter(|| cmd.run().unwrap());
+        });
+    }
+}
+
+criterion_group!(benches, evm);
+criterion_main!(benches);

bins/revme/src/cmd/bench.rs

@@ -13,11 +13,30 @@ pub enum BenchName {
     Transfer,
 }
 
+impl BenchName {
+    pub const ALL: &[BenchName] = &[
+        BenchName::Analysis,
+        // TODO: Fails with OutOfGas(InvalidOperand)
+        // BenchName::Burntpix,
+        BenchName::Snailtracer,
+        BenchName::Transfer,
+    ];
+
+    pub fn as_str(self) -> &'static str {
+        match self {
+            BenchName::Analysis => "analysis",
+            BenchName::Burntpix => "burntpix",
+            BenchName::Snailtracer => "snailtracer",
+            BenchName::Transfer => "transfer",
+        }
+    }
+}
+
 /// `bytecode` subcommand
 #[derive(Parser, Debug)]
 pub struct Cmd {
     #[arg(value_enum)]
-    name: BenchName,
+    pub name: BenchName,
 }
 
 impl Cmd {

bins/revme/src/cmd/bench/transfer.rs

@@ -5,7 +5,6 @@ use revm::{
     primitives::{TxKind, U256},
     Context, MainEvm,
 };
-use std::time::Duration;
 
 pub fn run() {
     let context = Context::builder()
@@ -24,10 +23,5 @@ pub fn run() {
         });
     let mut evm = MainEvm::new(context, EthHandler::default());
 
-    // Microbenchmark
-    let bench_options = microbench::Options::default().time(Duration::from_secs(3));
-
-    microbench::bench(&bench_options, "Simple value transfer", || {
-        let _ = evm.transact().unwrap();
-    });
+    let _ = evm.transact().unwrap();
 }

crates/database/Cargo.toml

@@ -49,7 +49,6 @@ alloy-transport = { version = "0.6", optional = true, default-features = false }
 [dev-dependencies]
 serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
 anyhow = "1.0.83"
-criterion = "0.5"
 indicatif = "0.17"
 rstest = "0.22.0"
 alloy-sol-types = "0.8"

crates/database/interface/Cargo.toml

@@ -41,7 +41,6 @@ tokio = { version = "1.40", optional = true }
 
 [dev-dependencies]
 anyhow = "1.0.83"
-criterion = "0.5"
 indicatif = "0.17"
 rstest = "0.22.0"
 alloy-sol-types = "0.8"

crates/optimism/Cargo.toml

@@ -40,7 +40,6 @@ serde = { version = "1.0", default-features = false, features = [
 [dev-dependencies]
 database.workspace = true
 anyhow = "1.0.89"
-criterion = "0.5"
 indicatif = "0.17"
 rstest = "0.23.0"
 alloy-sol-types = "0.8"

crates/precompile/Cargo.toml

@@ -69,7 +69,7 @@ p256 = { version = "0.13.2", optional = true, default-features = false, features
 cfg-if = { version = "1.0", default-features = false }
 
 [dev-dependencies]
-criterion = "0.5"
+criterion.workspace = true
 rand = { version = "0.8", features = ["std"] }
 eyre = "0.6.12"
 rstest = "0.22.0"