feat(benches): add Anvil IPC benchmark

Cargo.toml

@@ -16,7 +16,7 @@ arbiter-ethereum = { path = "arbiter-ethereum" }
 arbiter-macros   = { path = "arbiter-macros" }
 
 # Ethereum
-ethers          = { version = "2.0.14" }
+ethers          = { version = "2.0.14", features = ["ipc"] }
 revm            = { version = "8.0.0", features = ["ethersdb", "std", "serde"] }
 revm-primitives = "3.1.1"
 

README.md

@@ -36,7 +36,7 @@ If mechanism security interests you, please see the [Vulnerability Corpus](https
 
 The Arbiter workspace has five crates:
 - `arbiter`: The bin that exposes a command line interface for forking and binding contracts.
-- `arbiter-core`: A lib containing the core logic for the Arbiter framework, including the `ArbiterMiddleware` discussed before, and the `Environment`, our sandbox.
+- `arbiter-ethereum`: A lib containing the core logic for the Arbiter framework, including the `ArbiterMiddleware` discussed before, and the `Environment`, our sandbox.
 - `arbiter-engine`: A lib that provides abstractions for building simulations, agents, and behaviors.
 - `arbiter-macros`: A lib crate that contains the macros used to simplify development with Arbiter.
 - `arbiter-bindings`: A lib crate containing bindings for utility smart contracts used for testing and development.
@@ -136,14 +136,14 @@ To see the Cargo docs for the Arbiter crates, please visit the following:
 You will find each of these on crates.io.
 
 ## Benchmarks
-In `arbiter-core`, we have a a small benchmarking suite that compares the `ArbiterMiddleware` implementation over the `Environment` to the [Anvil](https://github.com/foundry-rs/foundry/tree/master/crates/anvil) local testnet chain implementation.
+In `arbiter-ethereum`, we have a a small benchmarking suite that compares the `ArbiterMiddleware` implementation over the `Environment` to the [Anvil](https://github.com/foundry-rs/foundry/tree/master/crates/anvil) local testnet chain implementation.
 The biggest reasons we chose to build Arbiter was to gain more control over the EVM environment and to have a more robust simulation framework. Still, we also wanted to gain speed, so we chose to build our own interface over `revm` instead of using Anvil (which uses `revm` under the hood). 
 For the following, Anvil was set to mine blocks for each transaction instead of setting an enforced block time. The `Environment` was configured with a block rate of 10.0.
 Preliminary benchmarks of the `ArbiterMiddleware` interface over `revm` against Anvil are given in the following table.
 
 To run the benchmarking code yourself, you can run:
 ```bash
-cargo bench --package arbiter-core
+cargo bench --package arbiter-ethereum
 ```
 
 | Operation       |  ArbiterMiddleware |    Anvil     | Relative Difference |
@@ -170,8 +170,8 @@ Divide by 100 to get the time to call a single stateless function.
 In this call, we called `ArbiterToken`'s `mint` function 100 times.
 Divide by 100 to get the time to call a single stateful function.
 
-The benchmarking code can be found in the `arbiter-core/benches/` directory, and these specific times were achieved over a 1000 run average. 
-The above was achieved by running `cargo bench --package arbiter-core`, which will automatically run with the release profile.
+The benchmarking code can be found in the `arbiter-ethereum/benches/` directory, and these specific times were achieved over a 1000 run average. 
+The above was achieved by running `cargo bench --package arbiter-ethereum`, which will automatically run with the release profile.
 Times were achieved on an Apple Macbook Pro M1 Max with 8 performance and 2 efficiency cores and 32GB of RAM.
 
 Of course, the use cases of Anvil and the `ArbiterMiddleware` can be different. 

arbiter-ethereum/benches/bench.rs

@@ -5,15 +5,15 @@ use std::{
   time::{Duration, Instant},
 };
 
-use arbiter_bindings::bindings::{
+use arbiter_ethereum::bindings::{
   arbiter_math::ArbiterMath,
   arbiter_token::{self, ArbiterToken},
 };
-use arbiter_core::{environment::Environment, middleware::ArbiterMiddleware};
+use arbiter_ethereum::{environment::Environment, middleware::ArbiterMiddleware};
 use ethers::{
   core::{k256::ecdsa::SigningKey, utils::Anvil},
   middleware::SignerMiddleware,
-  providers::{Http, Middleware, Provider},
+  providers::{Http, Ipc, Middleware, Provider},
   signers::{LocalWallet, Signer, Wallet},
   types::{Address, I256, U256},
   utils::AnvilInstance,
@@ -29,16 +29,16 @@ const NUM_LOOP_STEPS: usize = 10;
 
 #[derive(Debug)]
 struct BenchDurations {
-  deploy:         Duration,
-  lookup:         Duration,
+  deploy: Duration,
+  lookup: Duration,
   stateless_call: Duration,
-  stateful_call:  Duration,
+  stateful_call: Duration,
 }
 
 #[tokio::main]
 async fn main() {
   // Choose the benchmark group items by label.
-  let group = ["anvil", "arbiter"];
+  let group = ["anvil", "anvil-ipc", "arbiter"];
   let mut results: HashMap<&str, HashMap<&str, Duration>> = HashMap::new();
 
   // Set up for showing percentage done.
@@ -58,6 +58,12 @@ async fn main() {
           drop(_anvil_instance);
           duration
         },
+        label @ "anvil-ipc" => {
+          let (client, _anvil_instance) = anvil_ipc_startup().await;
+          let duration = bencher(client, label).await;
+          drop(_anvil_instance);
+          duration
+        },
         label @ "arbiter" => {
           let (_environment, client) = arbiter_startup();
           bencher(client, label).await
@@ -70,24 +76,24 @@ async fn main() {
     }
     let sum_durations = durations.iter().fold(
       BenchDurations {
-        deploy:         Duration::default(),
-        lookup:         Duration::default(),
+        deploy: Duration::default(),
+        lookup: Duration::default(),
         stateless_call: Duration::default(),
-        stateful_call:  Duration::default(),
+        stateful_call: Duration::default(),
       },
       |acc, duration| BenchDurations {
-        deploy:         acc.deploy + duration.deploy,
-        lookup:         acc.lookup + duration.lookup,
+        deploy: acc.deploy + duration.deploy,
+        lookup: acc.lookup + duration.lookup,
         stateless_call: acc.stateless_call + duration.stateless_call,
-        stateful_call:  acc.stateful_call + duration.stateful_call,
+        stateful_call: acc.stateful_call + duration.stateful_call,
       },
     );
 
     let average_durations = BenchDurations {
-      deploy:         sum_durations.deploy / NUM_BENCH_ITERATIONS as u32,
-      lookup:         sum_durations.lookup / NUM_BENCH_ITERATIONS as u32,
+      deploy: sum_durations.deploy / NUM_BENCH_ITERATIONS as u32,
+      lookup: sum_durations.lookup / NUM_BENCH_ITERATIONS as u32,
       stateless_call: sum_durations.stateless_call / NUM_BENCH_ITERATIONS as u32,
-      stateful_call:  sum_durations.stateful_call / NUM_BENCH_ITERATIONS as u32,
+      stateful_call: sum_durations.stateful_call / NUM_BENCH_ITERATIONS as u32,
     };
 
     item_results.insert("Deploy", average_durations.deploy);
@@ -100,8 +106,8 @@ async fn main() {
 
   let df = create_dataframe(&results, &group);
 
-  match get_version_of("arbiter-core") {
-    Some(version) => println!("arbiter-core version: {}", version),
+  match get_version_of("arbiter-ethereum") {
+    Some(version) => println!("arbiter-ethereum version: {}", version),
     None => println!("Could not find version for arbiter-core"),
   }
 
@@ -142,10 +148,10 @@ async fn bencher<M: Middleware + 'static>(client: Arc<M>, label: &str) -> BenchD
   total_stateful_call_duration += statefull_call_duration.as_micros();
 
   BenchDurations {
-    deploy:         Duration::from_micros(total_deploy_duration as u64),
-    lookup:         Duration::from_micros(total_lookup_duration as u64),
+    deploy: Duration::from_micros(total_deploy_duration as u64),
+    lookup: Duration::from_micros(total_lookup_duration as u64),
     stateless_call: Duration::from_micros(total_stateless_call_duration as u64),
-    stateful_call:  Duration::from_micros(total_stateful_call_duration as u64),
+    stateful_call: Duration::from_micros(total_stateful_call_duration as u64),
   }
 }
 
@@ -164,6 +170,33 @@ async fn anvil_startup(
   (client, anvil)
 }
 
+async fn anvil_ipc_startup(
+) -> (Arc<SignerMiddleware<Provider<Ipc>, Wallet<SigningKey>>>, AnvilInstance) {
+  // Create an Anvil IPC instance
+  // No blocktime mines a new block for each tx, which is fastest.
+
+  #[cfg(unix)]
+  let (anvil, provider) = {
+    let ipc_path = "/tmp/anvil.ipc";
+    let anvil = Anvil::new().arg("--ipc").arg(ipc_path).spawn();
+    let provider = Provider::connect_ipc(ipc_path).await.unwrap().interval(Duration::ZERO);
+    (anvil, provider)
+  };
+
+  #[cfg(windows)]
+  let (anvil, provider) = {
+    let ipc_path = r"\\.\pipe\anvil.ipc";
+    let anvil = Anvil::new().arg("--ipc").arg(ipc_path).spawn();
+    let provider = Provider::connect_ipc(ipc_path).await.unwrap().interval(Duration::ZERO);
+    (anvil, provider)
+  };
+
+  let wallet: LocalWallet = anvil.keys()[0].clone().into();
+  let client = Arc::new(SignerMiddleware::new(provider, wallet.with_chain_id(anvil.chain_id())));
+
+  (client, anvil)
+}
+
 fn arbiter_startup() -> (Environment, Arc<ArbiterMiddleware>) {
   let environment = Environment::builder().build();
 
@@ -236,30 +269,38 @@ async fn stateful_call_loop<M: Middleware + 'static>(
 
 fn create_dataframe(results: &HashMap<&str, HashMap<&str, Duration>>, group: &[&str]) -> DataFrame {
   let operations = ["Deploy", "Lookup", "Stateless Call", "Stateful Call"];
-  let mut df = DataFrame::new(vec![
-    Series::new("Operation", operations.to_vec()),
-    Series::new(
-      &format!("{} (μs)", group[0]),
-      operations
-        .iter()
-        .map(|&op| results.get(group[0]).unwrap().get(op).unwrap().as_micros() as f64)
-        .collect::<Vec<_>>(),
-    ),
-    Series::new(
-      &format!("{} (μs)", group[1]),
-      operations
-        .iter()
-        .map(|&op| results.get(group[1]).unwrap().get(op).unwrap().as_micros() as f64)
-        .collect::<Vec<_>>(),
-    ),
-  ])
-  .unwrap();
+  let series_columns: Vec<Series> = group
+    .iter()
+    .map(|group_name| {
+      Series::new(
+        &format!("{} (μs)", group_name),
+        operations
+          .iter()
+          .map(|&op| results.get(group_name).unwrap().get(&op).unwrap().as_micros() as f64)
+          .collect::<Vec<_>>(),
+      )
+    })
+    .collect();
+
+  let mut columns = vec![Series::new("Operation", operations.to_vec())];
+  columns.extend(series_columns);
+
+  let mut df = DataFrame::new(columns).unwrap();
 
   let s0 = df.column(&format!("{} (μs)", group[0])).unwrap().to_owned();
   let s1 = df.column(&format!("{} (μs)", group[1])).unwrap().to_owned();
-  let mut relative_difference = s0.divide(&s1).unwrap();
-
-  df.with_column::<Series>(relative_difference.rename("Relative Speedup").clone()).unwrap().clone()
+  let s2 = df.column(&format!("{} (μs)", group[2])).unwrap().to_owned();
+  let mut speedup_arb_anvil_rpc = s0.divide(&s2).unwrap();
+  let mut speedup_arb_anvil_ipc = s1.divide(&s2).unwrap();
+  let mut speedup_ipc_rpc = s0.divide(&s1).unwrap();
+
+  df.with_column::<Series>(speedup_arb_anvil_rpc.rename("Speedup - arbiter vs anvil RPC").clone())
+    .unwrap()
+    .with_column::<Series>(speedup_arb_anvil_ipc.rename("Speedup - arbiter vs anvil IPC").clone())
+    .unwrap()
+    .with_column::<Series>(speedup_ipc_rpc.rename("Speedup - anvil IPC vs anvil RPC").clone())
+    .unwrap()
+    .clone()
 }
 
 fn get_version_of(crate_name: &str) -> Option<String> {