diff --git a/crates/interpreter/src/gas.rs b/crates/interpreter/src/gas.rs index 8cc52c2026..e270224069 100644 --- a/crates/interpreter/src/gas.rs +++ b/crates/interpreter/src/gas.rs @@ -195,6 +195,7 @@ pub struct MemoryGas { impl MemoryGas { /// Creates a new `MemoryGas` instance with zero memory allocation. + #[inline] pub const fn new() -> Self { Self { words_num: 0, @@ -202,9 +203,9 @@ impl MemoryGas { } } - #[inline] /// Records a new memory length and calculates additional cost if memory is expanded. /// Returns the additional gas cost required, or None if no expansion is needed. + #[inline] pub fn record_new_len(&mut self, new_num: usize) -> Option { if new_num <= self.words_num { return None; diff --git a/crates/interpreter/src/instruction_context.rs b/crates/interpreter/src/instruction_context.rs index d7affc69c4..d8aa146829 100644 --- a/crates/interpreter/src/instruction_context.rs +++ b/crates/interpreter/src/instruction_context.rs @@ -6,10 +6,10 @@ use super::Instruction; /// This struct provides access to both the host interface for external state operations /// and the interpreter state for stack, memory, and gas operations. pub struct InstructionContext<'a, H: ?Sized, ITy: InterpreterTypes> { - /// Reference to the host interface for accessing external blockchain state. - pub host: &'a mut H, /// Reference to the interpreter containing execution state (stack, memory, gas, etc). pub interpreter: &'a mut Interpreter, + /// Reference to the host interface for accessing external blockchain state. + pub host: &'a mut H, } impl std::fmt::Debug for InstructionContext<'_, H, ITy> { diff --git a/crates/interpreter/src/instructions/control.rs b/crates/interpreter/src/instructions/control.rs index d28c12a283..58502560ca 100644 --- a/crates/interpreter/src/instructions/control.rs +++ b/crates/interpreter/src/instructions/control.rs @@ -29,10 +29,10 @@ pub fn jumpi(context: InstructionContext<'_, } } -#[inline(always)] /// Internal helper function for jump operations. /// /// Validates jump target and performs the actual jump. +#[inline(always)] fn jump_inner(interpreter: &mut Interpreter, target: U256) { let target = as_usize_or_fail!(interpreter, target, InstructionResult::InvalidJump); if !interpreter.bytecode.is_valid_legacy_jump(target) { diff --git a/crates/interpreter/src/instructions/macros.rs b/crates/interpreter/src/instructions/macros.rs index 39a2e5c083..b361c0369a 100644 --- a/crates/interpreter/src/instructions/macros.rs +++ b/crates/interpreter/src/instructions/macros.rs @@ -99,17 +99,15 @@ macro_rules! resize_memory { $crate::resize_memory!($interpreter, $offset, $len, ()) }; ($interpreter:expr, $offset:expr, $len:expr, $ret:expr) => { - let words_num = $crate::interpreter::num_words($offset.saturating_add($len)); - match $interpreter.gas.record_memory_expansion(words_num) { - $crate::gas::MemoryExtensionResult::Extended => { - $interpreter.memory.resize(words_num * 32); - } - $crate::gas::MemoryExtensionResult::OutOfGas => { - $interpreter.halt($crate::InstructionResult::MemoryOOG); - return $ret; - } - $crate::gas::MemoryExtensionResult::Same => (), // no action - }; + if !$crate::interpreter::resize_memory( + &mut $interpreter.gas, + &mut $interpreter.memory, + $offset, + $len, + ) { + $interpreter.halt($crate::InstructionResult::MemoryOOG); + return $ret; + } }; } @@ -124,14 +122,31 @@ macro_rules! popn { }; } +#[doc(hidden)] +#[macro_export] +macro_rules! _count { + (@count) => { 0 }; + (@count $head:tt $($tail:tt)*) => { 1 + _count!(@count $($tail)*) }; + ($($arg:tt)*) => { _count!(@count $($arg)*) }; +} + /// Pops n values from the stack and returns the top value. Fails the instruction if n values can't be popped. #[macro_export] macro_rules! popn_top { ([ $($x:ident),* ], $top:ident, $interpreter:expr $(,$ret:expr)? ) => { + /* let Some(([$( $x ),*], $top)) = $interpreter.stack.popn_top() else { $interpreter.halt($crate::InstructionResult::StackUnderflow); return $($ret)?; }; + */ + + // Workaround for https://github.com/rust-lang/rust/issues/144329. + if $interpreter.stack.len() < (1 + $crate::_count!($($x)*)) { + $interpreter.halt($crate::InstructionResult::StackUnderflow); + return $($ret)?; + } + let ([$( $x ),*], $top) = unsafe { $interpreter.stack.popn_top().unwrap_unchecked() }; }; } diff --git a/crates/interpreter/src/instructions/stack.rs b/crates/interpreter/src/instructions/stack.rs index 1c84ea4b83..cdfdceccd0 100644 --- a/crates/interpreter/src/instructions/stack.rs +++ b/crates/interpreter/src/instructions/stack.rs @@ -1,6 +1,5 @@ use crate::{ gas, - instructions::utility::cast_slice_to_u256, interpreter_types::{Immediates, InterpreterTypes, Jumps, RuntimeFlag, StackTr}, InstructionResult, }; @@ -33,11 +32,12 @@ pub fn push( context: InstructionContext<'_, H, WIRE>, ) { gas!(context.interpreter, gas::VERYLOW); - push!(context.interpreter, U256::ZERO); - popn_top!([], top, context.interpreter); - let imm = context.interpreter.bytecode.read_slice(N); - cast_slice_to_u256(imm, top); + let slice = context.interpreter.bytecode.read_slice(N); + if !context.interpreter.stack.push_slice(slice) { + context.interpreter.halt(InstructionResult::StackOverflow); + return; + } // Can ignore return. as relative N jump is safe operation context.interpreter.bytecode.relative_jump(N as isize); diff --git a/crates/interpreter/src/instructions/utility.rs b/crates/interpreter/src/instructions/utility.rs index 993e3e2d73..8a51ce9575 100644 --- a/crates/interpreter/src/instructions/utility.rs +++ b/crates/interpreter/src/instructions/utility.rs @@ -1,70 +1,5 @@ use primitives::{Address, B256, U256}; -/// Pushes an arbitrary length slice of bytes onto the stack, padding the last word with zeros -/// if necessary. -/// -/// # Panics -/// -/// Panics if slice is longer than 32 bytes. -#[inline] -pub fn cast_slice_to_u256(slice: &[u8], dest: &mut U256) { - if slice.is_empty() { - return; - } - assert!(slice.len() <= 32, "slice too long"); - - let n_words = slice.len().div_ceil(32); - - // SAFETY: Length checked above. - unsafe { - //let dst = self.data.as_mut_ptr().add(self.data.len()).cast::(); - //self.data.set_len(new_len); - let dst = dest.as_limbs_mut().as_mut_ptr(); - - let mut i = 0; - - // Write full words - let words = slice.chunks_exact(32); - let partial_last_word = words.remainder(); - for word in words { - // Note: We unroll `U256::from_be_bytes` here to write directly into the buffer, - // instead of creating a 32 byte array on the stack and then copying it over. - for l in word.rchunks_exact(8) { - dst.add(i).write(u64::from_be_bytes(l.try_into().unwrap())); - i += 1; - } - } - - if partial_last_word.is_empty() { - return; - } - - // Write limbs of partial last word - let limbs = partial_last_word.rchunks_exact(8); - let partial_last_limb = limbs.remainder(); - for l in limbs { - dst.add(i).write(u64::from_be_bytes(l.try_into().unwrap())); - i += 1; - } - - // Write partial last limb by padding with zeros - if !partial_last_limb.is_empty() { - let mut tmp = [0u8; 8]; - tmp[8 - partial_last_limb.len()..].copy_from_slice(partial_last_limb); - dst.add(i).write(u64::from_be_bytes(tmp)); - i += 1; - } - - debug_assert_eq!(i.div_ceil(4), n_words, "wrote too much"); - - // Zero out upper bytes of last word - let m = i % 4; // 32 / 8 - if m != 0 { - dst.add(i).write_bytes(0, 4 - m); - } - } -} - /// Trait for converting types into U256 values. pub trait IntoU256 { /// Converts the implementing type into a U256 value. diff --git a/crates/interpreter/src/interpreter.rs b/crates/interpreter/src/interpreter.rs index dee1b766a5..db0ae972f2 100644 --- a/crates/interpreter/src/interpreter.rs +++ b/crates/interpreter/src/interpreter.rs @@ -14,7 +14,7 @@ pub use ext_bytecode::ExtBytecode; pub use input::InputsImpl; pub use return_data::ReturnDataImpl; pub use runtime_flags::RuntimeFlags; -pub use shared_memory::{num_words, SharedMemory}; +pub use shared_memory::{num_words, resize_memory, SharedMemory}; pub use stack::{Stack, STACK_LIMIT}; // imports @@ -183,6 +183,13 @@ impl InterpreterTypes for EthInterpreter { } impl Interpreter { + /// Performs EVM memory resize. + #[inline] + #[must_use] + pub fn resize_memory(&mut self, offset: usize, len: usize) -> bool { + resize_memory(&mut self.gas, &mut self.memory, offset, len) + } + /// Takes the next action from the control and returns it. #[inline] pub fn take_next_action(&mut self) -> InterpreterAction { @@ -193,6 +200,8 @@ impl Interpreter { /// Halt the interpreter with the given result. /// /// This will set the action to [`InterpreterAction::Return`] and set the gas to the current gas. + #[cold] + #[inline(never)] pub fn halt(&mut self, result: InstructionResult) { self.bytecode .set_action(InterpreterAction::new_halt(result, self.gas)); diff --git a/crates/interpreter/src/interpreter/shared_memory.rs b/crates/interpreter/src/interpreter/shared_memory.rs index a366b2d0d4..8e1b503954 100644 --- a/crates/interpreter/src/interpreter/shared_memory.rs +++ b/crates/interpreter/src/interpreter/shared_memory.rs @@ -8,6 +8,29 @@ use core::{ use primitives::{hex, B256, U256}; use std::{rc::Rc, vec::Vec}; +trait RefcellExt { + fn dbg_borrow(&self) -> Ref<'_, T>; + fn dbg_borrow_mut(&self) -> RefMut<'_, T>; +} + +impl RefcellExt for RefCell { + #[inline] + fn dbg_borrow(&self) -> Ref<'_, T> { + match self.try_borrow() { + Ok(b) => b, + Err(e) => debug_unreachable!("{e}"), + } + } + + #[inline] + fn dbg_borrow_mut(&self) -> RefMut<'_, T> { + match self.try_borrow_mut() { + Ok(b) => b, + Err(e) => debug_unreachable!("{e}"), + } + } +} + /// A sequential memory shared between calls, which uses /// a `Vec` for internal representation. /// A [SharedMemory] instance should always be obtained using @@ -91,7 +114,7 @@ impl MemoryTr for SharedMemory { #[inline] #[cfg_attr(debug_assertions, track_caller)] fn global_slice(&self, range: Range) -> Ref<'_, [u8]> { - let buffer = self.buffer().borrow(); // Borrow the inner Vec + let buffer = self.buffer_ref(); Ref::map(buffer, |b| match b.get(range) { Some(slice) => slice, None => debug_unreachable!("slice OOB: range; len: {}", self.len()), @@ -167,6 +190,16 @@ impl SharedMemory { unsafe { self.buffer.as_ref().unwrap_unchecked() } } + #[inline] + fn buffer_ref(&self) -> Ref<'_, Vec> { + self.buffer().dbg_borrow() + } + + #[inline] + fn buffer_ref_mut(&self) -> RefMut<'_, Vec> { + self.buffer().dbg_borrow_mut() + } + /// Returns `true` if the `new_size` for the current context memory will /// make the shared buffer length exceed the `memory_limit`. #[cfg(feature = "memory_limit")] @@ -185,7 +218,7 @@ impl SharedMemory { if self.child_checkpoint.is_some() { panic!("new_child_context was already called without freeing child context"); } - let new_checkpoint = self.buffer().borrow().len(); + let new_checkpoint = self.full_len(); self.child_checkpoint = Some(new_checkpoint); SharedMemory { buffer: Some(self.buffer().clone()), @@ -204,14 +237,18 @@ impl SharedMemory { return; }; unsafe { - self.buffer().borrow_mut().set_len(child_checkpoint); + self.buffer_ref_mut().set_len(child_checkpoint); } } /// Returns the length of the current memory range. #[inline] pub fn len(&self) -> usize { - self.buffer().borrow().len() - self.my_checkpoint + self.full_len() - self.my_checkpoint + } + + fn full_len(&self) -> usize { + self.buffer_ref().len() } /// Returns `true` if the current memory range is empty. @@ -224,7 +261,7 @@ impl SharedMemory { #[inline] pub fn resize(&mut self, new_size: usize) { self.buffer() - .borrow_mut() + .dbg_borrow_mut() .resize(self.my_checkpoint + new_size, 0); } @@ -253,7 +290,7 @@ impl SharedMemory { #[inline] #[cfg_attr(debug_assertions, track_caller)] pub fn slice_range(&self, range: Range) -> Ref<'_, [u8]> { - let buffer = self.buffer().borrow(); // Borrow the inner Vec + let buffer = self.buffer_ref(); Ref::map(buffer, |b| { match b.get(range.start + self.my_checkpoint..range.end + self.my_checkpoint) { Some(slice) => slice, @@ -275,7 +312,7 @@ impl SharedMemory { #[inline] #[cfg_attr(debug_assertions, track_caller)] pub fn global_slice_range(&self, range: Range) -> Ref<'_, [u8]> { - let buffer = self.buffer().borrow(); // Borrow the inner Vec + let buffer = self.buffer_ref(); Ref::map(buffer, |b| match b.get(range) { Some(slice) => slice, None => debug_unreachable!("slice OOB: range; len: {}", self.len()), @@ -296,7 +333,7 @@ impl SharedMemory { #[inline] #[cfg_attr(debug_assertions, track_caller)] pub fn slice_mut(&mut self, offset: usize, size: usize) -> RefMut<'_, [u8]> { - let buffer = self.buffer().borrow_mut(); // Borrow the inner Vec mutably + let buffer = self.buffer_ref_mut(); RefMut::map(buffer, |b| { match b.get_mut(self.my_checkpoint + offset..self.my_checkpoint + offset + size) { Some(slice) => slice, @@ -404,7 +441,7 @@ impl SharedMemory { len: usize, data_range: Range, ) { - let mut buffer = self.buffer().borrow_mut(); // Borrow the inner Vec mutably + let mut buffer = self.buffer_ref_mut(); let (src, dst) = buffer.split_at_mut(self.my_checkpoint); let src = if data_range.is_empty() { &mut [] @@ -437,7 +474,7 @@ impl SharedMemory { /// behavior. The checkpoint must be within the bounds of the buffer. #[inline] pub fn context_memory(&self) -> Ref<'_, [u8]> { - let buffer = self.buffer().borrow(); + let buffer = self.buffer_ref(); Ref::map(buffer, |b| match b.get(self.my_checkpoint..) { Some(slice) => slice, None => debug_unreachable!("Context memory should be always valid"), @@ -456,7 +493,7 @@ impl SharedMemory { /// behavior. The checkpoint must be within the bounds of the buffer. #[inline] pub fn context_memory_mut(&mut self) -> RefMut<'_, [u8]> { - let buffer = self.buffer().borrow_mut(); // Borrow the inner Vec mutably + let buffer = self.buffer_ref_mut(); RefMut::map(buffer, |b| match b.get_mut(self.my_checkpoint..) { Some(slice) => slice, None => debug_unreachable!("Context memory should be always valid"), @@ -505,6 +542,42 @@ pub const fn num_words(len: usize) -> usize { len.saturating_add(31) / 32 } +/// Performs EVM memory resize. +#[inline] +#[must_use] +pub fn resize_memory( + gas: &mut crate::Gas, + memory: &mut Memory, + offset: usize, + len: usize, +) -> bool { + let new_num_words = num_words(offset.saturating_add(len)); + if new_num_words > gas.memory().words_num { + resize_memory_cold(gas, memory, new_num_words) + } else { + true + } +} + +#[cold] +#[inline(never)] +fn resize_memory_cold( + gas: &mut crate::Gas, + memory: &mut Memory, + new_num_words: usize, +) -> bool { + let cost = unsafe { + gas.memory_mut() + .record_new_len(new_num_words) + .unwrap_unchecked() + }; + if !gas.record_cost(cost) { + return false; + } + memory.resize(new_num_words * 32); + true +} + #[cfg(test)] mod tests { use super::*; @@ -526,45 +599,45 @@ mod tests { fn new_free_child_context() { let mut sm1 = SharedMemory::new(); - assert_eq!(sm1.buffer().borrow().len(), 0); + assert_eq!(sm1.buffer_ref().len(), 0); assert_eq!(sm1.my_checkpoint, 0); - unsafe { sm1.buffer().borrow_mut().set_len(32) }; + unsafe { sm1.buffer_ref_mut().set_len(32) }; assert_eq!(sm1.len(), 32); let mut sm2 = sm1.new_child_context(); - assert_eq!(sm2.buffer().borrow().len(), 32); + assert_eq!(sm2.buffer_ref().len(), 32); assert_eq!(sm2.my_checkpoint, 32); assert_eq!(sm2.len(), 0); - unsafe { sm2.buffer().borrow_mut().set_len(96) }; + unsafe { sm2.buffer_ref_mut().set_len(96) }; assert_eq!(sm2.len(), 64); let mut sm3 = sm2.new_child_context(); - assert_eq!(sm3.buffer().borrow().len(), 96); + assert_eq!(sm3.buffer_ref().len(), 96); assert_eq!(sm3.my_checkpoint, 96); assert_eq!(sm3.len(), 0); - unsafe { sm3.buffer().borrow_mut().set_len(128) }; + unsafe { sm3.buffer_ref_mut().set_len(128) }; let sm4 = sm3.new_child_context(); - assert_eq!(sm4.buffer().borrow().len(), 128); + assert_eq!(sm4.buffer_ref().len(), 128); assert_eq!(sm4.my_checkpoint, 128); assert_eq!(sm4.len(), 0); // Free contexts drop(sm4); sm3.free_child_context(); - assert_eq!(sm3.buffer().borrow().len(), 128); + assert_eq!(sm3.buffer_ref().len(), 128); assert_eq!(sm3.my_checkpoint, 96); assert_eq!(sm3.len(), 32); sm2.free_child_context(); - assert_eq!(sm2.buffer().borrow().len(), 96); + assert_eq!(sm2.buffer_ref().len(), 96); assert_eq!(sm2.my_checkpoint, 32); assert_eq!(sm2.len(), 64); sm1.free_child_context(); - assert_eq!(sm1.buffer().borrow().len(), 32); + assert_eq!(sm1.buffer_ref().len(), 32); assert_eq!(sm1.my_checkpoint, 0); assert_eq!(sm1.len(), 32); } @@ -573,22 +646,19 @@ mod tests { fn resize() { let mut sm1 = SharedMemory::new(); sm1.resize(32); - assert_eq!(sm1.buffer().borrow().len(), 32); + assert_eq!(sm1.buffer_ref().len(), 32); assert_eq!(sm1.len(), 32); - assert_eq!(sm1.buffer().borrow().get(0..32), Some(&[0_u8; 32] as &[u8])); + assert_eq!(sm1.buffer_ref().get(0..32), Some(&[0_u8; 32] as &[u8])); let mut sm2 = sm1.new_child_context(); sm2.resize(96); - assert_eq!(sm2.buffer().borrow().len(), 128); + assert_eq!(sm2.buffer_ref().len(), 128); assert_eq!(sm2.len(), 96); - assert_eq!( - sm2.buffer().borrow().get(32..128), - Some(&[0_u8; 96] as &[u8]) - ); + assert_eq!(sm2.buffer_ref().get(32..128), Some(&[0_u8; 96] as &[u8])); sm1.free_child_context(); - assert_eq!(sm1.buffer().borrow().len(), 32); + assert_eq!(sm1.buffer_ref().len(), 32); assert_eq!(sm1.len(), 32); - assert_eq!(sm1.buffer().borrow().get(0..32), Some(&[0_u8; 32] as &[u8])); + assert_eq!(sm1.buffer_ref().get(0..32), Some(&[0_u8; 32] as &[u8])); } } diff --git a/crates/interpreter/src/interpreter/stack.rs b/crates/interpreter/src/interpreter/stack.rs index b7b182458f..c835a0202a 100644 --- a/crates/interpreter/src/interpreter/stack.rs +++ b/crates/interpreter/src/interpreter/stack.rs @@ -48,6 +48,7 @@ impl Clone for Stack { } impl StackTr for Stack { + #[inline] fn len(&self) -> usize { self.len() } @@ -75,17 +76,25 @@ impl StackTr for Stack { Some(unsafe { self.popn_top::() }) } + #[inline] fn exchange(&mut self, n: usize, m: usize) -> bool { self.exchange(n, m) } + #[inline] fn dup(&mut self, n: usize) -> bool { self.dup(n) } + #[inline] fn push(&mut self, value: U256) -> bool { self.push(value) } + + #[inline] + fn push_slice(&mut self, slice: &[u8]) -> bool { + self.push_slice_(slice) + } } impl Stack { @@ -150,6 +159,7 @@ impl Stack { #[inline] #[cfg_attr(debug_assertions, track_caller)] pub unsafe fn pop_unsafe(&mut self) -> U256 { + assume!(!self.data.is_empty()); self.data.pop().unwrap_unchecked() } @@ -161,8 +171,8 @@ impl Stack { #[inline] #[cfg_attr(debug_assertions, track_caller)] pub unsafe fn top_unsafe(&mut self) -> &mut U256 { - let len = self.data.len(); - self.data.get_unchecked_mut(len - 1) + assume!(!self.data.is_empty()); + self.data.last_mut().unwrap_unchecked() } /// Pops `N` values from the stack. @@ -173,14 +183,8 @@ impl Stack { #[inline] #[cfg_attr(debug_assertions, track_caller)] pub unsafe fn popn(&mut self) -> [U256; N] { - if N == 0 { - return [U256::ZERO; N]; - } - let mut result = [U256::ZERO; N]; - for v in &mut result { - *v = self.data.pop().unwrap_unchecked(); - } - result + assume!(self.data.len() >= N); + core::array::from_fn(|_| unsafe { self.pop_unsafe() }) } /// Pops `N` values from the stack and returns the top of the stack. @@ -292,14 +296,25 @@ impl Stack { /// if necessary. #[inline] pub fn push_slice(&mut self, slice: &[u8]) -> Result<(), InstructionResult> { + if self.push_slice_(slice) { + Ok(()) + } else { + Err(InstructionResult::StackOverflow) + } + } + + /// Pushes an arbitrary length slice of bytes onto the stack, padding the last word with zeros + /// if necessary. + #[inline] + fn push_slice_(&mut self, slice: &[u8]) -> bool { if slice.is_empty() { - return Ok(()); + return true; } let n_words = slice.len().div_ceil(32); let new_len = self.data.len() + n_words; if new_len > STACK_LIMIT { - return Err(InstructionResult::StackOverflow); + return false; } // SAFETY: Length checked above. @@ -322,7 +337,7 @@ impl Stack { } if partial_last_word.is_empty() { - return Ok(()); + return true; } // Write limbs of partial last word @@ -350,7 +365,7 @@ impl Stack { } } - Ok(()) + true } /// Set a value at given index for the stack, where the top of the diff --git a/crates/interpreter/src/interpreter_types.rs b/crates/interpreter/src/interpreter_types.rs index 4ee3e053f0..810d1eec62 100644 --- a/crates/interpreter/src/interpreter_types.rs +++ b/crates/interpreter/src/interpreter_types.rs @@ -170,6 +170,14 @@ pub trait StackTr { #[must_use] fn push(&mut self, value: U256) -> bool; + /// Pushes slice to the stack. + /// + /// Returns `true` if push was successful, `false` if stack overflow. + /// + /// # Note + /// Error is internally set in interpreter. + fn push_slice(&mut self, slice: &[u8]) -> bool; + /// Pushes B256 value to the stack. /// /// Internally converts B256 to U256 and then calls [`StackTr::push`]. @@ -189,7 +197,7 @@ pub trait StackTr { /// Returns top value from the stack. #[must_use] fn top(&mut self) -> Option<&mut U256> { - self.popn_top::<0>().map(|(_, top)| top) + self.popn_top().map(|([], top)| top) } /// Pops one value from the stack.