Chaos mode MVP: Skip branch optimization in MachBuffer

Cargo.toml

@@ -162,6 +162,7 @@ cranelift-object = { path = "cranelift/object", version = "0.96.0" }
 cranelift-jit = { path = "cranelift/jit", version = "0.96.0" }
 cranelift-fuzzgen = { path = "cranelift/fuzzgen" }
 cranelift-bforest = { path = "cranelift/bforest", version = "0.96.0" }
+cranelift-control = { path = "cranelift/control", version = "0.96.0" }
 cranelift = { path = "cranelift/umbrella", version = "0.96.0" }
 
 winch-codegen = { path = "winch/codegen", version = "=0.7.0" }

cranelift/codegen/Cargo.toml

@@ -19,6 +19,7 @@ capstone = { workspace = true, optional = true }
 cranelift-codegen-shared = { path = "./shared", version = "0.96.0" }
 cranelift-entity = { workspace = true }
 cranelift-bforest = { workspace = true }
+cranelift-control = { workspace = true }
 hashbrown = { workspace = true, features = ["raw"] }
 target-lexicon = { workspace = true }
 log = { workspace = true }

cranelift/codegen/src/context.rs

@@ -33,6 +33,7 @@ use crate::{timing, CompileError};
 #[cfg(feature = "souper-harvest")]
 use alloc::string::String;
 use alloc::vec::Vec;
+use cranelift_control::ControlPlane;
 
 #[cfg(feature = "souper-harvest")]
 use crate::souper_harvest::do_souper_harvest;
@@ -124,23 +125,28 @@ impl Context {
         &mut self,
         isa: &dyn TargetIsa,
         mem: &mut Vec<u8>,
+        ctrl_plane: &mut ControlPlane,
     ) -> CompileResult<&CompiledCode> {
-        let compiled_code = self.compile(isa)?;
+        let compiled_code = self.compile(isa, ctrl_plane)?;
         mem.extend_from_slice(compiled_code.code_buffer());
         Ok(compiled_code)
     }
 
     /// Internally compiles the function into a stencil.
     ///
     /// Public only for testing and fuzzing purposes.
-    pub fn compile_stencil(&mut self, isa: &dyn TargetIsa) -> CodegenResult<CompiledCodeStencil> {
+    pub fn compile_stencil(
+        &mut self,
+        isa: &dyn TargetIsa,
+        ctrl_plane: &mut ControlPlane,
+    ) -> CodegenResult<CompiledCodeStencil> {
         let _tt = timing::compile();
 
         self.verify_if(isa)?;
 
         self.optimize(isa)?;
 
-        isa.compile_function(&self.func, &self.domtree, self.want_disasm)
+        isa.compile_function(&self.func, &self.domtree, self.want_disasm, ctrl_plane)
     }
 
     /// Optimize the function, performing all compilation steps up to
@@ -212,11 +218,17 @@ impl Context {
     /// code sink.
     ///
     /// Returns information about the function's code and read-only data.
-    pub fn compile(&mut self, isa: &dyn TargetIsa) -> CompileResult<&CompiledCode> {
-        let stencil = self.compile_stencil(isa).map_err(|error| CompileError {
-            inner: error,
-            func: &self.func,
-        })?;
+    pub fn compile(
+        &mut self,
+        isa: &dyn TargetIsa,
+        ctrl_plane: &mut ControlPlane,
+    ) -> CompileResult<&CompiledCode> {
+        let stencil = self
+            .compile_stencil(isa, ctrl_plane)
+            .map_err(|error| CompileError {
+                inner: error,
+                func: &self.func,
+            })?;
         Ok(self
             .compiled_code
             .insert(stencil.apply_params(&self.func.params)))

cranelift/codegen/src/incremental_cache.rs

@@ -32,6 +32,7 @@ use crate::{isa::TargetIsa, timing};
 use crate::{trace, CompileError, Context};
 use alloc::borrow::{Cow, ToOwned as _};
 use alloc::string::ToString as _;
+use cranelift_control::ControlPlane;
 
 impl Context {
     /// Compile the function, as in `compile`, but tries to reuse compiled artifacts from former
@@ -40,6 +41,7 @@ impl Context {
         &mut self,
         isa: &dyn TargetIsa,
         cache_store: &mut dyn CacheKvStore,
+        ctrl_plane: &mut ControlPlane,
     ) -> CompileResult<(&CompiledCode, bool)> {
         let cache_key_hash = {
             let _tt = timing::try_incremental_cache();
@@ -52,7 +54,7 @@ impl Context {
                         let info = compiled_code.code_info();
 
                         if isa.flags().enable_incremental_compilation_cache_checks() {
-                            let actual_result = self.compile(isa)?;
+                            let actual_result = self.compile(isa, ctrl_plane)?;
                             assert_eq!(*actual_result, compiled_code);
                             assert_eq!(actual_result.code_info(), info);
                             // no need to set `compiled_code` here, it's set by `compile()`.
@@ -71,10 +73,12 @@ impl Context {
             cache_key_hash
         };
 
-        let stencil = self.compile_stencil(isa).map_err(|err| CompileError {
-            inner: err,
-            func: &self.func,
-        })?;
+        let stencil = self
+            .compile_stencil(isa, ctrl_plane)
+            .map_err(|err| CompileError {
+                inner: err,
+                func: &self.func,
+            })?;
 
         let stencil = {
             let _tt = timing::store_incremental_cache();

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -1,5 +1,6 @@
 //! AArch64 ISA: binary code emission.
 
+use cranelift_control::ControlPlane;
 use regalloc2::Allocation;
 
 use crate::binemit::{Reloc, StackMap};
@@ -638,15 +639,19 @@ pub struct EmitState {
     stack_map: Option<StackMap>,
     /// Current source-code location corresponding to instruction to be emitted.
     cur_srcloc: RelSourceLoc,
+    /// Only used during fuzz-testing. Otherwise, it is a zero-sized struct and
+    /// optimized away at compiletime. See [cranelift_control].
+    ctrl_plane: ControlPlane,
 }
 
 impl MachInstEmitState<Inst> for EmitState {
-    fn new(abi: &Callee<AArch64MachineDeps>) -> Self {
+    fn new(abi: &Callee<AArch64MachineDeps>, ctrl_plane: ControlPlane) -> Self {
         EmitState {
             virtual_sp_offset: 0,
             nominal_sp_to_fp: abi.frame_size() as i64,
             stack_map: None,
             cur_srcloc: Default::default(),
+            ctrl_plane,
         }
     }
 
@@ -657,6 +662,14 @@ impl MachInstEmitState<Inst> for EmitState {
     fn pre_sourceloc(&mut self, srcloc: RelSourceLoc) {
         self.cur_srcloc = srcloc;
     }
+
+    fn ctrl_plane_mut(&mut self) -> &mut ControlPlane {
+        &mut self.ctrl_plane
+    }
+
+    fn take_ctrl_plane(self) -> ControlPlane {
+        self.ctrl_plane
+    }
 }
 
 impl EmitState {
@@ -1511,7 +1524,7 @@ impl MachInstEmit for Inst {
                 let again_label = sink.get_label();
 
                 // again:
-                sink.bind_label(again_label);
+                sink.bind_label(again_label, &mut state.ctrl_plane);
 
                 let srcloc = state.cur_srcloc();
                 if !srcloc.is_default() && !flags.notrap() {
@@ -1713,7 +1726,7 @@ impl MachInstEmit for Inst {
                 let out_label = sink.get_label();
 
                 // again:
-                sink.bind_label(again_label);
+                sink.bind_label(again_label, &mut state.ctrl_plane);
 
                 let srcloc = state.cur_srcloc();
                 if !srcloc.is_default() && !flags.notrap() {
@@ -1762,7 +1775,7 @@ impl MachInstEmit for Inst {
                 sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
 
                 // out:
-                sink.bind_label(out_label);
+                sink.bind_label(out_label, &mut state.ctrl_plane);
             }
             &Inst::LoadAcquire {
                 access_ty,
@@ -3007,13 +3020,13 @@ impl MachInstEmit for Inst {
                 sink.put4(enc_jump26(0b000101, 0 /* will be fixed up later */));
 
                 // else:
-                sink.bind_label(else_label);
+                sink.bind_label(else_label, &mut state.ctrl_plane);
 
                 // mov rd, rn
                 sink.put4(enc_vecmov(/* 16b = */ true, rd, rn));
 
                 // out:
-                sink.bind_label(out_label);
+                sink.bind_label(out_label, &mut state.ctrl_plane);
             }
             &Inst::MovToNZCV { rn } => {
                 let rn = allocs.next(rn);
@@ -3464,8 +3477,8 @@ impl MachInstEmit for Inst {
                         dest: BranchTarget::Label(jump_around_label),
                     };
                     jmp.emit(&[], sink, emit_info, state);
-                    sink.emit_island(needed_space + 4);
-                    sink.bind_label(jump_around_label);
+                    sink.emit_island(needed_space + 4, &mut state.ctrl_plane);
+                    sink.bind_label(jump_around_label, &mut state.ctrl_plane);
                 }
             }
 
@@ -3579,7 +3592,7 @@ impl MachInstEmit for Inst {
                 // out at this time.
 
                 let loop_start = sink.get_label();
-                sink.bind_label(loop_start);
+                sink.bind_label(loop_start, &mut state.ctrl_plane);
 
                 Inst::AluRRImm12 {
                     alu_op: ALUOp::Sub,
@@ -3614,7 +3627,7 @@ impl MachInstEmit for Inst {
                     kind: CondBrKind::Cond(Cond::Gt),
                 }
                 .emit(&[], sink, emit_info, state);
-                sink.bind_label(loop_end);
+                sink.bind_label(loop_end, &mut state.ctrl_plane);
             }
         }
 

cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs

@@ -7811,7 +7811,7 @@ fn test_aarch64_binemit() {
 
         let mut buffer = MachBuffer::new();
         insn.emit(&[], &mut buffer, &emit_info, &mut Default::default());
-        let buffer = buffer.finish();
+        let buffer = buffer.finish(&mut Default::default());
         let actual_encoding = &buffer.stringify_code_bytes();
         assert_eq!(expected_encoding, actual_encoding);
     }

cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs

@@ -91,7 +91,9 @@ mod tests {
             Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)),
         ));
 
-        let code = context.compile(&*isa).expect("expected compilation");
+        let code = context
+            .compile(&*isa, &mut Default::default())
+            .expect("expected compilation");
 
         let fde = match code
             .create_unwind_info(isa.as_ref())
@@ -130,7 +132,9 @@ mod tests {
 
         let mut context = Context::for_function(create_multi_return_function(CallConv::SystemV));
 
-        let code = context.compile(&*isa).expect("expected compilation");
+        let code = context
+            .compile(&*isa, &mut Default::default())
+            .expect("expected compilation");
 
         let fde = match code
             .create_unwind_info(isa.as_ref())

cranelift/codegen/src/isa/aarch64/mod.rs

@@ -15,6 +15,7 @@ use crate::result::CodegenResult;
 use crate::settings as shared_settings;
 use alloc::{boxed::Box, vec::Vec};
 use core::fmt;
+use cranelift_control::ControlPlane;
 use regalloc2::MachineEnv;
 use target_lexicon::{Aarch64Architecture, Architecture, OperatingSystem, Triple};
 
@@ -72,17 +73,19 @@ impl TargetIsa for AArch64Backend {
         func: &Function,
         domtree: &DominatorTree,
         want_disasm: bool,
+        ctrl_plane: &mut ControlPlane,
     ) -> CodegenResult<CompiledCodeStencil> {
         let (vcode, regalloc_result) = self.compile_vcode(func, domtree)?;
 
         let emit_result = vcode.emit(
             &regalloc_result,
             want_disasm,
             self.flags.machine_code_cfg_info(),
+            ctrl_plane,
         );
         let frame_size = emit_result.frame_size;
         let value_labels_ranges = emit_result.value_labels_ranges;
-        let buffer = emit_result.buffer.finish();
+        let buffer = emit_result.buffer.finish(ctrl_plane);
         let sized_stackslot_offsets = emit_result.sized_stackslot_offsets;
         let dynamic_stackslot_offsets = emit_result.dynamic_stackslot_offsets;
 

cranelift/codegen/src/isa/mod.rs

@@ -57,6 +57,7 @@ use crate::CodegenResult;
 use alloc::{boxed::Box, sync::Arc, vec::Vec};
 use core::fmt;
 use core::fmt::{Debug, Formatter};
+use cranelift_control::ControlPlane;
 use target_lexicon::{triple, Architecture, PointerWidth, Triple};
 
 // This module is made public here for benchmarking purposes. No guarantees are
@@ -273,6 +274,7 @@ pub trait TargetIsa: fmt::Display + Send + Sync {
         func: &Function,
         domtree: &DominatorTree,
         want_disasm: bool,
+        ctrl_plane: &mut ControlPlane,
     ) -> CodegenResult<CompiledCodeStencil>;
 
     #[cfg(feature = "unwind")]