diff --git a/crates/oxc_ast/src/generated/assert_layouts.rs b/crates/oxc_ast/src/generated/assert_unordered_layouts.rs similarity index 100% rename from crates/oxc_ast/src/generated/assert_layouts.rs rename to crates/oxc_ast/src/generated/assert_unordered_layouts.rs diff --git a/crates/oxc_ast/src/lib.rs b/crates/oxc_ast/src/lib.rs index 41ec931cab407..efaa0ebf52d38 100644 --- a/crates/oxc_ast/src/lib.rs +++ b/crates/oxc_ast/src/lib.rs @@ -38,7 +38,7 @@ mod trivia; mod generated { #[cfg(debug_assertions)] - pub mod assert_layouts; + pub mod assert_unordered_layouts; pub mod ast_builder; pub mod ast_kind; pub mod span; diff --git a/tasks/ast_codegen/src/generators/assert_layouts.rs b/tasks/ast_codegen/src/generators/assert_layouts.rs index 56d60153f0fa4..584de71c74f1f 100644 --- a/tasks/ast_codegen/src/generators/assert_layouts.rs +++ b/tasks/ast_codegen/src/generators/assert_layouts.rs @@ -9,16 +9,17 @@ use crate::{ use super::generated_header; -pub struct AssertLayouts; +pub struct AssertLayouts(pub &'static str); impl Generator for AssertLayouts { fn name(&self) -> &'static str { - stringify!(AssertLayouts) + self.0 } fn generate(&mut self, ctx: &CodegenCtx) -> GeneratorOutput { let (assertions_64, assertions_32) = ctx .schema + .borrow() .definitions .iter() .map(|def| { @@ -52,7 +53,7 @@ impl Generator for AssertLayouts { let header = generated_header!(); GeneratorOutput::Stream(( - output(crate::AST_CRATE, "assert_layouts.rs"), + output(crate::AST_CRATE, self.0), quote! { #header diff --git a/tasks/ast_codegen/src/layout.rs b/tasks/ast_codegen/src/layout.rs index 4c9b52bb46775..1bae287e31807 100644 --- a/tasks/ast_codegen/src/layout.rs +++ b/tasks/ast_codegen/src/layout.rs @@ -1,39 +1,5 @@ -/// We use compiler to infer 64bit type layouts. -#[cfg(not(target_pointer_width = "64"))] -compile_error!("`oxc_ast_codegen::layout` only supports 64 architectures."); use std::collections::HashMap; -use itertools::Itertools; -use lazy_static::lazy_static; -use quote::ToTokens; -use syn::Type; - -use crate::{ - schema::{REnum, RStruct, RType}, - util::{NormalizeError, TypeAnalyzeResult, TypeExt, TypeWrapper}, - CodegenCtx, Result, TypeRef, -}; - -/// Calculates the layout of `ty` by mutating it. -/// Returns `false` if the layout is unknown at this point. -pub fn calc_layout(ty: &mut RType, ctx: &CodegenCtx) -> Result { - let unknown_layout = ty - .layout_32() - .and_then(|x32| ty.layout_64().map(|x64| PlatformLayout(x64, x32))) - .is_ok_and(|pl| pl.is_unknown()); - let layout = match ty { - RType::Enum(enum_) if unknown_layout => calc_enum_layout(enum_, ctx), - RType::Struct(struct_) if unknown_layout => calc_struct_layout(struct_, ctx), - _ => return Ok(true), - }?; - if layout.is_unknown() { - Ok(false) - } else { - ty.set_layout(layout.x64(), layout.x32())?; - Ok(true) - } -} - #[derive(Debug, Default, Clone, Copy)] pub enum Layout { #[default] @@ -42,7 +8,7 @@ pub enum Layout { } impl Layout { - const fn known(size: usize, align: usize, niches: u128) -> Self { + pub const fn known(size: usize, align: usize, niches: u128) -> Self { Self::Layout(KnownLayout { size, align, niches }) } @@ -70,6 +36,9 @@ pub struct KnownLayout { } impl KnownLayout { + pub const fn new(size: usize, align: usize, niches: u128) -> Self { + Self { size, align, niches } + } #[inline] pub fn size(&self) -> usize { self.size @@ -86,6 +55,18 @@ impl KnownLayout { self.niches } + pub unsafe fn set_size_unchecked(&mut self, size: usize) { + self.size = size; + } + + pub unsafe fn set_align_unchecked(&mut self, align: usize) { + self.align = align; + } + + pub unsafe fn set_niches_unchecked(&mut self, niches: u128) { + self.niches = niches; + } + /// Panics /// if doesn't have enough viable space and `can_resize` is false pub fn consume_niches(&mut self, n: u128, can_resize: bool) { @@ -117,7 +98,7 @@ impl KnownLayout { impl Layout { /// # Panics /// If alignment of `T` is higher than one byte. - const fn of() -> Self { + pub const fn of() -> Self { // TODO: find a better way of calculating this. struct N1(Option); struct N2(N1>); @@ -159,337 +140,29 @@ impl Layout { Self::known(size, align, niches) } - const fn zero() -> Self { + pub const fn zero() -> Self { #[repr(C)] struct Empty; Self::of::() } - const fn ptr_32() -> Self { + pub const fn ptr_32() -> Self { Layout::known(4, 4, 0) } - const fn ptr_64() -> Self { + pub const fn ptr_64() -> Self { Layout::known(8, 8, 0) } - const fn wide_ptr_32() -> Self { + pub const fn wide_ptr_32() -> Self { Layout::known(8, 4, 1) } - const fn wide_ptr_64() -> Self { + pub const fn wide_ptr_64() -> Self { Layout::of::<&str>() } - fn is_unknown(&self) -> bool { + pub fn is_unknown(&self) -> bool { matches!(self, Self::Unknown) } } - -#[derive(Debug, Clone)] -struct PlatformLayout(/* 64 */ Layout, /* 32 */ Layout); - -impl PlatformLayout { - const UNKNOWN: Self = Self(Layout::Unknown, Layout::Unknown); - - const fn zero() -> Self { - Self(Layout::zero(), Layout::zero()) - } - - const fn ptr() -> Self { - Self(Layout::ptr_64(), Layout::ptr_32()) - } - - const fn wide_ptr() -> Self { - Self(Layout::wide_ptr_64(), Layout::wide_ptr_32()) - } - - fn x64(&self) -> Layout { - self.0 - } - - fn x32(&self) -> Layout { - self.1 - } - - /// Return `true` if either of platform layouts is unknown. - fn is_unknown(&self) -> bool { - self.0.is_unknown() || self.1.is_unknown() - } -} - -impl From<(Layout, Layout)> for PlatformLayout { - fn from((x64, x32): (Layout, Layout)) -> Self { - Self(x64, x32) - } -} - -type WellKnown = HashMap<&'static str, PlatformLayout>; - -fn calc_enum_layout(ty: &mut REnum, ctx: &CodegenCtx) -> Result { - fn collect_variant_layouts(ty: &REnum, ctx: &CodegenCtx) -> Result> { - // all unit variants? - if ty.item.variants.iter().all(|var| var.fields.is_empty()) { - // all AST enums are `repr(C)` so it would have a 4 byte layout/alignment, - let layout = Layout::known(0, 4, 0); - Ok(vec![PlatformLayout(layout, layout)]) - } else { - ty.item - .variants - .iter() - .map(|var| { - let typ = - var.fields.iter().exactly_one().map(|f| f.ty.analyze(ctx)).normalize()?; - calc_type_layout(&typ, ctx) - }) - .collect() - } - } - - fn fold_layout(mut acc: KnownLayout, layout: KnownLayout) -> KnownLayout { - // max alignment - if layout.align > acc.align { - acc.align = layout.align; - } - // max size - if layout.size > acc.size { - acc.size = layout.size; - } - // min niches - if layout.niches() < acc.niches() { - acc.niches = layout.niches(); - } - acc - } - - let with_tag = |mut acc: KnownLayout| -> KnownLayout { - acc.consume_niches(ty.item.variants.len() as u128, true); - acc - }; - - let layouts = collect_variant_layouts(ty, ctx)?; - let (layouts_x64, layouts_x32): (Vec, Vec) = layouts - .into_iter() - .map(|PlatformLayout(x64, x32)| { - x64.layout().and_then(|x64| x32.layout().map(|x32| (x64, x32))) - }) - .collect::>() - .expect("already checked."); - - let x32 = with_tag(layouts_x32.into_iter().fold(KnownLayout::default(), fold_layout)); - let x64 = with_tag(layouts_x64.into_iter().fold(KnownLayout::default(), fold_layout)); - Ok(PlatformLayout(Layout::from(x64), Layout::from(x32))) -} - -fn calc_struct_layout(ty: &mut RStruct, ctx: &CodegenCtx) -> Result { - fn collect_field_layouts(ty: &RStruct, ctx: &CodegenCtx) -> Result> { - if ty.item.fields.is_empty() { - Ok(vec![PlatformLayout::zero()]) - } else { - ty.item - .fields - .iter() - .map(|field| { - let typ = field.ty.analyze(ctx); - calc_type_layout(&typ, ctx) - }) - .collect() - } - } - - fn with_padding( - layouts: &[KnownLayout], - ) -> std::result::Result { - // TODO: store `offsets` in the layout - let mut offsets = Vec::new(); - let mut output = std::alloc::Layout::from_size_align(0, 1)?; - let mut niches = 0; - for layout in layouts { - let (new_layout, offset) = - output.extend(std::alloc::Layout::from_size_align(layout.size, layout.align)?)?; - output = new_layout; - offsets.push(offset); - niches += layout.niches(); - } - let output = output.pad_to_align(); - Ok(KnownLayout { size: output.size(), align: output.align(), niches }) - } - - let layouts = collect_field_layouts(ty, ctx)?; - - if layouts.iter().any(PlatformLayout::is_unknown) { - return Ok(PlatformLayout::UNKNOWN); - } - - let (layouts_x64, layouts_x32): (Vec, Vec) = layouts - .into_iter() - .map(|PlatformLayout(x64, x32)| { - x64.layout().and_then(|x64| x32.layout().map(|x32| (x64, x32))) - }) - .collect::>() - .expect("already checked."); - - let x32 = with_padding(&layouts_x32).normalize()?; - let x64 = with_padding(&layouts_x64).normalize()?; - - Ok(PlatformLayout(Layout::from(x64), Layout::from(x32))) -} - -fn calc_type_layout(ty: &TypeAnalyzeResult, ctx: &CodegenCtx) -> Result { - fn is_slice(ty: &TypeAnalyzeResult) -> bool { - if let Type::Reference(typ) = &ty.typ { - // TODO: support for &[T] slices. - typ.elem.get_ident().as_ident().is_some_and(|id| id == "str") - } else { - false - } - } - - fn try_fold_option(layout: Layout) -> Layout { - let Layout::Layout(mut known) = layout else { return layout }; - // option needs only one bit to store its tag and it can be in a fragmented offset - known.consume_niches(1, true); - Layout::Layout(known) - } - - let get_layout = |type_ref: Option<&TypeRef>| -> Result { - let result = if let Some(type_ref) = &type_ref { - if calc_layout(&mut type_ref.borrow_mut(), ctx)? { - type_ref.borrow().layouts().map(PlatformLayout::from)? - } else { - PlatformLayout::UNKNOWN - } - } else if let Some(well_known) = - WELL_KNOWN.get(ty.typ.get_ident().inner_ident().to_string().as_str()) - { - well_known.clone() - } else { - let Type::Path(typ) = &ty.typ else { - panic!(); - }; - - let typ = typ - .path - .segments - .first() - .map(|it| it.to_token_stream().to_string().replace(' ', "")) - .expect("We only accept single segment types."); - - if let Some(typ) = WELL_KNOWN.get(typ.as_str()) { - typ.clone() - } else { - panic!("Unsupported type: {:#?}", ty.typ.to_token_stream().to_string()) - } - }; - Ok(result) - }; - - let layout = match ty.wrapper { - TypeWrapper::Vec | TypeWrapper::VecBox | TypeWrapper::VecOpt | TypeWrapper::OptVec => { - WELL_KNOWN[stringify!(Vec)].clone() - } - TypeWrapper::Ref if is_slice(ty) => PlatformLayout::wide_ptr(), - TypeWrapper::Ref | TypeWrapper::Box | TypeWrapper::OptBox => PlatformLayout::ptr(), - TypeWrapper::None => get_layout(ty.type_ref.as_ref())?, - TypeWrapper::Opt => { - let PlatformLayout(x64, x32) = get_layout(ty.type_ref.as_ref())?; - PlatformLayout(try_fold_option(x64), try_fold_option(x32)) - } - TypeWrapper::Complex => { - let PlatformLayout(x64, x32) = get_layout(ty.type_ref.as_ref())?; - PlatformLayout(x64, x32) - } - }; - Ok(layout) -} - -macro_rules! well_known { - ($($typ:ty: { $($platform:tt => $layout:expr,)*},)*) => { - WellKnown::from([ - $(( - stringify!($typ), - well_known!(@ $( $platform => $layout,)*) - )),* - ]) - }; - - // entries - (@ _ => $layout:expr,) => { - PlatformLayout($layout, $layout) - }; - (@ 64 => $layout_64:expr, 32 => $layout_32:expr,) => { - PlatformLayout($layout_64, $layout_32) - }; - (@ 32 => $layout_32:expr, 64 => $layout_64:expr,) => { - well_known!(@ 64 => $layout_64, 32 => $layout_32) - }; - - // compile errors - (@ 32 => $layout:expr,) => { - ::std::compile_error!("non_exhaustive well known type, `64` target isn't covered.") - }; - (@ 64 => $layout:expr,) => { - ::std::compile_error!("non_exhaustive well known type, `32` target isn't covered.") - }; -} - -lazy_static! { - static ref WELL_KNOWN: WellKnown = well_known! { - // builtins - // types smaller or equal to 4bytes have the same layout on most platforms. - char: { _ => Layout::of::(), }, - bool: { _ => Layout::of::(), }, - u8: { _ => Layout::of::(), }, - i8: { _ => Layout::of::(), }, - u16: { _ => Layout::of::(), }, - i16: { _ => Layout::of::(), }, - u32: { _ => Layout::of::(), }, - i32: { _ => Layout::of::(), }, - f32: { _ => Layout::of::(), }, - // 32bit layouts are based on WASM - u64: { - 64 => Layout::of::(), - 32 => Layout::known(8, 8, 0), - }, - i64: { - 64 => Layout::of::(), - 32 => Layout::known(8, 8, 0), - }, - f64: { - 64 => Layout::of::(), - 32 => Layout::known(8, 8, 0), - }, - usize: { - 64 => Layout::of::(), - 32 => Layout::known(4, 4, 0), - }, - isize: { - 64 => Layout::of::(), - 32 => Layout::known(4, 4, 0), - }, - // well known types - // TODO: generate const assertions for these in the ast crate - Span: { _ => Layout::known(8, 4, 0), }, - Atom: { - 64 => Layout::wide_ptr_64(), - 32 => Layout::wide_ptr_32(), - }, - Vec: { - 64 => Layout::known(32, 8, 1), - 32 => Layout::known(16, 4, 1), - }, - Cell>: { _ => Layout::known(4, 4, 1), }, - Cell>: { _ => Layout::known(4, 4, 1), }, - Cell>: { _ => Layout::known(4, 4, 1), }, - ReferenceFlag: { _ => Layout::known(1, 1, 0), }, - AssignmentOperator: { _ => Layout::known(1, 1, 1), }, - LogicalOperator: { _ => Layout::known(1, 1, 1), }, - UnaryOperator: { _ => Layout::known(1, 1, 1), }, - BinaryOperator: { _ => Layout::known(1, 1, 1), }, - UpdateOperator: { _ => Layout::known(1, 1, 1), }, - SourceType: { _ => Layout::known(4, 1, 1), }, - RegExpFlags: { _ => Layout::known(1, 1, 0), }, - BigintBase: { _ => Layout::known(1, 1, 1), }, - NumberBase: { _ => Layout::known(1, 1, 1), }, - }; -} diff --git a/tasks/ast_codegen/src/linker.rs b/tasks/ast_codegen/src/linker.rs deleted file mode 100644 index 0a6a5af730263..0000000000000 --- a/tasks/ast_codegen/src/linker.rs +++ /dev/null @@ -1,82 +0,0 @@ -use syn::parse_quote; - -use crate::util::NormalizeError; - -use super::{CodegenCtx, Cow, Inherit, RType, Result}; - -pub trait Unresolved { - fn unresolved(&self) -> bool; - - // TODO: remove me - #[allow(dead_code)] - fn resolved(&self) -> bool { - !self.unresolved() - } -} - -impl Unresolved for Inherit { - fn unresolved(&self) -> bool { - matches!(self, Self::Unlinked(_)) - } -} - -impl Unresolved for Vec { - fn unresolved(&self) -> bool { - self.iter().any(Unresolved::unresolved) - } -} - -/// Returns false if can't resolve -/// # Panics -/// On invalid inheritance. -pub fn linker(ty: &mut RType, ctx: &CodegenCtx) -> Result { - // Exit early if it isn't an enum, We only link to resolve enum inheritance! - let RType::Enum(ty) = ty else { - return Ok(true); - }; - - // Exit early if there is this enum doesn't use enum inheritance - if ty.meta.inherits.is_empty() { - return Ok(true); - } - - let inherits = ty - .meta - .inherits - .drain(..) - .map(|it| match it { - Inherit::Unlinked(ref sup) => { - let linkee = ctx - .find(&Cow::Owned(sup.to_string())) - .normalize_with(format!("Unknown type {sup:?}"))?; - let linkee = linkee.borrow(); - let inherit_value = format!(r#""{}""#, linkee.ident().unwrap()); - let variants = match &*linkee { - RType::Enum(enum_) => { - if enum_.meta.inherits.unresolved() { - return Ok(Err(it)); - } - enum_.item.variants.clone().into_iter().map(|mut v| { - v.attrs = vec![parse_quote!(#[inherit = #inherit_value])]; - v - }) - } - _ => { - panic!("invalid inheritance, you can only inherit from enums and in enums.") - } - }; - ty.item.variants.extend(variants.clone()); - Ok(Ok(Inherit::Linked { - super_: linkee.as_type().unwrap(), - variants: variants.collect(), - })) - } - Inherit::Linked { .. } => Ok(Ok(it)), - }) - .collect::>>>()?; - let unresolved = inherits.iter().any(std::result::Result::is_err); - - ty.meta.inherits = inherits.into_iter().map(|it| it.unwrap_or_else(|it| it)).collect(); - - Ok(!unresolved) -} diff --git a/tasks/ast_codegen/src/main.rs b/tasks/ast_codegen/src/main.rs index da3a2543b548e..88de6f46254c8 100644 --- a/tasks/ast_codegen/src/main.rs +++ b/tasks/ast_codegen/src/main.rs @@ -6,18 +6,18 @@ mod defs; mod fmt; mod generators; mod layout; -mod linker; mod markers; -mod pass; +mod passes; mod schema; mod util; -use std::{borrow::Cow, cell::RefCell, collections::HashMap, io::Read, path::PathBuf, rc::Rc}; +use std::{cell::RefCell, collections::HashMap, io::Read, path::PathBuf, rc::Rc}; use bpaf::{Bpaf, Parser}; use fmt::{cargo_fmt, pprint}; use itertools::Itertools; -use layout::calc_layout; +// use layout::calc_layout; +use passes::{BuildSchema, CalcLayout, Linker, Pass, PassGenerator}; use proc_macro2::TokenStream; use syn::parse_file; @@ -25,8 +25,7 @@ use defs::TypeDef; use generators::{ AssertLayouts, AstBuilderGenerator, AstKindGenerator, VisitGenerator, VisitMutGenerator, }; -use linker::linker; -use schema::{Inherit, Module, REnum, RStruct, RType, Schema}; +use schema::{Module, REnum, RStruct, RType, Schema}; use util::{write_all_to, NormalizeError}; use crate::generators::ImplGetSpanGenerator; @@ -50,13 +49,16 @@ trait Generator { } type GeneratedStream = (/* output path */ PathBuf, TokenStream); +type DataStream = (/* output path */ PathBuf, Vec); // TODO: remove me #[allow(dead_code)] #[derive(Debug, Clone)] enum GeneratorOutput { None, - Info(String), + Err(String), + Info(Vec), + Data(DataStream), Stream(GeneratedStream), } @@ -71,7 +73,7 @@ impl GeneratorOutput { assert!(self.is_none()); } - pub fn to_info(&self) -> &String { + pub fn to_info(&self) -> &[u8] { if let Self::Info(it) = self { it } else { @@ -79,6 +81,14 @@ impl GeneratorOutput { } } + pub fn to_data(&self) -> &DataStream { + if let Self::Data(it) = self { + it + } else { + panic!(); + } + } + pub fn to_stream(&self) -> &GeneratedStream { if let Self::Stream(it) = self { it @@ -87,7 +97,7 @@ impl GeneratorOutput { } } - pub fn into_info(self) -> String { + pub fn into_info(self) -> Vec { if let Self::Info(it) = self { it } else { @@ -95,6 +105,14 @@ impl GeneratorOutput { } } + pub fn into_data(self) -> DataStream { + if let Self::Data(it) = self { + it + } else { + panic!(); + } + } + pub fn into_stream(self) -> GeneratedStream { if let Self::Stream(it) = self { it @@ -107,7 +125,8 @@ impl GeneratorOutput { struct CodegenCtx { ty_table: TypeTable, ident_table: IdentTable, - schema: Schema, + schema: RefCell, + mods: RefCell>, } struct CodegenResult { @@ -116,7 +135,7 @@ struct CodegenResult { } impl CodegenCtx { - fn new(mods: Vec) -> Result { + fn new(mods: Vec) -> Self { // worst case len let len = mods.iter().fold(0, |acc, it| acc + it.items.len()); let defs = mods.iter().flat_map(|it| it.items.iter()); @@ -132,13 +151,7 @@ impl CodegenCtx { } } - let mut me = Self { ty_table, ident_table, schema: Schema::default() } - .pass(linker)? - .pass(calc_layout)?; - for m in mods { - m.build_in(&mut me.schema)?; - } - Ok(me) + Self { ty_table, ident_table, mods: RefCell::new(mods), schema: RefCell::default() } } fn find(&self, key: &TypeName) -> Option { @@ -161,7 +174,15 @@ impl AstCodegen { } #[must_use] - fn with(mut self, generator: G) -> Self + fn pass

(self, name: &'static str, pass: P) -> Self + where + P: Pass + 'static, + { + self.gen(PassGenerator::new(name, pass)) + } + + #[must_use] + fn gen(mut self, generator: G) -> Self where G: Generator + 'static, { @@ -180,7 +201,7 @@ impl AstCodegen { .map_ok(Module::analyze) .collect::>>>()??; - let ctx = CodegenCtx::new(modules)?; + let ctx = CodegenCtx::new(modules); let outputs = self .generators @@ -188,7 +209,7 @@ impl AstCodegen { .map(|mut gen| (gen.name(), gen.generate(&ctx))) .collect_vec(); - Ok(CodegenResult { outputs, schema: ctx.schema }) + Ok(CodegenResult { outputs, schema: ctx.schema.into_inner() }) } } @@ -210,6 +231,13 @@ fn write_generated_streams( Ok(()) } +fn write_data_streams(streams: impl IntoIterator) -> std::io::Result<()> { + for (path, content) in streams { + write_all_to(&content, path.into_os_string().to_str().unwrap())?; + } + Ok(()) +} + #[derive(Debug, Bpaf)] pub struct CliOptions { /// Runs all generators but won't write anything down. @@ -228,19 +256,26 @@ fn main() -> std::result::Result<(), Box> { let CodegenResult { outputs, schema } = files() .fold(AstCodegen::default(), AstCodegen::add_file) - .with(AssertLayouts) - .with(AstKindGenerator) - .with(AstBuilderGenerator) - .with(ImplGetSpanGenerator) - .with(VisitGenerator) - .with(VisitMutGenerator) + .pass("link", Linker) + .pass("early layout", CalcLayout) + .pass("build early schema", BuildSchema) + .gen(AssertLayouts("assert_unordered_layouts.rs")) + .gen(AstKindGenerator) + .gen(AstBuilderGenerator) + .gen(ImplGetSpanGenerator) + .gen(VisitGenerator) + .gen(VisitMutGenerator) .generate()?; - let (streams, _): (Vec<_>, Vec<_>) = + let (streams, outputs): (Vec<_>, Vec<_>) = outputs.into_iter().partition(|it| matches!(it.1, GeneratorOutput::Stream(_))); + let (binaries, _): (Vec<_>, Vec<_>) = + outputs.into_iter().partition(|it| matches!(it.1, GeneratorOutput::Data(_))); + if !cli_options.dry_run { write_generated_streams(streams.into_iter().map(|it| it.1.into_stream()))?; + write_data_streams(binaries.into_iter().map(|it| it.1.into_data()))?; } if !cli_options.no_fmt { diff --git a/tasks/ast_codegen/src/pass.rs b/tasks/ast_codegen/src/pass.rs deleted file mode 100644 index cc8a769dd7b49..0000000000000 --- a/tasks/ast_codegen/src/pass.rs +++ /dev/null @@ -1,31 +0,0 @@ -use std::collections::VecDeque; - -use itertools::Itertools; - -use crate::{schema::RType, CodegenCtx, Result}; - -impl CodegenCtx { - pub fn pass

(self, mut pass: P) -> Result - where - P: FnMut(&mut RType, &Self) -> Result, - { - // we sort by `TypeId` so we always have the same ordering as how it is written in the rust. - let mut unresolved = self - .ident_table - .iter() - .sorted_by_key(|it| it.1) - .map(|it| it.0) - .collect::>(); - - while let Some(next) = unresolved.pop_back() { - let next_id = *self.type_id(next).unwrap(); - - let val = &mut self.ty_table[next_id].borrow_mut(); - - if !pass(val, &self)? { - unresolved.push_front(next); - } - } - Ok(self) - } -} diff --git a/tasks/ast_codegen/src/passes/build_schema.rs b/tasks/ast_codegen/src/passes/build_schema.rs new file mode 100644 index 0000000000000..c54520e44750a --- /dev/null +++ b/tasks/ast_codegen/src/passes/build_schema.rs @@ -0,0 +1,12 @@ +use super::Pass; + +pub struct BuildSchema; + +impl Pass for BuildSchema { + fn once(&mut self, ctx: &crate::CodegenCtx) -> crate::Result { + for m in ctx.mods.borrow().iter() { + m.build_in(&mut ctx.schema.borrow_mut())?; + } + Ok(true) + } +} diff --git a/tasks/ast_codegen/src/passes/calc_layout.rs b/tasks/ast_codegen/src/passes/calc_layout.rs new file mode 100644 index 0000000000000..b0b66348f776a --- /dev/null +++ b/tasks/ast_codegen/src/passes/calc_layout.rs @@ -0,0 +1,358 @@ +/// We use compiler to infer 64bit type layouts. +#[cfg(not(target_pointer_width = "64"))] +compile_error!("`oxc_ast_codegen::layout` only supports 64 architectures."); +use std::collections::HashMap; + +use itertools::Itertools; +use lazy_static::lazy_static; +use quote::ToTokens; +use syn::Type; + +use crate::{ + layout::{KnownLayout, Layout}, + schema::{REnum, RStruct, RType}, + util::{NormalizeError, TypeAnalyzeResult, TypeExt, TypeWrapper}, + CodegenCtx, Result, TypeRef, +}; + +use super::Pass; + +type WellKnown = HashMap<&'static str, PlatformLayout>; + +pub struct CalcLayout; + +impl Pass for CalcLayout { + fn each(&mut self, ty: &mut RType, ctx: &CodegenCtx) -> crate::Result { + calc_layout(ty, ctx) + } +} + +#[derive(Debug, Clone)] +struct PlatformLayout(/* 64 */ Layout, /* 32 */ Layout); + +impl PlatformLayout { + const UNKNOWN: Self = Self(Layout::Unknown, Layout::Unknown); + + const fn zero() -> Self { + Self(Layout::zero(), Layout::zero()) + } + + const fn ptr() -> Self { + Self(Layout::ptr_64(), Layout::ptr_32()) + } + + const fn wide_ptr() -> Self { + Self(Layout::wide_ptr_64(), Layout::wide_ptr_32()) + } + + fn x64(&self) -> Layout { + self.0 + } + + fn x32(&self) -> Layout { + self.1 + } + + /// Return `true` if either of platform layouts is unknown. + fn is_unknown(&self) -> bool { + self.0.is_unknown() || self.1.is_unknown() + } +} + +impl From<(Layout, Layout)> for PlatformLayout { + fn from((x64, x32): (Layout, Layout)) -> Self { + Self(x64, x32) + } +} + +/// Calculates the layout of `ty` by mutating it. +/// Returns `false` if the layout is unknown at this point. +pub fn calc_layout(ty: &mut RType, ctx: &CodegenCtx) -> Result { + let unknown_layout = ty + .layout_32() + .and_then(|x32| ty.layout_64().map(|x64| PlatformLayout(x64, x32))) + .is_ok_and(|pl| pl.is_unknown()); + let layout = match ty { + RType::Enum(enum_) if unknown_layout => calc_enum_layout(enum_, ctx), + RType::Struct(struct_) if unknown_layout => calc_struct_layout(struct_, ctx), + _ => return Ok(true), + }?; + if layout.is_unknown() { + Ok(false) + } else { + ty.set_layout(layout.x64(), layout.x32())?; + Ok(true) + } +} + +fn calc_enum_layout(ty: &mut REnum, ctx: &CodegenCtx) -> Result { + fn collect_variant_layouts(ty: &REnum, ctx: &CodegenCtx) -> Result> { + // all unit variants? + if ty.item.variants.iter().all(|var| var.fields.is_empty()) { + // all AST enums are `repr(C)` so it would have a 4 byte layout/alignment, + let layout = KnownLayout::new(0, 4, 0); + let layout = Layout::Layout(layout); + Ok(vec![PlatformLayout(layout, layout)]) + } else { + ty.item + .variants + .iter() + .map(|var| { + let typ = + var.fields.iter().exactly_one().map(|f| f.ty.analyze(ctx)).normalize()?; + calc_type_layout(&typ, ctx) + }) + .collect() + } + } + + fn fold_layout(mut acc: KnownLayout, layout: KnownLayout) -> KnownLayout { + // SAFETY: we are folding valid layouts so it is safe. + unsafe { + // max alignment + if layout.align() > acc.align() { + acc.set_align_unchecked(layout.align()); + } + // max size + if layout.size() > acc.size() { + acc.set_size_unchecked(layout.size()); + } + // min niches + if layout.niches() < acc.niches() { + acc.set_niches_unchecked(layout.niches()); + } + } + acc + } + + let with_tag = |mut acc: KnownLayout| -> KnownLayout { + acc.consume_niches(ty.item.variants.len() as u128, true); + acc + }; + + let layouts = collect_variant_layouts(ty, ctx)?; + let (layouts_x64, layouts_x32): (Vec, Vec) = layouts + .into_iter() + .map(|PlatformLayout(x64, x32)| { + x64.layout().and_then(|x64| x32.layout().map(|x32| (x64, x32))) + }) + .collect::>() + .expect("already checked."); + + let x32 = with_tag(layouts_x32.into_iter().fold(KnownLayout::default(), fold_layout)); + let x64 = with_tag(layouts_x64.into_iter().fold(KnownLayout::default(), fold_layout)); + Ok(PlatformLayout(Layout::from(x64), Layout::from(x32))) +} + +fn calc_struct_layout(ty: &mut RStruct, ctx: &CodegenCtx) -> Result { + fn collect_field_layouts(ty: &RStruct, ctx: &CodegenCtx) -> Result> { + if ty.item.fields.is_empty() { + Ok(vec![PlatformLayout::zero()]) + } else { + ty.item + .fields + .iter() + .map(|field| { + let typ = field.ty.analyze(ctx); + calc_type_layout(&typ, ctx) + }) + .collect() + } + } + + fn with_padding( + layouts: &[KnownLayout], + ) -> std::result::Result { + // TODO: store `offsets` in the layout + let mut offsets = Vec::new(); + let mut output = std::alloc::Layout::from_size_align(0, 1)?; + let mut niches = 0; + for layout in layouts { + let (new_layout, offset) = output + .extend(std::alloc::Layout::from_size_align(layout.size(), layout.align())?)?; + output = new_layout; + offsets.push(offset); + niches += layout.niches(); + } + let output = output.pad_to_align(); + Ok(KnownLayout::new(output.size(), output.align(), niches)) + } + + let layouts = collect_field_layouts(ty, ctx)?; + + if layouts.iter().any(PlatformLayout::is_unknown) { + return Ok(PlatformLayout::UNKNOWN); + } + + let (layouts_x64, layouts_x32): (Vec, Vec) = layouts + .into_iter() + .map(|PlatformLayout(x64, x32)| { + x64.layout().and_then(|x64| x32.layout().map(|x32| (x64, x32))) + }) + .collect::>() + .expect("already checked."); + + let x32 = with_padding(&layouts_x32).normalize()?; + let x64 = with_padding(&layouts_x64).normalize()?; + + Ok(PlatformLayout(Layout::from(x64), Layout::from(x32))) +} + +fn calc_type_layout(ty: &TypeAnalyzeResult, ctx: &CodegenCtx) -> Result { + fn is_slice(ty: &TypeAnalyzeResult) -> bool { + if let Type::Reference(typ) = &ty.typ { + // TODO: support for &[T] slices. + typ.elem.get_ident().as_ident().is_some_and(|id| id == "str") + } else { + false + } + } + + fn try_fold_option(layout: Layout) -> Layout { + let Layout::Layout(mut known) = layout else { return layout }; + // option needs only one bit to store its tag and it can be in a fragmented offset + known.consume_niches(1, true); + Layout::Layout(known) + } + + let get_layout = |type_ref: Option<&TypeRef>| -> Result { + let result = if let Some(type_ref) = &type_ref { + if calc_layout(&mut type_ref.borrow_mut(), ctx)? { + type_ref.borrow().layouts().map(PlatformLayout::from)? + } else { + PlatformLayout::UNKNOWN + } + } else if let Some(well_known) = + WELL_KNOWN.get(ty.typ.get_ident().inner_ident().to_string().as_str()) + { + well_known.clone() + } else { + let Type::Path(typ) = &ty.typ else { + panic!(); + }; + + let typ = typ + .path + .segments + .first() + .map(|it| it.to_token_stream().to_string().replace(' ', "")) + .expect("We only accept single segment types."); + + if let Some(typ) = WELL_KNOWN.get(typ.as_str()) { + typ.clone() + } else { + panic!("Unsupported type: {:#?}", ty.typ.to_token_stream().to_string()) + } + }; + Ok(result) + }; + + let layout = match ty.wrapper { + TypeWrapper::Vec | TypeWrapper::VecBox | TypeWrapper::VecOpt | TypeWrapper::OptVec => { + WELL_KNOWN[stringify!(Vec)].clone() + } + TypeWrapper::Ref if is_slice(ty) => PlatformLayout::wide_ptr(), + TypeWrapper::Ref | TypeWrapper::Box | TypeWrapper::OptBox => PlatformLayout::ptr(), + TypeWrapper::None => get_layout(ty.type_ref.as_ref())?, + TypeWrapper::Opt => { + let PlatformLayout(x64, x32) = get_layout(ty.type_ref.as_ref())?; + PlatformLayout(try_fold_option(x64), try_fold_option(x32)) + } + TypeWrapper::Complex => { + let PlatformLayout(x64, x32) = get_layout(ty.type_ref.as_ref())?; + PlatformLayout(x64, x32) + } + }; + Ok(layout) +} + +macro_rules! well_known { + ($($typ:ty: { $($platform:tt => $layout:expr,)*},)*) => { + WellKnown::from([ + $(( + stringify!($typ), + well_known!(@ $( $platform => $layout,)*) + )),* + ]) + }; + + // entries + (@ _ => $layout:expr,) => { + PlatformLayout($layout, $layout) + }; + (@ 64 => $layout_64:expr, 32 => $layout_32:expr,) => { + PlatformLayout($layout_64, $layout_32) + }; + (@ 32 => $layout_32:expr, 64 => $layout_64:expr,) => { + well_known!(@ 64 => $layout_64, 32 => $layout_32) + }; + + // compile errors + (@ 32 => $layout:expr,) => { + ::std::compile_error!("non_exhaustive well known type, `64` target isn't covered.") + }; + (@ 64 => $layout:expr,) => { + ::std::compile_error!("non_exhaustive well known type, `32` target isn't covered.") + }; +} + +lazy_static! { + static ref WELL_KNOWN: WellKnown = well_known! { + // builtins + // types smaller or equal to 4bytes have the same layout on most platforms. + char: { _ => Layout::of::(), }, + bool: { _ => Layout::of::(), }, + u8: { _ => Layout::of::(), }, + i8: { _ => Layout::of::(), }, + u16: { _ => Layout::of::(), }, + i16: { _ => Layout::of::(), }, + u32: { _ => Layout::of::(), }, + i32: { _ => Layout::of::(), }, + f32: { _ => Layout::of::(), }, + // 32bit layouts are based on WASM + u64: { + 64 => Layout::of::(), + 32 => Layout::known(8, 8, 0), + }, + i64: { + 64 => Layout::of::(), + 32 => Layout::known(8, 8, 0), + }, + f64: { + 64 => Layout::of::(), + 32 => Layout::known(8, 8, 0), + }, + usize: { + 64 => Layout::of::(), + 32 => Layout::known(4, 4, 0), + }, + isize: { + 64 => Layout::of::(), + 32 => Layout::known(4, 4, 0), + }, + // well known types + // TODO: generate const assertions for these in the ast crate + Span: { _ => Layout::known(8, 4, 0), }, + Atom: { + 64 => Layout::wide_ptr_64(), + 32 => Layout::wide_ptr_32(), + }, + Vec: { + 64 => Layout::known(32, 8, 1), + 32 => Layout::known(16, 4, 1), + }, + Cell>: { _ => Layout::known(4, 4, 1), }, + Cell>: { _ => Layout::known(4, 4, 1), }, + Cell>: { _ => Layout::known(4, 4, 1), }, + ReferenceFlag: { _ => Layout::known(1, 1, 0), }, + AssignmentOperator: { _ => Layout::known(1, 1, 1), }, + LogicalOperator: { _ => Layout::known(1, 1, 1), }, + UnaryOperator: { _ => Layout::known(1, 1, 1), }, + BinaryOperator: { _ => Layout::known(1, 1, 1), }, + UpdateOperator: { _ => Layout::known(1, 1, 1), }, + SourceType: { _ => Layout::known(4, 1, 1), }, + RegExpFlags: { _ => Layout::known(1, 1, 0), }, + BigintBase: { _ => Layout::known(1, 1, 1), }, + NumberBase: { _ => Layout::known(1, 1, 1), }, + }; +} diff --git a/tasks/ast_codegen/src/passes/linker.rs b/tasks/ast_codegen/src/passes/linker.rs new file mode 100644 index 0000000000000..9706eaf8ba32b --- /dev/null +++ b/tasks/ast_codegen/src/passes/linker.rs @@ -0,0 +1,87 @@ +use std::borrow::Cow; + +use syn::parse_quote; + +use crate::{schema::Inherit, util::NormalizeError}; + +use super::{CodegenCtx, Pass, RType, Result}; + +pub trait Unresolved { + fn unresolved(&self) -> bool; + + // TODO: remove me + #[allow(dead_code)] + fn resolved(&self) -> bool { + !self.unresolved() + } +} + +impl Unresolved for Inherit { + fn unresolved(&self) -> bool { + matches!(self, Self::Unlinked(_)) + } +} + +impl Unresolved for Vec { + fn unresolved(&self) -> bool { + self.iter().any(Unresolved::unresolved) + } +} + +pub struct Linker; + +impl Pass for Linker { + /// # Panics + /// On invalid inheritance. + fn each(&mut self, ty: &mut RType, ctx: &CodegenCtx) -> crate::Result { + // Exit early if it isn't an enum, We only link to resolve enum inheritance! + let RType::Enum(ty) = ty else { + return Ok(true); + }; + + // Exit early if there is this enum doesn't use enum inheritance + if ty.meta.inherits.is_empty() { + return Ok(true); + } + + let inherits = ty + .meta + .inherits + .drain(..) + .map(|it| match it { + Inherit::Unlinked(ref sup) => { + let linkee = ctx + .find(&Cow::Owned(sup.to_string())) + .normalize_with(format!("Unknown type {sup:?}"))?; + let linkee = linkee.borrow(); + let inherit_value = format!(r#""{}""#, linkee.ident().unwrap()); + let variants = match &*linkee { + RType::Enum(enum_) => { + if enum_.meta.inherits.unresolved() { + return Ok(Err(it)); + } + enum_.item.variants.clone().into_iter().map(|mut v| { + v.attrs = vec![parse_quote!(#[inherit = #inherit_value])]; + v + }) + } + _ => { + panic!("invalid inheritance, you can only inherit from enums and in enums.") + } + }; + ty.item.variants.extend(variants.clone()); + Ok(Ok(Inherit::Linked { + super_: linkee.as_type().unwrap(), + variants: variants.collect(), + })) + } + Inherit::Linked { .. } => Ok(Ok(it)), + }) + .collect::>>>()?; + let unresolved = inherits.iter().any(std::result::Result::is_err); + + ty.meta.inherits = inherits.into_iter().map(|it| it.unwrap_or_else(|it| it)).collect(); + + Ok(!unresolved) + } +} diff --git a/tasks/ast_codegen/src/passes/mod.rs b/tasks/ast_codegen/src/passes/mod.rs new file mode 100644 index 0000000000000..347be8c836119 --- /dev/null +++ b/tasks/ast_codegen/src/passes/mod.rs @@ -0,0 +1,71 @@ +mod build_schema; +mod calc_layout; +mod linker; + +use std::collections::VecDeque; + +use itertools::Itertools; + +use crate::{schema::RType, CodegenCtx, Generator, GeneratorOutput, Result}; + +pub use build_schema::BuildSchema; +pub use calc_layout::CalcLayout; +pub use linker::Linker; + +pub trait Pass { + /// Returns false if can't resolve + fn once(&mut self, _ctx: &CodegenCtx) -> Result { + Ok(true) + } + + /// Returns false if can't resolve + fn each(&mut self, _ty: &mut RType, _ctx: &CodegenCtx) -> Result { + Ok(true) + } +} + +pub struct PassGenerator(&'static str, Box); + +impl PassGenerator { + pub fn new(name: &'static str, pass: P) -> Self { + Self(name, Box::new(pass)) + } +} + +impl Generator for PassGenerator { + fn name(&self) -> &'static str { + self.0 + } + + fn generate(&mut self, ctx: &CodegenCtx) -> GeneratorOutput { + match self.call(ctx) { + Ok(val) => GeneratorOutput::Info(vec![val.into()]), + Err(err) => GeneratorOutput::Err(err), + } + } +} + +impl PassGenerator { + fn call(&mut self, ctx: &CodegenCtx) -> Result { + // call once + if !self.1.once(ctx)? { + return Ok(false); + } + + // call each + // we sort by `TypeId` so we always have the same ordering as how it is written in the rust. + let mut unresolved = + ctx.ident_table.iter().sorted_by_key(|it| it.1).map(|it| it.0).collect::>(); + + while let Some(next) = unresolved.pop_back() { + let next_id = *ctx.type_id(next).unwrap(); + + let val = &mut ctx.ty_table[next_id].borrow_mut(); + + if !self.1.each(val, ctx)? { + unresolved.push_front(next); + } + } + Ok(unresolved.is_empty()) + } +} diff --git a/tasks/ast_codegen/src/schema.rs b/tasks/ast_codegen/src/schema.rs index d5fcfdfa7ac0e..75bcacdf57f26 100644 --- a/tasks/ast_codegen/src/schema.rs +++ b/tasks/ast_codegen/src/schema.rs @@ -298,12 +298,12 @@ impl Module { Ok(self) } - pub fn build_in(self, schema: &mut Schema) -> Result<()> { + pub fn build_in(&self, schema: &mut Schema) -> Result<()> { if !self.loaded { return Err(String::from(LOAD_ERROR)); } - schema.definitions.extend(self.items.into_iter().filter_map(|it| (&*it.borrow()).into())); + schema.definitions.extend(self.items.iter().filter_map(|it| (&*it.borrow()).into())); Ok(()) } } diff --git a/tasks/ast_codegen/src/util.rs b/tasks/ast_codegen/src/util.rs index 9cbdba337059f..ed924a0762de5 100644 --- a/tasks/ast_codegen/src/util.rs +++ b/tasks/ast_codegen/src/util.rs @@ -279,9 +279,13 @@ impl TokenStreamExt for TokenStream { } } -pub fn write_all_to>(data: &[u8], path: S) -> std::io::Result<()> { +pub fn write_all_to>(data: &[u8], path: S) -> std::io::Result<()> { use std::{fs, io::Write}; - let mut file = fs::File::create(path.as_ref())?; + let path = path.as_ref(); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let mut file = fs::File::create(path)?; file.write_all(data)?; Ok(()) }