From 01de16bdd6be9143641445839fa0929bc71219fd Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Fri, 17 May 2024 15:16:06 +0200 Subject: [PATCH 1/8] Appended bytecode --- Makefile | 1 + embed.mjs | 118 +++++++++++++++++++++++ llrt/src/main.c | 40 ++++---- llrt_core/build.rs | 34 +++---- llrt_core/src/bytecode.rs | 1 + llrt_core/src/lib.rs | 1 + llrt_core/src/vm.rs | 193 ++++++++++++++++++++++++++++++++++---- 7 files changed, 330 insertions(+), 58 deletions(-) create mode 100644 embed.mjs diff --git a/Makefile b/Makefile index 5899c42e13..a78352c4a3 100644 --- a/Makefile +++ b/Makefile @@ -73,6 +73,7 @@ define release_template release-${1}: | clean-js js cargo $$(BUILD_ARG) --target $$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1})) --features lambda -vv ./pack target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/bootstrap + node embed.mjs target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/bootstrap target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/bootstrap cargo $$(BUILD_ARG) --target $$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1})) --features lambda,uncompressed -vv mv target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt llrt-container-${1} @rm -rf llrt-lambda-${1}.zip diff --git a/embed.mjs b/embed.mjs new file mode 100644 index 0000000000..420d331805 --- /dev/null +++ b/embed.mjs @@ -0,0 +1,118 @@ +import fs from "fs/promises"; +import path from "path"; + +const BUNDLE_DIR = "./bundle/lrt"; + +//read all files in ./bundle/js that ends with .lrt +async function readFiles() { + const fileEntries = await fs.readdir(BUNDLE_DIR, { + recursive: true, + withFileTypes: true, + }); + const files = fileEntries.reduce((acc, { name, parentPath }) => { + if (name.endsWith(".lrt")) { + acc.push(path.join(parentPath, name)); + } + + return acc; + }, []); + files.sort((a, b) => a.localeCompare(b)); + return files; +} + +async function readFileData(files) { + return await Promise.all( + files.map(async (file) => { + const data = await fs.readFile(file); + const { name, dir } = path.parse(path.relative(BUNDLE_DIR, file)); + return [`${dir ? `${dir}/` : ""}${name}`, data]; + }) + ); +} + +async function buildFileIndex(source, target, fileData) { + const uint32Buffer = (length) => { + const buffer = Buffer.alloc(4); + buffer.writeUInt32LE(length); + return buffer; + }; + + const uint16Buffer = (length) => { + const buffer = Buffer.alloc(2); + buffer.writeUInt16LE(length); + return buffer; + }; + + const sourceData = await fs.readFile(source); + const dataBuffers = []; + let offset = 0; + let indexBuffers = []; + for (let [name, data] of fileData) { + dataBuffers.push(data); + + if(name.startsWith("llrt-chunk-")){ + name = `${name}.js` + } + + const nameLengthBuffer = uint16Buffer(name.length); + const nameBuffer = Buffer.from(name); + const bytecodeSizeBuffer = uint32Buffer(data.length); + const bytecodeOffsetBuffer = uint32Buffer(offset); + + indexBuffers.push( + Buffer.concat([ + nameLengthBuffer, + nameBuffer, + bytecodeSizeBuffer, + bytecodeOffsetBuffer, + ]) + ); + + offset += data.length; + } + + const dataBuffer = Buffer.concat(dataBuffers); + + const packageCount = fileData.length; + const bytecodePosition = sourceData.length; + const packageIndexPosition = sourceData.length + dataBuffer.length; + + //[u32 package_count][u32 bytecode_pos][u32 package_index_pos] + const metadataBuffer = Buffer.concat([ + uint32Buffer(packageCount), + uint32Buffer(bytecodePosition), + uint32Buffer(packageIndexPosition), + ]); + + const signatureBuffer = Buffer.from("lrt"); + + let allIndexBuffers = Buffer.concat(indexBuffers); + + console.log("index buffer size: ", allIndexBuffers.length); + + const finalBuffer = Buffer.concat([ + sourceData, + dataBuffer, + allIndexBuffers, + metadataBuffer, + signatureBuffer, + ]); + + await fs.writeFile(target, finalBuffer); + await fs.chmod(target, 0o755); +} + +const source = process.argv[2]; +const target = process.argv[3]; +if (!source || !target) { + console.error( + `No source or target specified, use:\n${path.basename(process.argv[0])} ${path.basename(process.argv[1])} {input_target} {output_target}` + ); + process.exit(1); +} + +console.log("Reading files..."); +const files = await readFiles(); +console.log("Reading file data..."); +const filesContents = await readFileData(files); +await buildFileIndex(source, target, filesContents); diff --git a/llrt/src/main.c b/llrt/src/main.c index 018ae39608..87327c55f0 100644 --- a/llrt/src/main.c +++ b/llrt/src/main.c @@ -280,25 +280,25 @@ int main(int argc, char *argv[]) double t2 = micro_seconds(); logInfo("Extraction + write time: %10.4f ms\n", (t2 - t0) / 1000.0); - char **new_argv = malloc((size_t)(argc + 1) * sizeof *new_argv); - for (uint8_t i = 0; i < argc; ++i) - { - if (i == 0) - { - size_t length = strlen(appname) + 2; - new_argv[i] = malloc(length); - memcpy(new_argv[i], "/", 1); - memcpy(new_argv[i] + 1, appname, length); - setenv("_", new_argv[i], true); - } - else - { - size_t length = strlen(argv[i]) + 1; - new_argv[i] = malloc(length); - memcpy(new_argv[i], argv[i], length); - } - } - new_argv[argc] = NULL; + // char **new_argv = malloc((size_t)(argc + 1) * sizeof *new_argv); + // for (uint8_t i = 0; i < argc; ++i) + // { + // if (i == 0) + // { + // size_t length = strlen(appname) + 2; + // new_argv[i] = malloc(length); + // memcpy(new_argv[i], "/", 1); + // memcpy(new_argv[i] + 1, appname, length); + // setenv("_", new_argv[i], true); + // } + // else + // { + // size_t length = strlen(argv[i]) + 1; + // new_argv[i] = malloc(length); + // memcpy(new_argv[i], argv[i], length); + // } + // } + // new_argv[argc] = NULL; unsigned long startTime = (unsigned long)(micro_seconds() / 1000.0); @@ -330,7 +330,7 @@ int main(int argc, char *argv[]) logInfo("Starting app\n"); - fexecve(outputFd, new_argv, environ); + fexecve(outputFd, argv, environ); logError("Failed to start executable"); diff --git a/llrt_core/build.rs b/llrt_core/build.rs index ccc79be43f..715599e4b2 100644 --- a/llrt_core/build.rs +++ b/llrt_core/build.rs @@ -52,10 +52,6 @@ async fn main() -> StdResult<(), Box> { rt.set_loader(resolver, loader); let ctx = Context::full(&rt)?; - let sdk_bytecode_path = Path::new("src").join("bytecode_cache.rs"); - let mut sdk_bytecode_file = BufWriter::new(File::create(sdk_bytecode_path)?); - - let mut ph_map = phf_codegen::Map::::new(); let mut lrt_filenames = vec![]; let mut total_bytes: usize = 0; @@ -98,7 +94,7 @@ async fn main() -> StdResult<(), Box> { path.to_string_lossy().to_string() }; - info!("Compiling module: {}", module_name); + info!("Compiling modules: {}", module_name); let lrt_path = PathBuf::from(BUNDLE_LRT_DIR).join(path.with_extension(BYTECODE_EXT)); let lrt_filename = lrt_path.to_string_lossy().to_string(); @@ -128,25 +124,25 @@ async fn main() -> StdResult<(), Box> { info!("Done!"); - ph_map.entry( - module_name, - &format!( - "include_bytes!(\"..{}{}\")", - MAIN_SEPARATOR_STR, &lrt_filename - ), - ); + // ph_map.entry( + // module_name, + // &format!( + // "include_bytes!(\"..{}{}\")", + // MAIN_SEPARATOR_STR, &lrt_filename + // ), + // ); } StdResult::<_, Box>::Ok(()) })?; - write!( - &mut sdk_bytecode_file, - "// @generated by build.rs\n\npub static BYTECODE_CACHE: phf::Map<&'static str, &[u8]> = {}", - ph_map.build() - )?; - writeln!(&mut sdk_bytecode_file, ";")?; - sdk_bytecode_file.flush()?; + // write!( + // &mut sdk_bytecode_file, + // "// @generated by build.rs\n\npub static BYTECODE_CACHE: phf::Map<&'static str, &[u8]> = {}", + // ph_map.build() + // )?; + // writeln!(&mut sdk_bytecode_file, ";")?; + // sdk_bytecode_file.flush()?; info!( "\n===============================\nUncompressed bytecode size: {}\n===============================", diff --git a/llrt_core/src/bytecode.rs b/llrt_core/src/bytecode.rs index f0c9c5c754..3d9f0959d1 100644 --- a/llrt_core/src/bytecode.rs +++ b/llrt_core/src/bytecode.rs @@ -1,6 +1,7 @@ pub const BYTECODE_VERSION: &str = "lrt01"; pub const BYTECODE_COMPRESSED: u8 = b'c'; pub const BYTECODE_UNCOMPRESSED: u8 = b'u'; +pub const BYTECODE_EMBEDDED_SIGNATURE: &[u8] = b"lrt"; #[allow(dead_code)] pub const BYTECODE_EXT: &str = "lrt"; pub const SIGNATURE_LENGTH: usize = BYTECODE_VERSION.len() + 1; diff --git a/llrt_core/src/lib.rs b/llrt_core/src/lib.rs index 8c87f0d5f9..a076ad4016 100644 --- a/llrt_core/src/lib.rs +++ b/llrt_core/src/lib.rs @@ -1,6 +1,7 @@ #![allow(clippy::new_without_default)] #![allow(clippy::inherent_to_string)] #![cfg_attr(rust_nightly, feature(portable_simd))] +#![cfg_attr(rust_nightly, feature(unsized_locals))] #[macro_use] mod macros; diff --git a/llrt_core/src/vm.rs b/llrt_core/src/vm.rs index 0c3f44f249..268bbc99ff 100644 --- a/llrt_core/src/vm.rs +++ b/llrt_core/src/vm.rs @@ -1,17 +1,25 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 use std::{ + cell::RefCell, cmp::min, collections::{HashMap, HashSet}, env, ffi::CStr, fmt::Write, - future::{poll_fn, Future}, - io, + fs::File, + future::Future, + io::{self, Read, Seek, SeekFrom}, + mem::{size_of, MaybeUninit}, path::{Component, Path, PathBuf}, pin::pin, process::exit, result::Result as StdResult, + slice, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Mutex, Once, RwLock, + }, sync::{Arc, Mutex}, task::Poll, }; @@ -30,16 +38,20 @@ use rquickjs::{ qjs, AsyncContext, AsyncRuntime, CatchResultExt, CaughtError, Ctx, Error, Function, IntoJs, Module, Object, Result, Value, }; +use tokio::{ + fs, + sync::oneshot::{self, Receiver}, +}; use tracing::trace; use zstd::{bulk::Decompressor, dict::DecoderDictionary}; -include!("./bytecode_cache.rs"); - -use crate::modules::{ - console, - crypto::SYSTEM_RANDOM, - path::{dirname, join_path, resolve_path}, - timers::{self, poll_timers, TimerPoller}, +use crate::{ + bytecode::BYTECODE_EMBEDDED_SIGNATURE, + modules::{ + console, + crypto::SYSTEM_RANDOM, + path::{dirname, join_path, resolve_path}, + }, }; use crate::{ @@ -53,6 +65,20 @@ use crate::{ object::{get_bytes, ObjectExt}, }, }; + +#[derive(Default)] +pub struct EmbeddedBytecodeData { + argv_0: String, + index: HashMap, +} + +thread_local! { + static SELF_FILE: RefCell> = RefCell::new(None); +} + +pub static EMBEDDED_BYTECODE_DATA: Lazy> = + Lazy::new(|| RwLock::new(EmbeddedBytecodeData::default())); + #[inline] pub fn uncompressed_size(input: &[u8]) -> StdResult<(usize, &[u8]), io::Error> { let size = input.get(..4).ok_or(io::ErrorKind::InvalidInput)?; @@ -127,7 +153,9 @@ impl Resolver for BinaryResolver { fn resolve(&mut self, _ctx: &Ctx, base: &str, name: &str) -> Result { trace!("Try resolve \"{}\" from \"{}\"", name, base); - if BYTECODE_CACHE.contains_key(name) { + let embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.read().unwrap(); + + if embedded_bytecode_data.index.contains_key(name) { return Ok(name.to_string()); } @@ -157,11 +185,11 @@ impl Resolver for BinaryResolver { trace!("Normalized path: {}, key: {}", normalized_path, cache_key); - if BYTECODE_CACHE.contains_key(cache_key) { + if embedded_bytecode_data.index.contains_key(cache_key) { return Ok(cache_key.to_string()); } - if BYTECODE_CACHE.contains_key(base) { + if embedded_bytecode_data.index.contains_key(base) { normalized_path = name; if Path::new(normalized_path).exists() { return Ok(normalized_path.to_string()); @@ -246,12 +274,29 @@ where impl Loader for BinaryLoader { fn load<'js>(&mut self, ctx: &Ctx<'js>, name: &str) -> Result> { trace!("Loading module: {}", name); - let ctx = ctx.clone(); - if let Some(bytes) = BYTECODE_CACHE.get(name) { + let embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.read().unwrap(); + if let Some((bytes_size, bytes_pos)) = embedded_bytecode_data.index.get(name) { trace!("Loading embedded module: {}", name); - return load_bytecode_module(ctx, name, bytes); + let mut buf = SELF_FILE.with(|self_file| { + let mut buf = vec![0; *bytes_size]; + let mut borrow = self_file.borrow_mut(); + if let Some(file) = borrow.as_mut() { + file.seek(SeekFrom::Start((*bytes_pos) as u64))?; + file.read_exact(&mut buf)?; + } else { + let mut file = File::open(&embedded_bytecode_data.argv_0)?; + file.seek(SeekFrom::Start((*bytes_pos) as u64))?; + file.read_exact(&mut buf)?; + *borrow = Some(file); + } + + Ok::<_, Error>(buf) + })?; + + return load_bytecode_module(name, &mut buf); } + drop(embedded_bytecode_data); let path = PathBuf::from(name); let mut bytes: &[u8] = &std::fs::read(path)?; @@ -356,6 +401,8 @@ impl Default for VmOptions { } } +static SYNC_OBJ: Once = Once::new(); + impl Vm { pub const ENV_LAMBDA_TASK_ROOT: &'static str = "LAMBDA_TASK_ROOT"; @@ -364,9 +411,13 @@ impl Vm { ) -> StdResult> { llrt_modules::time::init(); - SYSTEM_RANDOM - .fill(&mut [0; 8]) - .expect("Failed to initialize SystemRandom"); + SYNC_OBJ.call_once(|| { + SYSTEM_RANDOM + .fill(&mut [0; 8]) + .expect("Failed to initialize SystemRandom"); + + init_embedded_bytecode().expect("Error reading embedded bytecode"); + }); let mut file_resolver = FileResolver::default(); let mut binary_resolver = BinaryResolver::default(); @@ -516,6 +567,106 @@ impl Vm { } } +fn init_embedded_bytecode() -> std::io::Result<()> { + let argv_0 = env::args().nth(0).expect("Failed to get argv0"); + let mut file = File::open(&argv_0)?; + + let mut embedded_bytecode_signature_buf = [0; BYTECODE_EMBEDDED_SIGNATURE.len()]; + let meta = file.metadata()?; + let total_file_size = meta.len(); + let signature_len = BYTECODE_EMBEDDED_SIGNATURE.len(); + let signed_signature_len = signature_len as i64; + file.seek(SeekFrom::End(-signed_signature_len))?; + file.read_exact(&mut embedded_bytecode_signature_buf)?; + + if embedded_bytecode_signature_buf != BYTECODE_EMBEDDED_SIGNATURE { + return Ok(()); + } + + struct EmbeddedMeta { + package_count: u32, + bytecode_pos: u32, + package_index_pos: u32, + } + + impl EmbeddedMeta { + fn read(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { + Ok(EmbeddedMeta { + package_count: read_u32(file, buf)?, + bytecode_pos: read_u32(file, buf)?, + package_index_pos: read_u32(file, buf)?, + }) + } + } + + let embedded_meta_size = size_of::(); + + let meta_and_signature_size = embedded_meta_size + signature_len; + + file.seek(SeekFrom::Current(-(meta_and_signature_size as i64)))?; + + let mut u32_buf = [0; size_of::()]; + let mut u16_buf = [0; size_of::()]; + + let embedded_metadata = EmbeddedMeta::read(&mut file, &mut u32_buf)?; + + file.seek(SeekFrom::Current( + -(total_file_size as i64 + - (embedded_metadata.package_index_pos as i64 + signed_signature_len)), + ))?; + + let mut embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.write().unwrap(); + embedded_bytecode_data + .index + .reserve(embedded_metadata.package_count as usize); + embedded_bytecode_data.argv_0 = argv_0; + + let mut current_pos = embedded_metadata.package_index_pos as usize; + + let end_pos = total_file_size as usize - meta_and_signature_size; + + loop { + let name_len = read_u16(&mut file, &mut u16_buf)?; + let mut buf: Vec = vec![0; name_len as usize]; + file.read_exact(&mut buf)?; + + let name = unsafe { String::from_utf8_unchecked(buf) }; + + let bytecode_size = read_u32(&mut file, &mut u32_buf)?; + let bytecode_offset = read_u32(&mut file, &mut u32_buf)?; + + current_pos = current_pos + name_len as usize + u16_buf.len() + (u32_buf.len() * 2); + + embedded_bytecode_data.index.insert( + name, + ( + bytecode_size as usize, + embedded_metadata.bytecode_pos as usize + bytecode_offset as usize, + ), + ); + + if current_pos == end_pos { + break; + } + } + + drop(embedded_bytecode_data); + + Ok(()) +} + +#[inline(always)] +fn read_u32(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { + file.read_exact(buf)?; + Ok(u32::from_le_bytes(*buf)) +} + +#[inline(always)] +fn read_u16(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { + file.read_exact(buf)?; + Ok(u16::from_le_bytes(*buf)) +} + fn json_parse_string<'js>(ctx: Ctx<'js>, value: Value<'js>) -> Result> { let bytes = get_bytes(&ctx, value)?; json_parse(&ctx, bytes) @@ -623,7 +774,11 @@ fn init(ctx: &Ctx<'_>, module_names: HashSet<&'static str>) -> Result<()> { specifier }; let import_name = if module_names.contains(specifier.as_str()) - || BYTECODE_CACHE.contains_key(&specifier) + || EMBEDDED_BYTECODE_DATA + .read() + .unwrap() + .index + .contains_key(&specifier) || specifier.starts_with('/') { specifier From 94230e07e7255d8a19979f56378584eca9716ab8 Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Sat, 8 Jun 2024 12:55:04 +0200 Subject: [PATCH 2/8] Fix and format --- llrt_core/build.rs | 8 +++----- llrt_core/src/vm.rs | 12 ++++-------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/llrt_core/build.rs b/llrt_core/build.rs index 715599e4b2..9d1a71909c 100644 --- a/llrt_core/build.rs +++ b/llrt_core/build.rs @@ -4,15 +4,13 @@ use std::{ collections::HashSet, env, error::Error, - fs::{self, File}, - io::{self, BufWriter}, - path::{Path, PathBuf, MAIN_SEPARATOR_STR}, + fs::{self}, + io::{self}, + path::{Path, PathBuf}, process::Command, result::Result as StdResult, }; -use std::io::Write; - use jwalk::WalkDir; use rquickjs::{CatchResultExt, CaughtError, Context, Module, Runtime}; diff --git a/llrt_core/src/vm.rs b/llrt_core/src/vm.rs index 268bbc99ff..4697dc23ce 100644 --- a/llrt_core/src/vm.rs +++ b/llrt_core/src/vm.rs @@ -10,12 +10,11 @@ use std::{ fs::File, future::Future, io::{self, Read, Seek, SeekFrom}, - mem::{size_of, MaybeUninit}, + mem::size_of, path::{Component, Path, PathBuf}, pin::pin, process::exit, result::Result as StdResult, - slice, sync::{ atomic::{AtomicUsize, Ordering}, Arc, Mutex, Once, RwLock, @@ -38,10 +37,7 @@ use rquickjs::{ qjs, AsyncContext, AsyncRuntime, CatchResultExt, CaughtError, Ctx, Error, Function, IntoJs, Module, Object, Result, Value, }; -use tokio::{ - fs, - sync::oneshot::{self, Receiver}, -}; +use tokio::sync::oneshot::{self, Receiver}; use tracing::trace; use zstd::{bulk::Decompressor, dict::DecoderDictionary}; @@ -73,7 +69,7 @@ pub struct EmbeddedBytecodeData { } thread_local! { - static SELF_FILE: RefCell> = RefCell::new(None); + static SELF_FILE: RefCell> = const { RefCell::new(None) }; } pub static EMBEDDED_BYTECODE_DATA: Lazy> = @@ -568,7 +564,7 @@ impl Vm { } fn init_embedded_bytecode() -> std::io::Result<()> { - let argv_0 = env::args().nth(0).expect("Failed to get argv0"); + let argv_0 = env::args().next().expect("Failed to get argv0"); let mut file = File::open(&argv_0)?; let mut embedded_bytecode_signature_buf = [0; BYTECODE_EMBEDDED_SIGNATURE.len()]; From c2d25f8f9cb98cd38899a62e57c30e454efb1cab Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Sat, 8 Jun 2024 22:20:30 +0200 Subject: [PATCH 3/8] Use mmap instead opened files --- llrt_core/src/vm.rs | 111 +++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 68 deletions(-) diff --git a/llrt_core/src/vm.rs b/llrt_core/src/vm.rs index 4697dc23ce..24aa49e235 100644 --- a/llrt_core/src/vm.rs +++ b/llrt_core/src/vm.rs @@ -1,7 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 use std::{ - cell::RefCell, cmp::min, collections::{HashMap, HashSet}, env, @@ -9,7 +8,7 @@ use std::{ fmt::Write, fs::File, future::Future, - io::{self, Read, Seek, SeekFrom}, + io::{self, Read}, mem::size_of, path::{Component, Path, PathBuf}, pin::pin, @@ -23,6 +22,7 @@ use std::{ task::Poll, }; +use memmap2::{Advice, Mmap}; use once_cell::sync::Lazy; pub use llrt_utils::{ctx::CtxExtension, error::ErrorExtensions}; @@ -68,9 +68,11 @@ pub struct EmbeddedBytecodeData { index: HashMap, } -thread_local! { - static SELF_FILE: RefCell> = const { RefCell::new(None) }; -} +pub static SELF_MEM_MAP: Lazy>> = Lazy::new(|| RwLock::new(None)); + +// thread_local! { +// static SELF_FILE: RefCell> = const { RefCell::new(None) }; +// } pub static EMBEDDED_BYTECODE_DATA: Lazy> = Lazy::new(|| RwLock::new(EmbeddedBytecodeData::default())); @@ -274,23 +276,10 @@ impl Loader for BinaryLoader { if let Some((bytes_size, bytes_pos)) = embedded_bytecode_data.index.get(name) { trace!("Loading embedded module: {}", name); - let mut buf = SELF_FILE.with(|self_file| { - let mut buf = vec![0; *bytes_size]; - let mut borrow = self_file.borrow_mut(); - if let Some(file) = borrow.as_mut() { - file.seek(SeekFrom::Start((*bytes_pos) as u64))?; - file.read_exact(&mut buf)?; - } else { - let mut file = File::open(&embedded_bytecode_data.argv_0)?; - file.seek(SeekFrom::Start((*bytes_pos) as u64))?; - file.read_exact(&mut buf)?; - *borrow = Some(file); - } - - Ok::<_, Error>(buf) - })?; - - return load_bytecode_module(name, &mut buf); + if let Some(mmap) = SELF_MEM_MAP.read().unwrap().as_ref() { + let buf = &mmap[*bytes_pos..*bytes_pos + *bytes_size]; + return load_bytecode_module(name, buf); + } } drop(embedded_bytecode_data); let path = PathBuf::from(name); @@ -565,51 +554,37 @@ impl Vm { fn init_embedded_bytecode() -> std::io::Result<()> { let argv_0 = env::args().next().expect("Failed to get argv0"); - let mut file = File::open(&argv_0)?; + let file = File::open(&argv_0)?; + + let mmap = unsafe { Mmap::map(&file)? }; + mmap.advise(Advice::Sequential).unwrap(); + + let total_file_size = mmap.len(); - let mut embedded_bytecode_signature_buf = [0; BYTECODE_EMBEDDED_SIGNATURE.len()]; - let meta = file.metadata()?; - let total_file_size = meta.len(); let signature_len = BYTECODE_EMBEDDED_SIGNATURE.len(); - let signed_signature_len = signature_len as i64; - file.seek(SeekFrom::End(-signed_signature_len))?; - file.read_exact(&mut embedded_bytecode_signature_buf)?; + let signed_signature_len = signature_len as isize; - if embedded_bytecode_signature_buf != BYTECODE_EMBEDDED_SIGNATURE { + if &mmap[(total_file_size as isize - signed_signature_len) as usize..] + != BYTECODE_EMBEDDED_SIGNATURE + { return Ok(()); } + #[repr(C)] struct EmbeddedMeta { package_count: u32, bytecode_pos: u32, package_index_pos: u32, } - impl EmbeddedMeta { - fn read(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { - Ok(EmbeddedMeta { - package_count: read_u32(file, buf)?, - bytecode_pos: read_u32(file, buf)?, - package_index_pos: read_u32(file, buf)?, - }) - } - } - let embedded_meta_size = size_of::(); - let meta_and_signature_size = embedded_meta_size + signature_len; - file.seek(SeekFrom::Current(-(meta_and_signature_size as i64)))?; - - let mut u32_buf = [0; size_of::()]; - let mut u16_buf = [0; size_of::()]; + let meta_start = (total_file_size as isize - meta_and_signature_size as isize) as usize; + let meta_end = (total_file_size as isize - signature_len as isize) as usize; - let embedded_metadata = EmbeddedMeta::read(&mut file, &mut u32_buf)?; - - file.seek(SeekFrom::Current( - -(total_file_size as i64 - - (embedded_metadata.package_index_pos as i64 + signed_signature_len)), - ))?; + let embedded_metadata: EmbeddedMeta = + unsafe { std::ptr::read(mmap[meta_start..meta_end].as_ptr() as *const _) }; let mut embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.write().unwrap(); embedded_bytecode_data @@ -618,20 +593,16 @@ fn init_embedded_bytecode() -> std::io::Result<()> { embedded_bytecode_data.argv_0 = argv_0; let mut current_pos = embedded_metadata.package_index_pos as usize; - let end_pos = total_file_size as usize - meta_and_signature_size; loop { - let name_len = read_u16(&mut file, &mut u16_buf)?; - let mut buf: Vec = vec![0; name_len as usize]; - file.read_exact(&mut buf)?; - - let name = unsafe { String::from_utf8_unchecked(buf) }; + let name_len = read_u16(&mmap, &mut current_pos); + let name = String::from_utf8_lossy(&mmap[current_pos..current_pos + name_len as usize]) + .into_owned(); + current_pos += name_len as usize; - let bytecode_size = read_u32(&mut file, &mut u32_buf)?; - let bytecode_offset = read_u32(&mut file, &mut u32_buf)?; - - current_pos = current_pos + name_len as usize + u16_buf.len() + (u32_buf.len() * 2); + let bytecode_size = read_u32(&mmap, &mut current_pos); + let bytecode_offset = read_u32(&mmap, &mut current_pos); embedded_bytecode_data.index.insert( name, @@ -646,21 +617,25 @@ fn init_embedded_bytecode() -> std::io::Result<()> { } } - drop(embedded_bytecode_data); + mmap.advise(Advice::Random).unwrap(); + + SELF_MEM_MAP.write().unwrap().replace(mmap); Ok(()) } #[inline(always)] -fn read_u32(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { - file.read_exact(buf)?; - Ok(u32::from_le_bytes(*buf)) +fn read_u32(data: &[u8], pos: &mut usize) -> u32 { + let value = unsafe { std::ptr::read_unaligned(data.as_ptr().add(*pos) as *const u32) }; + *pos += 4; + u32::from_le(value) } #[inline(always)] -fn read_u16(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { - file.read_exact(buf)?; - Ok(u16::from_le_bytes(*buf)) +fn read_u16(data: &[u8], pos: &mut usize) -> u16 { + let value = unsafe { std::ptr::read_unaligned(data.as_ptr().add(*pos) as *const u16) }; + *pos += 2; + u16::from_le(value) } fn json_parse_string<'js>(ctx: Ctx<'js>, value: Value<'js>) -> Result> { From 3b1d2185262ec2ec663f5e377d4ca73bbbda7efd Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Tue, 2 Jul 2024 13:13:15 +0200 Subject: [PATCH 4/8] Use embed in compressed binary --- Makefile | 6 +- embed.mjs | 6 +- llrt/src/main.c | 4 + llrt_core/src/vm.rs | 194 +++++++++++++++++++++++++++++--------------- 4 files changed, 139 insertions(+), 71 deletions(-) diff --git a/Makefile b/Makefile index a78352c4a3..7cc13e6ef3 100644 --- a/Makefile +++ b/Makefile @@ -72,10 +72,10 @@ llrt-darwin-x86_64.zip: llrt-darwin-x64.zip define release_template release-${1}: | clean-js js cargo $$(BUILD_ARG) --target $$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1})) --features lambda -vv + node embed.mjs target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt ./pack target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/bootstrap - node embed.mjs target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/bootstrap target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/bootstrap - cargo $$(BUILD_ARG) --target $$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1})) --features lambda,uncompressed -vv - mv target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt llrt-container-${1} +# cargo $$(BUILD_ARG) --target $$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1})) --features lambda,uncompressed -vv +# mv target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt llrt-container-${1} @rm -rf llrt-lambda-${1}.zip zip -j llrt-lambda-${1}.zip target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/bootstrap diff --git a/embed.mjs b/embed.mjs index 420d331805..7aaf85f484 100644 --- a/embed.mjs +++ b/embed.mjs @@ -50,8 +50,8 @@ async function buildFileIndex(source, target, fileData) { for (let [name, data] of fileData) { dataBuffers.push(data); - if(name.startsWith("llrt-chunk-")){ - name = `${name}.js` + if (name.startsWith("llrt-chunk-")) { + name = `${name}.js`; } const nameLengthBuffer = uint16Buffer(name.length); @@ -88,8 +88,6 @@ async function buildFileIndex(source, target, fileData) { let allIndexBuffers = Buffer.concat(indexBuffers); - console.log("index buffer size: ", allIndexBuffers.length); - const finalBuffer = Buffer.concat([ sourceData, dataBuffer, diff --git a/llrt/src/main.c b/llrt/src/main.c index 87327c55f0..3aa16b8dfa 100644 --- a/llrt/src/main.c +++ b/llrt/src/main.c @@ -324,9 +324,13 @@ int main(int argc, char *argv[]) char mimallocReserveMemoryMb[16]; sprintf(mimallocReserveMemoryMb, "%iMiB", (int)(memorySize * memoryFactor)); + char outputFdStr[10]; + sprintf(outputFdStr, "%i", outputFd); + setenv("_START_TIME", startTimeStr, false); setenv("MIMALLOC_RESERVE_OS_MEMORY", mimallocReserveMemoryMb, false); setenv("MIMALLOC_LIMIT_OS_ALLOC", "1", false); + setenv("LLRT_MEMFD", outputFdStr, false); logInfo("Starting app\n"); diff --git a/llrt_core/src/vm.rs b/llrt_core/src/vm.rs index 24aa49e235..9167dec97c 100644 --- a/llrt_core/src/vm.rs +++ b/llrt_core/src/vm.rs @@ -1,15 +1,17 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 use std::{ + cell::RefCell, cmp::min, collections::{HashMap, HashSet}, env, ffi::CStr, fmt::Write, fs::File, - future::Future, - io::{self, Read}, + future::{poll_fn, Future}, + io::{self, Read, Seek, SeekFrom}, mem::size_of, + os::fd::{FromRawFd, OwnedFd, RawFd}, path::{Component, Path, PathBuf}, pin::pin, process::exit, @@ -18,11 +20,9 @@ use std::{ atomic::{AtomicUsize, Ordering}, Arc, Mutex, Once, RwLock, }, - sync::{Arc, Mutex}, task::Poll, }; -use memmap2::{Advice, Mmap}; use once_cell::sync::Lazy; pub use llrt_utils::{ctx::CtxExtension, error::ErrorExtensions}; @@ -47,6 +47,7 @@ use crate::{ console, crypto::SYSTEM_RANDOM, path::{dirname, join_path, resolve_path}, + timers::{self, poll_timers, TimerPoller}, }, }; @@ -64,15 +65,33 @@ use crate::{ #[derive(Default)] pub struct EmbeddedBytecodeData { - argv_0: String, + argv_0: Option, + mem_fd: Option, index: HashMap, } -pub static SELF_MEM_MAP: Lazy>> = Lazy::new(|| RwLock::new(None)); +#[cfg(unix)] +fn file_from_raw_fd(raw_fd: i32) -> std::io::Result { + #[cfg(not(unix))] + { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Unsupported on non-unix platforms", + )); + } + #[cfg(unix)] + { + let dup_fd = unsafe { libc::dup(raw_fd) }; + if dup_fd == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(unsafe { File::from_raw_fd(dup_fd) }) + } +} -// thread_local! { -// static SELF_FILE: RefCell> = const { RefCell::new(None) }; -// } +thread_local! { + static SELF_FILE: RefCell> = const { RefCell::new(None) }; +} pub static EMBEDDED_BYTECODE_DATA: Lazy> = Lazy::new(|| RwLock::new(EmbeddedBytecodeData::default())); @@ -272,35 +291,57 @@ where impl Loader for BinaryLoader { fn load<'js>(&mut self, ctx: &Ctx<'js>, name: &str) -> Result> { trace!("Loading module: {}", name); - let embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.read().unwrap(); - if let Some((bytes_size, bytes_pos)) = embedded_bytecode_data.index.get(name) { - trace!("Loading embedded module: {}", name); + { + let embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.read().unwrap(); + if let Some((bytes_size, bytes_pos)) = embedded_bytecode_data.index.get(name) { + trace!("Loading embedded module: {}", name); + let mut buf = SELF_FILE.with(|self_file| { + let mut buf = vec![0; *bytes_size]; + let mut borrow = self_file.borrow_mut(); + if let Some(file) = borrow.as_mut() { + file.seek(SeekFrom::Start((*bytes_pos) as u64))?; + file.read_exact(&mut buf)?; + } else { + let mut file = if let Some(mem_fd) = embedded_bytecode_data.mem_fd { + file_from_raw_fd(mem_fd) + } else if let Some(argv_0) = &embedded_bytecode_data.argv_0 { + File::open(argv_0) + } else { + unreachable!() + }?; + + file.seek(SeekFrom::Start((*bytes_pos) as u64))?; + file.read_exact(&mut buf)?; + + *borrow = Some(file); + } - if let Some(mmap) = SELF_MEM_MAP.read().unwrap().as_ref() { - let buf = &mmap[*bytes_pos..*bytes_pos + *bytes_size]; - return load_bytecode_module(name, buf); + Ok::<_, Error>(buf) + })?; + + return load_bytecode_module(ctx.clone(), &mut buf); } - } - drop(embedded_bytecode_data); - let path = PathBuf::from(name); - let mut bytes: &[u8] = &std::fs::read(path)?; + drop(embedded_bytecode_data); + let path = PathBuf::from(name); + let mut bytes: &[u8] = &std::fs::read(path)?; - if name.ends_with(".lrt") { - trace!("Loading binary module: {}", name); - return load_bytecode_module(ctx, name, bytes); - } - if bytes.starts_with(b"#!") { - bytes = bytes.splitn(2, |&c| c == b'\n').nth(1).unwrap_or(bytes); + if name.ends_with(".lrt") { + trace!("Loading binary module: {}", name); + return load_bytecode_module(ctx.clone(), bytes); + } + if bytes.starts_with(b"#!") { + bytes = bytes.splitn(2, |&c| c == b'\n').nth(1).unwrap_or(bytes); + } + Module::declare(ctx.clone(), name, bytes) } - Module::declare(ctx, name, bytes) + .map_err(|err| { + trace!("Failed to load module: {}", err); + err + }) } } -pub fn load_bytecode_module<'js>( - ctx: Ctx<'js>, - _name: &str, - buf: &[u8], -) -> Result> { +pub fn load_bytecode_module<'js>(ctx: Ctx<'js>, buf: &[u8]) -> Result> { let bytes = get_module_bytecode(buf)?; unsafe { Module::load(ctx, &bytes) } } @@ -554,55 +595,84 @@ impl Vm { fn init_embedded_bytecode() -> std::io::Result<()> { let argv_0 = env::args().next().expect("Failed to get argv0"); - let file = File::open(&argv_0)?; - - let mmap = unsafe { Mmap::map(&file)? }; - mmap.advise(Advice::Sequential).unwrap(); + let mut embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.write().unwrap(); - let total_file_size = mmap.len(); + let mut file = if let Ok(fd_string) = env::var("LLRT_MEMFD") { + let mem_fd: i32 = fd_string + .parse() + .map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, "Invalid memfd"))?; + embedded_bytecode_data.mem_fd = Some(mem_fd); + file_from_raw_fd(mem_fd)? + } else { + let file = File::open(&argv_0)?; + embedded_bytecode_data.argv_0 = Some(argv_0); + file + }; + + let mut embedded_bytecode_signature_buf = [0; BYTECODE_EMBEDDED_SIGNATURE.len()]; + let meta = file.metadata()?; + let total_file_size = meta.len(); let signature_len = BYTECODE_EMBEDDED_SIGNATURE.len(); - let signed_signature_len = signature_len as isize; + let signed_signature_len = signature_len as i64; + file.seek(SeekFrom::End(-signed_signature_len))?; + file.read_exact(&mut embedded_bytecode_signature_buf)?; - if &mmap[(total_file_size as isize - signed_signature_len) as usize..] - != BYTECODE_EMBEDDED_SIGNATURE - { + if embedded_bytecode_signature_buf != BYTECODE_EMBEDDED_SIGNATURE { return Ok(()); } - #[repr(C)] struct EmbeddedMeta { package_count: u32, bytecode_pos: u32, package_index_pos: u32, } + impl EmbeddedMeta { + fn read(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { + Ok(EmbeddedMeta { + package_count: read_u32(file, buf)?, + bytecode_pos: read_u32(file, buf)?, + package_index_pos: read_u32(file, buf)?, + }) + } + } + let embedded_meta_size = size_of::(); + let meta_and_signature_size = embedded_meta_size + signature_len; - let meta_start = (total_file_size as isize - meta_and_signature_size as isize) as usize; - let meta_end = (total_file_size as isize - signature_len as isize) as usize; + file.seek(SeekFrom::Current(-(meta_and_signature_size as i64)))?; - let embedded_metadata: EmbeddedMeta = - unsafe { std::ptr::read(mmap[meta_start..meta_end].as_ptr() as *const _) }; + let mut u32_buf = [0; size_of::()]; + let mut u16_buf = [0; size_of::()]; + + let embedded_metadata = EmbeddedMeta::read(&mut file, &mut u32_buf)?; + + file.seek(SeekFrom::Current( + -(total_file_size as i64 + - (embedded_metadata.package_index_pos as i64 + signed_signature_len)), + ))?; - let mut embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.write().unwrap(); embedded_bytecode_data .index .reserve(embedded_metadata.package_count as usize); - embedded_bytecode_data.argv_0 = argv_0; let mut current_pos = embedded_metadata.package_index_pos as usize; + let end_pos = total_file_size as usize - meta_and_signature_size; loop { - let name_len = read_u16(&mmap, &mut current_pos); - let name = String::from_utf8_lossy(&mmap[current_pos..current_pos + name_len as usize]) - .into_owned(); - current_pos += name_len as usize; + let name_len = read_u16(&mut file, &mut u16_buf)?; + let mut buf: Vec = vec![0; name_len as usize]; + file.read_exact(&mut buf)?; + + let name = unsafe { String::from_utf8_unchecked(buf) }; - let bytecode_size = read_u32(&mmap, &mut current_pos); - let bytecode_offset = read_u32(&mmap, &mut current_pos); + let bytecode_size = read_u32(&mut file, &mut u32_buf)?; + let bytecode_offset = read_u32(&mut file, &mut u32_buf)?; + + current_pos = current_pos + name_len as usize + u16_buf.len() + (u32_buf.len() * 2); embedded_bytecode_data.index.insert( name, @@ -617,25 +687,21 @@ fn init_embedded_bytecode() -> std::io::Result<()> { } } - mmap.advise(Advice::Random).unwrap(); - - SELF_MEM_MAP.write().unwrap().replace(mmap); + drop(embedded_bytecode_data); Ok(()) } #[inline(always)] -fn read_u32(data: &[u8], pos: &mut usize) -> u32 { - let value = unsafe { std::ptr::read_unaligned(data.as_ptr().add(*pos) as *const u32) }; - *pos += 4; - u32::from_le(value) +fn read_u32(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { + file.read_exact(buf)?; + Ok(u32::from_le_bytes(*buf)) } #[inline(always)] -fn read_u16(data: &[u8], pos: &mut usize) -> u16 { - let value = unsafe { std::ptr::read_unaligned(data.as_ptr().add(*pos) as *const u16) }; - *pos += 2; - u16::from_le(value) +fn read_u16(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { + file.read_exact(buf)?; + Ok(u16::from_le_bytes(*buf)) } fn json_parse_string<'js>(ctx: Ctx<'js>, value: Value<'js>) -> Result> { From 37d59e57e3d5cbe912af15a951b6b928a0dc71ca Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Tue, 2 Jul 2024 13:20:33 +0200 Subject: [PATCH 5/8] Remove mut --- llrt_core/src/vm.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llrt_core/src/vm.rs b/llrt_core/src/vm.rs index 9167dec97c..d8e9f27bed 100644 --- a/llrt_core/src/vm.rs +++ b/llrt_core/src/vm.rs @@ -11,7 +11,7 @@ use std::{ future::{poll_fn, Future}, io::{self, Read, Seek, SeekFrom}, mem::size_of, - os::fd::{FromRawFd, OwnedFd, RawFd}, + os::fd::FromRawFd, path::{Component, Path, PathBuf}, pin::pin, process::exit, @@ -295,7 +295,7 @@ impl Loader for BinaryLoader { let embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.read().unwrap(); if let Some((bytes_size, bytes_pos)) = embedded_bytecode_data.index.get(name) { trace!("Loading embedded module: {}", name); - let mut buf = SELF_FILE.with(|self_file| { + let buf = SELF_FILE.with(|self_file| { let mut buf = vec![0; *bytes_size]; let mut borrow = self_file.borrow_mut(); if let Some(file) = borrow.as_mut() { @@ -319,7 +319,7 @@ impl Loader for BinaryLoader { Ok::<_, Error>(buf) })?; - return load_bytecode_module(ctx.clone(), &mut buf); + return load_bytecode_module(ctx.clone(), &buf); } drop(embedded_bytecode_data); let path = PathBuf::from(name); From 2ca1af4eec5d72182cf511d04371c1e90b0ed733 Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Sat, 6 Jul 2024 12:29:41 +0200 Subject: [PATCH 6/8] Update bytecode --- Makefile | 1 + embed.mjs | 45 +++------- llrt/src/main.c | 2 +- llrt_core/src/vm.rs | 213 ++++++++++++++++++++++---------------------- 4 files changed, 120 insertions(+), 141 deletions(-) diff --git a/Makefile b/Makefile index 7cc13e6ef3..1eeff3afa4 100644 --- a/Makefile +++ b/Makefile @@ -74,6 +74,7 @@ release-${1}: | clean-js js cargo $$(BUILD_ARG) --target $$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1})) --features lambda -vv node embed.mjs target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt ./pack target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/bootstrap + # cargo $$(BUILD_ARG) --target $$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1})) --features lambda,uncompressed -vv # mv target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt llrt-container-${1} @rm -rf llrt-lambda-${1}.zip diff --git a/embed.mjs b/embed.mjs index 7aaf85f484..6cf725ea81 100644 --- a/embed.mjs +++ b/embed.mjs @@ -44,57 +44,36 @@ async function buildFileIndex(source, target, fileData) { }; const sourceData = await fs.readFile(source); - const dataBuffers = []; - let offset = 0; - let indexBuffers = []; - for (let [name, data] of fileData) { - dataBuffers.push(data); + const cacheBuffers = []; + for (let [name, data] of fileData) { if (name.startsWith("llrt-chunk-")) { name = `${name}.js`; } - const nameLengthBuffer = uint16Buffer(name.length); const nameBuffer = Buffer.from(name); + const bytecodeSizeBuffer = uint32Buffer(data.length); - const bytecodeOffsetBuffer = uint32Buffer(offset); - - indexBuffers.push( - Buffer.concat([ - nameLengthBuffer, - nameBuffer, - bytecodeSizeBuffer, - bytecodeOffsetBuffer, - ]) - ); - offset += data.length; + cacheBuffers.push( + Buffer.concat([nameLengthBuffer, nameBuffer, bytecodeSizeBuffer, data]) + ); } - const dataBuffer = Buffer.concat(dataBuffers); - const packageCount = fileData.length; - const bytecodePosition = sourceData.length; - const packageIndexPosition = sourceData.length + dataBuffer.length; + const cachePosition = sourceData.length; - //[u32 package_count][u32 bytecode_pos][u32 package_index_pos] const metadataBuffer = Buffer.concat([ uint32Buffer(packageCount), - uint32Buffer(bytecodePosition), - uint32Buffer(packageIndexPosition), + uint32Buffer(cachePosition), + Buffer.from("lrt"), ]); - const signatureBuffer = Buffer.from("lrt"); + const cacheBuffer = Buffer.concat(cacheBuffers); - let allIndexBuffers = Buffer.concat(indexBuffers); + console.log("Embedded size:", cacheBuffer.length / 1024, "kB"); - const finalBuffer = Buffer.concat([ - sourceData, - dataBuffer, - allIndexBuffers, - metadataBuffer, - signatureBuffer, - ]); + const finalBuffer = Buffer.concat([sourceData, cacheBuffer, metadataBuffer]); await fs.writeFile(target, finalBuffer); await fs.chmod(target, 0o755); diff --git a/llrt/src/main.c b/llrt/src/main.c index 3aa16b8dfa..542c062bd2 100644 --- a/llrt/src/main.c +++ b/llrt/src/main.c @@ -330,7 +330,7 @@ int main(int argc, char *argv[]) setenv("_START_TIME", startTimeStr, false); setenv("MIMALLOC_RESERVE_OS_MEMORY", mimallocReserveMemoryMb, false); setenv("MIMALLOC_LIMIT_OS_ALLOC", "1", false); - setenv("LLRT_MEMFD", outputFdStr, false); + setenv("LLRT_MEM_FD", outputFdStr, false); logInfo("Starting app\n"); diff --git a/llrt_core/src/vm.rs b/llrt_core/src/vm.rs index d8e9f27bed..369c9d48b9 100644 --- a/llrt_core/src/vm.rs +++ b/llrt_core/src/vm.rs @@ -1,7 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 use std::{ - cell::RefCell, cmp::min, collections::{HashMap, HashSet}, env, @@ -64,10 +63,57 @@ use crate::{ }; #[derive(Default)] -pub struct EmbeddedBytecodeData { - argv_0: Option, - mem_fd: Option, - index: HashMap, +pub struct BytecodeCache { + data: Box<[u8]>, + map: HashMap, Arc<[u8]>>, +} + +impl BytecodeCache { + pub fn new(data: Vec, package_count: usize) -> Self { + let data = data.into_boxed_slice(); + let mut cache = BytecodeCache { + data, + map: HashMap::with_capacity(package_count), + }; + let mut offset = 0; + let data = &cache.data; + let mut entries = Vec::new(); + + loop { + let name_len_start = offset; + let name_len_end = offset + size_of::(); + let name_len = + u16_from_le_byte_slice_unchecked(&data[name_len_start..name_len_end]) as usize; + + let bytecode_size_start = name_len_end + name_len; + + let name = + unsafe { std::str::from_utf8_unchecked(&data[name_len_end..bytecode_size_start]) }; + + let bytecode_start = bytecode_size_start + size_of::(); + + let bytecode_size = + u32_from_le_byte_slice_unchecked(&data[bytecode_size_start..bytecode_start]) + as usize; + + let bytecode_end = bytecode_start + bytecode_size; + + entries.push((name.to_owned(), bytecode_start..bytecode_end)); + + offset = bytecode_start + bytecode_size; + + if offset >= data.len() { + break; + } + } + + for (name, range) in entries { + let key = Arc::from(name); + let value = Arc::from(&cache.data[range]); + cache.map.insert(key, value); + } + cache + } } #[cfg(unix)] @@ -89,12 +135,8 @@ fn file_from_raw_fd(raw_fd: i32) -> std::io::Result { } } -thread_local! { - static SELF_FILE: RefCell> = const { RefCell::new(None) }; -} - -pub static EMBEDDED_BYTECODE_DATA: Lazy> = - Lazy::new(|| RwLock::new(EmbeddedBytecodeData::default())); +pub static EMBEDDED_BYTECODE_DATA: Lazy> = + Lazy::new(|| RwLock::new(BytecodeCache::default())); #[inline] pub fn uncompressed_size(input: &[u8]) -> StdResult<(usize, &[u8]), io::Error> { @@ -172,7 +214,7 @@ impl Resolver for BinaryResolver { let embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.read().unwrap(); - if embedded_bytecode_data.index.contains_key(name) { + if embedded_bytecode_data.map.contains_key(name) { return Ok(name.to_string()); } @@ -202,11 +244,11 @@ impl Resolver for BinaryResolver { trace!("Normalized path: {}, key: {}", normalized_path, cache_key); - if embedded_bytecode_data.index.contains_key(cache_key) { + if embedded_bytecode_data.map.contains_key(cache_key) { return Ok(cache_key.to_string()); } - if embedded_bytecode_data.index.contains_key(base) { + if embedded_bytecode_data.map.contains_key(base) { normalized_path = name; if Path::new(normalized_path).exists() { return Ok(normalized_path.to_string()); @@ -293,33 +335,9 @@ impl Loader for BinaryLoader { trace!("Loading module: {}", name); { let embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.read().unwrap(); - if let Some((bytes_size, bytes_pos)) = embedded_bytecode_data.index.get(name) { + if let Some(bytes) = embedded_bytecode_data.map.get(name) { trace!("Loading embedded module: {}", name); - let buf = SELF_FILE.with(|self_file| { - let mut buf = vec![0; *bytes_size]; - let mut borrow = self_file.borrow_mut(); - if let Some(file) = borrow.as_mut() { - file.seek(SeekFrom::Start((*bytes_pos) as u64))?; - file.read_exact(&mut buf)?; - } else { - let mut file = if let Some(mem_fd) = embedded_bytecode_data.mem_fd { - file_from_raw_fd(mem_fd) - } else if let Some(argv_0) = &embedded_bytecode_data.argv_0 { - File::open(argv_0) - } else { - unreachable!() - }?; - - file.seek(SeekFrom::Start((*bytes_pos) as u64))?; - file.read_exact(&mut buf)?; - - *borrow = Some(file); - } - - Ok::<_, Error>(buf) - })?; - - return load_bytecode_module(ctx.clone(), &buf); + return load_bytecode_module(ctx.clone(), bytes); } drop(embedded_bytecode_data); let path = PathBuf::from(name); @@ -593,26 +611,34 @@ impl Vm { } } +fn u32_from_le_byte_slice_unchecked(bytes: &[u8]) -> u32 { + (bytes[0] as u32) + | ((bytes[1] as u32) << 8) + | ((bytes[2] as u32) << 16) + | ((bytes[3] as u32) << 24) +} + +fn u16_from_le_byte_slice_unchecked(bytes: &[u8]) -> u16 { + (bytes[0] as u16) | ((bytes[1] as u16) << 8) +} + fn init_embedded_bytecode() -> std::io::Result<()> { + trace!("Loading embedded bytecode"); let argv_0 = env::args().next().expect("Failed to get argv0"); let mut embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.write().unwrap(); - let mut file = if let Ok(fd_string) = env::var("LLRT_MEMFD") { - let mem_fd: i32 = fd_string - .parse() - .map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, "Invalid memfd"))?; - - embedded_bytecode_data.mem_fd = Some(mem_fd); - file_from_raw_fd(mem_fd)? + let mut file = if let Ok(fd_string) = env::var("LLRT_MEM_FD") { + let mem_fd: i32 = fd_string.parse().map_err(|_| { + std::io::Error::new(std::io::ErrorKind::Other, "Invalid bytecode-cache fd") + })?; + file_from_raw_fd(mem_fd) } else { - let file = File::open(&argv_0)?; - embedded_bytecode_data.argv_0 = Some(argv_0); - file - }; + File::open(argv_0) + }?; let mut embedded_bytecode_signature_buf = [0; BYTECODE_EMBEDDED_SIGNATURE.len()]; - let meta = file.metadata()?; - let total_file_size = meta.len(); + let file_metadata = file.metadata()?; + let total_file_size = file_metadata.len(); let signature_len = BYTECODE_EMBEDDED_SIGNATURE.len(); let signed_signature_len = signature_len as i64; file.seek(SeekFrom::End(-signed_signature_len))?; @@ -625,15 +651,15 @@ fn init_embedded_bytecode() -> std::io::Result<()> { struct EmbeddedMeta { package_count: u32, bytecode_pos: u32, - package_index_pos: u32, } impl EmbeddedMeta { - fn read(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { + fn read(file: &mut File) -> std::io::Result { + let mut buf = [0; size_of::() * 2]; + file.read_exact(&mut buf)?; Ok(EmbeddedMeta { - package_count: read_u32(file, buf)?, - bytecode_pos: read_u32(file, buf)?, - package_index_pos: read_u32(file, buf)?, + package_count: u32_from_le_byte_slice_unchecked(&buf[..size_of::()]), + bytecode_pos: u32_from_le_byte_slice_unchecked(&buf[size_of::()..]), }) } } @@ -644,65 +670,38 @@ fn init_embedded_bytecode() -> std::io::Result<()> { file.seek(SeekFrom::Current(-(meta_and_signature_size as i64)))?; - let mut u32_buf = [0; size_of::()]; - let mut u16_buf = [0; size_of::()]; - - let embedded_metadata = EmbeddedMeta::read(&mut file, &mut u32_buf)?; + let embedded_metadata = EmbeddedMeta::read(&mut file)?; file.seek(SeekFrom::Current( - -(total_file_size as i64 - - (embedded_metadata.package_index_pos as i64 + signed_signature_len)), + -(total_file_size as i64 - (embedded_metadata.bytecode_pos as i64 + signed_signature_len)), ))?; - embedded_bytecode_data - .index - .reserve(embedded_metadata.package_count as usize); - - let mut current_pos = embedded_metadata.package_index_pos as usize; + let total_cache_size = total_file_size as usize + - embedded_metadata.bytecode_pos as usize + - meta_and_signature_size; - let end_pos = total_file_size as usize - meta_and_signature_size; + let mut buf = vec![0; total_cache_size]; + trace!("Loading bytecode cache of {} kB", total_cache_size / 1024); + file.read_exact(&mut buf)?; - loop { - let name_len = read_u16(&mut file, &mut u16_buf)?; - let mut buf: Vec = vec![0; name_len as usize]; - file.read_exact(&mut buf)?; + let bytecode_cache = BytecodeCache::new(buf, embedded_metadata.package_count as usize); - let name = unsafe { String::from_utf8_unchecked(buf) }; - - let bytecode_size = read_u32(&mut file, &mut u32_buf)?; - let bytecode_offset = read_u32(&mut file, &mut u32_buf)?; - - current_pos = current_pos + name_len as usize + u16_buf.len() + (u32_buf.len() * 2); - - embedded_bytecode_data.index.insert( - name, - ( - bytecode_size as usize, - embedded_metadata.bytecode_pos as usize + bytecode_offset as usize, - ), - ); - - if current_pos == end_pos { - break; - } - } - - drop(embedded_bytecode_data); + *embedded_bytecode_data = bytecode_cache; Ok(()) } -#[inline(always)] -fn read_u32(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { - file.read_exact(buf)?; - Ok(u32::from_le_bytes(*buf)) -} +// #[inline(always)] +// fn read_u32(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { +// file.read_exact(buf)?; +// Ok(u32::from_le_bytes(*buf)) +// } -#[inline(always)] -fn read_u16(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { - file.read_exact(buf)?; - Ok(u16::from_le_bytes(*buf)) -} +// #[inline(always)] +// fn read_u16(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { +// file.read_exact(buf)?; +// Ok(u16::from_le_bytes(*buf)) +// } fn json_parse_string<'js>(ctx: Ctx<'js>, value: Value<'js>) -> Result> { let bytes = get_bytes(&ctx, value)?; @@ -814,8 +813,8 @@ fn init(ctx: &Ctx<'_>, module_names: HashSet<&'static str>) -> Result<()> { || EMBEDDED_BYTECODE_DATA .read() .unwrap() - .index - .contains_key(&specifier) + .map + .contains_key(&Arc::from(specifier.as_str())) || specifier.starts_with('/') { specifier From b1e5fd37879700f652ae2321202086a29a4d3246 Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Sun, 4 Aug 2024 22:55:54 +0200 Subject: [PATCH 7/8] Use external bytecode file --- Makefile | 9 ++--- embed.mjs | 21 +++++++---- llrt/src/main.c | 87 +++++++++++++++++++++++++++++---------------- llrt_core/src/vm.rs | 5 +++ pack | 24 +++++++------ 5 files changed, 95 insertions(+), 51 deletions(-) diff --git a/Makefile b/Makefile index 1eeff3afa4..2b94ae8eec 100644 --- a/Makefile +++ b/Makefile @@ -72,9 +72,10 @@ llrt-darwin-x86_64.zip: llrt-darwin-x64.zip define release_template release-${1}: | clean-js js cargo $$(BUILD_ARG) --target $$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1})) --features lambda -vv - node embed.mjs target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt - ./pack target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/bootstrap - + + node embed.mjs target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/bytecode -r + ./pack target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/bootstrap target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/bytecode + # cargo $$(BUILD_ARG) --target $$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1})) --features lambda,uncompressed -vv # mv target/$$(TARGET_linux_$$(RELEASE_ARCH_NAME_${1}))/release/llrt llrt-container-${1} @rm -rf llrt-lambda-${1}.zip @@ -131,7 +132,7 @@ run: export _EXIT_ITERATIONS = 1 run: export JS_MINIFY = 0 run: export RUST_LOG = llrt=trace run: export _HANDLER = index.handler -run: +run: cargo run -vv run-ssr: export AWS_LAMBDA_RUNTIME_API = localhost:3000 diff --git a/embed.mjs b/embed.mjs index 6cf725ea81..a9d61e5f56 100644 --- a/embed.mjs +++ b/embed.mjs @@ -30,7 +30,7 @@ async function readFileData(files) { ); } -async function buildFileIndex(source, target, fileData) { +async function buildFileIndex(source, target, fileData, writeRaw) { const uint32Buffer = (length) => { const buffer = Buffer.alloc(4); buffer.writeUInt32LE(length); @@ -61,7 +61,7 @@ async function buildFileIndex(source, target, fileData) { } const packageCount = fileData.length; - const cachePosition = sourceData.length; + const cachePosition = writeRaw ? 0 : sourceData.length; const metadataBuffer = Buffer.concat([ uint32Buffer(packageCount), @@ -73,14 +73,21 @@ async function buildFileIndex(source, target, fileData) { console.log("Embedded size:", cacheBuffer.length / 1024, "kB"); - const finalBuffer = Buffer.concat([sourceData, cacheBuffer, metadataBuffer]); + const finalBuffer = Buffer.concat([ + ...(writeRaw ? [] : [sourceData]), + cacheBuffer, + metadataBuffer, + ]); await fs.writeFile(target, finalBuffer); - await fs.chmod(target, 0o755); + if (!writeRaw) { + await fs.chmod(target, 0o755); + } } -const source = process.argv[2]; -const target = process.argv[3]; +const [source, target, rawArg] = process.argv.slice(2); +const writeRaw = rawArg == "-r" || rawArg == "--raw"; + if (!source || !target) { console.error( `No source or target specified, use:\n${path.basename(process.argv[0])} ${path.basename(process.argv[1])} {input_target} {output_target}` @@ -92,4 +99,4 @@ console.log("Reading files..."); const files = await readFiles(); console.log("Reading file data..."); const filesContents = await readFileData(files); -await buildFileIndex(source, target, filesContents); +await buildFileIndex(source, target, filesContents, writeRaw); diff --git a/llrt/src/main.c b/llrt/src/main.c index 542c062bd2..c34a7f62a1 100644 --- a/llrt/src/main.c +++ b/llrt/src/main.c @@ -116,7 +116,7 @@ static uint32_t calculateSum(uint32_t *array, uint8_t size) return sum; } -static double micro_seconds() +static double microSeconds() { struct timeval tv; gettimeofday(&tv, NULL); @@ -128,6 +128,9 @@ typedef struct uint32_t srcSize; uint32_t dstSize; uint32_t id; + uint32_t extraFd; + uint32_t extraDataSize; + const void *extraData; const void *inputBuffer; const void *outputBuffer; } DecompressThreadArgs; @@ -135,6 +138,12 @@ typedef struct static void *decompressPartial(void *arg) { DecompressThreadArgs *args = (DecompressThreadArgs *)arg; + + if (args->extraData != NULL) + { + write(args->extraFd, args->extraData, args->extraDataSize); + } + size_t srcSize = args->srcSize; size_t dstSize = args->dstSize; @@ -148,6 +157,20 @@ static void *decompressPartial(void *arg) return (void *)0; } +typedef struct +{ + int fd; + char *data; + size_t size; +} WriteFdThreadArgs; + +void *writeFdThread(void *arg) +{ + WriteFdThreadArgs *args = (WriteFdThreadArgs *)arg; + write(args->fd, args->data, args->size); + return (void *)0; +} + extern char **environ; static void readData( @@ -174,7 +197,7 @@ static void readData( *compressedData = (uint8_t *)&data[dataOffset]; } -static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int outputFd) +static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int outputFd, int extraFd) { #include "data.c" @@ -184,9 +207,16 @@ static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int uint32_t *outputSizes; uint32_t inputOffset = 0; uint32_t outputOffset = 0; + uint32_t extraDataSize = *(uint32_t *)extra; + char *uncompressed; uint8_t *compressedData; + char *extraData = (char *)extra + sizeof(uint32_t); + pthread_t writeFdThreadId; + + logInfo("Extra data size %lu bytes \n", extraDataSize); + pthread_t threads[parts]; if (parts > 1) @@ -219,6 +249,12 @@ static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int args[i].srcSize = inputSizes[i]; args[i].dstSize = outputSizes[i]; args[i].id = i; + if (i == 0 && extraDataSize > 0) + { + args[i].extraFd = extraFd; + args[i].extraData = extraData; + args[i].extraDataSize = extraDataSize; + } inputOffset += inputSizes[i]; outputOffset += outputSizes[i]; if (parts > 1) @@ -243,7 +279,7 @@ static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int } } - *uncompressedData = uncompressed; + *uncompressedData = uncompressed; } int main(int argc, char *argv[]) @@ -255,7 +291,7 @@ int main(int argc, char *argv[]) char *tmpAppname = strrchr(argv[0], '/'); char *appname = tmpAppname ? ++tmpAppname : argv[0]; - double t0 = micro_seconds(); + double t0 = microSeconds(); int outputFd = memfd_create_syscall(appname, 0); if (outputFd == -1) @@ -263,12 +299,18 @@ int main(int argc, char *argv[]) err(1, "Could not create memfd"); } + int extraFd = memfd_create_syscall("__bootstrap_extra", 0); + if (extraFd == -1) + { + err(1, "Could not create memfd"); + } + char *uncompressedData; uint32_t uncompressedSize; - decompress(&uncompressedData, &uncompressedSize, outputFd); + decompress(&uncompressedData, &uncompressedSize, outputFd, extraFd); - double t1 = micro_seconds(); + double t1 = microSeconds(); logInfo("Runtime starting\n"); logInfo("Extraction time: %10.4f ms\n", (t1 - t0) / 1000.0); @@ -277,30 +319,15 @@ int main(int argc, char *argv[]) err(1, "Failed to unmap memory"); } - double t2 = micro_seconds(); + if () + { + err(1, "Failed to unmap memory"); + } + + double t2 = microSeconds(); logInfo("Extraction + write time: %10.4f ms\n", (t2 - t0) / 1000.0); - // char **new_argv = malloc((size_t)(argc + 1) * sizeof *new_argv); - // for (uint8_t i = 0; i < argc; ++i) - // { - // if (i == 0) - // { - // size_t length = strlen(appname) + 2; - // new_argv[i] = malloc(length); - // memcpy(new_argv[i], "/", 1); - // memcpy(new_argv[i] + 1, appname, length); - // setenv("_", new_argv[i], true); - // } - // else - // { - // size_t length = strlen(argv[i]) + 1; - // new_argv[i] = malloc(length); - // memcpy(new_argv[i], argv[i], length); - // } - // } - // new_argv[argc] = NULL; - - unsigned long startTime = (unsigned long)(micro_seconds() / 1000.0); + unsigned long startTime = (unsigned long)(microSeconds() / 1000.0); char startTimeStr[16]; sprintf(startTimeStr, "%lu", startTime); @@ -325,7 +352,7 @@ int main(int argc, char *argv[]) sprintf(mimallocReserveMemoryMb, "%iMiB", (int)(memorySize * memoryFactor)); char outputFdStr[10]; - sprintf(outputFdStr, "%i", outputFd); + sprintf(outputFdStr, "%i", extraFd); setenv("_START_TIME", startTimeStr, false); setenv("MIMALLOC_RESERVE_OS_MEMORY", mimallocReserveMemoryMb, false); @@ -341,4 +368,4 @@ int main(int argc, char *argv[]) err(1, "fexecve failed"); return 1; -} \ No newline at end of file +} diff --git a/llrt_core/src/vm.rs b/llrt_core/src/vm.rs index 369c9d48b9..81053b0923 100644 --- a/llrt_core/src/vm.rs +++ b/llrt_core/src/vm.rs @@ -20,6 +20,7 @@ use std::{ Arc, Mutex, Once, RwLock, }, task::Poll, + time::Instant, }; use once_cell::sync::Lazy; @@ -623,6 +624,7 @@ fn u16_from_le_byte_slice_unchecked(bytes: &[u8]) -> u16 { } fn init_embedded_bytecode() -> std::io::Result<()> { + let now = Instant::now(); trace!("Loading embedded bytecode"); let argv_0 = env::args().next().expect("Failed to get argv0"); let mut embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.write().unwrap(); @@ -631,6 +633,7 @@ fn init_embedded_bytecode() -> std::io::Result<()> { let mem_fd: i32 = fd_string.parse().map_err(|_| { std::io::Error::new(std::io::ErrorKind::Other, "Invalid bytecode-cache fd") })?; + trace!("Using raw memfd"); file_from_raw_fd(mem_fd) } else { File::open(argv_0) @@ -688,6 +691,8 @@ fn init_embedded_bytecode() -> std::io::Result<()> { *embedded_bytecode_data = bytecode_cache; + trace!("Building cache took: {:?}", now.elapsed()); + Ok(()) } diff --git a/pack b/pack index d0beac0549..308e8d8ba8 100755 --- a/pack +++ b/pack @@ -4,19 +4,15 @@ set -e -if [ "$#" -ne 2 ]; then +if [ "$#" -lt 2 ]; then echo "Error: Two file arguments are required." - echo "Usage: $0 input_file output_file" + echo "Usage: $0 input_file output_file {uncompressed_data}" exit 1 fi - -# Set the number of parts to split the file into -num_parts=2 -data_size=$(wc -c < "$1") - srcfile="$1" dstfile="$2" +extrafile="$3" # Check if the bytes start with \x7fELF if [[ $(hexdump -n 4 -e '4/1 "%02x"' $srcfile) != "7f454c46" ]]; then @@ -37,6 +33,12 @@ else exit 1 fi +# Set the number of parts to split the file into +num_parts=2 +data_size=$(wc -c < "$srcfile") +extra_data_size=$(wc -c < "$extrafile") +extra_data_hexed=$(hexdump -v -e '/1 " %02x"' "$extrafile" | sed 's/ /\\\\x/g') + # Create a temporary directory to store the parts temp_folder=$(mktemp -d) @@ -70,7 +72,7 @@ for file in ${temp_folder}/*; do rm "${file}" done -function little_endian_hex() +function to_u32_le_hex() { local hex_code="" local number @@ -86,11 +88,13 @@ function little_endian_hex() echo "$hex_code" } -INPUT_SIZES_STR=$(little_endian_hex ${input_sizes[@]}) -OUTPUT_SIZES_STR=$(little_endian_hex ${output_sizes[@]}) +INPUT_SIZES_STR=$(to_u32_le_hex ${input_sizes[@]}) +OUTPUT_SIZES_STR=$(to_u32_le_hex ${output_sizes[@]}) +EXTRA_SIZE_STR=$(to_u32_le_hex ${extra_data_size}) cat < ${temp_folder}/data.c const char *data = "$(printf "\\\x%02x" $num_parts)${INPUT_SIZES_STR}${OUTPUT_SIZES_STR}$(printf "$compressed_data")"; +const char *extra = "${EXTRA_SIZE_STR}$(printf "$extra_data_hexed")"; EOF cp "$1" "${1}.bak" From d00dbf043317e4537da73c79ecc7fea45f074183 Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Wed, 7 Aug 2024 22:43:47 +0200 Subject: [PATCH 8/8] use separate file with mmap --- Cargo.lock | 16 ++++- embed.mjs | 36 +++++++--- llrt/src/main.c | 37 +++++----- llrt_core/Cargo.toml | 1 + llrt_core/build.rs | 21 +----- llrt_core/src/vm.rs | 159 ++++++++++++++++++++----------------------- 6 files changed, 136 insertions(+), 134 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23c7a2cba5..9fbebb4adc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1530,7 +1530,7 @@ dependencies = [ "gix-object", "gix-traverse", "itoa", - "memmap2", + "memmap2 0.5.10", "smallvec", "thiserror", ] @@ -1610,7 +1610,7 @@ dependencies = [ "gix-path", "gix-tempfile", "gix-traverse", - "memmap2", + "memmap2 0.5.10", "parking_lot", "smallvec", "thiserror", @@ -1696,7 +1696,7 @@ dependencies = [ "gix-path", "gix-tempfile", "gix-validate", - "memmap2", + "memmap2 0.5.10", "nom", "thiserror", ] @@ -2589,6 +2589,7 @@ dependencies = [ "llrt_modules", "llrt_utils", "md-5", + "memmap2 0.9.4", "nanoid", "once_cell", "phf", @@ -2706,6 +2707,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memmap2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" +dependencies = [ + "libc", +] + [[package]] name = "minimal-lexical" version = "0.2.1" diff --git a/embed.mjs b/embed.mjs index a9d61e5f56..8fd9ff41c6 100644 --- a/embed.mjs +++ b/embed.mjs @@ -44,7 +44,9 @@ async function buildFileIndex(source, target, fileData, writeRaw) { }; const sourceData = await fs.readFile(source); - const cacheBuffers = []; + const packageIndexList = []; + let offset = 0; + const bytecodeData = []; for (let [name, data] of fileData) { if (name.startsWith("llrt-chunk-")) { @@ -54,31 +56,45 @@ async function buildFileIndex(source, target, fileData, writeRaw) { const nameBuffer = Buffer.from(name); const bytecodeSizeBuffer = uint32Buffer(data.length); - - cacheBuffers.push( - Buffer.concat([nameLengthBuffer, nameBuffer, bytecodeSizeBuffer, data]) + const bytecodeOffsetBuffer = uint32Buffer(offset); + + packageIndexList.push( + Buffer.concat([ + nameLengthBuffer, + nameBuffer, + bytecodeOffsetBuffer, + bytecodeSizeBuffer, + ]) ); + + offset += data.length; + bytecodeData.push(data); } + const allBytecodeData = Buffer.concat(bytecodeData); + const packageCount = fileData.length; - const cachePosition = writeRaw ? 0 : sourceData.length; + const bytecodePosition = writeRaw ? 0 : sourceData.length; + const packageIndexPosition = bytecodePosition + allBytecodeData.length; const metadataBuffer = Buffer.concat([ uint32Buffer(packageCount), - uint32Buffer(cachePosition), + uint32Buffer(bytecodePosition), + uint32Buffer(packageIndexPosition), Buffer.from("lrt"), ]); - const cacheBuffer = Buffer.concat(cacheBuffers); - - console.log("Embedded size:", cacheBuffer.length / 1024, "kB"); + const packageIndexBuffer = Buffer.concat(packageIndexList); const finalBuffer = Buffer.concat([ ...(writeRaw ? [] : [sourceData]), - cacheBuffer, + allBytecodeData, + packageIndexBuffer, metadataBuffer, ]); + console.log("Embedded size:", allBytecodeData.length / 1024, "kB"); + await fs.writeFile(target, finalBuffer); if (!writeRaw) { await fs.chmod(target, 0o755); diff --git a/llrt/src/main.c b/llrt/src/main.c index c34a7f62a1..1966af56be 100644 --- a/llrt/src/main.c +++ b/llrt/src/main.c @@ -128,9 +128,9 @@ typedef struct uint32_t srcSize; uint32_t dstSize; uint32_t id; - uint32_t extraFd; uint32_t extraDataSize; - const void *extraData; + const void *extraSrc; + const void *extraDst; const void *inputBuffer; const void *outputBuffer; } DecompressThreadArgs; @@ -139,9 +139,10 @@ static void *decompressPartial(void *arg) { DecompressThreadArgs *args = (DecompressThreadArgs *)arg; - if (args->extraData != NULL) + if (args->extraSrc != NULL) { - write(args->extraFd, args->extraData, args->extraDataSize); + memcpy((void *)args->extraDst, args->extraSrc, args->extraDataSize); + munmap((void *)args->extraDst, args->extraDataSize); } size_t srcSize = args->srcSize; @@ -208,12 +209,12 @@ static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int uint32_t inputOffset = 0; uint32_t outputOffset = 0; uint32_t extraDataSize = *(uint32_t *)extra; + char *extraDataMap; char *uncompressed; uint8_t *compressedData; char *extraData = (char *)extra + sizeof(uint32_t); - pthread_t writeFdThreadId; logInfo("Extra data size %lu bytes \n", extraDataSize); @@ -235,6 +236,15 @@ static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int err(1, "Failed to set file size"); } + if (extraDataSize > 0) + { + if (ftruncate(extraFd, extraDataSize) == -1) + { + err(1, "Failed to set file size"); + } + extraDataMap = (char *)mmap(NULL, extraDataSize, PROT_WRITE | PROT_READ, MAP_SHARED, extraFd, 0); + } + uncompressed = mmap(NULL, *uncompressedSize, PROT_READ | PROT_WRITE, MAP_SHARED, outputFd, 0); if (uncompressed == MAP_FAILED || !uncompressed) { @@ -251,8 +261,8 @@ static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int args[i].id = i; if (i == 0 && extraDataSize > 0) { - args[i].extraFd = extraFd; - args[i].extraData = extraData; + args[i].extraSrc = extraData; + args[i].extraDst = extraDataMap; args[i].extraDataSize = extraDataSize; } inputOffset += inputSizes[i]; @@ -279,7 +289,7 @@ static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int } } - *uncompressedData = uncompressed; + *uncompressedData = uncompressed; } int main(int argc, char *argv[]) @@ -319,11 +329,6 @@ int main(int argc, char *argv[]) err(1, "Failed to unmap memory"); } - if () - { - err(1, "Failed to unmap memory"); - } - double t2 = microSeconds(); logInfo("Extraction + write time: %10.4f ms\n", (t2 - t0) / 1000.0); @@ -351,13 +356,13 @@ int main(int argc, char *argv[]) char mimallocReserveMemoryMb[16]; sprintf(mimallocReserveMemoryMb, "%iMiB", (int)(memorySize * memoryFactor)); - char outputFdStr[10]; - sprintf(outputFdStr, "%i", extraFd); + char extraFdStr[14]; + sprintf(extraFdStr, "%i", extraFd); setenv("_START_TIME", startTimeStr, false); setenv("MIMALLOC_RESERVE_OS_MEMORY", mimallocReserveMemoryMb, false); setenv("MIMALLOC_LIMIT_OS_ALLOC", "1", false); - setenv("LLRT_MEM_FD", outputFdStr, false); + setenv("LLRT_MEM_FD", extraFdStr, false); logInfo("Starting app\n"); diff --git a/llrt_core/Cargo.toml b/llrt_core/Cargo.toml index 93ffe39883..84e231ff3d 100644 --- a/llrt_core/Cargo.toml +++ b/llrt_core/Cargo.toml @@ -80,6 +80,7 @@ flate2 = { version = "1.0.30", features = [ brotlic = "0.8.2" rustls-pemfile = "2.1.2" url = "=2.5.1" +memmap2 = "0.9.4" [build-dependencies] rquickjs = { version = "0.6.2", features = [ diff --git a/llrt_core/build.rs b/llrt_core/build.rs index 9d1a71909c..7d237ffe47 100644 --- a/llrt_core/build.rs +++ b/llrt_core/build.rs @@ -38,8 +38,6 @@ async fn main() -> StdResult<(), Box> { set_nightly_cfg(); rerun_if_changed!(BUNDLE_JS_DIR); - rerun_if_changed!("Cargo.toml"); - rerun_if_changed!("patches"); cargo_patch::patch()?; @@ -68,7 +66,9 @@ async fn main() -> StdResult<(), Box> { #[cfg(feature = "lambda")] { + info!("Path is: {:?}", path); if path == PathBuf::new().join("@llrt").join("test.js") { + info!("SKIPPING TEST!!!"); continue; } } @@ -113,6 +113,7 @@ async fn main() -> StdResult<(), Box> { total_bytes += bytes.len(); fs::create_dir_all(lrt_path.parent().unwrap())?; + println!("Writing: {:?}", lrt_path); if cfg!(feature = "uncompressed") { let uncompressed = add_bytecode_header(bytes, None); fs::write(&lrt_path, uncompressed)?; @@ -121,27 +122,11 @@ async fn main() -> StdResult<(), Box> { } info!("Done!"); - - // ph_map.entry( - // module_name, - // &format!( - // "include_bytes!(\"..{}{}\")", - // MAIN_SEPARATOR_STR, &lrt_filename - // ), - // ); } StdResult::<_, Box>::Ok(()) })?; - // write!( - // &mut sdk_bytecode_file, - // "// @generated by build.rs\n\npub static BYTECODE_CACHE: phf::Map<&'static str, &[u8]> = {}", - // ph_map.build() - // )?; - // writeln!(&mut sdk_bytecode_file, ";")?; - // sdk_bytecode_file.flush()?; - info!( "\n===============================\nUncompressed bytecode size: {}\n===============================", human_file_size(total_bytes) diff --git a/llrt_core/src/vm.rs b/llrt_core/src/vm.rs index 81053b0923..c5383b41de 100644 --- a/llrt_core/src/vm.rs +++ b/llrt_core/src/vm.rs @@ -8,21 +8,20 @@ use std::{ fmt::Write, fs::File, future::{poll_fn, Future}, - io::{self, Read, Seek, SeekFrom}, + io::{self}, mem::size_of, + ops::Range, os::fd::FromRawFd, path::{Component, Path, PathBuf}, pin::pin, process::exit, result::Result as StdResult, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, Mutex, Once, RwLock, - }, + sync::{Arc, Mutex, Once, RwLock}, task::Poll, time::Instant, }; +use memmap2::{Advice, Mmap}; use once_cell::sync::Lazy; pub use llrt_utils::{ctx::CtxExtension, error::ErrorExtensions}; @@ -37,12 +36,12 @@ use rquickjs::{ qjs, AsyncContext, AsyncRuntime, CatchResultExt, CaughtError, Ctx, Error, Function, IntoJs, Module, Object, Result, Value, }; -use tokio::sync::oneshot::{self, Receiver}; + use tracing::trace; use zstd::{bulk::Decompressor, dict::DecoderDictionary}; use crate::{ - bytecode::BYTECODE_EMBEDDED_SIGNATURE, + bytecode::{self, BYTECODE_EMBEDDED_SIGNATURE}, modules::{ console, crypto::SYSTEM_RANDOM, @@ -63,57 +62,51 @@ use crate::{ }, }; -#[derive(Default)] -pub struct BytecodeCache { - data: Box<[u8]>, - map: HashMap, Arc<[u8]>>, +struct BytecodeCache { + data: Mmap, + map: HashMap, Range>, } impl BytecodeCache { - pub fn new(data: Vec, package_count: usize) -> Self { - let data = data.into_boxed_slice(); - let mut cache = BytecodeCache { - data, - map: HashMap::with_capacity(package_count), - }; - let mut offset = 0; - let data = &cache.data; - let mut entries = Vec::new(); + pub fn new(data: Mmap, start_position: usize, length: usize, package_count: usize) -> Self { + let mut offset = start_position; + let mut map = HashMap::with_capacity(package_count); loop { let name_len_start = offset; let name_len_end = offset + size_of::(); let name_len = u16_from_le_byte_slice_unchecked(&data[name_len_start..name_len_end]) as usize; - - let bytecode_size_start = name_len_end + name_len; + let bytecode_pos_start = name_len_end + name_len; let name = - unsafe { std::str::from_utf8_unchecked(&data[name_len_end..bytecode_size_start]) }; + unsafe { std::str::from_utf8_unchecked(&data[name_len_end..bytecode_pos_start]) }; - let bytecode_start = bytecode_size_start + size_of::(); + let bytecode_pos_end = bytecode_pos_start + size_of::(); + let bytecode_pos = + u32_from_le_byte_slice_unchecked(&data[bytecode_pos_start..bytecode_pos_end]) + as usize; + let bytecode_size_start = bytecode_pos_end; + let bytecode_size_end = bytecode_size_start + size_of::(); let bytecode_size = - u32_from_le_byte_slice_unchecked(&data[bytecode_size_start..bytecode_start]) + u32_from_le_byte_slice_unchecked(&data[bytecode_size_start..bytecode_size_end]) as usize; - let bytecode_end = bytecode_start + bytecode_size; - - entries.push((name.to_owned(), bytecode_start..bytecode_end)); + map.insert(name.into(), bytecode_pos..bytecode_pos + bytecode_size); - offset = bytecode_start + bytecode_size; + offset = bytecode_size_end; - if offset >= data.len() { + if offset >= length - 1 { break; } } - for (name, range) in entries { - let key = Arc::from(name); - let value = Arc::from(&cache.data[range]); - cache.map.insert(key, value); - } - cache + Self { data, map } + } + + fn get(&self, name: &str) -> Option<&[u8]> { + self.map.get(name).map(|range| &self.data[range.clone()]) } } @@ -136,8 +129,8 @@ fn file_from_raw_fd(raw_fd: i32) -> std::io::Result { } } -pub static EMBEDDED_BYTECODE_DATA: Lazy> = - Lazy::new(|| RwLock::new(BytecodeCache::default())); +static EMBEDDED_BYTECODE_DATA: Lazy>> = + Lazy::new(|| RwLock::new(None)); #[inline] pub fn uncompressed_size(input: &[u8]) -> StdResult<(usize, &[u8]), io::Error> { @@ -214,6 +207,7 @@ impl Resolver for BinaryResolver { trace!("Try resolve \"{}\" from \"{}\"", name, base); let embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.read().unwrap(); + let embedded_bytecode_data = embedded_bytecode_data.as_ref().unwrap(); if embedded_bytecode_data.map.contains_key(name) { return Ok(name.to_string()); @@ -336,11 +330,12 @@ impl Loader for BinaryLoader { trace!("Loading module: {}", name); { let embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.read().unwrap(); - if let Some(bytes) = embedded_bytecode_data.map.get(name) { + let embedded_bytecode_data = embedded_bytecode_data.as_ref().unwrap(); + if let Some(bytes) = embedded_bytecode_data.get(name) { trace!("Loading embedded module: {}", name); return load_bytecode_module(ctx.clone(), bytes); } - drop(embedded_bytecode_data); + let path = PathBuf::from(name); let mut bytes: &[u8] = &std::fs::read(path)?; @@ -629,85 +624,73 @@ fn init_embedded_bytecode() -> std::io::Result<()> { let argv_0 = env::args().next().expect("Failed to get argv0"); let mut embedded_bytecode_data = EMBEDDED_BYTECODE_DATA.write().unwrap(); - let mut file = if let Ok(fd_string) = env::var("LLRT_MEM_FD") { + let file = if let Ok(fd_string) = env::var("LLRT_MEM_FD") { let mem_fd: i32 = fd_string.parse().map_err(|_| { std::io::Error::new(std::io::ErrorKind::Other, "Invalid bytecode-cache fd") })?; - trace!("Using raw memfd"); + trace!("Using raw memfd bytecode cache"); file_from_raw_fd(mem_fd) } else { File::open(argv_0) }?; - let mut embedded_bytecode_signature_buf = [0; BYTECODE_EMBEDDED_SIGNATURE.len()]; - let file_metadata = file.metadata()?; - let total_file_size = file_metadata.len(); + let mmap = unsafe { Mmap::map(&file)? }; + mmap.advise(Advice::Sequential).unwrap(); + + let total_file_size = mmap.len(); + let signature_len = BYTECODE_EMBEDDED_SIGNATURE.len(); - let signed_signature_len = signature_len as i64; - file.seek(SeekFrom::End(-signed_signature_len))?; - file.read_exact(&mut embedded_bytecode_signature_buf)?; + let signed_signature_len = signature_len as isize; - if embedded_bytecode_signature_buf != BYTECODE_EMBEDDED_SIGNATURE { + if &mmap[(total_file_size as isize - signed_signature_len) as usize..] + != BYTECODE_EMBEDDED_SIGNATURE + { return Ok(()); } + #[repr(C)] + #[derive(Debug)] struct EmbeddedMeta { package_count: u32, bytecode_pos: u32, - } - - impl EmbeddedMeta { - fn read(file: &mut File) -> std::io::Result { - let mut buf = [0; size_of::() * 2]; - file.read_exact(&mut buf)?; - Ok(EmbeddedMeta { - package_count: u32_from_le_byte_slice_unchecked(&buf[..size_of::()]), - bytecode_pos: u32_from_le_byte_slice_unchecked(&buf[size_of::()..]), - }) - } + package_index_pos: u32, } let embedded_meta_size = size_of::(); - let meta_and_signature_size = embedded_meta_size + signature_len; - file.seek(SeekFrom::Current(-(meta_and_signature_size as i64)))?; + let meta_start = (total_file_size as isize - meta_and_signature_size as isize) as usize; + let meta_end = (total_file_size as isize - signature_len as isize) as usize; - let embedded_metadata = EmbeddedMeta::read(&mut file)?; + let embedded_metadata: EmbeddedMeta = + unsafe { std::ptr::read(mmap[meta_start..meta_end].as_ptr() as *const _) }; - file.seek(SeekFrom::Current( - -(total_file_size as i64 - (embedded_metadata.bytecode_pos as i64 + signed_signature_len)), - ))?; + println!("Metadata: {:?}", embedded_metadata); - let total_cache_size = total_file_size as usize - - embedded_metadata.bytecode_pos as usize - - meta_and_signature_size; + let bytecode_pos = embedded_metadata.bytecode_pos as usize; + let start_position = embedded_metadata.package_index_pos as usize; + let end_pos = total_file_size as usize - meta_and_signature_size; + let length = end_pos - bytecode_pos; - let mut buf = vec![0; total_cache_size]; - trace!("Loading bytecode cache of {} kB", total_cache_size / 1024); - file.read_exact(&mut buf)?; + trace!( + "Loading bytecode cache of {} kB", + (end_pos - bytecode_pos) / 1024 + ); - let bytecode_cache = BytecodeCache::new(buf, embedded_metadata.package_count as usize); + let bytecode_cache = BytecodeCache::new( + mmap, + start_position, + length, + embedded_metadata.package_count as usize, + ); - *embedded_bytecode_data = bytecode_cache; + *embedded_bytecode_data = Some(bytecode_cache); trace!("Building cache took: {:?}", now.elapsed()); Ok(()) } -// #[inline(always)] -// fn read_u32(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { -// file.read_exact(buf)?; -// Ok(u32::from_le_bytes(*buf)) -// } - -// #[inline(always)] -// fn read_u16(file: &mut File, buf: &mut [u8; size_of::()]) -> std::io::Result { -// file.read_exact(buf)?; -// Ok(u16::from_le_bytes(*buf)) -// } - fn json_parse_string<'js>(ctx: Ctx<'js>, value: Value<'js>) -> Result> { let bytes = get_bytes(&ctx, value)?; json_parse(&ctx, bytes) @@ -818,8 +801,10 @@ fn init(ctx: &Ctx<'_>, module_names: HashSet<&'static str>) -> Result<()> { || EMBEDDED_BYTECODE_DATA .read() .unwrap() + .as_ref() + .unwrap() .map - .contains_key(&Arc::from(specifier.as_str())) + .contains_key(specifier.as_str()) || specifier.starts_with('/') { specifier