use separate file with mmap

Author: richarddavison

Cargo.lock

@@ -44,7 +44,9 @@ async function buildFileIndex(source, target, fileData, writeRaw) {
   };
 
   const sourceData = await fs.readFile(source);
-  const cacheBuffers = [];
+  const packageIndexList = [];
+  let offset = 0;
+  const bytecodeData = [];
 
   for (let [name, data] of fileData) {
     if (name.startsWith("llrt-chunk-")) {
@@ -54,31 +56,45 @@ async function buildFileIndex(source, target, fileData, writeRaw) {
     const nameBuffer = Buffer.from(name);
 
     const bytecodeSizeBuffer = uint32Buffer(data.length);
-
-    cacheBuffers.push(
-      Buffer.concat([nameLengthBuffer, nameBuffer, bytecodeSizeBuffer, data])
+    const bytecodeOffsetBuffer = uint32Buffer(offset);
+
+    packageIndexList.push(
+      Buffer.concat([
+        nameLengthBuffer,
+        nameBuffer,
+        bytecodeOffsetBuffer,
+        bytecodeSizeBuffer,
+      ])
     );
+
+    offset += data.length;
+    bytecodeData.push(data);
   }
 
+  const allBytecodeData = Buffer.concat(bytecodeData);
+
   const packageCount = fileData.length;
-  const cachePosition = writeRaw ? 0 : sourceData.length;
+  const bytecodePosition = writeRaw ? 0 : sourceData.length;
+  const packageIndexPosition = bytecodePosition + allBytecodeData.length;
 
   const metadataBuffer = Buffer.concat([
     uint32Buffer(packageCount),
-    uint32Buffer(cachePosition),
+    uint32Buffer(bytecodePosition),
+    uint32Buffer(packageIndexPosition),
     Buffer.from("lrt"),
   ]);
 
-  const cacheBuffer = Buffer.concat(cacheBuffers);
-
-  console.log("Embedded size:", cacheBuffer.length / 1024, "kB");
+  const packageIndexBuffer = Buffer.concat(packageIndexList);
 
   const finalBuffer = Buffer.concat([
     ...(writeRaw ? [] : [sourceData]),
-    cacheBuffer,
+    allBytecodeData,
+    packageIndexBuffer,
     metadataBuffer,
   ]);
 
+  console.log("Embedded size:", allBytecodeData.length / 1024, "kB");
+
   await fs.writeFile(target, finalBuffer);
   if (!writeRaw) {
     await fs.chmod(target, 0o755);

embed.mjs

@@ -128,9 +128,9 @@ typedef struct
   uint32_t srcSize;
   uint32_t dstSize;
   uint32_t id;
-  uint32_t extraFd;
   uint32_t extraDataSize;
-  const void *extraData;
+  const void *extraSrc;
+  const void *extraDst;
   const void *inputBuffer;
   const void *outputBuffer;
 } DecompressThreadArgs;
@@ -139,9 +139,10 @@ static void *decompressPartial(void *arg)
 {
   DecompressThreadArgs *args = (DecompressThreadArgs *)arg;
 
-  if (args->extraData != NULL)
+  if (args->extraSrc != NULL)
   {
-    write(args->extraFd, args->extraData, args->extraDataSize);
+    memcpy((void *)args->extraDst, args->extraSrc, args->extraDataSize);
+    munmap((void *)args->extraDst, args->extraDataSize);
   }
 
   size_t srcSize = args->srcSize;
@@ -208,12 +209,12 @@ static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int
   uint32_t inputOffset = 0;
   uint32_t outputOffset = 0;
   uint32_t extraDataSize = *(uint32_t *)extra;
+  char *extraDataMap;
 
   char *uncompressed;
   uint8_t *compressedData;
 
   char *extraData = (char *)extra + sizeof(uint32_t);
-  pthread_t writeFdThreadId;
 
   logInfo("Extra data size %lu bytes \n", extraDataSize);
 
@@ -235,6 +236,15 @@ static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int
     err(1, "Failed to set file size");
   }
 
+  if (extraDataSize > 0)
+  {
+    if (ftruncate(extraFd, extraDataSize) == -1)
+    {
+      err(1, "Failed to set file size");
+    }
+    extraDataMap = (char *)mmap(NULL, extraDataSize, PROT_WRITE | PROT_READ, MAP_SHARED, extraFd, 0);
+  }
+
   uncompressed = mmap(NULL, *uncompressedSize, PROT_READ | PROT_WRITE, MAP_SHARED, outputFd, 0);
   if (uncompressed == MAP_FAILED || !uncompressed)
   {
@@ -251,8 +261,8 @@ static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int
     args[i].id = i;
     if (i == 0 && extraDataSize > 0)
     {
-      args[i].extraFd = extraFd;
-      args[i].extraData = extraData;
+      args[i].extraSrc = extraData;
+      args[i].extraDst = extraDataMap;
       args[i].extraDataSize = extraDataSize;
     }
     inputOffset += inputSizes[i];
@@ -279,7 +289,7 @@ static void decompress(char **uncompressedData, uint32_t *uncompressedSize, int
     }
   }
 
-   *uncompressedData = uncompressed;
+  *uncompressedData = uncompressed;
 }
 
 int main(int argc, char *argv[])
@@ -319,11 +329,6 @@ int main(int argc, char *argv[])
     err(1, "Failed to unmap memory");
   }
 
-  if ()
-  {
-    err(1, "Failed to unmap memory");
-  }
-
   double t2 = microSeconds();
   logInfo("Extraction + write time: %10.4f ms\n", (t2 - t0) / 1000.0);
 
@@ -351,13 +356,13 @@ int main(int argc, char *argv[])
   char mimallocReserveMemoryMb[16];
   sprintf(mimallocReserveMemoryMb, "%iMiB", (int)(memorySize * memoryFactor));
 
-  char outputFdStr[10];
-  sprintf(outputFdStr, "%i", extraFd);
+  char extraFdStr[14];
+  sprintf(extraFdStr, "%i", extraFd);
 
   setenv("_START_TIME", startTimeStr, false);
   setenv("MIMALLOC_RESERVE_OS_MEMORY", mimallocReserveMemoryMb, false);
   setenv("MIMALLOC_LIMIT_OS_ALLOC", "1", false);
-  setenv("LLRT_MEM_FD", outputFdStr, false);
+  setenv("LLRT_MEM_FD", extraFdStr, false);
 
   logInfo("Starting app\n");
 

llrt/src/main.c

@@ -80,6 +80,7 @@ flate2 = { version = "1.0.30", features = [
 brotlic = "0.8.2"
 rustls-pemfile = "2.1.2"
 url = "=2.5.1"
+memmap2 = "0.9.4"
 
 [build-dependencies]
 rquickjs = { version = "0.6.2", features = [

llrt_core/Cargo.toml

@@ -38,8 +38,6 @@ async fn main() -> StdResult<(), Box<dyn Error>> {
     set_nightly_cfg();
 
     rerun_if_changed!(BUNDLE_JS_DIR);
-    rerun_if_changed!("Cargo.toml");
-    rerun_if_changed!("patches");
 
     cargo_patch::patch()?;
 
@@ -68,7 +66,9 @@ async fn main() -> StdResult<(), Box<dyn Error>> {
 
             #[cfg(feature = "lambda")]
             {
+                info!("Path is: {:?}", path);
                 if path == PathBuf::new().join("@llrt").join("test.js") {
+                    info!("SKIPPING TEST!!!");
                     continue;
                 }
             }
@@ -113,6 +113,7 @@ async fn main() -> StdResult<(), Box<dyn Error>> {
             total_bytes += bytes.len();
 
             fs::create_dir_all(lrt_path.parent().unwrap())?;
+            println!("Writing: {:?}", lrt_path);
             if cfg!(feature = "uncompressed") {
                 let uncompressed = add_bytecode_header(bytes, None);
                 fs::write(&lrt_path, uncompressed)?;
@@ -121,27 +122,11 @@ async fn main() -> StdResult<(), Box<dyn Error>> {
             }
 
             info!("Done!");
-
-            // ph_map.entry(
-            //     module_name,
-            //     &format!(
-            //         "include_bytes!(\"..{}{}\")",
-            //         MAIN_SEPARATOR_STR, &lrt_filename
-            //     ),
-            // );
         }
 
         StdResult::<_, Box<dyn Error>>::Ok(())
     })?;
 
-    // write!(
-    //     &mut sdk_bytecode_file,
-    //     "// @generated by build.rs\n\npub static BYTECODE_CACHE: phf::Map<&'static str, &[u8]> = {}",
-    //     ph_map.build()
-    // )?;
-    // writeln!(&mut sdk_bytecode_file, ";")?;
-    // sdk_bytecode_file.flush()?;
-
     info!(
         "\n===============================\nUncompressed bytecode size: {}\n===============================",
         human_file_size(total_bytes)