From 71c41784c60002f83216a90c491becf680265383 Mon Sep 17 00:00:00 2001 From: Nikita Rakitin Date: Wed, 14 Oct 2026 01:51:40 +0300 Subject: [PATCH 1/4] Export: MP4 and MOV video of the deck, with our own H.264 encoder file.export writes .mp4/.mov: each slide's transition, then the slide held for its "advance after" time (else secondsPerSlide), at a chosen height (480p to 4K) and frame rate, hidden slides left out unless asked. Frames come from the slide and transition renderers and are encoded by a new Constrained Baseline H.264 encoder in deckcraft-h264 (CAVLC, Intra 16x16 with V/H/DC prediction, unchanged macroblocks skipped, deblocking off), checked bit-exact against our decoder, and muxed with deckcraft-isobmff (faststart). A key frame starts every slide and transition and at least every 2 s. No external tool is needed. Co-authored-by: Cursor --- Cargo.lock | 2 + crates/engine/Cargo.toml | 2 + crates/engine/src/cmd/file.rs | 24 +- crates/engine/src/lib.rs | 1 + crates/engine/src/tests.rs | 2 + crates/engine/src/video.rs | 272 ++++++++++ crates/h264/src/encoder.rs | 920 ++++++++++++++++++++++++++++++++++ crates/h264/src/lib.rs | 2 + docs/file-format-parity.md | 2 +- docs/gaps.md | 15 +- 10 files changed, 1232 insertions(+), 10 deletions(-) create mode 100644 crates/engine/src/video.rs create mode 100644 crates/h264/src/encoder.rs diff --git a/Cargo.lock b/Cargo.lock index e2bc79659f..19de932433 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -959,6 +959,8 @@ dependencies = [ "deckcraft-fonts", "deckcraft-format", "deckcraft-geom", + "deckcraft-h264", + "deckcraft-isobmff", "deckcraft-media", "deckcraft-model", "deckcraft-pdf", diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 778664a7da..222ed6159f 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -19,6 +19,8 @@ deckcraft-pdf = { workspace = true } deckcraft-format = { workspace = true } deckcraft-pptx = { workspace = true } deckcraft-media = { workspace = true } +deckcraft-h264 = { workspace = true } +deckcraft-isobmff = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } i_overlay = { workspace = true } diff --git a/crates/engine/src/cmd/file.rs b/crates/engine/src/cmd/file.rs index 4761dc103b..6e13320bfa 100644 --- a/crates/engine/src/cmd/file.rs +++ b/crates/engine/src/cmd/file.rs @@ -33,7 +33,7 @@ pub fn specs() -> Vec { cmd!(noundo "file.saveAs", "Save As…", ["File"], Some("Cmd+Shift+S"), "{path, format?}", has_doc, save_as), cmd!(noundo "file.saveTemplate", "Save as Template…", ["File"], None, "{path}", has_doc, save_as), cmd!(query "file.saveBytes", "Save to Data", [], None, "{format?: deckcraft|pptx} → {data: base64}", has_doc, save_bytes_cmd), - cmd!(noundo "file.export", "Export…", ["File"], None, "{path, format?: png|jpeg|pptx|deckcraft|outline|pdf|gif (else by path), slide?: index, all?: bool, scale?: px per pt, layout?: slides|notes|handouts (pdf), perPage?: 1|2|3|4|6|9, paper?: letter|a4|legal|a3 (pdf), orientation?: portrait|landscape (pdf), dpi?, slides?: [index]|range (\"1-3,5\"), includeHidden?, textLayer?, frame?, width?: px 16-3840|small|medium|large|extraLarge (gif), secondsPerSlide?: 0.1-600 (gif; 5), loop?: bool (gif; true: forever)} → {path, bytes} or {files} (png, jpeg); gif adds frames, width, height, secondsPerSlide", has_doc, export), + cmd!(noundo "file.export", "Export…", ["File"], None, "{path, format?: png|jpeg|pptx|deckcraft|outline|pdf|gif|mp4|mov (else by path), slide?: index, all?: bool, scale?: px per pt, layout?: slides|notes|handouts (pdf), perPage?: 1|2|3|4|6|9, paper?: letter|a4|legal|a3 (pdf), orientation?: portrait|landscape (pdf), dpi?, slides?: [index]|range (\"1-3,5\"), includeHidden?, textLayer?, frame?, width?: px 16-3840|small|medium|large|extraLarge (gif), secondsPerSlide?: 0.1-600 (gif; 5), loop?: bool (gif; true: forever), height?: 480|720|1080|2160 (mp4/mov), fps? (mp4/mov)} → {path, bytes} or {files} (png, jpeg); gif adds frames, width, height, secondsPerSlide", has_doc, export), cmd!( query "file.info", "Info", @@ -337,7 +337,7 @@ pub fn pdf_bytes(doc: &Presentation, p: &Value) -> Result> { } /// The output extensions [`format_for_path`] maps to a writer of that format. -pub const WRITABLE_EXTENSIONS: &[&str] = &["deckcraft", "pptx", "potx", "ppsx", "pdf", "png", "jpg", "jpeg", "gif", "txt", "md"]; +pub const WRITABLE_EXTENSIONS: &[&str] = &["deckcraft", "pptx", "potx", "ppsx", "pdf", "png", "jpg", "jpeg", "gif", "mp4", "m4v", "mov", "txt", "md"]; /// The format a file named `path` is written in, or `None` when its extension names no format /// DeckCraft writes (`.odp`, `.svg`, no extension …): [`format_for_path`] would save @@ -368,6 +368,10 @@ pub fn format_for_path(path: &str) -> &'static str { "pdf" } else if l.ends_with(".gif") { "gif" + } else if l.ends_with(".mp4") || l.ends_with(".m4v") { + "mp4" + } else if l.ends_with(".mov") { + "mov" } else if l.ends_with(".txt") || l.ends_with(".md") { "outline" } else { @@ -514,6 +518,7 @@ fn export(s: &mut Session, p: &Value) -> Result { } Ok(json!({"files": written})) } + "mp4" | "mov" => video_export(&st.doc, &path, &format, p), "pdf" => { let bytes = pdf_bytes(&st.doc, p)?; write_file(&path, &bytes)?; @@ -545,6 +550,21 @@ fn export(s: &mut Session, p: &Value) -> Result { } } +/// File › Export › Create a Video. +fn video_export(doc: &deckcraft_model::Presentation, path: &str, format: &str, p: &Value) -> Result { + let d = crate::video::VideoOpts::default(); + let opts = crate::video::VideoOpts { + height: usize_param(p, "height").map(|h| h.clamp(2, 4320) as u32).unwrap_or(d.height), + fps: usize_param(p, "fps").map(|f| f.clamp(1, 60) as u32).unwrap_or(d.fps), + seconds_per_slide: f64_or(p, "secondsPerSlide", d.seconds_per_slide).clamp(0.1, 3600.0), + include_hidden: bool_or(p, "includeHidden", false), + }; + let (bytes, frames, (w, h)) = crate::video::encode(doc, format, &opts).map_err(|e| bad("file.export", e))?; + write_file(path, &bytes)?; + let fps = opts.fps.clamp(1, 60); + Ok(json!({"path": path, "bytes": bytes.len(), "frames": frames, "width": w, "height": h, "fps": fps, "seconds": frames as f64 / fps as f64})) +} + /// File › Info: what the Info page shows about the active presentation. fn info(s: &mut Session, _p: &Value) -> Result { let st = s.doc()?; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ac25d6669a..22c2da3b56 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -28,6 +28,7 @@ pub mod spell; pub mod templates; pub mod thesaurus; pub mod tools; +pub mod video; use std::sync::Arc; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index f7022eab81..934427d707 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -326,6 +326,8 @@ fn writable_format_for_path_rejects_formats_deckcraft_does_not_write() { assert_eq!(f("a.jpg"), Some("jpeg")); assert_eq!(f("a.txt"), Some("outline")); assert_eq!(f("a.gif"), Some("gif")); + assert_eq!(f("a.mp4"), Some("mp4")); + assert_eq!(f("a.mov"), Some("mov")); for path in ["a.odp", "a.bmp", "a.svg", "a.outline", "a", "dir.v2/a"] { assert_eq!(f(path), None, "{path}"); } diff --git a/crates/engine/src/video.rs b/crates/engine/src/video.rs new file mode 100644 index 0000000000..dbb30f705e --- /dev/null +++ b/crates/engine/src/video.rs @@ -0,0 +1,272 @@ +//! Video export (File › Export › Create a Video): the deck as an MP4 or MOV, one frame stream +//! of each slide's transition then the slide held for its timing. Frames come from the slide +//! renderer and are encoded with DeckCraft's own H.264 encoder ([`deckcraft_h264::Encoder`]: +//! intra-coded changes, skipped unchanged macroblocks) into an MP4 or QuickTime file +//! ([`deckcraft_isobmff::Mp4Writer`]); no external tool is needed. Animations within a slide are +//! shown in their final state, and there is no audio track. + +use deckcraft_model::Presentation; + +/// The most frames one export writes (two hours at 30 fps), against runaway timings. +pub const MAX_FRAMES: u64 = 2 * 60 * 60 * 30; + +/// Export options, as `file.export` takes them. +#[derive(Clone, Debug, PartialEq)] +pub struct VideoOpts { + /// Frame height in pixels (PowerPoint's 480p / 720p / 1080p / 4K); the width follows the + /// slide's aspect ratio. Both are rounded to even numbers, as H.264 needs. + pub height: u32, + pub fps: u32, + /// How long a slide stays up when it has no "advance after" timing of its own. + pub seconds_per_slide: f64, + pub include_hidden: bool, +} + +impl Default for VideoOpts { + fn default() -> Self { + VideoOpts { height: 1080, fps: 30, seconds_per_slide: 5.0, include_hidden: false } + } +} + +/// One step of the timeline: a transition frame at progress `t`, or slide `index` held for +/// `frames` frames. +#[derive(Clone, Debug, PartialEq)] +pub enum Step { + Transition { index: usize, t: f64 }, + Hold { index: usize, frames: u64 }, +} + +/// The frame size for `opts` on this deck's slide size: even, at least 2×2. +pub fn frame_size(pres: &Presentation, opts: &VideoOpts) -> (u32, u32) { + let h = opts.height.clamp(2, 4320) & !1; + let aspect = pres.slide_size.width / pres.slide_size.height.max(1.0); + let aspect = if aspect.is_finite() && aspect > 0.0 { aspect.clamp(0.1, 10.0) } else { 16.0 / 9.0 }; + let w = ((h as f64 * aspect).round() as u32).clamp(2, 8192) & !1; + (w, h) +} + +/// Seconds as frames at `fps`, at least one. +fn frames_for(seconds: f64, fps: u32) -> u64 { + let s = if seconds.is_finite() { seconds.clamp(0.0, 3600.0) } else { 0.0 }; + ((s * fps as f64).round() as u64).max(1) +} + +/// The timeline: for each slide shown, its transition's frames (none for "none", a cut or a +/// zero duration) then the slide held for its "advance after" time, else `seconds_per_slide`. +/// The total is capped at [`MAX_FRAMES`]. +pub fn timeline(pres: &Presentation, opts: &VideoOpts) -> Vec { + let fps = opts.fps.clamp(1, 60); + let mut out = vec![]; + let mut total: u64 = 0; + for (index, slide) in pres.slides.iter().enumerate() { + if slide.hidden && !opts.include_hidden { + continue; + } + if let Some(tr) = slide.transition.as_ref().filter(|t| !matches!(t.kind.as_str(), "none" | "cut" | "") && t.duration_ms > 0) { + let n = frames_for(tr.duration_ms as f64 / 1000.0, fps); + for k in 0..n { + if total >= MAX_FRAMES { + return out; + } + out.push(Step::Transition { index, t: k as f64 / n as f64 }); + total = total.saturating_add(1); + } + } + let secs = slide.transition.as_ref().and_then(|t| t.advance_after_ms).map(|ms| ms as f64 / 1000.0).unwrap_or(opts.seconds_per_slide); + let frames = frames_for(secs, fps).min(MAX_FRAMES.saturating_sub(total)); + if frames == 0 { + return out; + } + out.push(Step::Hold { index, frames }); + total = total.saturating_add(frames); + } + out +} + +/// An image fitted to exactly `w`×`h` RGBA (the renderer's size can differ by a pixel): copied +/// into a black frame, cropped where it overhangs. +fn fit(img: &deckcraft_render::Image, w: u32, h: u32) -> Vec { + let mut out = vec![0u8; (w as usize).saturating_mul(h as usize).saturating_mul(4)]; + for px in out.as_chunks_mut::<4>().0 { + px[3] = 255; + } + let cw = img.width.min(w) as usize; + for y in 0..img.height.min(h) as usize { + let src = y.saturating_mul(img.width as usize).saturating_mul(4); + let dst = y.saturating_mul(w as usize).saturating_mul(4); + if let (Some(s), Some(d)) = (img.pixels.get(src..src + cw * 4), out.get_mut(dst..dst + cw * 4)) { + d.copy_from_slice(s); + } + } + out +} + +/// The renderer options for a frame `h` pixels high. +fn render_opts(pres: &Presentation, h: u32) -> deckcraft_render::RenderOpts<'static> { + let scale = h as f64 / pres.slide_size.height.max(1.0); + deckcraft_render::RenderOpts { scale: if scale.is_finite() { scale.clamp(0.01, 16.0) } else { 1.0 }, ..Default::default() } +} + +/// Every frame of `steps`, in order, handed to `sink` (RGBA, `w`×`h`). A held slide is rendered +/// once. Stops at the first error `sink` returns. +pub fn frames(pres: &Presentation, steps: &[Step], (w, h): (u32, u32), sink: &mut dyn FnMut(&[u8]) -> Result<(), String>) -> Result { + let opts = render_opts(pres, h); + let mut n: u64 = 0; + for step in steps { + match step { + Step::Transition { index, t } => { + let img = deckcraft_render::render_transition(pres, *index, *t, &opts); + sink(&fit(&img, w, h))?; + n = n.saturating_add(1); + } + Step::Hold { index, frames } => { + let img = fit(&deckcraft_render::render_slide(pres, *index, &opts), w, h); + for _ in 0..*frames { + sink(&img)?; + n = n.saturating_add(1); + } + } + } + } + Ok(n) +} + +/// A key frame at least this often (seconds), so players can seek. +const KEY_EVERY_S: u32 = 2; + +/// Encode the deck to `path` as `mp4` or `mov`, at quality `qp` (H.264 QP, lower is better). +/// The file is assembled in memory and written through a temporary sibling, so a failed export +/// leaves no half file. Returns the frame count and size. +pub fn encode(pres: &Presentation, format: &str, opts: &VideoOpts) -> Result<(Vec, u64, (u32, u32)), String> { + use deckcraft_isobmff::{AvcConfig, Brand, CodecConfig, FourCc, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + let steps = timeline(pres, opts); + if steps.is_empty() { + return Err("there are no slides to put in the video".into()); + } + let (w, h) = frame_size(pres, opts); + let fps = opts.fps.clamp(1, 60); + let mut enc = deckcraft_h264::Encoder::new(deckcraft_h264::EncoderConfig { width: w, height: h, fps, qp: 24 }).map_err(|e| e.to_string())?; + let avc = AvcConfig::new(vec![enc.sps()], vec![enc.pps()], 4); + let entry = SampleEntry::video(FourCc::new(b"avc1"), CodecConfig::Avc(avc), w.min(u32::from(u16::MAX)) as u16, h.min(u32::from(u16::MAX)) as u16); + let brand = if format == "mov" { Brand::Mov } else { Brand::Mp4 }; + let mut mux = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(brand)).map_err(|e| e.to_string())?; + let track = mux.add_track(TrackConfig::new(entry, fps)).map_err(|e| e.to_string())?; + let key_every = u64::from(fps * KEY_EVERY_S); + let mut since_key: u64 = 0; + let mut total: u64 = 0; + for (i, step) in steps.iter().enumerate() { + // A key frame starts each slide and each transition, so the file seeks by slide. + let continues = matches!((i.checked_sub(1).and_then(|p| steps.get(p)), step), (Some(Step::Transition { index: a, .. }), Step::Transition { index: b, .. }) if a == b); + let mut start = !continues; + let mut sink = |px: &[u8]| -> Result<(), String> { + let pic = enc.encode_rgba(px, start || since_key >= key_every); + start = false; + since_key = if pic.key { 1 } else { since_key.saturating_add(1) }; + let data = pic.length_prefixed(); + mux.write_sample(track, WriteSample { data: &data, duration: 1, composition_offset: 0, is_sync: pic.key }).map_err(|e| e.to_string()) + }; + total = total.saturating_add(frames(pres, std::slice::from_ref(step), (w, h), &mut sink)?); + } + let n = total; + let bytes = mux.finish_faststart().map_err(|e| e.to_string())?.into_inner(); + Ok((bytes, n, (w, h))) +} + +#[cfg(test)] +mod tests { + use super::*; + use deckcraft_model::anim::Transition; + + fn deck(n: usize) -> Presentation { + let mut p = deckcraft_model::defaults::new_presentation(None); + while p.slides.len() < n { + let s = p.slides[0].clone(); + p.slides.push(s); + } + p.slides.truncate(n); + p + } + + #[test] + fn timeline_holds_slides_and_plays_transitions() { + let mut p = deck(3); + std::sync::Arc::make_mut(&mut p.slides[1]).transition = + Some(Transition { kind: "fade".into(), duration_ms: 500, advance_after_ms: Some(2000), ..Default::default() }); + std::sync::Arc::make_mut(&mut p.slides[2]).hidden = true; + let o = VideoOpts { fps: 10, seconds_per_slide: 3.0, ..Default::default() }; + let t = timeline(&p, &o); + assert_eq!(t.first(), Some(&Step::Hold { index: 0, frames: 30 })); + assert_eq!(t.iter().filter(|s| matches!(s, Step::Transition { index: 1, .. })).count(), 5); + assert_eq!(t.last(), Some(&Step::Hold { index: 1, frames: 20 }), "the hidden slide is left out"); + let with_hidden = timeline(&p, &VideoOpts { include_hidden: true, ..o }); + assert_eq!(with_hidden.last(), Some(&Step::Hold { index: 2, frames: 30 })); + } + + #[test] + fn timeline_is_capped_and_sizes_are_even() { + let mut p = deck(2); + for s in &mut p.slides { + std::sync::Arc::make_mut(s).transition = Some(Transition { advance_after_ms: Some(u32::MAX), ..Default::default() }); + } + let total: u64 = timeline(&p, &VideoOpts { fps: 60, ..Default::default() }) + .iter() + .map(|s| match s { + Step::Hold { frames, .. } => *frames, + Step::Transition { .. } => 1, + }) + .sum(); + assert!(total <= MAX_FRAMES); + let (w, h) = frame_size(&p, &VideoOpts { height: 721, ..Default::default() }); + assert_eq!((w % 2, h), (0, 720)); + } + + #[test] + fn frames_are_exactly_the_frame_size() { + let p = deck(1); + let size = frame_size(&p, &VideoOpts { height: 90, ..Default::default() }); + let mut seen = vec![]; + let n = frames(&p, &[Step::Hold { index: 0, frames: 2 }], size, &mut |px| { + seen.push(px.len()); + Ok(()) + }); + assert_eq!(n, Ok(2)); + assert!(seen.iter().all(|l| *l == size.0 as usize * size.1 as usize * 4)); + } + + #[test] + fn encodes_a_playable_mp4() { + let mut p = deck(2); + std::sync::Arc::make_mut(&mut p.slides[1]).transition = Some(Transition { kind: "fade".into(), duration_ms: 200, ..Default::default() }); + let o = VideoOpts { height: 72, fps: 10, seconds_per_slide: 0.5, include_hidden: false }; + let (bytes, n, (w, h)) = encode(&p, "mp4", &o).unwrap(); + assert_eq!(n, 5 + 2 + 5); + let f = deckcraft_isobmff::open(bytes.as_slice()).unwrap(); + let t = &f.tracks[0]; + assert_eq!(t.samples.len(), n as usize); + // Key frames start each slide and each transition. + let keys: Vec = t.samples.iter().enumerate().filter(|(_, s)| s.is_sync).map(|(i, _)| i).collect(); + assert_eq!(keys, vec![0, 5, 7]); + let mut dec = deckcraft_h264::Decoder::new(); + // Rebuild an Annex B stream from the length-prefixed samples and the avcC parameter sets. + let Some(deckcraft_isobmff::CodecConfig::Avc(avc)) = t.entries.first().map(|e| &e.codec) else { panic!("not AVC") }; + let mut stream = vec![]; + for ps in avc.sps.iter().chain(&avc.pps) { + stream.extend([0, 0, 0, 1]); + stream.extend(ps); + } + for i in 0..t.samples.len() { + let data = f.read_sample(bytes.as_slice(), 0, i).unwrap(); + let mut k = 0; + while k + 4 <= data.len() { + let len = u32::from_be_bytes([data[k], data[k + 1], data[k + 2], data[k + 3]]) as usize; + stream.extend([0, 0, 0, 1]); + stream.extend(&data[k + 4..k + 4 + len]); + k += 4 + len; + } + } + let mut pics = dec.decode(&stream, 0).unwrap(); + pics.extend(dec.flush()); + assert_eq!(pics.len(), n as usize); + assert_eq!((pics[0].width, pics[0].height), (w, h)); + } +} diff --git a/crates/h264/src/encoder.rs b/crates/h264/src/encoder.rs new file mode 100644 index 0000000000..14e205a772 --- /dev/null +++ b/crates/h264/src/encoder.rs @@ -0,0 +1,920 @@ +//! A small H.264 encoder for synthetic pictures (slides): Constrained Baseline, CAVLC, 8-bit +//! 4:2:0, one slice per picture, deblocking off. Every coded macroblock is Intra 16x16 (vertical, +//! horizontal or DC prediction, chosen by SAD); in P pictures a macroblock whose source pixels are +//! unchanged since the previous picture is skipped, so a held slide costs a few bytes a frame. +//! Written from ITU-T Rec. H.264 (clauses 7.3, 8.3.3, 8.3.4, 8.5, 9.2) as the inverse of +//! [`crate::Decoder`], which the tests decode it with. + +use crate::cavlc_tables::{COEFF_TOKEN, RUN_BEFORE, TOTAL_ZEROS_2X2, TOTAL_ZEROS_4X4}; + +/// Encoder settings. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EncoderConfig { + /// Picture size in pixels; both even, at most 8192×4320. + pub width: u32, + pub height: u32, + /// Frame rate written to the VUI timing (1–240). + pub fps: u32, + /// Quantisation parameter, 10–51 (lower is better quality and larger). + pub qp: u8, +} + +/// One coded picture: its NAL units without start codes (SPS and PPS lead a key picture). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncodedPicture { + pub nals: Vec>, + pub key: bool, +} + +impl EncodedPicture { + /// Annex B byte stream (start codes). + pub fn annex_b(&self) -> Vec { + let mut out = vec![]; + for n in &self.nals { + out.extend_from_slice(&[0, 0, 0, 1]); + out.extend_from_slice(n); + } + out + } + /// The picture's NAL units with 4-byte big-endian lengths (MP4 samples), parameter sets + /// left out (they go in the `avcC` box). + pub fn length_prefixed(&self) -> Vec { + let mut out = vec![]; + for n in self.nals.iter().filter(|n| !matches!(n.first().map(|b| b & 0x1F), Some(7 | 8))) { + out.extend_from_slice(&(n.len() as u32).to_be_bytes()); + out.extend_from_slice(n); + } + out + } +} + +/// Bit writer for RBSPs. +#[derive(Default)] +struct Bits { + buf: Vec, + cur: u64, + n: u32, +} + +impl Bits { + fn put(&mut self, v: u32, bits: u32) { + if bits == 0 { + return; + } + let bits = bits.min(32); + let mask = if bits == 32 { u32::MAX } else { (1u32 << bits) - 1 }; + self.cur = (self.cur << bits) | u64::from(v & mask); + self.n += bits; + while self.n >= 8 { + self.n -= 8; + self.buf.push((self.cur >> self.n) as u8); + } + self.cur &= (1u64 << self.n) - 1; + } + fn flag(&mut self, b: bool) { + self.put(u32::from(b), 1); + } + /// A code from the tables ("0101"). + fn code(&mut self, s: &str) { + for c in s.bytes() { + self.put(u32::from(c == b'1'), 1); + } + } + fn ue(&mut self, v: u32) { + let x = u64::from(v) + 1; + let len = 64 - x.leading_zeros(); + self.put(0, len - 1); + // len ≤ 33: write the high bit and the rest separately. + self.put(1, 1); + if len > 1 { + self.put((x & ((1u64 << (len - 1)) - 1)) as u32, len - 1); + } + } + fn se(&mut self, v: i32) { + let k = if v > 0 { (v as u32) * 2 - 1 } else { v.unsigned_abs() * 2 }; + self.ue(k); + } + /// `rbsp_trailing_bits`, then the bytes. + fn finish(mut self) -> Vec { + self.put(1, 1); + if self.n > 0 { + let pad = 8 - self.n; + self.put(0, pad); + } + self.buf + } +} + +/// A NAL unit from its header byte and RBSP, with emulation prevention. +fn nal(header: u8, rbsp: &[u8]) -> Vec { + let mut out = Vec::with_capacity(rbsp.len() + rbsp.len() / 64 + 2); + out.push(header); + let mut zeros = 0; + for &b in rbsp { + if zeros >= 2 && b <= 3 { + out.push(3); + zeros = 0; + } + out.push(b); + zeros = if b == 0 { zeros + 1 } else { 0 }; + } + out +} + +/// 4x4 zig-zag scan (frame): scan index → raster index. +const ZIGZAG: [usize; 16] = [0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15]; +/// Quantisation multipliers by `qp % 6` for positions (even, even), (odd, odd) and the rest. +const MF: [[i32; 3]; 6] = [[13107, 5243, 8066], [11916, 4660, 7490], [10082, 4194, 6554], [9362, 3647, 5825], [8192, 3355, 5243], [7282, 2893, 4559]]; +/// Dequantisation scales (`normAdjust4x4`), same positions. +const V: [[i32; 3]; 6] = [[10, 16, 13], [11, 18, 14], [13, 20, 16], [14, 23, 18], [16, 25, 20], [18, 29, 23]]; +/// Chroma QP for luma QP 30..=51 (Table 8-15); below 30 they are equal. +const QPC: [u8; 22] = [29, 30, 31, 32, 32, 33, 34, 34, 35, 35, 36, 36, 37, 37, 37, 38, 38, 38, 39, 39, 39, 39]; + +fn chroma_qp(qp: u8) -> u8 { + if qp < 30 { qp } else { QPC.get(usize::from(qp - 30)).copied().unwrap_or(39) } +} + +fn pos_class(raster: usize) -> usize { + let (i, j) = (raster / 4, raster % 4); + match (i % 2, j % 2) { + (0, 0) => 0, + (1, 1) => 1, + _ => 2, + } +} + +/// Forward core transform of a 4x4 residual (raster), `Cf · X · Cfᵀ`. +fn fdct(x: &[i32; 16]) -> [i32; 16] { + let mut t = [0i32; 16]; + for i in 0..4 { + let r = &x[i * 4..i * 4 + 4]; + let (s03, d03, s12, d12) = (r[0] + r[3], r[0] - r[3], r[1] + r[2], r[1] - r[2]); + t[i * 4] = s03 + s12; + t[i * 4 + 1] = 2 * d03 + d12; + t[i * 4 + 2] = s03 - s12; + t[i * 4 + 3] = d03 - 2 * d12; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let (a, b, c, d) = (t[j], t[4 + j], t[8 + j], t[12 + j]); + let (s03, d03, s12, d12) = (a + d, a - d, b + c, b - c); + out[j] = s03 + s12; + out[4 + j] = 2 * d03 + d12; + out[8 + j] = s03 - s12; + out[12 + j] = d03 - 2 * d12; + } + out +} + +/// Inverse transform (8.5.12.2) of dequantised coefficients, giving the residual. +fn idct(d: &[i32; 16]) -> [i32; 16] { + let mut f = [0i32; 16]; + for i in 0..4 { + let r = &d[i * 4..i * 4 + 4]; + let e0 = r[0] + r[2]; + let e1 = r[0] - r[2]; + let e2 = (r[1] >> 1) - r[3]; + let e3 = r[1] + (r[3] >> 1); + f[i * 4] = e0 + e3; + f[i * 4 + 1] = e1 + e2; + f[i * 4 + 2] = e1 - e2; + f[i * 4 + 3] = e0 - e3; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let (a, b, c, dd) = (f[j], f[4 + j], f[8 + j], f[12 + j]); + let g0 = a + c; + let g1 = a - c; + let g2 = (b >> 1) - dd; + let g3 = b + (dd >> 1); + out[j] = (g0 + g3 + 32) >> 6; + out[4 + j] = (g1 + g2 + 32) >> 6; + out[8 + j] = (g1 - g2 + 32) >> 6; + out[12 + j] = (g0 - g3 + 32) >> 6; + } + out +} + +/// Largest level magnitude written (keeps every level within the 12-bit escape). +const MAX_LEVEL: i32 = 2063; + +fn quant(w: i32, mf: i32, qbits: u32, f: i32) -> i32 { + let z = ((i64::from(w.abs()) * i64::from(mf) + i64::from(f)) >> qbits) as i32; + z.min(MAX_LEVEL) * w.signum() +} + +/// Dequantise an AC (or chroma AC) coefficient (8.5.12.1, flat scaling). +fn dequant(c: i32, qp: u8, class: usize) -> i32 { + let ls = 16 * V[usize::from(qp % 6)][class]; + let q = u32::from(qp / 6); + if qp >= 24 { (c * ls) << (q - 4) } else { (c * ls + (1 << (3 - q))) >> (4 - q) } +} + +/// Write one CAVLC `residual_block` of `coeffs` (scan order) with context `nc` (−1: chroma DC). +/// Returns TotalCoeff. +fn residual_block(b: &mut Bits, coeffs: &[i32], nc: i32) -> u8 { + let max = coeffs.len(); + let nz: Vec = (0..max).filter(|&i| coeffs[i] != 0).collect(); + let tc = nz.len(); + let mut t1 = 0; + for &i in nz.iter().rev() { + if coeffs[i].abs() == 1 && t1 < 3 { + t1 += 1; + } else { + break; + } + } + let col = match nc { + -1 => 4, + n if n < 2 => 0, + n if n < 4 => 1, + n if n < 8 => 2, + _ => 3, + }; + let token = COEFF_TOKEN.iter().find(|(o, t, _)| usize::from(*o) == t1 && usize::from(*t) == tc).map(|e| e.2[col]).unwrap_or(""); + b.code(token); + if tc == 0 { + return 0; + } + // Highest frequency first. + let levels: Vec = nz.iter().rev().map(|&i| coeffs[i]).collect(); + for l in levels.iter().take(t1) { + b.flag(*l < 0); + } + let mut sl: u32 = if tc > 10 && t1 < 3 { 1 } else { 0 }; + for (k, &l) in levels.iter().enumerate().skip(t1) { + let mut code = if l > 0 { 2 * l - 2 } else { -2 * l - 1 }; + if k == t1 && t1 < 3 { + code -= 2; + } + let code = code.max(0) as u32; + if sl == 0 { + if code < 14 { + b.put(0, code); + b.put(1, 1); + } else if code < 30 { + b.put(0, 14); + b.put(1, 1); + b.put(code - 14, 4); + } else { + b.put(0, 15); + b.put(1, 1); + b.put((code - 30).min(4095), 12); + } + } else if code < (15 << sl) { + b.put(0, code >> sl); + b.put(1, 1); + b.put(code & ((1 << sl) - 1), sl); + } else { + b.put(0, 15); + b.put(1, 1); + b.put((code - (15 << sl)).min(4095), 12); + } + if sl == 0 { + sl = 1; + } + if l.unsigned_abs() > (3 << (sl - 1)) && sl < 6 { + sl += 1; + } + } + let last = nz.last().copied().unwrap_or(0); + let total_zeros = last + 1 - tc; + if tc < max { + let code = if max == 4 { + TOTAL_ZEROS_2X2.get(tc - 1).and_then(|r| r.get(total_zeros)) + } else { + TOTAL_ZEROS_4X4.get(tc - 1).and_then(|r| r.get(total_zeros)) + }; + b.code(code.copied().unwrap_or("")); + } + let mut zeros_left = total_zeros; + for w in 0..tc.saturating_sub(1) { + if zeros_left == 0 { + break; + } + // nz descending: run of zeros below the w-th highest. + let (hi, lo) = (nz[tc - 1 - w], nz[tc - 2 - w]); + let run = hi - lo - 1; + let row = zeros_left.min(7) - 1; + b.code(RUN_BEFORE.get(row).and_then(|r| r.get(run)).copied().unwrap_or("")); + zeros_left -= run; + } + tc as u8 +} + +/// A picture's planes, padded to whole macroblocks. +#[derive(Clone)] +struct Planes { + y: Vec, + u: Vec, + v: Vec, + w: usize, + h: usize, +} + +impl Planes { + fn new(w: usize, h: usize) -> Self { + Planes { y: vec![16; w * h], u: vec![128; w * h / 4], v: vec![128; w * h / 4], w, h } + } +} + +/// The encoder; feed it pictures in display order. +pub struct Encoder { + cfg: EncoderConfig, + mbw: usize, + mbh: usize, + recon: Planes, + prev: Option, + frame_num: u32, + idr_id: u32, + /// TotalCoeff per 4x4 block for this picture: luma (mbw*4 × mbh*4), Cb, Cr (mbw*2 × mbh*2). + nc_y: Vec, + nc_u: Vec, + nc_v: Vec, +} + +/// Why an encoder can't be made. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConfigError(pub &'static str); + +impl std::fmt::Display for ConfigError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0) + } +} + +impl Encoder { + pub fn new(cfg: EncoderConfig) -> Result { + if cfg.width < 16 || cfg.height < 16 || cfg.width > 8192 || cfg.height > 4320 || !cfg.width.is_multiple_of(2) || !cfg.height.is_multiple_of(2) + { + return Err(ConfigError("the picture must be 16×16 to 8192×4320 pixels, with even sides")); + } + let mbw = cfg.width.div_ceil(16) as usize; + let mbh = cfg.height.div_ceil(16) as usize; + let cfg = EncoderConfig { qp: cfg.qp.clamp(10, 51), fps: cfg.fps.clamp(1, 240), ..cfg }; + Ok(Encoder { + cfg, + mbw, + mbh, + recon: Planes::new(mbw * 16, mbh * 16), + prev: None, + frame_num: 0, + idr_id: 0, + nc_y: vec![0; mbw * mbh * 16], + nc_u: vec![0; mbw * mbh * 4], + nc_v: vec![0; mbw * mbh * 4], + }) + } + + fn level_idc(&self) -> u8 { + let mbs = (self.mbw * self.mbh) as u64; + let rate = mbs * u64::from(self.cfg.fps); + match (mbs, rate) { + (m, r) if m <= 1620 && r <= 40_500 => 30, + (m, r) if m <= 3600 && r <= 108_000 => 31, + (m, r) if m <= 8192 && r <= 245_760 => 40, + (m, r) if m <= 8704 && r <= 522_240 => 42, + (m, r) if m <= 22_080 && r <= 589_824 => 50, + _ => 51, + } + } + + /// The sequence parameter set NAL unit. + pub fn sps(&self) -> Vec { + let mut b = Bits::default(); + b.put(66, 8); // profile_idc: Baseline + b.put(0xC0, 8); // constraint_set0 and set1: Constrained Baseline + b.put(u32::from(self.level_idc()), 8); + b.ue(0); // seq_parameter_set_id + b.ue(12); // log2_max_frame_num_minus4: 16-bit frame_num + b.ue(2); // pic_order_cnt_type 2: output order is decode order + b.ue(1); // max_num_ref_frames + b.flag(false); // gaps_in_frame_num_value_allowed_flag + b.ue(self.mbw as u32 - 1); + b.ue(self.mbh as u32 - 1); + b.flag(true); // frame_mbs_only_flag + b.flag(true); // direct_8x8_inference_flag + let (cr, cb) = ((self.mbw as u32 * 16 - self.cfg.width) / 2, (self.mbh as u32 * 16 - self.cfg.height) / 2); + b.flag(cr > 0 || cb > 0); + if cr > 0 || cb > 0 { + b.ue(0); + b.ue(cr); + b.ue(0); + b.ue(cb); + } + b.flag(true); // vui_parameters_present_flag + b.flag(false); // aspect_ratio_info_present_flag + b.flag(false); // overscan_info_present_flag + b.flag(true); // video_signal_type_present_flag + b.put(5, 3); // video_format: unspecified + b.flag(false); // video_full_range_flag + b.flag(true); // colour_description_present_flag + b.put(1, 8); // BT.709 primaries, transfer and matrix + b.put(1, 8); + b.put(1, 8); + b.flag(false); // chroma_loc_info_present_flag + b.flag(true); // timing_info_present_flag + b.put(1, 32); // num_units_in_tick + b.put(self.cfg.fps * 2, 32); // time_scale + b.flag(true); // fixed_frame_rate_flag + b.flag(false); // nal_hrd_parameters_present_flag + b.flag(false); // vcl_hrd_parameters_present_flag + b.flag(false); // pic_struct_present_flag + b.flag(true); // bitstream_restriction_flag + b.flag(true); // motion_vectors_over_pic_boundaries_flag + b.ue(0); // max_bytes_per_pic_denom + b.ue(0); // max_bits_per_mb_denom + b.ue(0); // log2_max_mv_length_horizontal (every vector is zero) + b.ue(0); + b.ue(0); // max_num_reorder_frames + b.ue(1); // max_dec_frame_buffering + nal(0x67, &b.finish()) + } + + /// The picture parameter set NAL unit. + pub fn pps(&self) -> Vec { + let mut b = Bits::default(); + b.ue(0); // pic_parameter_set_id + b.ue(0); // seq_parameter_set_id + b.flag(false); // entropy_coding_mode_flag: CAVLC + b.flag(false); // bottom_field_pic_order_in_frame_present_flag + b.ue(0); // num_slice_groups_minus1 + b.ue(0); // num_ref_idx_l0_default_active_minus1 + b.ue(0); + b.flag(false); // weighted_pred_flag + b.put(0, 2); // weighted_bipred_idc + b.se(0); // pic_init_qp_minus26 + b.se(0); // pic_init_qs_minus26 + b.se(0); // chroma_qp_index_offset + b.flag(true); // deblocking_filter_control_present_flag + b.flag(false); // constrained_intra_pred_flag + b.flag(false); // redundant_pic_cnt_present_flag + nal(0x68, &b.finish()) + } + + /// Encode one RGBA picture (`width`×`height`, row-major). A key picture (IDR, with SPS and + /// PPS) is made when `key` is set and for the first picture. + pub fn encode_rgba(&mut self, rgba: &[u8], key: bool) -> EncodedPicture { + let src = self.to_yuv(rgba); + let key = key || self.prev.is_none(); + if key { + self.frame_num = 0; + self.idr_id = (self.idr_id + 1) % 65_536; + } else { + self.frame_num = (self.frame_num + 1) % 65_536; + } + self.nc_y.iter_mut().for_each(|n| *n = 0); + self.nc_u.iter_mut().for_each(|n| *n = 0); + self.nc_v.iter_mut().for_each(|n| *n = 0); + let qp = self.cfg.qp; + let mut b = Bits::default(); + b.ue(0); // first_mb_in_slice + b.ue(if key { 7 } else { 5 }); // slice_type: all I / all P + b.ue(0); // pic_parameter_set_id + b.put(self.frame_num, 16); + if key { + b.ue(self.idr_id); + } + if !key { + b.flag(false); // num_ref_idx_active_override_flag + b.flag(false); // ref_pic_list_modification_flag_l0 + } + // dec_ref_pic_marking + if key { + b.flag(false); // no_output_of_prior_pics_flag + b.flag(false); // long_term_reference_flag + } else { + b.flag(false); // adaptive_ref_pic_marking_mode_flag + } + b.se(i32::from(qp) - 26); // slice_qp_delta + b.ue(1); // disable_deblocking_filter_idc: off + let mut skip_run = 0u32; + for my in 0..self.mbh { + for mx in 0..self.mbw { + let unchanged = !key && self.prev.as_ref().is_some_and(|p| same_mb(p, &src, mx, my)); + if unchanged { + skip_run += 1; + continue; + } + if !key { + b.ue(skip_run); + skip_run = 0; + } + self.intra_mb(&mut b, &src, mx, my, !key); + } + } + if !key && skip_run > 0 { + b.ue(skip_run); + } + let slice = nal(if key { 0x65 } else { 0x41 }, &b.finish()); + self.prev = Some(src); + let nals = if key { vec![self.sps(), self.pps(), slice] } else { vec![slice] }; + EncodedPicture { nals, key } + } + + /// RGBA to padded BT.709 limited-range 4:2:0 (edges replicated into the padding). + fn to_yuv(&self, rgba: &[u8]) -> Planes { + let (w, h) = (self.cfg.width as usize, self.cfg.height as usize); + let mut p = Planes::new(self.mbw * 16, self.mbh * 16); + let px = |x: usize, y: usize| -> (i32, i32, i32) { + let i = (y.min(h - 1) * w + x.min(w - 1)) * 4; + match rgba.get(i..i + 3) { + Some(s) => (i32::from(s[0]), i32::from(s[1]), i32::from(s[2])), + None => (0, 0, 0), + } + }; + for y in 0..p.h { + for x in 0..p.w { + let (r, g, bl) = px(x, y); + p.y[y * p.w + x] = (((47 * r + 157 * g + 16 * bl + 128) >> 8) + 16).clamp(0, 255) as u8; + } + } + let cw = p.w / 2; + for y in 0..p.h / 2 { + for x in 0..cw { + let (mut su, mut sv) = (0, 0); + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let (r, g, bl) = px(x * 2 + dx, y * 2 + dy); + su += (-26 * r - 87 * g + 112 * bl + 128) >> 8; + sv += (112 * r - 102 * g - 10 * bl + 128) >> 8; + } + p.u[y * cw + x] = ((su + 2).div_euclid(4) + 128).clamp(0, 255) as u8; + p.v[y * cw + x] = ((sv + 2).div_euclid(4) + 128).clamp(0, 255) as u8; + } + } + p + } + + fn nc_luma(&self, bx: usize, by: usize) -> i32 { + let w = self.mbw * 4; + let a = (bx > 0).then(|| i32::from(self.nc_y[by * w + bx - 1])); + let t = (by > 0).then(|| i32::from(self.nc_y[(by - 1) * w + bx])); + combine(a, t) + } + + fn nc_chroma(&self, plane: &[u8], bx: usize, by: usize) -> i32 { + let w = self.mbw * 2; + let a = (bx > 0).then(|| i32::from(plane[by * w + bx - 1])); + let t = (by > 0).then(|| i32::from(plane[(by - 1) * w + bx])); + combine(a, t) + } + + fn intra_mb(&mut self, b: &mut Bits, src: &Planes, mx: usize, my: usize, in_p: bool) { + let qp = self.cfg.qp; + let stride = self.recon.w; + let (x0, y0) = (mx * 16, my * 16); + let (has_top, has_left) = (my > 0, mx > 0); + let top: Vec = if has_top { (0..16).map(|i| i32::from(self.recon.y[(y0 - 1) * stride + x0 + i])).collect() } else { vec![] }; + let left: Vec = if has_left { (0..16).map(|i| i32::from(self.recon.y[(y0 + i) * stride + x0 - 1])).collect() } else { vec![] }; + let dc = match (has_top, has_left) { + (true, true) => (top.iter().sum::() + left.iter().sum::() + 16) >> 5, + (true, false) => (top.iter().sum::() + 8) >> 4, + (false, true) => (left.iter().sum::() + 8) >> 4, + _ => 128, + }; + let s = |x: usize, y: usize| i32::from(src.y[(y0 + y) * stride + x0 + x]); + let pred_of = |mode: u32, x: usize, y: usize| match mode { + 0 => top[x], + 1 => left[y], + _ => dc, + }; + let mut modes = vec![2u32]; + if has_top { + modes.push(0); + } + if has_left { + modes.push(1); + } + let mode = + modes.iter().copied().min_by_key(|&m| (0..256).map(|i| (s(i % 16, i / 16) - pred_of(m, i % 16, i / 16)).abs()).sum::()).unwrap_or(2); + // Luma: transform each 4x4, DC through the Hadamard. + let qbits = 15 + u32::from(qp / 6); + let f = (1 << qbits) / 3; + let m6 = usize::from(qp % 6); + let mut ac = [[0i32; 16]; 16]; // [block raster (by*4+bx)][raster coeff], quantised + let mut wdc = [0i32; 16]; + for blk in 0..16 { + let (bx, by) = (blk % 4, blk / 4); + let mut r = [0i32; 16]; + for i in 0..16 { + let (x, y) = (bx * 4 + i % 4, by * 4 + i / 4); + r[i] = s(x, y) - pred_of(mode, x, y); + } + let w = fdct(&r); + wdc[blk] = w[0]; + for k in 1..16 { + ac[blk][k] = quant(w[k], MF[m6][pos_class(k)], qbits, f); + } + } + let ydc = hadamard4(&wdc); + let mut zdc = [0i32; 16]; + for k in 0..16 { + zdc[k] = quant(ydc[k] / 2, MF[m6][0], qbits + 1, 2 * f); + } + let cbp_luma = ac.iter().any(|blk| blk.iter().skip(1).any(|c| *c != 0)); + // Chroma. + let qpc = chroma_qp(qp); + let cq = qpc_planes(self, src, mx, my, qpc); + let cbp_chroma = if cq.iter().any(|c| c.ac.iter().any(|blk| blk.iter().skip(1).any(|v| *v != 0))) { + 2 + } else if cq.iter().any(|c| c.dc.iter().any(|v| *v != 0)) { + 1 + } else { + 0 + }; + // Syntax. + let mb_type = 1 + mode + 4 * cbp_chroma + if cbp_luma { 12 } else { 0 }; + b.ue(mb_type + if in_p { 5 } else { 0 }); + b.ue(0); // intra_chroma_pred_mode: DC + b.se(0); // mb_qp_delta + let (bx0, by0) = (mx * 4, my * 4); + let dc_scan: Vec = ZIGZAG.iter().map(|&r| zdc[r]).collect(); + let nc0 = self.nc_luma(bx0, by0); + residual_block(b, &dc_scan, nc0); + let lw = self.mbw * 4; + for idx in 0..16 { + let (bx, by) = blk_xy(idx); + let blk = by * 4 + bx; + let total = if cbp_luma { + let scan: Vec = ZIGZAG.iter().skip(1).map(|&r| ac[blk][r]).collect(); + let nc = self.nc_luma(bx0 + bx, by0 + by); + residual_block(b, &scan, nc) + } else { + 0 + }; + self.nc_y[(by0 + by) * lw + bx0 + bx] = total; + } + if cbp_chroma > 0 { + for c in &cq { + residual_block(b, &c.dc, -1); + } + } + let cw = self.mbw * 2; + for (ci, c) in cq.iter().enumerate() { + for blk in 0..4 { + let (bx, by) = (blk % 2, blk / 2); + let total = if cbp_chroma == 2 { + let scan: Vec = ZIGZAG.iter().skip(1).map(|&r| c.ac[blk][r]).collect(); + let plane = if ci == 0 { &self.nc_u } else { &self.nc_v }; + let nc = self.nc_chroma(plane, mx * 2 + bx, my * 2 + by); + residual_block(b, &scan, nc) + } else { + 0 + }; + let plane = if ci == 0 { &mut self.nc_u } else { &mut self.nc_v }; + plane[(my * 2 + by) * cw + mx * 2 + bx] = total; + } + } + // Reconstruction, as the decoder does it. + let fdc = hadamard4(&zdc); + let ls0 = 16 * V[m6][0]; + let q6 = u32::from(qp / 6); + for blk in 0..16 { + let (bx, by) = (blk % 4, blk / 4); + let mut d = [0i32; 16]; + d[0] = if qp >= 36 { (fdc[blk] * ls0) << (q6 - 6) } else { (fdc[blk] * ls0 + (1 << (5 - q6))) >> (6 - q6) }; + for k in 1..16 { + d[k] = dequant(ac[blk][k], qp, pos_class(k)); + } + let r = idct(&d); + for i in 0..16 { + let (x, y) = (bx * 4 + i % 4, by * 4 + i / 4); + self.recon.y[(y0 + y) * stride + x0 + x] = (pred_of(mode, x, y) + r[i]).clamp(0, 255) as u8; + } + } + for (ci, c) in cq.iter().enumerate() { + let cs = self.recon.w / 2; + let plane = if ci == 0 { &mut self.recon.u } else { &mut self.recon.v }; + // Chroma DC levels are in raster order; inverse 2x2 transform and scale. + let f2 = [ + c.dc[0] + c.dc[1] + c.dc[2] + c.dc[3], + c.dc[0] - c.dc[1] + c.dc[2] - c.dc[3], + c.dc[0] + c.dc[1] - c.dc[2] - c.dc[3], + c.dc[0] - c.dc[1] - c.dc[2] + c.dc[3], + ]; + let lsc = 16 * V[usize::from(qpc % 6)][0]; + for blk in 0..4 { + let (bx, by) = (blk % 2, blk / 2); + let mut d = [0i32; 16]; + d[0] = ((f2[blk] * lsc) << u32::from(qpc / 6)) >> 5; + for k in 1..16 { + d[k] = dequant(c.ac[blk][k], qpc, pos_class(k)); + } + let r = idct(&d); + for i in 0..16 { + let (x, y) = (bx * 4 + i % 4, by * 4 + i / 4); + plane[(my * 8 + y) * cs + mx * 8 + x] = (c.pred[blk] + r[i]).clamp(0, 255) as u8; + } + } + } + } +} + +/// One chroma component of a macroblock: DC prediction per 4x4 block, quantised DC (raster +/// 2x2) and AC levels. +struct ChromaMb { + pred: [i32; 4], + dc: [i32; 4], + ac: [[i32; 16]; 4], +} + +fn qpc_planes(e: &Encoder, src: &Planes, mx: usize, my: usize, qpc: u8) -> [ChromaMb; 2] { + let cs = e.recon.w / 2; + let (x0, y0) = (mx * 8, my * 8); + let qbits = 15 + u32::from(qpc / 6); + let f = (1 << qbits) / 3; + let m6 = usize::from(qpc % 6); + let one = |rec: &[u8], s: &[u8]| -> ChromaMb { + let top: Option> = (my > 0).then(|| (0..8).map(|i| i32::from(rec[(y0 - 1) * cs + x0 + i])).collect()); + let left: Option> = (mx > 0).then(|| (0..8).map(|i| i32::from(rec[(y0 + i) * cs + x0 - 1])).collect()); + let mut pred = [128; 4]; + for (blk, p) in pred.iter_mut().enumerate() { + let (xo, yo) = ((blk % 2) * 4, (blk / 2) * 4); + let st = top.as_ref().map(|t| t[xo..xo + 4].iter().sum::()); + let sl = left.as_ref().map(|l| l[yo..yo + 4].iter().sum::()); + *p = match ((xo, yo), st, sl) { + ((0, 0), Some(a), Some(c)) | ((4, 4), Some(a), Some(c)) => (a + c + 4) >> 3, + ((0, 0), _, Some(c)) | ((4, 4), _, Some(c)) => (c + 2) >> 2, + ((0, 0), Some(a), _) | ((4, 4), Some(a), _) => (a + 2) >> 2, + ((4, 0), Some(a), _) => (a + 2) >> 2, + ((4, 0), None, Some(c)) => (c + 2) >> 2, + ((0, 4), _, Some(c)) => (c + 2) >> 2, + ((0, 4), Some(a), None) => (a + 2) >> 2, + _ => 128, + }; + } + let mut wdc = [0i32; 4]; + let mut ac = [[0i32; 16]; 4]; + for blk in 0..4 { + let (bx, by) = (blk % 2, blk / 2); + let mut r = [0i32; 16]; + for i in 0..16 { + let (x, y) = (bx * 4 + i % 4, by * 4 + i / 4); + r[i] = i32::from(s[(y0 + y) * cs + x0 + x]) - pred[blk]; + } + let w = fdct(&r); + wdc[blk] = w[0]; + for k in 1..16 { + ac[blk][k] = quant(w[k], MF[m6][pos_class(k)], qbits, f); + } + } + let y2 = [ + wdc[0] + wdc[1] + wdc[2] + wdc[3], + wdc[0] - wdc[1] + wdc[2] - wdc[3], + wdc[0] + wdc[1] - wdc[2] - wdc[3], + wdc[0] - wdc[1] - wdc[2] + wdc[3], + ]; + let dc = y2.map(|v| quant(v, MF[m6][0], qbits + 1, 2 * f)); + ChromaMb { pred, dc, ac } + }; + [one(&e.recon.u, &src.u), one(&e.recon.v, &src.v)] +} + +/// nC from the left and top neighbours' TotalCoeff (9.2.1). +fn combine(a: Option, t: Option) -> i32 { + match (a, t) { + (Some(a), Some(t)) => (a + t + 1) >> 1, + (Some(a), None) => a, + (None, Some(t)) => t, + (None, None) => 0, + } +} + +/// `luma4x4BlkIdx` → 4x4 block position in the macroblock. +fn blk_xy(idx: usize) -> (usize, usize) { + let (b8, b4) = (idx / 4, idx % 4); + ((b8 % 2) * 2 + b4 % 2, (b8 / 2) * 2 + b4 / 2) +} + +/// 4x4 Hadamard `H · X · H` (raster, symmetric H). +fn hadamard4(x: &[i32; 16]) -> [i32; 16] { + let h = |a: i32, b: i32, c: i32, d: i32| [a + b + c + d, a + b - c - d, a - b - c + d, a - b + c - d]; + let mut t = [0i32; 16]; + for i in 0..4 { + let r = h(x[i * 4], x[i * 4 + 1], x[i * 4 + 2], x[i * 4 + 3]); + t[i * 4..i * 4 + 4].copy_from_slice(&r); + } + let mut out = [0i32; 16]; + for j in 0..4 { + let c = h(t[j], t[4 + j], t[8 + j], t[12 + j]); + for i in 0..4 { + out[i * 4 + j] = c[i]; + } + } + out +} + +/// The macroblock's source pixels are the same in both pictures. +fn same_mb(a: &Planes, b: &Planes, mx: usize, my: usize) -> bool { + let w = a.w; + let rows = |pa: &[u8], pb: &[u8], stride: usize, x0: usize, y0: usize, n: usize| { + (0..n).all(|y| pa[(y0 + y) * stride + x0..(y0 + y) * stride + x0 + n] == pb[(y0 + y) * stride + x0..(y0 + y) * stride + x0 + n]) + }; + rows(&a.y, &b.y, w, mx * 16, my * 16, 16) && rows(&a.u, &b.u, w / 2, mx * 8, my * 8, 8) && rows(&a.v, &b.v, w / 2, mx * 8, my * 8, 8) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A test card: gradients, a hard-edged box and text-like stripes. + fn card(w: u32, h: u32, shift: u32) -> Vec { + let mut out = vec![0u8; (w * h * 4) as usize]; + for y in 0..h { + for x in 0..w { + let i = ((y * w + x) * 4) as usize; + let inbox = (x + shift) % w > w / 4 && (x + shift) % w < w / 2 && y > h / 3 && y < h * 2 / 3; + let stripe = y > h * 3 / 4 && (x / 3) % 2 == 0; + let (r, g, b) = if inbox { + (230, 80, 30) + } else if stripe { + (20, 20, 20) + } else { + ((x * 255 / w) as u8, (y * 255 / h) as u8, 200) + }; + out[i..i + 4].copy_from_slice(&[r, g, b, 255]); + } + } + out + } + + fn psnr(a: &[u8], b: &[u8]) -> f64 { + let mse: f64 = a.iter().zip(b).map(|(x, y)| (f64::from(*x) - f64::from(*y)).powi(2)).sum::() / a.len().max(1) as f64; + if mse == 0.0 { 99.0 } else { 10.0 * (255.0f64 * 255.0 / mse).log10() } + } + + #[test] + fn decodes_with_our_decoder_at_good_quality() { + let (w, h) = (200, 120); // not a multiple of 16: cropping + let mut enc = Encoder::new(EncoderConfig { width: w, height: h, fps: 30, qp: 22 }).unwrap(); + let mut dec = crate::Decoder::new(); + let frames = [card(w, h, 0), card(w, h, 0), card(w, h, 17), card(w, h, 17)]; + let mut stream = vec![]; + let mut sizes = vec![]; + let mut recons = vec![]; + for (i, f) in frames.iter().enumerate() { + let p = enc.encode_rgba(f, i == 0); + sizes.push(p.length_prefixed().len()); + stream.extend(p.annex_b()); + recons.push(enc.recon.y.clone()); + } + let mut pics = dec.decode(&stream, 0).unwrap(); + pics.extend(dec.flush()); + assert_eq!(pics.len(), 4); + let src = enc.to_yuv(&frames[3]); + for (k, pic) in pics.iter().enumerate() { + assert_eq!((pic.width, pic.height), (w, h)); + // The decoder reconstructs exactly what the encoder predicted from. + let stride = enc.recon.w; + for y in 0..h as usize { + let d = &pic.y[y * pic.y_stride..y * pic.y_stride + w as usize]; + let e = &recons[k][y * stride..y * stride + w as usize]; + assert_eq!(d, e, "picture {k} row {y}"); + } + } + let last = &pics[3]; + let dy: Vec = (0..h as usize).flat_map(|y| last.y[y * last.y_stride..y * last.y_stride + w as usize].to_vec()).collect(); + let sy: Vec = (0..h as usize).flat_map(|y| src.y[y * src.w..y * src.w + w as usize].to_vec()).collect(); + assert!(psnr(&dy, &sy) > 38.0, "psnr {}", psnr(&dy, &sy)); + // An unchanged picture is all skipped: a handful of bytes. + assert!(sizes[1] < 16, "{sizes:?}"); + assert!(sizes[3] < 16 && sizes[2] < sizes[0], "{sizes:?}"); + } + + #[test] + fn rejects_bad_sizes_and_writes_levels_for_big_pictures() { + assert!(Encoder::new(EncoderConfig { width: 15, height: 16, fps: 30, qp: 26 }).is_err()); + assert!(Encoder::new(EncoderConfig { width: 101, height: 16, fps: 30, qp: 26 }).is_err()); + let e = Encoder::new(EncoderConfig { width: 1920, height: 1080, fps: 30, qp: 26 }).unwrap(); + assert_eq!(e.level_idc(), 40); + let e = Encoder::new(EncoderConfig { width: 3840, height: 2160, fps: 30, qp: 26 }).unwrap(); + assert_eq!(e.level_idc(), 51); + } + + #[test] + fn big_levels_and_flat_pictures_round_trip() { + // Black then white at a low QP: large DC levels and escapes. + let (w, h) = (64, 32); + let mut enc = Encoder::new(EncoderConfig { width: w, height: h, fps: 30, qp: 10 }).unwrap(); + let mut dec = crate::Decoder::new(); + let mut stream = vec![]; + for (i, v) in [0u8, 255, 128].iter().enumerate() { + let mut img = vec![*v; (w * h * 4) as usize]; + // Noise for many coefficients per block. + for (k, p) in img.iter_mut().enumerate() { + if k % 4 != 3 && (k * 7919) % 13 < 6 { + *p = p.wrapping_add(((k * 31) % 97) as u8); + } + } + stream.extend(enc.encode_rgba(&img, i == 0).annex_b()); + } + let mut pics = dec.decode(&stream, 0).unwrap(); + pics.extend(dec.flush()); + assert_eq!(pics.len(), 3); + let last = &pics[2]; + let stride = enc.recon.w; + for y in 0..h as usize { + assert_eq!(&last.y[y * last.y_stride..y * last.y_stride + w as usize], &enc.recon.y[y * stride..y * stride + w as usize]); + } + } +} diff --git a/crates/h264/src/lib.rs b/crates/h264/src/lib.rs index a65a18e342..7ac80f324a 100644 --- a/crates/h264/src/lib.rs +++ b/crates/h264/src/lib.rs @@ -27,6 +27,8 @@ mod cavlc; #[rustfmt::skip] mod cavlc_tables; mod deblock; +mod encoder; +pub use encoder::{ConfigError, EncodedPicture, Encoder, EncoderConfig}; mod decoder; mod dpb; mod error; diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index a5f72fe78f..8258237762 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -29,7 +29,7 @@ PowerPoint opens and saves; Viewer = opens only). | PNG / JPEG slide images | export | n/a | yes (`file.export`, `deckcraft-cli render`) | Matches the editor's renderer. PNG `pHYs` and JPEG JFIF density are `scale` × 72 dpi (default scale 2 is 144 dpi), so Preview shows the slide's real size | render and engine tests | | TIFF / BMP / GIF slide images | export | n/a | no | Easy (the `image` crate writes them) | none | | Animated GIF of the deck | export | n/a | yes (`file.export` format `gif`, Export…, `deckcraft-cli`, MCP `export`) | One frame per slide from the slide renderer; size small/medium/large/extra large or 16–3840 px wide, 0.1–600 s per slide, loop forever or play once, slide range, hidden slides skipped; a 256-colour palette per frame (exact, or NeuQuant above 256 colours). Transitions and build animations are not animated | `crates/engine/src/gif_export.rs` tests (decoded frames, size, delay, loop, hidden slides, hostile params), MCP, CLI and export dialog tests | -| MP4 / MOV video of the deck | export | n/a | no | Needs timeline render + H.264 encode | none | +| MP4 / MOV video of the deck | export | n/a | yes | `file.export` mp4/mov: transitions then each slide for its timing, 480p–4K, own H.264 encoder (intra + skip); no in-slide animations, no audio | none | | Print | yes | n/a | no | PR #66 (browser) open | none | ## Pictures and media inside decks diff --git a/docs/gaps.md b/docs/gaps.md index 4e56495123..e24ba23a92 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -138,13 +138,14 @@ Detail in [file-format-parity.md](file-format-parity.md). - **Impact:** handouts and speaker notes are printed in many workplaces. - **Estimate:** 5–8 h (print through the PDF path and each OS's print dialog). **Beta blocker.** -### No video export; the animated GIF has no transitions or builds -- **Evidence:** checklist row "Export video (MP4) / animated GIF" is partial: `file.export` writes - an animated GIF with one frame per slide (#171), but there is no MP4/MOV export, and the GIF - doesn't animate transitions or build animations. -- **Impact:** sharing a talk as a video is a common request. -- **Estimate:** 10–16 h (render the timeline frame by frame, which the GIF can then reuse for - transitions and builds; an H.264 encoder is the hard part, FilmCraft has one to copy). +### Video has no animations or audio; the animated GIF has no transitions or builds +- **Evidence:** `file.export` writes MP4/MOV: each slide's transition, then the slide held for its + "advance after" time (else a default), at 480p–4K, encoded by DeckCraft's own H.264 encoder + (`crates/h264/src/encoder.rs`). Animations inside a slide show in their final state and the video + has no audio track (narration, media sound). The animated GIF (#171) has one frame per slide. +- **Impact:** sharing a talk as a video is a common request; builds and narration are part of it. +- **Estimate:** 4–8 h (animation timeline frames, an AAC track from the media and narration, an + Export dialog page). ### Embedded MicroType Express font subsets aren't drawn - **Evidence:** `p:embeddedFontLst` is read and written. A raw TrueType/OpenType face, or one inside an Embedded OpenType wrapper, is loaded and used to draw. A MicroType Express subset (`application/x-fontdata` with neither form) is kept and written back, and the text still uses a stand-in. From 558bb64145ee434fc1be38d559364aec799e2154 Mon Sep 17 00:00:00 2001 From: Nikita Rakitin Date: Wed, 14 Oct 2026 02:07:25 +0300 Subject: [PATCH 2/4] Video: animations and sound in MP4/MOV export Exported video now plays each slide's in-slide animations on the slide-show timeline (a secondsPerClick pause before each click step; with/after-previous and timed effects play as in the show) and carries a sound track: embedded audio (autoplay and Media effects, trim, fades, volume, loop, play across slides), narration clips and transition sounds, mixed to 48 kHz stereo. Sound is AAC-LC by default via deckcraft-aac, a pure-Rust encoder ported from FilmCraft (layer 1, no new third-party deps), or 16-bit PCM in .mov (audio: pcm); audio: none drops it. Video clips still show their poster frame. Co-authored-by: Cursor --- Cargo.lock | 10 + Cargo.toml | 1 + crates/aac/Cargo.toml | 19 + crates/aac/README.md | 191 ++++ crates/aac/src/config.rs | 447 ++++++++++ crates/aac/src/decoder.rs | 584 +++++++++++++ crates/aac/src/encoder/mod.rs | 833 ++++++++++++++++++ crates/aac/src/encoder/psy.rs | 176 ++++ crates/aac/src/encoder/quant.rs | 382 ++++++++ crates/aac/src/huffman.rs | 306 +++++++ crates/aac/src/huffman_tables.rs | 1406 ++++++++++++++++++++++++++++++ crates/aac/src/ics.rs | 151 ++++ crates/aac/src/lib.rs | 79 ++ crates/aac/src/mdct.rs | 274 ++++++ crates/aac/src/tables.rs | 143 +++ crates/aac/src/tns.rs | 313 +++++++ crates/engine/Cargo.toml | 2 + crates/engine/src/cmd/file.rs | 19 +- crates/engine/src/video.rs | 597 +++++++++++-- docs/file-format-parity.md | 2 +- docs/gaps.md | 15 +- xtask/src/layers.rs | 1 + 22 files changed, 5866 insertions(+), 85 deletions(-) create mode 100644 crates/aac/Cargo.toml create mode 100644 crates/aac/README.md create mode 100644 crates/aac/src/config.rs create mode 100644 crates/aac/src/decoder.rs create mode 100644 crates/aac/src/encoder/mod.rs create mode 100644 crates/aac/src/encoder/psy.rs create mode 100644 crates/aac/src/encoder/quant.rs create mode 100644 crates/aac/src/huffman.rs create mode 100644 crates/aac/src/huffman_tables.rs create mode 100644 crates/aac/src/ics.rs create mode 100644 crates/aac/src/lib.rs create mode 100644 crates/aac/src/mdct.rs create mode 100644 crates/aac/src/tables.rs create mode 100644 crates/aac/src/tns.rs diff --git a/Cargo.lock b/Cargo.lock index 19de932433..167cfbd5f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -909,6 +909,15 @@ dependencies = [ "winresource", ] +[[package]] +name = "deckcraft-aac" +version = "0.7.0" +dependencies = [ + "deckcraft-bitstream", + "rayon", + "thiserror 2.0.21", +] + [[package]] name = "deckcraft-anim" version = "0.7.0" @@ -954,6 +963,7 @@ dependencies = [ name = "deckcraft-engine" version = "0.7.0" dependencies = [ + "deckcraft-aac", "deckcraft-anim", "deckcraft-color", "deckcraft-fonts", diff --git a/Cargo.toml b/Cargo.toml index c9cb818236..b815999fb8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ deckcraft-ui-egui = { path = "crates/ui-egui" } deckcraft-media = { path = "crates/media" } # codecs (ported from FilmCraft, same owner and licence) deckcraft-bitstream = { path = "crates/bitstream" } +deckcraft-aac = { path = "crates/aac" } deckcraft-h264 = { path = "crates/h264" } deckcraft-hevc = { path = "crates/hevc" } deckcraft-vp9 = { path = "crates/vp9" } diff --git a/crates/aac/Cargo.toml b/crates/aac/Cargo.toml new file mode 100644 index 0000000000..1df55220c7 --- /dev/null +++ b/crates/aac/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "deckcraft-aac" +description = "Clean-room AAC-LC encoder and decoder (ISO/IEC 14496-3), ported from FilmCraft" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[features] +default = [] +rayon = ["dep:rayon"] + +[dependencies] +deckcraft-bitstream = { workspace = true } +thiserror = { workspace = true } +rayon = { workspace = true, optional = true } diff --git a/crates/aac/README.md b/crates/aac/README.md new file mode 100644 index 0000000000..9132995baf --- /dev/null +++ b/crates/aac/README.md @@ -0,0 +1,191 @@ +# deckcraft-aac + +A clean-room AAC-LC encoder and decoder (ISO/IEC 14496-3, MPEG-4 Audio, Low Complexity profile), +written from the standard and the public psychoacoustics literature. Layer L0: it depends only on +`deckcraft-bitstream` and `thiserror` (plus optional `rayon`), has no `unsafe`, and builds for +`wasm32-unknown-unknown`. + +No FFmpeg/FAAC/FDK-AAC/libfaad or other GPL/LGPL code was read or ported. ffmpeg is used only as an +external **test oracle** (decoding our streams, producing reference streams). + +## API + +```rust +use deckcraft_aac::{Decoder, Encoder, EncoderConfig, BitrateMode, AdtsHeader, split_adts}; + +// Encoder: planar f32 (±1.0) → raw access units +let mut enc = Encoder::new(EncoderConfig::cbr(48_000, 2, 128_000))?; // or EncoderConfig::vbr(48_000, 2, 3.0) +let mut aus: Vec> = enc.encode(&[&left, &right]); // 0..n AUs per call, any chunk size +aus.extend(enc.flush()); // zero-pads and drains the look-ahead +let asc = enc.audio_specific_config(); // bytes for the MP4 `esds` +let delay = enc.priming_samples(); // 1024 → edit list media_time +let stats = enc.stats(); // frames, bytes, bitrate(), short blocks, M/S, TNS + +// Decoder: AudioSpecificConfig + raw AUs → planar f32, 1024 samples per channel per AU +let mut dec = Decoder::new(&asc)?; +let pcm: Vec> = dec.decode(&aus[0])?; + +// ADTS +let frames = split_adts(&adts_bytes)?; // [(AdtsHeader, raw payload)] +let mut dec = Decoder::from_adts(&frames[0].0)?; // channel config 0 → layout from the in-band PCE +``` + +`EncoderConfig` fields: `sample_rate` (the 13 AAC rates, 7.35–96 kHz), `channels` (1–8), `mode` +(`BitrateMode::Cbr(bps)` / `BitrateMode::Vbr(quality 1.0..=5.0)`), `bandwidth` (Hz, `None` = automatic +from bitrate/quality), `tns`, `ms_stereo`, `window_shape` (sine/KBD), `adts` (prefix each AU with a +7-byte ADTS header). + +**Channel order** is AAC element order, for both encoder input and decoder output: + +| channels | configuration | order | +|---|---|---| +| 1 | 1 | C | +| 2 | 2 | L R | +| 3 | 3 | C L R | +| 4 | 4 | C L R Cs | +| 5 | 5 | C L R Ls Rs | +| 6 | 6 | C L R Ls Rs LFE | +| 7 | 0 + PCE | C L R Ls Rs Cs LFE (PCE in the ASC; repeated in every AU when ADTS-framed) | +| 8 | 7 | C Lc Rc L R Ls Rs LFE | + +The decoder also reads configurations 11 (6.1: C L R Ls Rs Cs LFE) and 12 (7.1: C L R Ls Rs Lsr Rsr +LFE) from ISO/IEC 14496-3 Amd 4; ffmpeg 9 writes 7.1 AAC as configuration 12. + +`Decoder::layout()` reports the element layout (it follows a PCE for configuration 0). + +## Encoder + +Per 1024-sample frame and channel element: + +1. **Transient detection**: energy of the first-difference (high-passed) signal in 128-sample + sub-blocks against the mean of the previous eight; one frame of look-ahead selects + `ONLY_LONG` / `LONG_START` / `EIGHT_SHORT` / `LONG_STOP`. Short windows are grouped by energy + similarity (an attack starts a new group). LFE is always long. +2. **MDCT** (2048/256 points) via a DCT-IV on an N/4-point radix-2 complex FFT; sine or KBD + (α = 4 long, 6 short) windows. +3. **Psychoacoustic model** in the MDCT domain: band energies, tonality from the spectral flatness of + a ≥16-line neighbourhood, Schroeder's spreading function in the Bark domain, tonality-dependent + masking offset (30 dB tonal … 6 dB noise), Terhardt threshold in quiet (capped at 40 dB SPL) as a + floor, pre-echo control (threshold ≤ 2× previous frame), perceptual entropy. +4. **TNS** on long windows: Levinson–Durbin (order 8) over the spectrum above 1.4 kHz; applied when the + prediction gain exceeds 1.4, with 4-bit reflection coefficients. +5. **M/S** per band (and group) for channel pairs (always common window): chosen when the estimated + perceptual cost of M and S (with the stricter L/R threshold, halved) is below that of L and R. +6. **Rate–distortion loop**: one multiplier λ scales every band's allowed noise. Each band gets the + coarsest scalefactor whose *measured* quantisation error meets its target (binary search, + `0.4054` rounding), scalefactor differences are kept within ±60, codebooks and sections are chosen + by dynamic programming over all 12 codebook states, and λ is bracketed and bisected until the frame + fits its budget (CBR) or fixed from the quality (VBR). +7. **Bit reservoir** (CBR): frame budgets follow the perceptual entropy (short-window frames get extra), + bounded by the 6144-bit-per-channel decoder buffer; overflow is padded with fill elements and the + final frame drains the reservoir, so the stream is exactly `frames × bitrate × 1024 / rate`. + +Bandwidth: automatic from bitrate per full-bandwidth channel (e.g. 64 kbps/ch → 16 kHz, 96 → 19 kHz, +≥144 → 22 kHz) or from quality in VBR. + +Delay: 1024 priming samples (`priming_samples()`); after `flush()` the decoded stream minus the priming +covers every input sample (`ceil(n / 1024) + 1` access units). + +Performance (Apple Silicon, release, single thread): 10 s of 44.1 kHz stereo encodes at ~85× realtime +and decodes at ~1000× realtime. With the `rayon` feature channels are quantised in parallel inside the +rate loop. + +## Decoder + +Full AAC-LC syntax: SCE/CPE/LFE, DSE, FIL, PCE (reconfigures a configuration-0 stream), all window +sequences and both window shapes (shape of the previous frame for the left half), section data, +scalefactors, pulse data, TNS (all-pole synthesis, long and short), M/S (per band / all bands), +intensity stereo (in/out of phase, M/S-inverted), PNS (with correlated noise for `ms_used` noise bands +as specified), IMDCT and overlap-add. Output is planar f32 at ±1.0 full scale. All reads are bounds +checked: malformed access units return `Err`, never panic. + +## Test results + +All produced by `cargo test -p deckcraft-aac -- --nocapture` (ffmpeg 9.0 at `/opt/homebrew/bin` as +oracle; tests skip when it is absent). Fixtures go to `target/fixtures/aac/`. + +### Encoder quality / bitrate (2 s synthetic signals, decoded by ffmpeg) + +SNR is full-band; bwSNR is measured after low-passing both signals at the coded bandwidth; +segSNR averages 1024-sample segments above −50 dBFS (clamped to −10…60 dB). "us-vs-ff" is the +largest difference between our decoder and ffmpeg's on the same stream. Every stream decodes without +ffmpeg errors, no output clips (peak < 1.0), and CBR is within ±0.1% of target (test limit ±5%). + +| signal | kbps | actual | bw Hz | SNR | bwSNR | segSNR | short frames | us-vs-ff | +|---|---|---|---|---|---|---|---|---| +| sweep 2ch 44.1k | 64 | 64.00 | 11000 | 12.1 | 36.1 | 43.6 | 1 | 5e-7 | +| sweep 2ch 44.1k | 128 | 127.92 | 16000 | 26.1 | 27.3 | 54.2 | 1 | 3e-7 | +| sweep 2ch 44.1k | 320 | 319.90 | 21609 | 46.6 | 46.6 | 59.6 | 1 | 3e-7 | +| white noise 2ch 44.1k | 64 | 64.00 | 11000 | 1.3 | 3.3 | 1.3 | 1 | 5e-7 | +| white noise 2ch 44.1k | 320 | 319.91 | 21609 | 16.4 | 16.5 | 16.6 | 1 | 1.5e-6 | +| speech-like 2ch 48k | 64 | 64.00 | 11000 | 13.6 | 14.0 | 15.2 | 13 | 2e-7 | +| speech-like 2ch 48k | 128 | 127.98 | 16000 | 21.3 | 23.0 | 20.7 | 13 | 2e-7 | +| speech-like 2ch 48k | 320 | 319.96 | 22000 | 31.2 | 31.5 | 31.3 | 13 | 2e-7 | +| chords 2ch 44.1k | 64 | 64.01 | 11000 | 12.4 | 12.4 | 20.6 | 4 | 2e-7 | +| chords 2ch 44.1k | 128 | 128.00 | 16000 | 20.1 | 20.1 | 38.5 | 4 | 2e-7 | +| chords 2ch 44.1k | 320 | 319.99 | 21609 | 35.9 | 35.9 | 55.5 | 4 | 2e-7 | +| clicks 2ch 48k | 64 | 64.00 | 11000 | 10.4 | 11.2 | 48.6 | 11 | 9e-8 | +| clicks 2ch 48k | 128 | 128.00 | 16000 | 20.2 | 25.1 | 53.9 | 11 | 2e-7 | +| clicks 2ch 48k | 320 | 319.95 | 22000 | 39.6 | 39.6 | 58.0 | 11 | 1e-7 | +| speech-like 1ch 44.1k | 64 | 64.00 | 16000 | 18.4 | 19.2 | 18.0 | 15 | 2e-7 | +| chords 5.1 48k | 320 | 320.00 | 16000 | 27.2 | 27.2 | 37.9 | 9 | 2e-7 | +| chords 3/4/5/7/8ch 48k | 192–448 | exact | 16000 | 26–28 | 26–28 | 38 | 7–13 | 2e-7 | +| chords 2ch 8 k … 96 k (10 rates) | 24–256 | exact | 3.9–21 k | 13–38 | | 17–47 | | ≤3e-7 | + +Reference points from ffmpeg's own AAC encoder on the same signals (stereo 48 kHz, segSNR): +sweep 128k 39.7, speech 128k 21.2, chords 128k 32.8, clicks 128k 43.3 (ours: 52/20/39/54). Pure SNR +is not a perceptual measure: the model deliberately leaves noise under the masking threshold, and +full-band white noise is the worst case by design. + +VBR (`Vbr(q)`, 44.1 kHz stereo): chords q1 62 kbps / q3 139 / q5 225 (segSNR 18 / 39 / 51); +speech-like q1 42 / q3 166 / q5 272 kbps (segSNR 13 / 29 / 41). Bitrate and quality grow +monotonically with q. + +### Decoder accuracy vs ffmpeg (streams from `ffmpeg -c:a aac`, PNS off) + +| stream | max abs error | +|---|---| +| stereo 44.1k 128k / 48k 256k / 96k 256k | 1.2e-7 / 1.2e-7 / 1.3e-7 | +| mono 48k 64k / 22.05k 32k / 16k 24k / 8k 16k | 1.5e-7 / 1.2e-7 / 1.2e-7 / 8.9e-8 | +| stereo 44.1k 96k with intensity stereo + M/S + TNS | 1.2e-7 | +| stereo 32k 48k with intensity stereo | 1.2e-7 | +| 3.0, 5.1 (320k), 7.1 wide with PCE (448k) | 1.2e-7, 1.5e-7, 1.3e-7 | +| stereo 44.1k 160k `-aac_coder fast` | 1.2e-7 | + +Our own encoder's streams: ≤1.6e-6 against ffmpeg for every case above. PNS (ffmpeg's encoder does not +emit it, so streams are hand-built): total energy within 0.15 dB of ffmpeg; per-frame energy within +0.8 dB (random vectors differ). For noise bands flagged `ms_used` we produce identical L/R noise as +the standard specifies; ffmpeg produces independent noise there. + +### Robustness + +12 000 mutated access units (bit flips, byte overwrites, truncation, extension, splices, random +buffers) over mono/stereo/5.1/7-channel streams, 20 000 random `AudioSpecificConfig`s and 2 000 random +ADTS streams: no panics. The encoder accepts NaN/±inf/over-range input, any chunk size, and missing +channels. + +## Clean-room notes + +- Syntax, tables (scalefactor bands, TNS limits), filterbank, TNS, PNS, M/S and intensity stereo are + implemented from ISO/IEC 14496-3. +- The Huffman codebooks (`src/huffman_tables.rs`) are the normative values of the standard's Annex + 4.A, recovered by black-box probing of the ffmpeg *binary* as an oracle: `tools/` contains the + Python (stdlib-only) scripts that craft single-frame streams repeating a candidate bit string, fit the + decoded output to the IMDCT basis, and walk the code tree. Every table is verified to be a complete + prefix code (Kraft sum exactly 1) covering each index once (unit test), and the decoder matches + ffmpeg to ~1e-7 on real streams. +- Psychoacoustics follow textbook models (Bark scale, Schroeder spreading, Terhardt ATH, Johnston-style + tonality/SFM offsets, perceptual entropy). + +## Gaps / not implemented + +- HE-AAC v1/v2 (SBR/PS): explicit signalling is parsed and only the AAC-LC core is decoded (half-rate, + no SBR); implicit SBR fill elements are skipped. +- AAC Main (prediction), SSR (gain control), LTP, ER/LD/ELD object types, coupling channel elements + (CCE → `Err(Unsupported)`), 960-sample frames. +- Encoder: no PNS or intensity stereo (decoder supports both), no TNS on short windows, no pulse data; + window shape is fixed per stream (sine or KBD). Short blocks at onsets cost segmental SNR on + speech-like material relative to long windows + TNS; perceptually they avoid pre-echo. +- No error concealment: a corrupt AU returns `Err`; the caller may substitute silence. +- ADTS with CRC is parsed (header skipped, CRC not verified); multi-block ADTS frames are split per + frame only. diff --git a/crates/aac/src/config.rs b/crates/aac/src/config.rs new file mode 100644 index 0000000000..58126beaea --- /dev/null +++ b/crates/aac/src/config.rs @@ -0,0 +1,447 @@ +//! `AudioSpecificConfig` (ISO/IEC 14496-3 §1.6.2.1), `program_config_element` (§4.4.1.1) and ADTS +//! framing (§1.A.2.2). + +use deckcraft_bitstream::{BitReader, BitWriter}; + +use crate::tables::{SAMPLE_RATES, table_index_for_rate}; +use crate::{Error, Result}; + +/// Syntactic element types (`id_syn_ele`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ElementType { + /// single_channel_element + Sce, + /// channel_pair_element + Cpe, + /// lfe_channel_element + Lfe, +} + +impl ElementType { + pub fn channels(self) -> usize { + if self == ElementType::Cpe { 2 } else { 1 } + } +} + +/// A `program_config_element` (only the parts that affect channel mapping are interpreted). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ProgramConfig { + pub element_instance_tag: u8, + pub object_type: u8, + pub sf_index: u8, + /// (is_cpe, tag) for front, side and back channel elements. + pub front: Vec<(bool, u8)>, + pub side: Vec<(bool, u8)>, + pub back: Vec<(bool, u8)>, + pub lfe: Vec, + pub assoc_data: Vec, + /// (is_ind_sw, tag) + pub cc: Vec<(bool, u8)>, + pub comment: Vec, +} + +impl ProgramConfig { + pub fn parse(br: &mut BitReader) -> Result { + let mut p = ProgramConfig { + element_instance_tag: br.read_bits(4)? as u8, + object_type: br.read_bits(2)? as u8, + sf_index: br.read_bits(4)? as u8, + ..Default::default() + }; + let nf = br.read_bits(4)?; + let ns = br.read_bits(4)?; + let nb = br.read_bits(4)?; + let nl = br.read_bits(2)?; + let na = br.read_bits(3)?; + let nc = br.read_bits(4)?; + if br.read_bits(1)? == 1 { + br.skip(4)?; + } + if br.read_bits(1)? == 1 { + br.skip(4)?; + } + if br.read_bits(1)? == 1 { + br.skip(3)?; + } + for (n, v) in [(nf, &mut p.front), (ns, &mut p.side), (nb, &mut p.back)] { + for _ in 0..n { + let cpe = br.read_bits(1)? == 1; + v.push((cpe, br.read_bits(4)? as u8)); + } + } + for _ in 0..nl { + p.lfe.push(br.read_bits(4)? as u8); + } + for _ in 0..na { + p.assoc_data.push(br.read_bits(4)? as u8); + } + for _ in 0..nc { + let ind = br.read_bits(1)? == 1; + p.cc.push((ind, br.read_bits(4)? as u8)); + } + br.byte_align(); + let len = br.read_bits(8)?; + for _ in 0..len { + p.comment.push(br.read_bits(8)? as u8); + } + Ok(p) + } + + pub fn write(&self, bw: &mut BitWriter) { + bw.write_bits(self.element_instance_tag as u32, 4); + bw.write_bits(self.object_type as u32, 2); + bw.write_bits(self.sf_index as u32, 4); + bw.write_bits(self.front.len() as u32, 4); + bw.write_bits(self.side.len() as u32, 4); + bw.write_bits(self.back.len() as u32, 4); + bw.write_bits(self.lfe.len() as u32, 2); + bw.write_bits(self.assoc_data.len() as u32, 3); + bw.write_bits(self.cc.len() as u32, 4); + bw.write_bits(0, 3); // no mixdowns + for v in [&self.front, &self.side, &self.back] { + for &(cpe, tag) in v { + bw.write_bits(cpe as u32, 1); + bw.write_bits(tag as u32, 4); + } + } + for &t in &self.lfe { + bw.write_bits(t as u32, 4); + } + for &t in &self.assoc_data { + bw.write_bits(t as u32, 4); + } + for &(ind, t) in &self.cc { + bw.write_bits(ind as u32, 1); + bw.write_bits(t as u32, 4); + } + bw.align_zero(); + bw.write_bits(self.comment.len().min(255) as u32, 8); + for &b in self.comment.iter().take(255) { + bw.write_bits(b as u32, 8); + } + } + + /// Elements in output channel order (front, side, back, LFE). + pub fn layout(&self) -> Vec<(ElementType, u8)> { + let mut out = Vec::new(); + for v in [&self.front, &self.side, &self.back] { + for &(cpe, tag) in v { + out.push((if cpe { ElementType::Cpe } else { ElementType::Sce }, tag)); + } + } + for &t in &self.lfe { + out.push((ElementType::Lfe, t)); + } + out + } +} + +/// Element layout of `channelConfiguration` 1..=7, 11 and 12 (tags as conventionally assigned; 11 and +/// 12 from ISO/IEC 14496-3 Amd 4: 6.1 with a back centre, 7.1 with back surrounds). +pub fn config_layout(channel_config: u8) -> Option> { + use ElementType::*; + Some(match channel_config { + 1 => vec![(Sce, 0)], + 2 => vec![(Cpe, 0)], + 3 => vec![(Sce, 0), (Cpe, 0)], + 4 => vec![(Sce, 0), (Cpe, 0), (Sce, 1)], + 5 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1)], + 6 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Lfe, 0)], + 7 | 12 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Cpe, 2), (Lfe, 0)], + 11 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Sce, 1), (Lfe, 0)], + _ => return None, + }) +} + +/// Parsed `AudioSpecificConfig`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AudioSpecificConfig { + /// Audio object type of the core (2 = AAC LC). + pub object_type: u8, + /// Core sampling rate. + pub sample_rate: u32, + /// Index into the scalefactor-band tables (the signalled index, or derived from an explicit rate). + pub sf_index: u8, + pub channel_config: u8, + pub frame_length_flag: bool, + /// Explicitly signalled SBR/PS (HE-AAC v1/v2); only the AAC-LC core is decoded. + pub sbr: bool, + pub ps: bool, + /// Output (SBR) rate when `sbr` is set. + pub extension_sample_rate: Option, + pub pce: Option, +} + +fn read_aot(br: &mut BitReader) -> Result { + let a = br.read_bits(5)? as u8; + Ok(if a == 31 { 32 + br.read_bits(6)? as u8 } else { a }) +} + +fn read_rate(br: &mut BitReader) -> Result<(u8, u32)> { + let i = br.read_bits(4)? as u8; + if i == 15 { + let r = br.read_bits(24)?; + return Ok((table_index_for_rate(r), r)); + } + let r = *SAMPLE_RATES.get(i as usize).ok_or(Error::InvalidConfig("reserved sampling frequency index"))?; + Ok((i, r)) +} + +impl AudioSpecificConfig { + /// A plain AAC-LC configuration. + pub fn lc(sf_index: u8, channel_config: u8) -> AudioSpecificConfig { + AudioSpecificConfig { + object_type: 2, + sample_rate: SAMPLE_RATES[sf_index as usize], + sf_index, + channel_config, + frame_length_flag: false, + sbr: false, + ps: false, + extension_sample_rate: None, + pce: None, + } + } + + pub fn parse(data: &[u8]) -> Result { + let mut br = BitReader::new(data); + let mut aot = read_aot(&mut br)?; + let (sf_index, sample_rate) = read_rate(&mut br)?; + let channel_config = br.read_bits(4)? as u8; + let (mut sbr, mut ps, mut extension_sample_rate) = (false, false, None); + if aot == 5 || aot == 29 { + sbr = true; + ps = aot == 29; + let (_, ext) = read_rate(&mut br)?; + extension_sample_rate = Some(ext); + aot = read_aot(&mut br)?; + } + if aot != 2 { + return Err(Error::Unsupported(match aot { + 1 => "AAC Main profile", + 3 => "AAC SSR profile", + 4 => "AAC LTP profile", + 23 => "AAC-LD", + 39 => "AAC-ELD", + _ => "audio object type other than AAC LC", + })); + } + let frame_length_flag = br.read_bits(1)? == 1; + if br.read_bits(1)? == 1 { + br.skip(14)?; // coreCoderDelay + } + let _extension_flag = br.read_bits(1)?; + let pce = if channel_config == 0 { Some(ProgramConfig::parse(&mut br)?) } else { None }; + if frame_length_flag { + return Err(Error::Unsupported("960-sample frames")); + } + let cfg = + AudioSpecificConfig { object_type: aot, sample_rate, sf_index, channel_config, frame_length_flag, sbr, ps, extension_sample_rate, pce }; + if cfg.channels() == 0 { + return Err(Error::InvalidConfig("no channels")); + } + Ok(cfg) + } + + /// Serialise (AAC-LC, no SBR signalling). + pub fn to_bytes(&self) -> Vec { + let mut bw = BitWriter::new(); + bw.write_bits(self.object_type as u32, 5); + match crate::tables::sample_rate_index(self.sample_rate) { + Some(i) => bw.write_bits(i as u32, 4), + None => { + bw.write_bits(15, 4); + bw.write_bits(self.sample_rate, 24); + } + } + bw.write_bits(self.channel_config as u32, 4); + bw.write_bits(0, 3); // frameLengthFlag, dependsOnCoreCoder, extensionFlag + if self.channel_config == 0 + && let Some(p) = &self.pce + { + p.write(&mut bw); + } + bw.finish() + } + + /// Elements in output channel order. + pub fn layout(&self) -> Vec<(ElementType, u8)> { + match &self.pce { + Some(p) if self.channel_config == 0 => p.layout(), + _ => config_layout(self.channel_config).unwrap_or_default(), + } + } + + pub fn channels(&self) -> usize { + self.layout().iter().map(|e| e.0.channels()).sum() + } +} + +/// A parsed ADTS header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AdtsHeader { + /// Audio object type (`profile + 1`). + pub object_type: u8, + pub sf_index: u8, + pub channel_config: u8, + /// Whole frame length including the header. + pub frame_length: usize, + pub buffer_fullness: u16, + /// Header length (7, or 9 with CRC). + pub header_length: usize, + pub raw_data_blocks: u8, +} + +impl AdtsHeader { + pub fn parse(data: &[u8]) -> Result { + let mut br = BitReader::new(data); + if br.read_bits(12)? != 0xFFF { + return Err(Error::Bitstream("missing ADTS syncword")); + } + br.skip(1)?; // ID + if br.read_bits(2)? != 0 { + return Err(Error::Bitstream("ADTS layer must be 0")); + } + let protection_absent = br.read_bits(1)? == 1; + let object_type = br.read_bits(2)? as u8 + 1; + let sf_index = br.read_bits(4)? as u8; + if sf_index as usize >= SAMPLE_RATES.len() { + return Err(Error::Bitstream("invalid ADTS sampling frequency index")); + } + br.skip(1)?; + let channel_config = br.read_bits(3)? as u8; + br.skip(4)?; + let frame_length = br.read_bits(13)? as usize; + let buffer_fullness = br.read_bits(11)? as u16; + let raw_data_blocks = br.read_bits(2)? as u8 + 1; + let header_length = if protection_absent { 7 } else { 9 }; + if frame_length < header_length { + return Err(Error::Bitstream("ADTS frame shorter than its header")); + } + Ok(AdtsHeader { object_type, sf_index, channel_config, frame_length, buffer_fullness, header_length, raw_data_blocks }) + } + + /// The equivalent `AudioSpecificConfig`. Channel configuration 0 (PCE in-band) is resolved by the + /// decoder from the first frame; this returns stereo-less layout in that case. + pub fn to_asc(&self) -> AudioSpecificConfig { + AudioSpecificConfig { object_type: self.object_type, ..AudioSpecificConfig::lc(self.sf_index, self.channel_config) } + } + + /// Write a 7-byte header (no CRC) for a frame whose raw payload is `payload_len` bytes. + pub fn write(object_type: u8, sf_index: u8, channel_config: u8, payload_len: usize, buffer_fullness: u16) -> [u8; 7] { + let len = (payload_len + 7) as u32; + let mut bw = BitWriter::new(); + bw.write_bits(0xFFF, 12); + bw.write_bits(0, 1); // MPEG-4 + bw.write_bits(0, 2); + bw.write_bits(1, 1); // protection absent + bw.write_bits((object_type.max(1) - 1) as u32 & 3, 2); + bw.write_bits(sf_index as u32, 4); + bw.write_bits(0, 1); + bw.write_bits(channel_config as u32 & 7, 3); + bw.write_bits(0, 4); + bw.write_bits(len & 0x1FFF, 13); + bw.write_bits(buffer_fullness as u32 & 0x7FF, 11); + bw.write_bits(0, 2); + let v = bw.finish(); + let mut out = [0u8; 7]; + out.copy_from_slice(&v); + out + } +} + +/// Split an ADTS stream into `(header, raw payload)` frames. Stops at the first malformed or +/// truncated frame (returning what was parsed so far); errors only if nothing parses. +pub fn split_adts(mut data: &[u8]) -> Result> { + let mut out = Vec::new(); + while data.len() >= 7 { + let h = match AdtsHeader::parse(data) { + Ok(h) => h, + Err(e) => { + if out.is_empty() { + return Err(e); + } + break; + } + }; + if h.frame_length > data.len() { + break; + } + out.push((h, &data[h.header_length..h.frame_length])); + data = &data[h.frame_length..]; + } + if out.is_empty() { + return Err(Error::Bitstream("no ADTS frames")); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn asc_roundtrip() { + let a = AudioSpecificConfig::lc(4, 2); + let b = a.to_bytes(); + assert_eq!(b, vec![0x12, 0x10]); + assert_eq!(AudioSpecificConfig::parse(&b).unwrap(), a); + // 48 kHz 5.1 + let a = AudioSpecificConfig::lc(3, 6); + assert_eq!(AudioSpecificConfig::parse(&a.to_bytes()).unwrap().channels(), 6); + } + + #[test] + fn amd4_channel_configurations() { + // 6.1 and 7.1 back (#922); 22.2 and 7.1 with heights stay unsupported + let channels = |c| AudioSpecificConfig::parse(&AudioSpecificConfig::lc(3, c).to_bytes()).map(|a| a.channels()).ok(); + assert_eq!([7, 11, 12].map(channels), [Some(8), Some(7), Some(8)]); + assert_eq!([13, 14].map(config_layout), [None, None]); + } + + #[test] + fn asc_with_pce() { + let pce = ProgramConfig { + sf_index: 3, + object_type: 1, + front: vec![(false, 0), (true, 0)], + side: vec![(true, 1)], + back: vec![(false, 1)], + lfe: vec![0], + ..Default::default() + }; + let a = AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) }; + let p = AudioSpecificConfig::parse(&a.to_bytes()).unwrap(); + assert_eq!(p.channels(), 7); + assert_eq!(p, a); + } + + #[test] + fn he_aac_asc_parses_core() { + // AOT 5, 24 kHz core, stereo, ext 48 kHz, core AOT 2 + let mut bw = BitWriter::new(); + bw.write_bits(5, 5); + bw.write_bits(6, 4); + bw.write_bits(2, 4); + bw.write_bits(3, 4); + bw.write_bits(2, 5); + bw.write_bits(0, 3); + let a = AudioSpecificConfig::parse(&bw.finish()).unwrap(); + assert!(a.sbr); + assert_eq!(a.sample_rate, 24000); + assert_eq!(a.extension_sample_rate, Some(48000)); + } + + #[test] + fn adts_roundtrip() { + let h = AdtsHeader::write(2, 4, 2, 100, 0x7FF); + let mut frame = h.to_vec(); + frame.extend(std::iter::repeat_n(0u8, 100)); + let p = AdtsHeader::parse(&frame).unwrap(); + assert_eq!(p.frame_length, 107); + assert_eq!((p.object_type, p.sf_index, p.channel_config), (2, 4, 2)); + let frames = split_adts(&frame).unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].1.len(), 100); + } +} diff --git a/crates/aac/src/decoder.rs b/crates/aac/src/decoder.rs new file mode 100644 index 0000000000..7b3f5ec99f --- /dev/null +++ b/crates/aac/src/decoder.rs @@ -0,0 +1,584 @@ +//! AAC-LC decoder: raw access units (`raw_data_block`, ISO/IEC 14496-3 §4.4.2.1) → planar f32. + +use deckcraft_bitstream::BitReader; + +use crate::config::{AudioSpecificConfig, ElementType, ProgramConfig}; +use crate::huffman::{self, INTENSITY_HCB, INTENSITY_HCB2, NOISE_HCB, ZERO_HCB}; +use crate::ics::{IcsInfo, WindowSequence}; +use crate::mdct::{WindowShape, mdct_long, mdct_short, window}; +use crate::tables::{pow43, sf_gain}; +use crate::tns::{self, TnsData}; +use crate::{Error, Result}; + +const ID_SCE: u32 = 0; +const ID_CPE: u32 = 1; +const ID_CCE: u32 = 2; +const ID_LFE: u32 = 3; +const ID_DSE: u32 = 4; +const ID_PCE: u32 = 5; +const ID_FIL: u32 = 6; +const ID_END: u32 = 7; + +/// `extension_type` of a fill element carrying SBR data (ISO/IEC 14496-3 §4.5.2.8.2, Table 4.121). +const EXT_SBR_DATA: u32 = 13; +const EXT_SBR_DATA_CRC: u32 = 14; + +/// One decoded `individual_channel_stream` before stereo processing and synthesis. +struct Ics { + info: IcsInfo, + /// Band type per (group, sfb). + band_type: [[u8; 64]; 8], + /// Scalefactor / noise energy / intensity position per (group, sfb). + sf: [[i32; 64]; 8], + tns: Option, + /// Dequantised spectrum (short windows at stride 128). + spec: Vec, +} + +#[derive(Clone)] +struct ChannelState { + overlap: Vec, + prev_shape: WindowShape, +} + +/// AAC-LC decoder. +pub struct Decoder { + asc: AudioSpecificConfig, + layout: Vec<(ElementType, u8)>, + /// First output channel of each layout entry. + offsets: Vec, + channels: Vec, + noise_state: u32, + /// A fill element carried SBR data (implicitly signalled HE-AAC). + sbr_seen: bool, +} + +impl Decoder { + /// Create a decoder from an `AudioSpecificConfig`. + pub fn new(asc: &[u8]) -> Result { + Self::from_config(AudioSpecificConfig::parse(asc)?) + } + + /// Create a decoder for an ADTS stream from its first header. With channel configuration 0 the + /// channel layout is taken from the program config element carried in the first frame. + pub fn from_adts(header: &crate::config::AdtsHeader) -> Result { + Self::from_config(header.to_asc()) + } + + pub fn from_config(asc: AudioSpecificConfig) -> Result { + if asc.object_type != 2 { + return Err(Error::Unsupported("audio object type other than AAC LC")); + } + let layout = asc.layout(); + if layout.is_empty() && asc.channel_config != 0 { + return Err(Error::InvalidConfig("unsupported channel configuration")); + } + let mut d = Decoder { asc, layout: Vec::new(), offsets: Vec::new(), channels: Vec::new(), noise_state: 0x2545_F491, sbr_seen: false }; + d.set_layout(layout); + Ok(d) + } + + fn set_layout(&mut self, layout: Vec<(ElementType, u8)>) { + let mut offsets = Vec::with_capacity(layout.len()); + let mut n = 0; + for e in &layout { + offsets.push(n); + n += e.0.channels(); + } + self.layout = layout; + self.offsets = offsets; + self.channels = vec![ChannelState { overlap: vec![0.0; 1024], prev_shape: WindowShape::Sine }; n]; + } + + pub fn config(&self) -> &AudioSpecificConfig { + &self.asc + } + pub fn sample_rate(&self) -> u32 { + self.asc.sample_rate + } + /// Whether the stream is HE-AAC: SBR signalled in the `AudioSpecificConfig` (explicit) or SBR + /// data seen in a fill element of a decoded access unit (implicit). Only the AAC-LC core is + /// decoded, at [`Self::sample_rate`]; the SBR high band is not reconstructed. + pub fn sbr(&self) -> bool { + self.asc.sbr || self.sbr_seen + } + pub fn channels(&self) -> usize { + self.channels.len() + } + /// Element layout in output channel order (C, L/R, surround pairs, LFE for configurations 1–7). + pub fn layout(&self) -> &[(ElementType, u8)] { + &self.layout + } + + /// Clear the overlap state (e.g. after a seek). + pub fn reset(&mut self) { + for c in &mut self.channels { + c.overlap.fill(0.0); + c.prev_shape = WindowShape::Sine; + } + } + + fn sf_index(&self) -> u8 { + self.asc.sf_index + } + + /// Decode one access unit (one `raw_data_block`) into 1024 samples per channel. + pub fn decode(&mut self, au: &[u8]) -> Result>> { + self.decode_block(au, true) + } + + fn decode_block(&mut self, au: &[u8], allow_reconfig: bool) -> Result>> { + let mut br = BitReader::new(au); + let nch = self.channels.len(); + let mut saw_element = false; + let mut out: Vec>> = vec![None; nch]; + let mut used = vec![false; self.layout.len()]; + loop { + let id = br.read_bits(3)?; + saw_element |= matches!(id, ID_SCE | ID_CPE | ID_LFE); + match id { + ID_SCE | ID_LFE => { + let tag = br.read_bits(4)? as u8; + let ty = if id == ID_SCE { ElementType::Sce } else { ElementType::Lfe }; + let slot = self.find_slot(ty, tag, &used)?; + used[slot] = true; + let mut ics = self.decode_ics(&mut br, None)?; + let ch = self.offsets[slot]; + self.fill_noise(&mut ics, None); + out[ch] = Some(self.finish_channel(ch, &mut ics)); + } + ID_CPE => { + let tag = br.read_bits(4)? as u8; + let slot = self.find_slot(ElementType::Cpe, tag, &used)?; + used[slot] = true; + let ch = self.offsets[slot]; + let [a, b] = self.decode_cpe(&mut br, ch)?; + out[ch] = Some(a); + out[ch + 1] = Some(b); + } + ID_CCE => return Err(Error::Unsupported("coupling channel element")), + ID_DSE => { + br.skip(4)?; + let align = br.read_bits(1)? == 1; + let mut count = br.read_bits(8)? as usize; + if count == 255 { + count += br.read_bits(8)? as usize; + } + if align { + br.byte_align(); + } + br.skip(count * 8)?; + } + ID_PCE => { + let pce = ProgramConfig::parse(&mut br)?; + if allow_reconfig && !saw_element && self.asc.channel_config == 0 && self.asc.pce.as_ref() != Some(&pce) { + let layout = pce.layout(); + if !layout.is_empty() && layout.iter().map(|e| e.0.channels()).sum::() <= 64 { + self.asc.pce = Some(pce); + self.set_layout(layout); + return self.decode_block(au, false); + } + } + } + ID_FIL => { + let mut count = br.read_bits(4)? as usize; + if count == 15 { + count += br.read_bits(8)? as usize; + count -= 1; + } + if count > 0 { + let ext = br.read_bits(4)?; + self.sbr_seen |= matches!(ext, EXT_SBR_DATA | EXT_SBR_DATA_CRC); + br.skip(count * 8 - 4)?; + } + } + _ => break, // ID_END + } + if id == ID_END { + break; + } + } + if nch == 0 && !saw_element { + return Err(Error::InvalidConfig("channel configuration 0 without a program config element")); + } + let mut res = Vec::with_capacity(nch); + for o in out { + res.push(o.ok_or(Error::Bitstream("access unit is missing a channel element"))?); + } + Ok(res) + } + + fn find_slot(&self, ty: ElementType, tag: u8, used: &[bool]) -> Result { + let compatible = |t: ElementType| t == ty || (ty != ElementType::Cpe && t != ElementType::Cpe); + if let Some(i) = self.layout.iter().enumerate().position(|(i, &(t, g))| t == ty && g == tag && !used[i]) { + return Ok(i); + } + if let Some(i) = self.layout.iter().enumerate().position(|(i, &(t, _))| t == ty && !used[i]) { + return Ok(i); + } + self.layout + .iter() + .enumerate() + .position(|(i, &(t, _))| compatible(t) && !used[i]) + .ok_or(Error::Bitstream("unexpected channel element for the channel configuration")) + } + + fn decode_cpe(&mut self, br: &mut BitReader, ch: usize) -> Result<[Vec; 2]> { + let common_window = br.read_bits(1)? == 1; + let mut ms_mask_present = 0; + let mut ms_used = [[false; 64]; 8]; + let mut info = None; + if common_window { + let i = IcsInfo::parse(br, self.sf_index())?; + ms_mask_present = br.read_bits(2)?; + if ms_mask_present == 3 { + return Err(Error::Bitstream("reserved ms_mask_present")); + } + if ms_mask_present == 1 { + for g in 0..i.num_groups { + for sfb in 0..i.max_sfb { + ms_used[g][sfb] = br.read_bits(1)? == 1; + } + } + } else if ms_mask_present == 2 { + ms_used = [[true; 64]; 8]; + } + info = Some(i); + } + let mut l = self.decode_ics(br, info)?; + let mut r = self.decode_ics(br, info)?; + let ms = if ms_mask_present > 0 { Some(&ms_used) } else { None }; + self.fill_noise(&mut l, None); + self.fill_noise(&mut r, if common_window { Some((&l, ms)) } else { None }); + if common_window { + let swb = l.info.swb_offsets(self.sf_index()); + for g in 0..l.info.num_groups { + let w0 = l.info.group_start(g); + for sfb in 0..l.info.max_sfb { + let (bl, br_) = (l.band_type[g][sfb], r.band_type[g][sfb]); + let range = swb[sfb] as usize..swb[sfb + 1] as usize; + if br_ == INTENSITY_HCB || br_ == INTENSITY_HCB2 { + let mut c: f32 = if br_ == INTENSITY_HCB { 1.0 } else { -1.0 }; + if ms_mask_present > 0 && ms_used[g][sfb] { + c = -c; + } + let scale = c * (-0.25 * r.sf[g][sfb].clamp(-400, 400) as f64).exp2() as f32; + for w in w0..w0 + l.info.group_len[g] as usize { + for k in range.clone() { + r.spec[w * 128 + k] = l.spec[w * 128 + k] * scale; + } + } + } else if ms_mask_present > 0 && ms_used[g][sfb] && bl < NOISE_HCB && br_ < NOISE_HCB { + for w in w0..w0 + l.info.group_len[g] as usize { + for k in range.clone() { + let (m, s) = (l.spec[w * 128 + k], r.spec[w * 128 + k]); + l.spec[w * 128 + k] = m + s; + r.spec[w * 128 + k] = m - s; + } + } + } + } + } + } + Ok([self.finish_channel(ch, &mut l), self.finish_channel(ch + 1, &mut r)]) + } + + fn decode_ics(&mut self, br: &mut BitReader, common: Option) -> Result { + let sfi = self.sf_index(); + let global_gain = br.read_bits(8)? as i32; + let info = match common { + Some(i) => i, + None => IcsInfo::parse(br, sfi)?, + }; + let swb = info.swb_offsets(sfi); + let num_swb = swb.len() - 1; + let mut ics = Ics { info, band_type: [[0; 64]; 8], sf: [[0; 64]; 8], tns: None, spec: vec![0.0; 1024] }; + + // section_data + let (sect_bits, esc) = if info.is_short() { (3, 7) } else { (5, 31) }; + for g in 0..info.num_groups { + let mut k = 0; + while k < info.max_sfb { + let cb = br.read_bits(4)? as u8; + if cb == 12 { + return Err(Error::Bitstream("reserved codebook 12")); + } + let mut len = 0usize; + loop { + let inc = br.read_bits(sect_bits)?; + len += inc as usize; + if inc != esc { + break; + } + } + if k + len > info.max_sfb { + return Err(Error::Bitstream("section exceeds max_sfb")); + } + if len == 0 { + // zero-length sections are legal but useless; guard against endless loops + if br.bits_left() == 0 { + return Err(Error::Eof); + } + } + for sfb in k..k + len { + ics.band_type[g][sfb] = cb; + } + k += len; + } + } + + // scale_factor_data + let mut sf = global_gain; + let mut noise = global_gain - 90; + let mut is_pos = 0i32; + let mut noise_first = true; + for g in 0..info.num_groups { + for sfb in 0..info.max_sfb { + match ics.band_type[g][sfb] { + ZERO_HCB => ics.sf[g][sfb] = 0, + INTENSITY_HCB | INTENSITY_HCB2 => { + is_pos += huffman::decode_sf(br)?; + ics.sf[g][sfb] = is_pos; + } + NOISE_HCB => { + if noise_first { + noise += br.read_bits(9)? as i32 - 256; + noise_first = false; + } else { + noise += huffman::decode_sf(br)?; + } + ics.sf[g][sfb] = noise; + } + _ => { + sf += huffman::decode_sf(br)?; + if !(0..=255).contains(&sf) { + return Err(Error::Bitstream("scalefactor out of range")); + } + ics.sf[g][sfb] = sf; + } + } + } + } + + // pulse_data + let mut pulses: Vec<(usize, i32)> = Vec::new(); + if br.read_bits(1)? == 1 { + if info.is_short() { + return Err(Error::Bitstream("pulse data in short window")); + } + let n = br.read_bits(2)? as usize + 1; + let start = br.read_bits(6)? as usize; + if start >= num_swb { + return Err(Error::Bitstream("pulse_start_sfb out of range")); + } + let mut k = swb[start] as usize; + for _ in 0..n { + k += br.read_bits(5)? as usize; + let amp = br.read_bits(4)? as i32; + if k >= 1024 { + return Err(Error::Bitstream("pulse position out of range")); + } + pulses.push((k, amp)); + } + } + if br.read_bits(1)? == 1 { + ics.tns = Some(TnsData::parse(br, &info)?); + } + if br.read_bits(1)? == 1 { + return Err(Error::Unsupported("gain control data (AAC SSR)")); + } + + // spectral_data + let mut q = vec![0i32; 1024]; + let mut vals = [0i32; 4]; + for g in 0..info.num_groups { + let w0 = info.group_start(g); + let glen = info.group_len[g] as usize; + for sfb in 0..info.max_sfb { + let cb = ics.band_type[g][sfb]; + if cb == ZERO_HCB || cb >= NOISE_HCB { + continue; + } + let dim = huffman::book(cb).dim; + let (lo, hi) = (swb[sfb] as usize, swb[sfb + 1] as usize); + for w in w0..w0 + glen { + let base = w * 128; + let mut k = lo; + while k < hi { + huffman::decode_spectral(br, cb, &mut vals)?; + q[base + k..base + k + dim].copy_from_slice(&vals[..dim]); + k += dim; + } + } + } + } + for (k, amp) in pulses { + if q[k] > 0 { + q[k] += amp; + } else { + q[k] -= amp; + } + } + + // dequantise + for g in 0..info.num_groups { + let w0 = info.group_start(g); + for sfb in 0..info.max_sfb { + let cb = ics.band_type[g][sfb]; + if cb == ZERO_HCB || cb >= NOISE_HCB { + continue; + } + let gain = sf_gain(ics.sf[g][sfb]); + for w in w0..w0 + info.group_len[g] as usize { + for k in swb[sfb] as usize..swb[sfb + 1] as usize { + let v = q[w * 128 + k]; + let m = pow43(v.unsigned_abs()) * gain; + ics.spec[w * 128 + k] = if v < 0 { -m } else { m }; + } + } + } + } + Ok(ics) + } + + fn next_random(&mut self) -> f32 { + self.noise_state = self.noise_state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (self.noise_state as i32) as f32 + } + + /// Perceptual noise substitution. `pair` = (left channel, ms mask) for the right channel of a CPE: + /// bands that are noise in both channels with `ms_used` reuse the left channel's noise vector. + fn fill_noise(&mut self, ics: &mut Ics, pair: Option<(&Ics, Option<&[[bool; 64]; 8]>)>) { + let info = ics.info; + let swb = info.swb_offsets(self.sf_index()); + for g in 0..info.num_groups { + let w0 = info.group_start(g); + for sfb in 0..info.max_sfb { + if ics.band_type[g][sfb] != NOISE_HCB { + continue; + } + let (lo, hi) = (swb[sfb] as usize, swb[sfb + 1] as usize); + let target = (0.25 * ics.sf[g][sfb].clamp(-400, 400) as f64).exp2(); + let correlated = match pair { + Some((l, Some(ms))) => ms[g][sfb] && l.band_type[g][sfb] == NOISE_HCB, + _ => false, + }; + for w in w0..w0 + info.group_len[g] as usize { + let band = w * 128 + lo..w * 128 + hi; + if correlated && let Some((l, _)) = pair { + ics.spec[band.clone()].copy_from_slice(&l.spec[band.clone()]); + } else { + for k in band.clone() { + ics.spec[k] = self.next_random(); + } + } + let energy: f64 = ics.spec[band.clone()].iter().map(|&v| v as f64 * v as f64).sum(); + let scale = if energy > 0.0 { (target / energy.sqrt()) as f32 } else { 0.0 }; + for k in band { + ics.spec[k] *= scale; + } + } + } + } + } + + /// TNS + filterbank for a single channel element; returns output samples. + fn finish_channel(&mut self, ch: usize, ics: &mut Ics) -> Vec { + if let Some(t) = &ics.tns { + tns::apply_decoder(&mut ics.spec, &ics.info, self.sf_index(), t); + } + synth(&mut self.channels[ch], &ics.info, &ics.spec) + } +} + +/// Inverse MDCT, windowing and overlap-add (§4.6.18), output scaled to ±1.0 full scale. +fn synth(state: &mut ChannelState, info: &IcsInfo, spec: &[f32]) -> Vec { + let mut buf = vec![0f32; 2048]; + let prev = state.prev_shape; + let cur = info.window_shape; + let norm = 1.0 / 32768.0; + match info.window_sequence { + WindowSequence::EightShort => { + let t = mdct_short(); + let mut y = vec![0f32; 256]; + let wc = window(cur, false); + for w in 0..8 { + t.inverse(&spec[w * 128..(w + 1) * 128], &mut y); + let wl = window(if w == 0 { prev } else { cur }, false); + let off = 448 + 128 * w; + let s = norm / 128.0; + for n in 0..128 { + buf[off + n] += y[n] * wl[n] * s; + buf[off + 128 + n] += y[128 + n] * wc[127 - n] * s; + } + } + } + seq => { + let mut y = vec![0f32; 2048]; + mdct_long().inverse(&spec[..1024], &mut y); + let s = norm / 1024.0; + match seq { + WindowSequence::LongStop => { + let ws = window(prev, false); + for n in 448..576 { + buf[n] = y[n] * ws[n - 448] * s; + } + for n in 576..1024 { + buf[n] = y[n] * s; + } + } + _ => { + let wl = window(prev, true); + for n in 0..1024 { + buf[n] = y[n] * wl[n] * s; + } + } + } + match seq { + WindowSequence::LongStart => { + let ws = window(cur, false); + for n in 1024..1472 { + buf[n] = y[n] * s; + } + for n in 1472..1600 { + buf[n] = y[n] * ws[127 - (n - 1472)] * s; + } + } + _ => { + let wl = window(cur, true); + for n in 1024..2048 { + buf[n] = y[n] * wl[2047 - n] * s; + } + } + } + } + } + let out: Vec = (0..1024).map(|n| state.overlap[n] + buf[n]).collect(); + state.overlap.copy_from_slice(&buf[1024..]); + state.prev_shape = cur; + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Implicitly signalled HE-AAC (an AAC-LC config) is recognised from the SBR fill element. + #[test] + fn sbr_fill_element_marks_implicit_he_aac() { + // AAC-LC, 22.05 kHz, stereo + let mut dec = Decoder::new(&[0x13, 0x90]).unwrap(); + assert!(!dec.sbr()); + // ID_FIL, count 1, extension_type EXT_SBR_DATA (+ 4 bits of payload), ID_END + let _ = dec.decode(&[0b1100_0011, 0b1010_0001, 0b1100_0000]); + assert!(dec.sbr()); + // other fill payloads (EXT_FILL) don't + let mut dec = Decoder::new(&[0x13, 0x90]).unwrap(); + let _ = dec.decode(&[0b1100_0010, 0b0000_0001, 0b1100_0000]); + assert!(!dec.sbr()); + // explicit signalling: AOT 5, core 22.05 kHz, stereo, extension 44.1 kHz, core AOT 2 + let dec = Decoder::new(&[0x2B, 0x92, 0x08, 0x00]).unwrap(); + assert!(dec.sbr()); + assert_eq!(dec.sample_rate(), 22_050); + } +} diff --git a/crates/aac/src/encoder/mod.rs b/crates/aac/src/encoder/mod.rs new file mode 100644 index 0000000000..17b0e27433 --- /dev/null +++ b/crates/aac/src/encoder/mod.rs @@ -0,0 +1,833 @@ +//! AAC-LC encoder. +//! +//! Pipeline per 1024-sample frame and channel element: +//! 1. transient detection on high-passed 128-sample sub-blocks (one frame of look-ahead) → +//! window sequence (`ONLY_LONG` / `LONG_START` / `EIGHT_SHORT` / `LONG_STOP`) and short-window grouping; +//! 2. windowed MDCT (sine or KBD); +//! 3. psychoacoustic model → per-band masking thresholds and perceptual entropy; +//! 4. TNS analysis (long windows; Levinson–Durbin, applied when the prediction gain is high); +//! 5. per-band M/S decision for channel pairs (common window); +//! 6. rate–distortion loop: a single multiplier λ scales every band's allowed noise; each band gets the +//! coarsest scalefactor meeting its noise target, sections/codebooks are chosen by dynamic +//! programming, and λ is bisected until the frame fits its bit budget (CBR) or fixed (VBR); +//! 7. bit reservoir (CBR): budget follows the frame's PE, bounded by the decoder buffer +//! (6144 bits/channel); overflow is padded with fill elements so the rate stays constant. + +mod psy; +mod quant; + +use deckcraft_bitstream::BitWriter; + +use crate::config::{AdtsHeader, AudioSpecificConfig, ElementType, ProgramConfig, config_layout}; +use crate::ics::{IcsInfo, WindowSequence}; +use crate::mdct::{WindowShape, mdct_long, mdct_short, window}; +use crate::tables::{sample_rate_index, swb_offsets_long, swb_offsets_short, tns_max_bands}; +use crate::tns::{self, TnsData, TnsFilter}; +use crate::{Error, FRAME_LEN, Result}; +use psy::Psy; +use quant::{ChannelInput, QuantChannel, quantise_channel, write_ics}; + +/// Rate control mode. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum BitrateMode { + /// Constant bitrate in bits per second (bit reservoir + fill elements). + Cbr(u32), + /// Constant quality, 1.0 (small) ..= 5.0 (transparent-ish). 3.0 targets the masking threshold. + Vbr(f32), +} + +/// Encoder settings. +#[derive(Debug, Clone, PartialEq)] +pub struct EncoderConfig { + /// One of the 13 AAC sampling rates (8–96 kHz). + pub sample_rate: u32, + /// 1–8. Channel order is the AAC element order: C; L R; C L R; C L R Cs; C L R Ls Rs; + /// C L R Ls Rs LFE; (7) C L R Ls Rs Cs LFE; (8) C Lc Rc L R Ls Rs LFE. + pub channels: usize, + pub mode: BitrateMode, + /// Audio bandwidth in Hz (`None` = chosen from the bitrate / quality). + pub bandwidth: Option, + /// Temporal noise shaping on long windows. + pub tns: bool, + /// Per-band mid/side stereo for channel pairs. + pub ms_stereo: bool, + pub window_shape: WindowShape, + /// Prefix every access unit with an ADTS header. + pub adts: bool, +} + +impl EncoderConfig { + pub fn cbr(sample_rate: u32, channels: usize, bitrate_bps: u32) -> EncoderConfig { + EncoderConfig { + sample_rate, + channels, + mode: BitrateMode::Cbr(bitrate_bps), + bandwidth: None, + tns: true, + ms_stereo: true, + window_shape: WindowShape::Sine, + adts: false, + } + } + pub fn vbr(sample_rate: u32, channels: usize, quality: f32) -> EncoderConfig { + EncoderConfig { mode: BitrateMode::Vbr(quality), ..Self::cbr(sample_rate, channels, 0) } + } +} + +/// Running statistics. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct EncoderStats { + pub frames: u64, + /// Raw access-unit bytes (without ADTS headers). + pub bytes: u64, + /// Element-frames coded with eight short windows. + pub short_blocks: u64, + pub ms_bands: u64, + pub stereo_bands: u64, + pub tns_filters: u64, + /// Bits spent on fill elements (CBR padding). + pub fill_bits: u64, +} + +impl EncoderStats { + /// Average bitrate of the raw access units. + pub fn bitrate(&self, sample_rate: u32) -> f64 { + if self.frames == 0 { 0.0 } else { self.bytes as f64 * 8.0 * sample_rate as f64 / (self.frames as f64 * FRAME_LEN as f64) } + } +} + +struct ElementState { + ty: ElementType, + tag: u8, + first_ch: usize, + prev_seq: WindowSequence, + short_next: Option, + /// Previous long-window thresholds per channel (pre-echo control). + prev_thr: Vec>, +} + +/// Per-frame analysis of one element. +struct ElementFrame { + info: IcsInfo, + chans: Vec, + ms_mask: Option<(u32, Vec)>, +} + +const HISTORY: usize = 2048; + +/// AAC-LC encoder. +pub struct Encoder { + cfg: EncoderConfig, + sf_index: u8, + asc: AudioSpecificConfig, + psy: Psy, + elements: Vec, + /// Input history per channel; `hist[c][0]` is absolute sample `origin`. + hist: Vec>, + origin: i64, + total_in: u64, + frames_done: u64, + flushed: bool, + bandwidth: u32, + // rate control + mean_acc: u64, + level: i64, + res_max: i64, + max_frame_bits: i64, + pe_avg: f32, + lambda: f32, + stats: EncoderStats, +} + +impl Encoder { + pub fn new(cfg: EncoderConfig) -> Result { + let sf_index = + sample_rate_index(cfg.sample_rate).ok_or(Error::InvalidConfig("sample rate must be one of the 13 AAC rates (7350–96000 Hz)"))?; + if !(1..=8).contains(&cfg.channels) { + return Err(Error::InvalidConfig("channels must be 1..=8")); + } + use ElementType::*; + let (channel_config, layout) = match cfg.channels { + 7 => (0u8, vec![(Sce, 0u8), (Cpe, 0), (Cpe, 1), (Sce, 1), (Lfe, 0)]), + 8 => (7, config_layout(7).unwrap_or_default()), + n => (n as u8, config_layout(n as u8).unwrap_or_default()), + }; + let mut asc = AudioSpecificConfig::lc(sf_index, channel_config); + if channel_config == 0 { + asc.pce = Some(ProgramConfig { + element_instance_tag: 0, + object_type: 1, + sf_index, + front: vec![(false, 0), (true, 0)], + side: vec![(true, 1)], + back: vec![(false, 1)], + lfe: vec![0], + ..Default::default() + }); + } + let mut elements = Vec::new(); + let mut ch = 0; + for (ty, tag) in layout { + elements.push(ElementState { + ty, + tag, + first_ch: ch, + prev_seq: WindowSequence::OnlyLong, + short_next: None, + prev_thr: vec![Vec::new(); ty.channels()], + }); + ch += ty.channels(); + } + let nfull = cfg.channels - elements.iter().filter(|e| e.ty == Lfe).count(); + let nyq = cfg.sample_rate / 2; + let auto_bw = match cfg.mode { + BitrateMode::Cbr(b) => { + let per = b as f32 / nfull.max(1) as f32 / 1000.0; + let pts = [ + (12.0f32, 4000.0f32), + (16.0, 5500.0), + (24.0, 8000.0), + (32.0, 11000.0), + (48.0, 14000.0), + (64.0, 16000.0), + (80.0, 17500.0), + (96.0, 19000.0), + (112.0, 20000.0), + (144.0, 22000.0), + ]; + interp(&pts, per) as u32 + } + BitrateMode::Vbr(q) => interp(&[(1.0, 13000.0), (2.0, 15000.0), (3.0, 17000.0), (4.0, 19000.0), (5.0, 20000.0)], q) as u32, + }; + let bandwidth = cfg.bandwidth.unwrap_or(auto_bw).clamp(1000, nyq.saturating_sub(nyq / 50).max(1000)); + let max_frame_bits = 6144 * cfg.channels as i64; + let mean = match cfg.mode { + BitrateMode::Cbr(b) => b as i64 * FRAME_LEN as i64 / cfg.sample_rate as i64, + BitrateMode::Vbr(_) => 0, + }; + if let BitrateMode::Cbr(b) = cfg.mode + && (b < 1000 * cfg.channels as u32 || mean > max_frame_bits) + { + return Err(Error::InvalidConfig("bitrate out of range for the channel count and sample rate")); + } + let res_max = (max_frame_bits - mean).max(0); + let hist = vec![vec![0f32; HISTORY]; cfg.channels]; + Ok(Encoder { + psy: Psy::new(cfg.sample_rate, sf_index), + sf_index, + asc, + elements, + hist, + origin: -(HISTORY as i64), + total_in: 0, + frames_done: 0, + flushed: false, + bandwidth, + mean_acc: 0, + level: 0, + res_max, + max_frame_bits, + pe_avg: 0.0, + lambda: 1.0, + stats: EncoderStats::default(), + cfg, + }) + } + + pub fn config(&self) -> &EncoderConfig { + &self.cfg + } + + /// `AudioSpecificConfig` bytes for the MP4 `esds` decoder-specific info. + pub fn audio_specific_config(&self) -> Vec { + self.asc.to_bytes() + } + + /// Encoder delay: the first 1024 decoded samples are priming and should be skipped (MP4 edit + /// list `media_time`, or iTunSMPB). + pub fn priming_samples(&self) -> u32 { + FRAME_LEN as u32 + } + + /// Audio bandwidth in Hz. + pub fn bandwidth(&self) -> u32 { + self.bandwidth + } + + pub fn stats(&self) -> EncoderStats { + self.stats + } + + /// Feed planar samples (±1.0 full scale); returns any complete access units. Missing channels are + /// treated as silence; all channels are truncated to the shortest slice. + pub fn encode(&mut self, planar: &[&[f32]]) -> Vec> { + if self.flushed { + return Vec::new(); + } + let n = planar.iter().take(self.cfg.channels).map(|p| p.len()).min().unwrap_or(0); + for c in 0..self.cfg.channels { + match planar.get(c) { + Some(p) => self.hist[c].extend(p[..n].iter().map(|v| if v.is_finite() { v.clamp(-8.0, 8.0) } else { 0.0 })), + None => self.hist[c].extend(std::iter::repeat_n(0.0, n)), + } + } + self.total_in += n as u64; + let mut out = Vec::new(); + while self.available() >= self.needed_for(self.frames_done) { + out.push(self.encode_frame(false)); + } + out + } + + /// Encode the remaining input (zero-padded). Afterwards the decoded stream, minus + /// [`priming_samples`](Self::priming_samples), covers every input sample. + pub fn flush(&mut self) -> Vec> { + if self.flushed { + return Vec::new(); + } + let frames = if self.total_in == 0 { 0 } else { self.total_in.div_ceil(FRAME_LEN as u64) + 1 }; + let mut out = Vec::new(); + while self.frames_done < frames { + let need = self.needed_for(self.frames_done); + let have = self.available(); + if have < need { + for h in &mut self.hist { + h.extend(std::iter::repeat_n(0.0, (need - have) as usize)); + } + } + let last = self.frames_done + 1 == frames; + out.push(self.encode_frame(last)); + } + self.flushed = true; + out + } + + /// Absolute sample index one past the buffered input. + fn available(&self) -> i64 { + self.origin + self.hist[0].len() as i64 + } + /// Input needed (absolute end) before frame `t` can be encoded (window + transient look-ahead). + fn needed_for(&self, t: u64) -> i64 { + (t as i64 + 1) * FRAME_LEN as i64 + 640 + } + + fn sample(&self, ch: usize, abs: i64) -> f32 { + let i = abs - self.origin; + if i < 0 { 0.0 } else { self.hist[ch].get(i as usize).copied().unwrap_or(0.0) } + } + + fn subblock_energy(&self, ch: usize, j: i64) -> f32 { + let s = j * 128; + let mut e = 0f32; + let mut prev = self.sample(ch, s - 1); + for n in s..s + 128 { + let x = self.sample(ch, n); + let d = x - prev; + e += d * d; + prev = x; + } + e + } + + /// Transient in the region covered by frame `t`'s short windows. + fn transient(&self, first_ch: usize, nch: usize, t: i64) -> bool { + for ch in first_ch..first_ch + nch { + let base = 8 * t; + let mut hist: Vec = (base - 13..base - 5).map(|j| self.subblock_energy(ch, j)).collect(); + for j in base - 5..=base + 4 { + let e = self.subblock_energy(ch, j); + let mean = hist.iter().sum::() / hist.len() as f32; + if e > 1.5e-4 && e > 10.0 * mean { + return true; + } + hist.remove(0); + hist.push(e); + } + } + false + } + + fn max_sfb(&self, short: bool, lfe: bool) -> usize { + let bw = if lfe { 240.min(self.bandwidth) } else { self.bandwidth }; + let (swb, m) = if short { (swb_offsets_short(self.sf_index), 128f64) } else { (swb_offsets_long(self.sf_index), 1024f64) }; + let bin = (bw as f64 / (self.cfg.sample_rate as f64 / 2.0) * m).ceil() as u16; + swb.iter().take(swb.len() - 1).filter(|&&o| o < bin).count().max(1) + } + + fn mdct(&self, ch: usize, seq: WindowSequence) -> Vec { + let shape = self.cfg.window_shape; + let seg = &self.hist[ch][HISTORY - FRAME_LEN..HISTORY + FRAME_LEN]; + let mut spec = vec![0f32; 1024]; + let scale = 32768.0 * 2.0; + if seq == WindowSequence::EightShort { + let ws = window(shape, false); + let mut buf = vec![0f32; 256]; + for w in 0..8 { + let off = 448 + 128 * w; + for n in 0..128 { + buf[n] = seg[off + n] * ws[n] * scale; + buf[128 + n] = seg[off + 128 + n] * ws[127 - n] * scale; + } + mdct_short().forward(&buf, &mut spec[w * 128..(w + 1) * 128]); + } + return spec; + } + let wl = window(shape, true); + let ws = window(shape, false); + let mut buf = vec![0f32; 2048]; + for n in 0..1024 { + let w = match seq { + WindowSequence::LongStop => { + if n < 448 { + 0.0 + } else if n < 576 { + ws[n - 448] + } else { + 1.0 + } + } + _ => wl[n], + }; + buf[n] = seg[n] * w * scale; + } + for n in 1024..2048 { + let w = match seq { + WindowSequence::LongStart => { + if n < 1472 { + 1.0 + } else if n < 1600 { + ws[127 - (n - 1472)] + } else { + 0.0 + } + } + _ => wl[2047 - n], + }; + buf[n] = seg[n] * w * scale; + } + mdct_long().forward(&buf, &mut spec); + spec + } + + fn tns_analyse(&self, spec: &mut [f32], info: &IcsInfo) -> Option { + let swb = swb_offsets_long(self.sf_index); + let lim = tns_max_bands(self.sf_index, false).min(info.max_sfb); + let hz = self.cfg.sample_rate as f64 / 2048.0; + let start = swb.iter().position(|&o| o as f64 * hz >= 1400.0).unwrap_or(swb.len() - 1); + if lim < start + 4 { + return None; + } + let (lo, hi) = (swb[start] as usize, swb[lim] as usize); + let x = &spec[lo..hi]; + const ORDER: usize = 8; + let mut r = [0f64; ORDER + 1]; + for (lag, rv) in r.iter_mut().enumerate() { + let mut acc = 0f64; + for n in lag..x.len() { + acc += x[n] as f64 * x[n - lag] as f64; + } + // Gaussian lag window for robustness + *rv = acc * (-0.5 * (0.02 * lag as f64).powi(2)).exp(); + } + if r[0] <= 1e-3 { + return None; + } + let (ks, gain) = tns::levinson(&r, ORDER); + if gain < 1.4 { + return None; + } + let mut coef = [0i8; 12]; + let mut order = 0; + for (i, &k) in ks.iter().enumerate() { + coef[i] = tns::quant_coef(k, true); + if coef[i] != 0 { + order = i + 1; + } + } + if order == 0 { + return None; + } + let mut t = TnsData::default(); + t.windows[0].n_filt = 1; + t.windows[0].coef_res = true; + t.windows[0].filt[0] = TnsFilter { length: (swb.len() - 1 - start) as u8, order: order as u8, direction: false, coef_compress: false, coef }; + tns::apply_encoder(spec, info, self.sf_index, &t); + Some(t) + } + + fn analyse_element(&mut self, ei: usize, t: i64) -> (ElementFrame, f32) { + let (ty, first, nch, prev_seq, cached) = { + let e = &self.elements[ei]; + (e.ty, e.first_ch, e.ty.channels(), e.prev_seq, e.short_next) + }; + let lfe = ty == ElementType::Lfe; + let (short_now, short_next) = + if lfe { (false, false) } else { (cached.unwrap_or_else(|| self.transient(first, nch, t)), self.transient(first, nch, t + 1)) }; + use WindowSequence::*; + let seq = if short_now { + EightShort + } else if prev_seq == EightShort || prev_seq == LongStart { + if short_next { EightShort } else { LongStop } + } else if short_next { + LongStart + } else { + OnlyLong + }; + let specs: Vec> = (first..first + nch).map(|c| self.mdct(c, seq)).collect(); + let short = seq == EightShort; + let mut info = IcsInfo { window_sequence: seq, window_shape: self.cfg.window_shape, max_sfb: self.max_sfb(short, lfe), ..Default::default() }; + if short { + // group windows of similar energy; an attack starts a new group + let en: Vec = + (0..8).map(|w| specs.iter().map(|s| s[w * 128..(w + 1) * 128].iter().map(|v| v * v).sum::()).sum::() + 1.0).collect(); + info.num_groups = 1; + info.group_len = [1, 0, 0, 0, 0, 0, 0, 0]; + for w in 1..8 { + let r = en[w] / en[w - 1]; + if !(1.0 / 6.0..=3.0).contains(&r) { + info.num_groups += 1; + info.group_len[info.num_groups - 1] = 1; + } else { + info.group_len[info.num_groups - 1] += 1; + } + } + } + let mut pe = 0f32; + let mut specs = specs; + let mut thrs: Vec>> = Vec::with_capacity(nch); + let mut tnss = Vec::with_capacity(nch); + for (i, spec) in specs.iter_mut().enumerate() { + let mut r = self.psy.analyse(spec, short); + pe += r.pe; + let e = &mut self.elements[ei]; + if !short { + if prev_seq != EightShort && e.prev_thr[i].len() == r.thr[0].len() { + for (t, p) in r.thr[0].iter_mut().zip(&e.prev_thr[i]) { + *t = t.min(2.0 * p.max(1e-9)); + } + } + e.prev_thr[i] = r.thr[0].clone(); + } else { + e.prev_thr[i].clear(); + } + // group thresholds: the most sensitive window times the group length + let mut thr_g = vec![vec![0f32; 64]; info.num_groups]; + for g in 0..info.num_groups { + let w0 = info.group_start(g); + let gl = info.group_len[g] as usize; + for sfb in 0..info.max_sfb { + let m = (w0..w0 + gl).map(|w| r.thr[w][sfb]).fold(f32::MAX, f32::min); + thr_g[g][sfb] = m * gl as f32; + } + } + thrs.push(thr_g); + let tns = if self.cfg.tns && !short && !lfe { self.tns_analyse(spec, &info) } else { None }; + if let Some(t) = &tns { + self.stats.tns_filters += t.windows[0].n_filt as u64; + } + tnss.push(tns); + } + let swb = info.swb_offsets(self.sf_index); + let mut ms_mask = None; + if ty == ElementType::Cpe { + let mut used = Vec::with_capacity(info.num_groups * info.max_sfb); + let (l, r) = specs.split_at_mut(1); + let (l, r) = (&mut l[0], &mut r[0]); + for g in 0..info.num_groups { + let w0 = info.group_start(g); + let gl = info.group_len[g] as usize; + for sfb in 0..info.max_sfb { + let (mut el, mut er, mut em, mut es) = (0f32, 0f32, 0f32, 0f32); + for w in w0..w0 + gl { + for k in w * 128 + swb[sfb] as usize..w * 128 + swb[sfb + 1] as usize { + let (a, b) = (l[k], r[k]); + el += a * a; + er += b * b; + em += 0.25 * (a + b) * (a + b); + es += 0.25 * (a - b) * (a - b); + } + } + let (tl, tr) = (thrs[0][g][sfb].max(1e-9), thrs[1][g][sfb].max(1e-9)); + let tm = 0.5 * tl.min(tr); + let c = |e: f32, t: f32| (e / t).max(1.0).log2(); + let use_ms = self.cfg.ms_stereo && c(em, tm) + c(es, tm) < c(el, tl) + c(er, tr); + used.push(use_ms); + self.stats.stereo_bands += 1; + if use_ms { + self.stats.ms_bands += 1; + thrs[0][g][sfb] = tm; + thrs[1][g][sfb] = tm; + for w in w0..w0 + gl { + for k in w * 128 + swb[sfb] as usize..w * 128 + swb[sfb + 1] as usize { + let (a, b) = (l[k], r[k]); + l[k] = 0.5 * (a + b); + r[k] = 0.5 * (a - b); + } + } + } + } + } + let n_ms = used.iter().filter(|&&u| u).count(); + let present = if n_ms == 0 { + 0 + } else if n_ms == used.len() { + 2 + } else { + 1 + }; + ms_mask = Some((present, used)); + } + let chans = specs.iter().zip(thrs.iter()).zip(tnss).map(|((s, th), tn)| ChannelInput::new(info, swb, s, th, tn)).collect(); + let e = &mut self.elements[ei]; + e.prev_seq = seq; + e.short_next = Some(short_next); + if short { + self.stats.short_blocks += 1; + } + (ElementFrame { info, chans, ms_mask }, pe) + } + + fn element_overhead(&self, ei: usize, f: &ElementFrame) -> u32 { + let info_bits = if f.info.is_short() { 15 } else { 11 }; + match self.elements[ei].ty { + ElementType::Cpe => { + let ms = match &f.ms_mask { + Some((1, used)) => used.len() as u32, + _ => 0, + }; + 3 + 4 + 1 + info_bits + 2 + ms + } + _ => 3 + 4 + info_bits, + } + } + + fn quantise_all(frames: &[ElementFrame], lambda: f32) -> Vec> { + #[cfg(feature = "rayon")] + { + use rayon::prelude::*; + frames.par_iter().map(|f| f.chans.par_iter().map(|c| quantise_channel(c, lambda)).collect()).collect() + } + #[cfg(not(feature = "rayon"))] + { + frames.iter().map(|f| f.chans.iter().map(|c| quantise_channel(c, lambda)).collect()).collect() + } + } + + /// `last`: final frame of the stream — a CBR stream spends (or pads away) the whole reservoir so + /// the total size is exactly `frames × bitrate × 1024 / rate`. + fn encode_frame(&mut self, last: bool) -> Vec { + let t = self.frames_done as i64; + let mut frames = Vec::with_capacity(self.elements.len()); + let mut pe = 0f32; + for ei in 0..self.elements.len() { + let (f, p) = self.analyse_element(ei, t); + pe += p; + frames.push(f); + } + let pce_bits = if self.cfg.adts && self.asc.channel_config == 0 { + let mut bw = BitWriter::new(); + if let Some(p) = &self.asc.pce { + p.write(&mut bw); + } + 3 + bw.bit_len() as u32 + 8 + } else { + 0 + }; + let overhead: u32 = (0..frames.len()).map(|i| self.element_overhead(i, &frames[i])).sum::() + 3 + 7 + pce_bits; + let total_bits = |q: &Vec>| overhead + q.iter().flatten().map(|c| c.bits).sum::(); + + let any_short = frames.iter().any(|f| f.info.is_short()); + let (mean, target) = match self.cfg.mode { + BitrateMode::Cbr(b) => { + self.mean_acc += b as u64 * FRAME_LEN as u64; + let mean = (self.mean_acc / self.cfg.sample_rate as u64) as i64; + self.mean_acc %= self.cfg.sample_rate as u64; + self.pe_avg = if self.pe_avg == 0.0 { pe.max(1.0) } else { 0.95 * self.pe_avg + 0.05 * pe.max(1.0) }; + let mut ratio = (pe.max(1.0) / self.pe_avg).clamp(0.7, 1.6); + if any_short { + ratio = (ratio * 1.3).min(2.0); + } + let mut desired = mean as f32 * ratio + (self.level as f32 - 0.5 * self.res_max as f32) * 0.15; + desired = desired.min((mean + self.level) as f32).max((mean - (self.res_max - self.level)) as f32); + desired = desired.min(self.max_frame_bits as f32); + if last { + desired = (mean + self.level) as f32; + } + (mean, desired.max(0.0) as u32) + } + BitrateMode::Vbr(_) => (0, self.max_frame_bits as u32), + }; + + let quant = match self.cfg.mode { + BitrateMode::Vbr(q) => { + let d = q.clamp(0.5, 6.0) - 1.0; + let mut lambda = 10f32.powf(-0.2 - 1.2 * d + 0.09 * d * d); + let mut res = Self::quantise_all(&frames, lambda); + while total_bits(&res) > target && lambda < 1e9 { + lambda *= 2.0; + res = Self::quantise_all(&frames, lambda); + } + res + } + BitrateMode::Cbr(_) => { + let mut lam = self.lambda.clamp(1e-6, 1e9); + let mut cur = Self::quantise_all(&frames, lam); + let (mut lo, mut hi, mut best); + if total_bits(&cur) <= target { + hi = lam; + best = cur; + lo = lam; + loop { + lo /= 4.0; + if lo < 1e-6 { + break; + } + cur = Self::quantise_all(&frames, lo); + if total_bits(&cur) > target { + break; + } + hi = lo; + best = cur; + } + } else { + lo = lam; + loop { + lam *= 4.0; + cur = Self::quantise_all(&frames, lam); + if total_bits(&cur) <= target || lam > 1e9 { + break; + } + lo = lam; + } + hi = lam; + best = cur; + } + for _ in 0..6 { + if hi / lo < 1.03 { + break; + } + let mid = (lo * hi).sqrt(); + let r = Self::quantise_all(&frames, mid); + if total_bits(&r) <= target { + hi = mid; + best = r; + } else { + lo = mid; + } + } + self.lambda = hi; + best + } + }; + + // assemble the raw_data_block + let mut bw = BitWriter::new(); + if pce_bits > 0 + && let Some(p) = &self.asc.pce + { + bw.write_bits(5, 3); + p.write(&mut bw); + } + for (ei, (f, q)) in frames.iter().zip(&quant).enumerate() { + let e = &self.elements[ei]; + match e.ty { + ElementType::Cpe => { + bw.write_bits(1, 3); + bw.write_bits(e.tag as u32, 4); + bw.write_bits(1, 1); // common_window + f.info.write(&mut bw); + let (present, used) = f.ms_mask.clone().unwrap_or((0, Vec::new())); + bw.write_bits(present, 2); + if present == 1 { + for &u in &used { + bw.write_bits(u as u32, 1); + } + } + write_ics(&mut bw, &f.chans[0], &q[0], false); + write_ics(&mut bw, &f.chans[1], &q[1], false); + } + ty => { + bw.write_bits(if ty == ElementType::Lfe { 3 } else { 0 }, 3); + bw.write_bits(e.tag as u32, 4); + write_ics(&mut bw, &f.chans[0], &q[0], true); + } + } + } + if let BitrateMode::Cbr(_) = self.cfg.mode { + let projected = ((bw.bit_len() as i64 + 3 + 7) / 8) * 8; + let excess = self.level + mean - projected - if last { 0 } else { self.res_max }; + if excess > 0 { + self.stats.fill_bits += write_fill(&mut bw, excess as u64); + } + } + bw.write_bits(7, 3); // ID_END + let au = bw.finish(); + let used = au.len() as i64 * 8; + if let BitrateMode::Cbr(_) = self.cfg.mode { + self.level = (self.level + mean - used).clamp(0, self.res_max); + } + self.stats.frames += 1; + self.stats.bytes += au.len() as u64; + + // slide the input window + for h in &mut self.hist { + h.drain(..FRAME_LEN); + } + self.origin += FRAME_LEN as i64; + self.frames_done += 1; + + if self.cfg.adts { + let fullness = match self.cfg.mode { + BitrateMode::Cbr(_) => ((self.level / (32 * self.cfg.channels as i64)) as u16).min(0x7FE), + BitrateMode::Vbr(_) => 0x7FF, + }; + let mut out = AdtsHeader::write(2, self.sf_index, self.asc.channel_config, au.len(), fullness).to_vec(); + out.extend_from_slice(&au); + out + } else { + au + } + } +} + +/// Write fill elements totalling at least `bits - 7` bits (at most `bits + 8`); returns bits written. +fn write_fill(bw: &mut BitWriter, bits: u64) -> u64 { + let start = bw.bit_len() as u64; + let mut left = bits as i64; + while left >= 7 { + let n = (((left - 7) / 8) as usize).min(269); + let n = if n >= 15 && (left - 15) / 8 < 15 { 14 } else { n }; + bw.write_bits(6, 3); + if n >= 15 { + bw.write_bits(15, 4); + bw.write_bits((n - 14) as u32, 8); + } else { + bw.write_bits(n as u32, 4); + } + if n > 0 { + bw.write_bits(0, 8); // extension_type EXT_FILL + fill_nibble + for _ in 1..n { + bw.write_bits(0xA5, 8); + } + } + let el = 7 + if n >= 15 { 8 } else { 0 } + 8 * n as i64; + left -= el; + if n == 0 { + break; + } + } + bw.bit_len() as u64 - start +} + +fn interp(pts: &[(f32, f32)], x: f32) -> f32 { + if x <= pts[0].0 { + return pts[0].1; + } + for w in pts.windows(2) { + if x <= w[1].0 { + let t = (x - w[0].0) / (w[1].0 - w[0].0); + return w[0].1 + t * (w[1].1 - w[0].1); + } + } + pts[pts.len() - 1].1 +} diff --git a/crates/aac/src/encoder/psy.rs b/crates/aac/src/encoder/psy.rs new file mode 100644 index 0000000000..1101a76234 --- /dev/null +++ b/crates/aac/src/encoder/psy.rs @@ -0,0 +1,176 @@ +//! Psychoacoustic model operating on MDCT coefficients. +//! +//! Per scalefactor band: energy; tonality from the spectral flatness measure (SFM) of a +//! neighbourhood of ≥ 16 lines; masking-threshold PSD by spreading the band PSDs with Schroeder's +//! spreading function in the Bark domain; a tonality-dependent masking offset (≈18 dB for tonal, +//! ≈6 dB for noise-like maskers); the absolute threshold of hearing (Terhardt) as a floor; and +//! perceptual entropy (PE) as an estimate of the bits the frame needs. + +use crate::tables::{swb_offsets_long, swb_offsets_short}; + +/// Precomputed per-band data for one window length. +pub struct BandModel { + pub offsets: &'static [u16], + /// Linear spreading factor `[maskee][masker]` (power). + spread: Vec>, + /// ATH as band energy in encoder spectrum units. + ath: Vec, + /// Neighbourhood `[lo, hi)` in lines used for the tonality estimate. + tonal_span: Vec<(usize, usize)>, +} + +fn bark(f: f64) -> f64 { + 13.0 * (0.00076 * f).atan() + 3.5 * (f / 7500.0).powi(2).atan() +} + +/// Terhardt's threshold in quiet (dB SPL), capped at 40 dB so content near the band edge is not +/// discarded wholesale (the bandwidth setting limits the coded range instead). +fn ath_db(f: f64) -> f64 { + let f = (f / 1000.0).max(0.02); + (3.64 * f.powf(-0.8) - 6.5 * (-0.6 * (f - 3.3).powi(2)).exp() + 1e-3 * f.powi(4)).min(40.0) +} + +impl BandModel { + fn new(sample_rate: u32, offsets: &'static [u16], m: usize) -> BandModel { + let nb = offsets.len() - 1; + let hz_per_line = sample_rate as f64 / (2.0 * m as f64); + let centre: Vec = (0..nb).map(|b| bark((offsets[b] as f64 + offsets[b + 1] as f64) * 0.5 * hz_per_line)).collect(); + let spread = (0..nb) + .map(|i| { + (0..nb) + .map(|j| { + let dz = centre[i] - centre[j] + 0.474; + let db = 15.81 + 7.5 * dz - 17.5 * (1.0 + dz * dz).sqrt(); + if db < -60.0 { 0.0 } else { 10f64.powf(db / 10.0) as f32 } + }) + .collect() + }) + .collect(); + // A full-scale (16-bit) sine of power P_ref = 32768²/2 has MDCT energy ≈ 2·M²·P per frame + // with the encoder's scaling; it is taken as 96 dB SPL. 3 dB of safety margin. + let p_ref = 32768f64 * 32768.0 / 2.0; + let ath = (0..nb) + .map(|b| { + let mut lo = f64::MAX; + for k in offsets[b]..offsets[b + 1] { + lo = lo.min(ath_db((k as f64 + 0.5) * hz_per_line)); + } + (2.0 * (m * m) as f64 * p_ref * 10f64.powf((lo - 96.0 - 3.0) / 10.0)) as f32 + }) + .collect(); + let tonal_span = (0..nb) + .map(|b| { + let (lo, hi) = (offsets[b] as usize, offsets[b + 1] as usize); + if hi - lo >= 16 { + (lo, hi) + } else { + let c = (lo + hi) / 2; + let lo = c.saturating_sub(8); + let hi = (lo + 16).min(m); + (hi.saturating_sub(16), hi) + } + }) + .collect(); + BandModel { offsets, spread, ath, tonal_span } + } + + pub fn num_bands(&self) -> usize { + self.offsets.len() - 1 + } +} + +pub struct Psy { + pub long: BandModel, + pub short: BandModel, +} + +/// Per-window, per-band analysis of one channel. +#[derive(Clone, Default)] +pub struct PsyResult { + /// `[window][band]` energy. + pub en: Vec>, + /// `[window][band]` masking threshold (allowed noise energy). + pub thr: Vec>, + pub pe: f32, +} + +impl Psy { + pub fn new(sample_rate: u32, sf_index: u8) -> Psy { + Psy { + long: BandModel::new(sample_rate, swb_offsets_long(sf_index), 1024), + short: BandModel::new(sample_rate, swb_offsets_short(sf_index), 128), + } + } + + /// Analyse a spectrum: 1024 lines (long) or 8×128 (short). + pub fn analyse(&self, spec: &[f32], short: bool) -> PsyResult { + let (model, nwin, wlen) = if short { (&self.short, 8, 128) } else { (&self.long, 1, 1024) }; + let nb = model.num_bands(); + let mut res = PsyResult { en: Vec::with_capacity(nwin), thr: Vec::with_capacity(nwin), pe: 0.0 }; + let mut psd = vec![0f32; nb]; + let mut tonality = vec![0f32; nb]; + for w in 0..nwin { + let s = &spec[w * wlen..(w + 1) * wlen]; + let mut en = vec![0f32; nb]; + for b in 0..nb { + let (lo, hi) = (model.offsets[b] as usize, model.offsets[b + 1] as usize); + en[b] = s[lo..hi].iter().map(|v| v * v).sum(); + psd[b] = en[b] / (hi - lo) as f32; + // spectral flatness over the neighbourhood + let (tlo, thi) = model.tonal_span[b]; + let n = (thi - tlo) as f64; + let mut sum = 0f64; + let mut log_sum = 0f64; + for &v in &s[tlo..thi] { + let p = v as f64 * v as f64 + 1e-3; + sum += p; + log_sum += p.ln(); + } + let sfm_db = 10.0 * ((log_sum / n).exp() / (sum / n)).log10(); + // real-valued Gaussian noise gives ≈ -5.5 dB; a pure tone ≤ -25 dB + tonality[b] = ((-sfm_db - 6.0) / 19.0).clamp(0.0, 1.0) as f32; + } + let mut thr = vec![0f32; nb]; + for b in 0..nb { + let row = &model.spread[b]; + let mut sp = 0f32; + for j in 0..nb { + sp += psd[j] * row[j]; + } + let width = (model.offsets[b + 1] - model.offsets[b]) as f32; + let offset_db = tonality[b] * 30.0 + (1.0 - tonality[b]) * 6.0; + let t = sp * width * 10f32.powf(-offset_db / 10.0); + thr[b] = t.max(model.ath[b]); + if en[b] > thr[b] { + res.pe += width * 0.5 * (en[b] / thr[b]).log2(); + } + } + res.en.push(en); + res.thr.push(thr); + } + res + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tone_is_masked_less_than_noise() { + let psy = Psy::new(44100, 4); + let tone: Vec = (0..1024).map(|k| if k == 100 { 1e6 } else { 0.0 }).collect(); + let mut seed = 1u32; + let noise: Vec = (0..1024) + .map(|_| { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + ((seed >> 8) as f32 / (1 << 24) as f32 - 0.5) * 1e4 + }) + .collect(); + let rt = psy.analyse(&tone, false); + let rn = psy.analyse(&noise, false); + let b = psy.long.offsets.iter().position(|&o| o > 100).unwrap() - 1; + assert!(rt.thr[0][b] < rt.en[0][b] / 30.0); + assert!(rn.thr[0][b] > rn.en[0][b] / 10.0); + } +} diff --git a/crates/aac/src/encoder/quant.rs b/crates/aac/src/encoder/quant.rs new file mode 100644 index 0000000000..e07ac67d72 --- /dev/null +++ b/crates/aac/src/encoder/quant.rs @@ -0,0 +1,382 @@ +//! Quantisation, scalefactor selection, codebook/section selection, bit counting and writing of one +//! `individual_channel_stream`. +//! +//! Coefficients are kept in bitstream order ("interleaved": group → band → window → line), so every +//! (group, band) is a contiguous slice and short-window grouping needs no special casing. + +use deckcraft_bitstream::BitWriter; + +use crate::huffman::{self, ZERO_HCB}; +use crate::ics::IcsInfo; +use crate::tables::pow43; +use crate::tns::TnsData; + +/// Magic rounding offset for AAC quantisation (`0.5 - 0.0946`), minimising the error for the +/// `x^(3/4)` companding. +const ROUND: f32 = 0.4054; +const MAX_Q: i32 = 8191; + +/// One (group, band) of a channel, in bitstream order. +#[derive(Clone, Copy, Debug)] +pub struct Band { + /// Range in the interleaved coefficient array. + pub start: usize, + pub end: usize, + /// Allowed noise energy at λ = 1. + pub thr: f32, + /// Coefficient energy. + pub en: f32, + /// Largest `|x|^(3/4)`. + pub max34: f32, +} + +/// Everything the rate loop needs for one channel. +pub struct ChannelInput { + pub info: IcsInfo, + /// Interleaved (bitstream-order) coefficients. + pub x: Vec, + pub xabs: Vec, + pub x34: Vec, + pub bands: Vec, + pub tns: Option, +} + +impl ChannelInput { + /// `spec` is window-major (short windows at stride 128); `thr[g][sfb]` and the grouping come from + /// `info`. + pub fn new(info: IcsInfo, swb: &[u16], spec: &[f32], thr: &[Vec], tns: Option) -> ChannelInput { + let mut x = Vec::with_capacity(1024); + let mut bands = Vec::new(); + for g in 0..info.num_groups { + let w0 = info.group_start(g); + for sfb in 0..info.max_sfb { + let start = x.len(); + for w in w0..w0 + info.group_len[g] as usize { + x.extend_from_slice(&spec[w * 128 + swb[sfb] as usize..w * 128 + swb[sfb + 1] as usize]); + } + bands.push(Band { start, end: x.len(), thr: thr[g][sfb], en: 0.0, max34: 0.0 }); + } + } + let xabs: Vec = x.iter().map(|v| v.abs()).collect(); + let x34: Vec = xabs.iter().map(|v| v.powf(0.75)).collect(); + for b in &mut bands { + b.en = x[b.start..b.end].iter().map(|v| v * v).sum(); + b.max34 = x34[b.start..b.end].iter().cloned().fold(0.0, f32::max); + } + ChannelInput { info, x, xabs, x34, bands, tns } + } +} + +/// A quantised channel ready to be written. +#[derive(Clone, Default)] +pub struct QuantChannel { + pub q: Vec, + /// Scalefactor per band (only meaningful where `cb != 0`). + pub sf: Vec, + pub cb: Vec, + /// Per group: (codebook, number of bands). + pub sections: Vec>, + pub global_gain: i32, + /// Bits of the ICS excluding `ics_info`. + pub bits: u32, +} + +#[inline] +fn gain34(sf: i32) -> f32 { + (-0.1875 * (sf as f32 - 100.0)).exp2() +} +#[inline] +fn gain(sf: i32) -> f32 { + (0.25 * (sf as f32 - 100.0)).exp2() +} + +/// Quantise a slice at `sf`, returning the distortion; writes the (signed) values to `out` if given. +fn quantise(x: &[f32], xabs: &[f32], x34: &[f32], sf: i32, out: Option<&mut [i32]>) -> f32 { + let g34 = gain34(sf); + let g = gain(sf); + let mut d = 0f32; + match out { + Some(out) => { + for i in 0..x.len() { + let q = ((x34[i] * g34 + ROUND) as i32).min(MAX_Q); + let e = xabs[i] - pow43(q as u32) * g; + d += e * e; + out[i] = if x[i] < 0.0 { -q } else { q }; + } + } + None => { + for i in 0..x.len() { + let q = ((x34[i] * g34 + ROUND) as i32).min(MAX_Q); + let e = xabs[i] - pow43(q as u32) * g; + d += e * e; + } + } + } + d +} + +/// Largest scalefactor whose distortion stays within `allowed`. +fn search_sf(ch: &ChannelInput, b: &Band, allowed: f32) -> i32 { + let r = b.start..b.end; + let (x, xa, x34) = (&ch.x[r.clone()], &ch.xabs[r.clone()], &ch.x34[r]); + let l2 = b.max34.log2(); + let lo = ((100.0 - ((MAX_Q as f32 + 0.5).log2() - l2) / 0.1875).ceil() as i32).clamp(0, 255); + let zero = ((100.0 - ((1.0 - ROUND).log2() - l2) / 0.1875).floor() as i32 + 1).clamp(0, 256); + let mut a = lo; + let mut hi = (zero - 1).clamp(lo, 255); + if quantise(x, xa, x34, a, None) > allowed { + return a; + } + while a < hi { + let mid = (a + hi + 1) / 2; + if quantise(x, xa, x34, mid, None) <= allowed { + a = mid; + } else { + hi = mid - 1; + } + } + a +} + +/// Candidate codebooks for a band whose largest magnitude is `m` (0 for all-zero bands). +fn candidates(m: i32) -> &'static [u8] { + match m { + 0 => &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + 1 => &[1, 2, 3, 4, 5, 6], + 2 => &[3, 4, 5, 6, 7, 8], + 3..=4 => &[5, 6, 7, 8, 9, 10], + 5..=7 => &[7, 8, 9, 10, 11], + 8..=12 => &[9, 10, 11], + _ => &[11], + } +} + +/// Quantise one channel at rate-distortion multiplier `lambda` and select sections. +pub fn quantise_channel(ch: &ChannelInput, lambda: f32) -> QuantChannel { + let nb = ch.bands.len(); + let mut sf = vec![-1i32; nb]; + for (i, b) in ch.bands.iter().enumerate() { + let allowed = lambda * b.thr; + if b.max34 <= 0.0 || b.en <= allowed { + continue; + } + sf[i] = search_sf(ch, b, allowed); + } + // Scalefactor differences between coded bands must stay within ±60: lower (refine) offenders, + // then quantise; repeat if a band unexpectedly quantised to all zeros (it leaves the chain). + let mut q = vec![0i32; ch.x.len()]; + let mut maxq = vec![0i32; nb]; + loop { + for _ in 0..8 { + let mut changed = false; + let coded: Vec = (0..nb).filter(|&i| sf[i] >= 0).collect(); + for w in coded.windows(2) { + if sf[w[1]] > sf[w[0]] + 60 { + sf[w[1]] = sf[w[0]] + 60; + changed = true; + } + } + for w in coded.windows(2).rev() { + if sf[w[0]] > sf[w[1]] + 60 { + sf[w[0]] = sf[w[1]] + 60; + changed = true; + } + } + if !changed { + break; + } + } + let mut removed = false; + for (i, b) in ch.bands.iter().enumerate() { + let r = b.start..b.end; + if sf[i] < 0 { + q[r].fill(0); + maxq[i] = 0; + continue; + } + quantise(&ch.x[r.clone()], &ch.xabs[r.clone()], &ch.x34[r.clone()], sf[i], Some(&mut q[r.clone()])); + maxq[i] = q[r].iter().map(|v| v.abs()).max().unwrap_or(0); + if maxq[i] == 0 { + sf[i] = -1; + removed = true; + } + } + if !removed { + break; + } + } + + // Section selection: dynamic programming per group over codebooks. + let info = &ch.info; + let (sect_bits, esc) = if info.is_short() { (3u32, 7usize) } else { (5u32, 31usize) }; + let mut cb = vec![ZERO_HCB; nb]; + let mut sections = Vec::with_capacity(info.num_groups); + let mut spectral_bits = 0u32; + let mut section_bits = 0u32; + let per_group = info.max_sfb; + for g in 0..info.num_groups { + let base = g * per_group; + let n = per_group; + if n == 0 { + sections.push(Vec::new()); + continue; + } + // cost[i][c] + const INF: u32 = u32::MAX / 4; + let mut cost = vec![[INF; 12]; n]; + for i in 0..n { + let bi = base + i; + let b = &ch.bands[bi]; + let vals = &q[b.start..b.end]; + for &c in candidates(maxq[bi]) { + cost[i][c as usize] = if c == 0 { + 0 + } else { + // all-zero band inside a coded section also needs a (zero) scalefactor difference + huffman::count_bits(c, vals) + if maxq[bi] == 0 { 1 } else { 0 } + }; + } + } + let new_sec = 4 + sect_bits; + let mut best = [INF; 12]; + let mut back = vec![[0u8; 12]; n]; + for c in 0..12 { + if cost[0][c] < INF { + best[c] = new_sec + cost[0][c]; + } + } + for i in 1..n { + let (mut min_c, mut min_v) = (0usize, INF); + for c in 0..12 { + if best[c] < min_v { + min_v = best[c]; + min_c = c; + } + } + let mut nb_ = [INF; 12]; + for c in 0..12 { + if cost[i][c] >= INF { + continue; + } + let stay = best[c]; + let switch = min_v + new_sec; + if stay <= switch { + nb_[c] = stay + cost[i][c]; + back[i][c] = c as u8; + } else { + nb_[c] = switch + cost[i][c]; + back[i][c] = min_c as u8; + } + } + best = nb_; + } + let mut c = (0..12).min_by_key(|&c| best[c]).unwrap_or(0); + for i in (0..n).rev() { + cb[base + i] = c as u8; + c = back[i][c] as usize; + } + // run-length sections and exact costs + let mut secs: Vec<(u8, usize)> = Vec::new(); + for i in 0..n { + let c = cb[base + i]; + match secs.last_mut() { + Some((lc, len)) if *lc == c => *len += 1, + _ => secs.push((c, 1)), + } + if c != ZERO_HCB { + let b = &ch.bands[base + i]; + spectral_bits += huffman::count_bits(c, &q[b.start..b.end]); + } + } + for &(_, len) in &secs { + section_bits += 4 + sect_bits * (len / esc + 1) as u32; + } + sections.push(secs); + } + + // Scalefactors: global gain is the first coded band's value; bands coded with a non-zero + // codebook but all-zero values reuse the previous value (difference 0). + let first = (0..nb).find(|&i| sf[i] >= 0).map(|i| sf[i]).unwrap_or(100); + let mut prev = first; + let mut sf_bits = 0u32; + for i in 0..nb { + if cb[i] == ZERO_HCB { + continue; + } + if sf[i] < 0 { + sf[i] = prev; + } + sf_bits += huffman::sf_bits(sf[i] - prev); + prev = sf[i]; + } + let tns_bits = 1 + ch.tns.as_ref().map(|t| t.bit_count(info)).unwrap_or(0); + let bits = 8 + section_bits + sf_bits + 1 + tns_bits + 1 + spectral_bits; + QuantChannel { q, sf, cb, sections, global_gain: first, bits } +} + +/// Write an `individual_channel_stream`; `write_info` is false inside a common-window CPE. +pub fn write_ics(bw: &mut BitWriter, ch: &ChannelInput, qc: &QuantChannel, write_info: bool) { + let info = &ch.info; + bw.write_bits(qc.global_gain as u32, 8); + if write_info { + info.write(bw); + } + let (sect_bits, esc) = if info.is_short() { (3u32, 7usize) } else { (5u32, 31usize) }; + for secs in &qc.sections { + for &(c, len) in secs { + bw.write_bits(c as u32, 4); + let mut l = len; + while l >= esc { + bw.write_bits(esc as u32, sect_bits); + l -= esc; + } + bw.write_bits(l as u32, sect_bits); + } + } + let mut prev = qc.global_gain; + for i in 0..ch.bands.len() { + if qc.cb[i] == ZERO_HCB { + continue; + } + huffman::write_sf(bw, qc.sf[i] - prev); + prev = qc.sf[i]; + } + bw.write_bits(0, 1); // pulse_data_present + match &ch.tns { + Some(t) => { + bw.write_bits(1, 1); + t.write(bw, info); + } + None => bw.write_bits(0, 1), + } + bw.write_bits(0, 1); // gain_control_data_present + for (i, b) in ch.bands.iter().enumerate() { + if qc.cb[i] != ZERO_HCB { + huffman::write_spectral(bw, qc.cb[i], &qc.q[b.start..b.end]); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ics::WindowSequence; + use crate::mdct::WindowShape; + use crate::tables::swb_offsets_long; + use deckcraft_bitstream::BitWriter; + + #[test] + fn bit_count_matches_written_bits() { + let swb = swb_offsets_long(4); + let info = IcsInfo { window_sequence: WindowSequence::OnlyLong, window_shape: WindowShape::Sine, max_sfb: 40, ..Default::default() }; + let spec: Vec = (0..1024).map(|i| ((i * 7919 % 1000) as f32 - 500.0) * (1000.0 / (i as f32 + 10.0))).collect(); + let thr = vec![vec![1000.0f32; 64]]; + let ch = ChannelInput::new(info, swb, &spec, &thr, None); + for lambda in [0.01f32, 1.0, 100.0, 1e5] { + let qc = quantise_channel(&ch, lambda); + let mut bw = BitWriter::new(); + write_ics(&mut bw, &ch, &qc, false); + assert_eq!(bw.bit_len() as u32, qc.bits, "lambda {lambda}"); + } + } +} diff --git a/crates/aac/src/huffman.rs b/crates/aac/src/huffman.rs new file mode 100644 index 0000000000..2e116ed283 --- /dev/null +++ b/crates/aac/src/huffman.rs @@ -0,0 +1,306 @@ +//! Huffman coding of spectral values and scalefactors (ISO/IEC 14496-3 §4.6.3, Annex 4.A). + +use std::sync::OnceLock; + +use deckcraft_bitstream::{BitReader, BitWriter}; + +use crate::huffman_tables::{ + SCALEFACTOR, SPECTRUM_1, SPECTRUM_2, SPECTRUM_3, SPECTRUM_4, SPECTRUM_5, SPECTRUM_6, SPECTRUM_7, SPECTRUM_8, SPECTRUM_9, SPECTRUM_10, SPECTRUM_11, +}; +use crate::{Error, Result}; + +/// `ZERO_HCB`: band quantised to zero, nothing transmitted. +pub const ZERO_HCB: u8 = 0; +/// `ESC_HCB`: codebook 11 with escape sequences. +pub const ESC_HCB: u8 = 11; +/// `NOISE_HCB`: perceptual noise substitution. +pub const NOISE_HCB: u8 = 13; +/// `INTENSITY_HCB2`: intensity stereo, out of phase. +pub const INTENSITY_HCB2: u8 = 14; +/// `INTENSITY_HCB`: intensity stereo, in phase. +pub const INTENSITY_HCB: u8 = 15; + +/// Parameters of one spectrum codebook. +pub struct Book { + /// Values per codeword (4 = quad, 2 = pair). + pub dim: usize, + /// Signed values in the codeword (no separate sign bits). + pub signed: bool, + /// Largest absolute value (16 = escape for codebook 11). + pub lav: u32, + /// Radix of the index (`2*lav+1` signed, `lav+1` unsigned). + pub modulo: u32, + pub codes: &'static [(u32, u8)], +} + +static BOOKS: [Book; 11] = [ + Book { dim: 4, signed: true, lav: 1, modulo: 3, codes: &SPECTRUM_1 }, + Book { dim: 4, signed: true, lav: 1, modulo: 3, codes: &SPECTRUM_2 }, + Book { dim: 4, signed: false, lav: 2, modulo: 3, codes: &SPECTRUM_3 }, + Book { dim: 4, signed: false, lav: 2, modulo: 3, codes: &SPECTRUM_4 }, + Book { dim: 2, signed: true, lav: 4, modulo: 9, codes: &SPECTRUM_5 }, + Book { dim: 2, signed: true, lav: 4, modulo: 9, codes: &SPECTRUM_6 }, + Book { dim: 2, signed: false, lav: 7, modulo: 8, codes: &SPECTRUM_7 }, + Book { dim: 2, signed: false, lav: 7, modulo: 8, codes: &SPECTRUM_8 }, + Book { dim: 2, signed: false, lav: 12, modulo: 13, codes: &SPECTRUM_9 }, + Book { dim: 2, signed: false, lav: 12, modulo: 13, codes: &SPECTRUM_10 }, + Book { dim: 2, signed: false, lav: 16, modulo: 17, codes: &SPECTRUM_11 }, +]; + +/// Spectrum codebook 1..=11. +#[inline] +pub fn book(cb: u8) -> &'static Book { + &BOOKS[(cb as usize).clamp(1, 11) - 1] +} + +const LEAF: u32 = 1 << 31; + +/// Binary decoding tree for one codebook. +struct Tree { + nodes: Vec<[u32; 2]>, +} + +impl Tree { + fn build(codes: &[(u32, u8)]) -> Tree { + let mut nodes: Vec<[u32; 2]> = vec![[0, 0]]; + for (sym, &(code, len)) in codes.iter().enumerate() { + let mut node = 0usize; + for i in (0..len).rev() { + let bit = ((code >> i) & 1) as usize; + if i == 0 { + nodes[node][bit] = LEAF | sym as u32; + } else { + if nodes[node][bit] == 0 { + nodes.push([0, 0]); + nodes[node][bit] = (nodes.len() - 1) as u32; + } + node = nodes[node][bit] as usize; + } + } + } + Tree { nodes } + } + + #[inline] + fn decode(&self, br: &mut BitReader) -> Result { + let mut node = 0usize; + loop { + let bit = br.read_bits(1)? as usize; + let next = self.nodes[node][bit]; + if next & LEAF != 0 { + return Ok(next & !LEAF); + } + if next == 0 { + return Err(Error::Bitstream("invalid Huffman codeword")); + } + node = next as usize; + } + } +} + +fn trees() -> &'static [Tree; 12] { + static TREES: OnceLock<[Tree; 12]> = OnceLock::new(); + TREES.get_or_init(|| std::array::from_fn(|i| if i == 0 { Tree::build(&SCALEFACTOR) } else { Tree::build(BOOKS[i - 1].codes) })) +} + +/// Decode one scalefactor difference (`-60..=60`). +#[inline] +pub fn decode_sf(br: &mut BitReader) -> Result { + Ok(trees()[0].decode(br)? as i32 - 60) +} + +/// Write one scalefactor difference (`-60..=60`). +#[inline] +pub fn write_sf(bw: &mut BitWriter, diff: i32) { + let (code, len) = SCALEFACTOR[(diff + 60).clamp(0, 120) as usize]; + bw.write_bits(code, len as u32); +} + +/// Bits of one scalefactor difference. +#[inline] +pub fn sf_bits(diff: i32) -> u32 { + SCALEFACTOR[(diff + 60).clamp(0, 120) as usize].1 as u32 +} + +/// Decode one codeword of codebook `cb` (1..=11) into `out[..dim]`, including sign bits and escapes. +pub fn decode_spectral(br: &mut BitReader, cb: u8, out: &mut [i32]) -> Result<()> { + let b = book(cb); + let mut idx = trees()[cb as usize].decode(br)?; + for i in (0..b.dim).rev() { + out[i] = (idx % b.modulo) as i32; + idx /= b.modulo; + } + if b.signed { + for v in out[..b.dim].iter_mut() { + *v -= b.lav as i32; + } + return Ok(()); + } + for v in out[..b.dim].iter_mut() { + if *v != 0 && br.read_bits(1)? == 1 { + *v = -*v; + } + } + if cb == ESC_HCB { + for v in out[..b.dim].iter_mut() { + if v.abs() == 16 { + let mut n = 0u32; + while br.read_bits(1)? == 1 { + n += 1; + if n > 8 { + return Err(Error::Bitstream("escape sequence too long")); + } + } + let mag = (1i32 << (n + 4)) + br.read_bits(n + 4)? as i32; + *v = if *v < 0 { -mag } else { mag }; + } + } + } + Ok(()) +} + +/// Codebook index of the (sign-stripped for unsigned books) values. +#[inline] +fn index_of(b: &Book, vals: &[i32]) -> usize { + let mut idx = 0u32; + for &v in &vals[..b.dim] { + let d = if b.signed { (v + b.lav as i32) as u32 } else { v.unsigned_abs().min(b.lav) }; + idx = idx * b.modulo + d; + } + idx as usize +} + +#[inline] +fn escape_bits(mag: u32) -> u32 { + // mag >= 16: N ones, a zero, N+4 bits where 2^(N+4) <= mag < 2^(N+5). + let n = 31 - mag.leading_zeros() - 4; + 2 * n + 5 +} + +/// Bits needed to code `vals` (a multiple of the codebook dimension) with codebook `cb` (1..=11). +/// The values must fit the codebook (`|v| <= lav`, or `<= 8191` for codebook 11). +pub fn count_bits(cb: u8, vals: &[i32]) -> u32 { + let b = book(cb); + let mut bits = 0u32; + for chunk in vals.chunks_exact(b.dim) { + bits += b.codes[index_of(b, chunk)].1 as u32; + if !b.signed { + for &v in chunk { + if v != 0 { + bits += 1; + if cb == ESC_HCB && v.unsigned_abs() >= 16 { + bits += escape_bits(v.unsigned_abs()); + } + } + } + } + } + bits +} + +/// Write `vals` with codebook `cb` (1..=11). +pub fn write_spectral(bw: &mut BitWriter, cb: u8, vals: &[i32]) { + let b = book(cb); + for chunk in vals.chunks_exact(b.dim) { + let (code, len) = b.codes[index_of(b, chunk)]; + bw.write_bits(code, len as u32); + if b.signed { + continue; + } + for &v in chunk { + if v != 0 { + bw.write_bits((v < 0) as u32, 1); + } + } + if cb == ESC_HCB { + for &v in chunk { + let mag = v.unsigned_abs(); + if mag >= 16 { + let n = 31 - mag.leading_zeros() - 4; + for _ in 0..n { + bw.write_bits(1, 1); + } + bw.write_bits(0, 1); + bw.write_bits(mag - (1 << (n + 4)), n + 4); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn check_complete(codes: &[(u32, u8)]) { + // Kraft sum exactly one and prefix-free. + let max = codes.iter().map(|c| c.1).max().unwrap() as u32; + let mut kraft = 0u64; + for &(_, l) in codes { + kraft += 1u64 << (max - l as u32); + } + assert_eq!(kraft, 1u64 << max); + for (i, &(ci, li)) in codes.iter().enumerate() { + for (j, &(cj, lj)) in codes.iter().enumerate() { + if i != j && li <= lj { + assert_ne!(cj >> (lj - li), ci, "codeword {i} is a prefix of {j}"); + } + } + } + } + + #[test] + fn codebooks_are_complete_prefix_codes() { + check_complete(&SCALEFACTOR); + assert_eq!(SCALEFACTOR[60], (0, 1)); + for b in &BOOKS { + assert_eq!(b.codes.len() as u32, b.modulo.pow(b.dim as u32)); + check_complete(b.codes); + } + } + + #[test] + fn spectral_roundtrip_every_book() { + for cb in 1..=11u8 { + let b = book(cb); + let lim = if cb == 11 { 8191 } else { b.lav as i32 }; + let mut vals = Vec::new(); + let mut seed = 12345u32; + for _ in 0..400 { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + let r = (seed >> 8) as i32; + let v = if cb == 11 && r % 3 == 0 { r % (lim + 1) } else { r % (b.lav.min(16) as i32 + 1) }; + let v = v.min(lim); + vals.push(if b.signed || r & 1 == 0 { v } else { -v }); + } + if b.signed { + for v in vals.iter_mut() { + *v = (*v).clamp(-(b.lav as i32), b.lav as i32); + } + } + let mut bw = BitWriter::new(); + write_spectral(&mut bw, cb, &vals); + assert_eq!(bw.bit_len() as u32, count_bits(cb, &vals)); + let data = bw.finish(); + let mut br = BitReader::new(&data); + let mut out = vec![0i32; vals.len()]; + for c in out.chunks_mut(b.dim) { + decode_spectral(&mut br, cb, c).unwrap(); + } + assert_eq!(out, vals, "codebook {cb}"); + } + } + + #[test] + fn scalefactor_roundtrip() { + let mut bw = BitWriter::new(); + for d in -60..=60 { + write_sf(&mut bw, d); + } + let data = bw.finish(); + let mut br = BitReader::new(&data); + for d in -60..=60 { + assert_eq!(decode_sf(&mut br).unwrap(), d); + } + } +} diff --git a/crates/aac/src/huffman_tables.rs b/crates/aac/src/huffman_tables.rs new file mode 100644 index 0000000000..c65c7df609 --- /dev/null +++ b/crates/aac/src/huffman_tables.rs @@ -0,0 +1,1406 @@ +//! AAC Huffman codebooks (ISO/IEC 14496-3 §4.A.1, spectrum codebooks 1–11 and the scalefactor codebook). +//! +//! Each table is indexed by the codebook index defined in the standard and stores `(codeword, length)`. +//! Generated data: derived by black-box probing of an external decoder used purely as a test oracle +//! (every table is a complete prefix code — Kraft sum exactly 1 — covering every index once; see +//! `tests::codebooks_are_complete_prefix_codes`). + +#![allow(clippy::unreadable_literal)] + +pub(crate) static SCALEFACTOR: [(u32, u8); 121] = [ + (0x3ffe8, 18), + (0x3ffe6, 18), + (0x3ffe7, 18), + (0x3ffe5, 18), + (0x7fff5, 19), + (0x7fff1, 19), + (0x7ffed, 19), + (0x7fff6, 19), + (0x7ffee, 19), + (0x7ffef, 19), + (0x7fff0, 19), + (0x7fffc, 19), + (0x7fffd, 19), + (0x7ffff, 19), + (0x7fffe, 19), + (0x7fff7, 19), + (0x7fff8, 19), + (0x7fffb, 19), + (0x7fff9, 19), + (0x3ffe4, 18), + (0x7fffa, 19), + (0x3ffe3, 18), + (0x1ffef, 17), + (0x1fff0, 17), + (0xfff5, 16), + (0x1ffee, 17), + (0xfff2, 16), + (0xfff3, 16), + (0xfff4, 16), + (0xfff1, 16), + (0x7ff6, 15), + (0x7ff7, 15), + (0x3ff9, 14), + (0x3ff5, 14), + (0x3ff7, 14), + (0x3ff3, 14), + (0x3ff6, 14), + (0x3ff2, 14), + (0x1ff7, 13), + (0x1ff5, 13), + (0xff9, 12), + (0xff7, 12), + (0xff6, 12), + (0x7f9, 11), + (0xff4, 12), + (0x7f8, 11), + (0x3f9, 10), + (0x3f7, 10), + (0x3f5, 10), + (0x1f8, 9), + (0x1f7, 9), + (0xfa, 8), + (0xf8, 8), + (0xf6, 8), + (0x79, 7), + (0x3a, 6), + (0x38, 6), + (0x1a, 5), + (0xb, 4), + (0x4, 3), + (0x0, 1), + (0xa, 4), + (0xc, 4), + (0x1b, 5), + (0x39, 6), + (0x3b, 6), + (0x78, 7), + (0x7a, 7), + (0xf7, 8), + (0xf9, 8), + (0x1f6, 9), + (0x1f9, 9), + (0x3f4, 10), + (0x3f6, 10), + (0x3f8, 10), + (0x7f5, 11), + (0x7f4, 11), + (0x7f6, 11), + (0x7f7, 11), + (0xff5, 12), + (0xff8, 12), + (0x1ff4, 13), + (0x1ff6, 13), + (0x1ff8, 13), + (0x3ff8, 14), + (0x3ff4, 14), + (0xfff0, 16), + (0x7ff4, 15), + (0xfff6, 16), + (0x7ff5, 15), + (0x3ffe2, 18), + (0x7ffd9, 19), + (0x7ffda, 19), + (0x7ffdb, 19), + (0x7ffdc, 19), + (0x7ffdd, 19), + (0x7ffde, 19), + (0x7ffd8, 19), + (0x7ffd2, 19), + (0x7ffd3, 19), + (0x7ffd4, 19), + (0x7ffd5, 19), + (0x7ffd6, 19), + (0x7fff2, 19), + (0x7ffdf, 19), + (0x7ffe7, 19), + (0x7ffe8, 19), + (0x7ffe9, 19), + (0x7ffea, 19), + (0x7ffeb, 19), + (0x7ffe6, 19), + (0x7ffe0, 19), + (0x7ffe1, 19), + (0x7ffe2, 19), + (0x7ffe3, 19), + (0x7ffe4, 19), + (0x7ffe5, 19), + (0x7ffd7, 19), + (0x7ffec, 19), + (0x7fff4, 19), + (0x7fff3, 19), +]; + +pub(crate) static SPECTRUM_1: [(u32, u8); 81] = [ + (0x7f8, 11), + (0x1f1, 9), + (0x7fd, 11), + (0x3f5, 10), + (0x68, 7), + (0x3f0, 10), + (0x7f7, 11), + (0x1ec, 9), + (0x7f5, 11), + (0x3f1, 10), + (0x72, 7), + (0x3f4, 10), + (0x74, 7), + (0x11, 5), + (0x76, 7), + (0x1eb, 9), + (0x6c, 7), + (0x3f6, 10), + (0x7fc, 11), + (0x1e1, 9), + (0x7f1, 11), + (0x1f0, 9), + (0x61, 7), + (0x1f6, 9), + (0x7f2, 11), + (0x1ea, 9), + (0x7fb, 11), + (0x1f2, 9), + (0x69, 7), + (0x1ed, 9), + (0x77, 7), + (0x17, 5), + (0x6f, 7), + (0x1e6, 9), + (0x64, 7), + (0x1e5, 9), + (0x67, 7), + (0x15, 5), + (0x62, 7), + (0x12, 5), + (0x0, 1), + (0x14, 5), + (0x65, 7), + (0x16, 5), + (0x6d, 7), + (0x1e9, 9), + (0x63, 7), + (0x1e4, 9), + (0x6b, 7), + (0x13, 5), + (0x71, 7), + (0x1e3, 9), + (0x70, 7), + (0x1f3, 9), + (0x7fe, 11), + (0x1e7, 9), + (0x7f3, 11), + (0x1ef, 9), + (0x60, 7), + (0x1ee, 9), + (0x7f0, 11), + (0x1e2, 9), + (0x7fa, 11), + (0x3f3, 10), + (0x6a, 7), + (0x1e8, 9), + (0x75, 7), + (0x10, 5), + (0x73, 7), + (0x1f4, 9), + (0x6e, 7), + (0x3f7, 10), + (0x7f6, 11), + (0x1e0, 9), + (0x7f9, 11), + (0x3f2, 10), + (0x66, 7), + (0x1f5, 9), + (0x7ff, 11), + (0x1f7, 9), + (0x7f4, 11), +]; + +pub(crate) static SPECTRUM_2: [(u32, u8); 81] = [ + (0x1f3, 9), + (0x6f, 7), + (0x1fd, 9), + (0xeb, 8), + (0x23, 6), + (0xea, 8), + (0x1f7, 9), + (0xe8, 8), + (0x1fa, 9), + (0xf2, 8), + (0x2d, 6), + (0x70, 7), + (0x20, 6), + (0x6, 5), + (0x2b, 6), + (0x6e, 7), + (0x28, 6), + (0xe9, 8), + (0x1f9, 9), + (0x66, 7), + (0xf8, 8), + (0xe7, 8), + (0x1b, 6), + (0xf1, 8), + (0x1f4, 9), + (0x6b, 7), + (0x1f5, 9), + (0xec, 8), + (0x2a, 6), + (0x6c, 7), + (0x2c, 6), + (0xa, 5), + (0x27, 6), + (0x67, 7), + (0x1a, 6), + (0xf5, 8), + (0x24, 6), + (0x8, 5), + (0x1f, 6), + (0x9, 5), + (0x0, 3), + (0x7, 5), + (0x1d, 6), + (0xb, 5), + (0x30, 6), + (0xef, 8), + (0x1c, 6), + (0x64, 7), + (0x1e, 6), + (0xc, 5), + (0x29, 6), + (0xf3, 8), + (0x2f, 6), + (0xf0, 8), + (0x1fc, 9), + (0x71, 7), + (0x1f2, 9), + (0xf4, 8), + (0x21, 6), + (0xe6, 8), + (0xf7, 8), + (0x68, 7), + (0x1f8, 9), + (0xee, 8), + (0x22, 6), + (0x65, 7), + (0x31, 6), + (0x2, 4), + (0x26, 6), + (0xed, 8), + (0x25, 6), + (0x6a, 7), + (0x1fb, 9), + (0x72, 7), + (0x1fe, 9), + (0x69, 7), + (0x2e, 6), + (0xf6, 8), + (0x1ff, 9), + (0x6d, 7), + (0x1f6, 9), +]; + +pub(crate) static SPECTRUM_3: [(u32, u8); 81] = [ + (0x0, 1), + (0x9, 4), + (0xef, 8), + (0xb, 4), + (0x19, 5), + (0xf0, 8), + (0x1eb, 9), + (0x1e6, 9), + (0x3f2, 10), + (0xa, 4), + (0x35, 6), + (0x1ef, 9), + (0x34, 6), + (0x37, 6), + (0x1e9, 9), + (0x1ed, 9), + (0x1e7, 9), + (0x3f3, 10), + (0x1ee, 9), + (0x3ed, 10), + (0x1ffa, 13), + (0x1ec, 9), + (0x1f2, 9), + (0x7f9, 11), + (0x7f8, 11), + (0x3f8, 10), + (0xff8, 12), + (0x8, 4), + (0x38, 6), + (0x3f6, 10), + (0x36, 6), + (0x75, 7), + (0x3f1, 10), + (0x3eb, 10), + (0x3ec, 10), + (0xff4, 12), + (0x18, 5), + (0x76, 7), + (0x7f4, 11), + (0x39, 6), + (0x74, 7), + (0x3ef, 10), + (0x1f3, 9), + (0x1f4, 9), + (0x7f6, 11), + (0x1e8, 9), + (0x3ea, 10), + (0x1ffc, 13), + (0xf2, 8), + (0x1f1, 9), + (0xffb, 12), + (0x3f5, 10), + (0x7f3, 11), + (0xffc, 12), + (0xee, 8), + (0x3f7, 10), + (0x7ffe, 15), + (0x1f0, 9), + (0x7f5, 11), + (0x7ffd, 15), + (0x1ffb, 13), + (0x3ffa, 14), + (0xffff, 16), + (0xf1, 8), + (0x3f0, 10), + (0x3ffc, 14), + (0x1ea, 9), + (0x3ee, 10), + (0x3ffb, 14), + (0xff6, 12), + (0xffa, 12), + (0x7ffc, 15), + (0x7f2, 11), + (0xff5, 12), + (0xfffe, 16), + (0x3f4, 10), + (0x7f7, 11), + (0x7ffb, 15), + (0xff7, 12), + (0xff9, 12), + (0x7ffa, 15), +]; + +pub(crate) static SPECTRUM_4: [(u32, u8); 81] = [ + (0x7, 4), + (0x16, 5), + (0xf6, 8), + (0x18, 5), + (0x8, 4), + (0xef, 8), + (0x1ef, 9), + (0xf3, 8), + (0x7f8, 11), + (0x19, 5), + (0x17, 5), + (0xed, 8), + (0x15, 5), + (0x1, 4), + (0xe2, 8), + (0xf0, 8), + (0x70, 7), + (0x3f0, 10), + (0x1ee, 9), + (0xf1, 8), + (0x7fa, 11), + (0xee, 8), + (0xe4, 8), + (0x3f2, 10), + (0x7f6, 11), + (0x3ef, 10), + (0x7fd, 11), + (0x5, 4), + (0x14, 5), + (0xf2, 8), + (0x9, 4), + (0x4, 4), + (0xe5, 8), + (0xf4, 8), + (0xe8, 8), + (0x3f4, 10), + (0x6, 4), + (0x2, 4), + (0xe7, 8), + (0x3, 4), + (0x0, 4), + (0x6b, 7), + (0xe3, 8), + (0x69, 7), + (0x1f3, 9), + (0xeb, 8), + (0xe6, 8), + (0x3f6, 10), + (0x6e, 7), + (0x6a, 7), + (0x1f4, 9), + (0x3ec, 10), + (0x1f0, 9), + (0x3f9, 10), + (0xf5, 8), + (0xec, 8), + (0x7fb, 11), + (0xea, 8), + (0x6f, 7), + (0x3f7, 10), + (0x7f9, 11), + (0x3f3, 10), + (0xfff, 12), + (0xe9, 8), + (0x6d, 7), + (0x3f8, 10), + (0x6c, 7), + (0x68, 7), + (0x1f5, 9), + (0x3ee, 10), + (0x1f2, 9), + (0x7f4, 11), + (0x7f7, 11), + (0x3f1, 10), + (0xffe, 12), + (0x3ed, 10), + (0x1f1, 9), + (0x7f5, 11), + (0x7fe, 11), + (0x3f5, 10), + (0x7fc, 11), +]; + +pub(crate) static SPECTRUM_5: [(u32, u8); 81] = [ + (0x1fff, 13), + (0xff7, 12), + (0x7f4, 11), + (0x7e8, 11), + (0x3f1, 10), + (0x7ee, 11), + (0x7f9, 11), + (0xff8, 12), + (0x1ffd, 13), + (0xffd, 12), + (0x7f1, 11), + (0x3e8, 10), + (0x1e8, 9), + (0xf0, 8), + (0x1ec, 9), + (0x3ee, 10), + (0x7f2, 11), + (0xffa, 12), + (0xff4, 12), + (0x3ef, 10), + (0x1f2, 9), + (0xe8, 8), + (0x70, 7), + (0xec, 8), + (0x1f0, 9), + (0x3ea, 10), + (0x7f3, 11), + (0x7eb, 11), + (0x1eb, 9), + (0xea, 8), + (0x1a, 5), + (0x8, 4), + (0x19, 5), + (0xee, 8), + (0x1ef, 9), + (0x7ed, 11), + (0x3f0, 10), + (0xf2, 8), + (0x73, 7), + (0xb, 4), + (0x0, 1), + (0xa, 4), + (0x71, 7), + (0xf3, 8), + (0x7e9, 11), + (0x7ef, 11), + (0x1ee, 9), + (0xef, 8), + (0x18, 5), + (0x9, 4), + (0x1b, 5), + (0xeb, 8), + (0x1e9, 9), + (0x7ec, 11), + (0x7f6, 11), + (0x3eb, 10), + (0x1f3, 9), + (0xed, 8), + (0x72, 7), + (0xe9, 8), + (0x1f1, 9), + (0x3ed, 10), + (0x7f7, 11), + (0xff6, 12), + (0x7f0, 11), + (0x3e9, 10), + (0x1ed, 9), + (0xf1, 8), + (0x1ea, 9), + (0x3ec, 10), + (0x7f8, 11), + (0xff9, 12), + (0x1ffc, 13), + (0xffc, 12), + (0xff5, 12), + (0x7ea, 11), + (0x3f3, 10), + (0x3f2, 10), + (0x7f5, 11), + (0xffb, 12), + (0x1ffe, 13), +]; + +pub(crate) static SPECTRUM_6: [(u32, u8); 81] = [ + (0x7fe, 11), + (0x3fd, 10), + (0x1f1, 9), + (0x1eb, 9), + (0x1f4, 9), + (0x1ea, 9), + (0x1f0, 9), + (0x3fc, 10), + (0x7fd, 11), + (0x3f6, 10), + (0x1e5, 9), + (0xea, 8), + (0x6c, 7), + (0x71, 7), + (0x68, 7), + (0xf0, 8), + (0x1e6, 9), + (0x3f7, 10), + (0x1f3, 9), + (0xef, 8), + (0x32, 6), + (0x27, 6), + (0x28, 6), + (0x26, 6), + (0x31, 6), + (0xeb, 8), + (0x1f7, 9), + (0x1e8, 9), + (0x6f, 7), + (0x2e, 6), + (0x8, 4), + (0x4, 4), + (0x6, 4), + (0x29, 6), + (0x6b, 7), + (0x1ee, 9), + (0x1ef, 9), + (0x72, 7), + (0x2d, 6), + (0x2, 4), + (0x0, 4), + (0x3, 4), + (0x2f, 6), + (0x73, 7), + (0x1fa, 9), + (0x1e7, 9), + (0x6e, 7), + (0x2b, 6), + (0x7, 4), + (0x1, 4), + (0x5, 4), + (0x2c, 6), + (0x6d, 7), + (0x1ec, 9), + (0x1f9, 9), + (0xee, 8), + (0x30, 6), + (0x24, 6), + (0x2a, 6), + (0x25, 6), + (0x33, 6), + (0xec, 8), + (0x1f2, 9), + (0x3f8, 10), + (0x1e4, 9), + (0xed, 8), + (0x6a, 7), + (0x70, 7), + (0x69, 7), + (0x74, 7), + (0xf1, 8), + (0x3fa, 10), + (0x7ff, 11), + (0x3f9, 10), + (0x1f6, 9), + (0x1ed, 9), + (0x1f8, 9), + (0x1e9, 9), + (0x1f5, 9), + (0x3fb, 10), + (0x7fc, 11), +]; + +pub(crate) static SPECTRUM_7: [(u32, u8); 64] = [ + (0x0, 1), + (0x5, 3), + (0x37, 6), + (0x74, 7), + (0xf2, 8), + (0x1eb, 9), + (0x3ed, 10), + (0x7f7, 11), + (0x4, 3), + (0xc, 4), + (0x35, 6), + (0x71, 7), + (0xec, 8), + (0xee, 8), + (0x1ee, 9), + (0x1f5, 9), + (0x36, 6), + (0x34, 6), + (0x72, 7), + (0xea, 8), + (0xf1, 8), + (0x1e9, 9), + (0x1f3, 9), + (0x3f5, 10), + (0x73, 7), + (0x70, 7), + (0xeb, 8), + (0xf0, 8), + (0x1f1, 9), + (0x1f0, 9), + (0x3ec, 10), + (0x3fa, 10), + (0xf3, 8), + (0xed, 8), + (0x1e8, 9), + (0x1ef, 9), + (0x3ef, 10), + (0x3f1, 10), + (0x3f9, 10), + (0x7fb, 11), + (0x1ed, 9), + (0xef, 8), + (0x1ea, 9), + (0x1f2, 9), + (0x3f3, 10), + (0x3f8, 10), + (0x7f9, 11), + (0x7fc, 11), + (0x3ee, 10), + (0x1ec, 9), + (0x1f4, 9), + (0x3f4, 10), + (0x3f7, 10), + (0x7f8, 11), + (0xffd, 12), + (0xffe, 12), + (0x7f6, 11), + (0x3f0, 10), + (0x3f2, 10), + (0x3f6, 10), + (0x7fa, 11), + (0x7fd, 11), + (0xffc, 12), + (0xfff, 12), +]; + +pub(crate) static SPECTRUM_8: [(u32, u8); 64] = [ + (0xe, 5), + (0x5, 4), + (0x10, 5), + (0x30, 6), + (0x6f, 7), + (0xf1, 8), + (0x1fa, 9), + (0x3fe, 10), + (0x3, 4), + (0x0, 3), + (0x4, 4), + (0x12, 5), + (0x2c, 6), + (0x6a, 7), + (0x75, 7), + (0xf8, 8), + (0xf, 5), + (0x2, 4), + (0x6, 4), + (0x14, 5), + (0x2e, 6), + (0x69, 7), + (0x72, 7), + (0xf5, 8), + (0x2f, 6), + (0x11, 5), + (0x13, 5), + (0x2a, 6), + (0x32, 6), + (0x6c, 7), + (0xec, 8), + (0xfa, 8), + (0x71, 7), + (0x2b, 6), + (0x2d, 6), + (0x31, 6), + (0x6d, 7), + (0x70, 7), + (0xf2, 8), + (0x1f9, 9), + (0xef, 8), + (0x68, 7), + (0x33, 6), + (0x6b, 7), + (0x6e, 7), + (0xee, 8), + (0xf9, 8), + (0x3fc, 10), + (0x1f8, 9), + (0x74, 7), + (0x73, 7), + (0xed, 8), + (0xf0, 8), + (0xf6, 8), + (0x1f6, 9), + (0x1fd, 9), + (0x3fd, 10), + (0xf3, 8), + (0xf4, 8), + (0xf7, 8), + (0x1f7, 9), + (0x1fb, 9), + (0x1fc, 9), + (0x3ff, 10), +]; + +pub(crate) static SPECTRUM_9: [(u32, u8); 169] = [ + (0x0, 1), + (0x5, 3), + (0x37, 6), + (0xe7, 8), + (0x1de, 9), + (0x3ce, 10), + (0x3d9, 10), + (0x7c8, 11), + (0x7cd, 11), + (0xfc8, 12), + (0xfdd, 12), + (0x1fe4, 13), + (0x1fec, 13), + (0x4, 3), + (0xc, 4), + (0x35, 6), + (0x72, 7), + (0xea, 8), + (0xed, 8), + (0x1e2, 9), + (0x3d1, 10), + (0x3d3, 10), + (0x3e0, 10), + (0x7d8, 11), + (0xfcf, 12), + (0xfd5, 12), + (0x36, 6), + (0x34, 6), + (0x71, 7), + (0xe8, 8), + (0xec, 8), + (0x1e1, 9), + (0x3cf, 10), + (0x3dd, 10), + (0x3db, 10), + (0x7d0, 11), + (0xfc7, 12), + (0xfd4, 12), + (0xfe4, 12), + (0xe6, 8), + (0x70, 7), + (0xe9, 8), + (0x1dd, 9), + (0x1e3, 9), + (0x3d2, 10), + (0x3dc, 10), + (0x7cc, 11), + (0x7ca, 11), + (0x7de, 11), + (0xfd8, 12), + (0xfea, 12), + (0x1fdb, 13), + (0x1df, 9), + (0xeb, 8), + (0x1dc, 9), + (0x1e6, 9), + (0x3d5, 10), + (0x3de, 10), + (0x7cb, 11), + (0x7dd, 11), + (0x7dc, 11), + (0xfcd, 12), + (0xfe2, 12), + (0xfe7, 12), + (0x1fe1, 13), + (0x3d0, 10), + (0x1e0, 9), + (0x1e4, 9), + (0x3d6, 10), + (0x7c5, 11), + (0x7d1, 11), + (0x7db, 11), + (0xfd2, 12), + (0x7e0, 11), + (0xfd9, 12), + (0xfeb, 12), + (0x1fe3, 13), + (0x1fe9, 13), + (0x7c4, 11), + (0x1e5, 9), + (0x3d7, 10), + (0x7c6, 11), + (0x7cf, 11), + (0x7da, 11), + (0xfcb, 12), + (0xfda, 12), + (0xfe3, 12), + (0xfe9, 12), + (0x1fe6, 13), + (0x1ff3, 13), + (0x1ff7, 13), + (0x7d3, 11), + (0x3d8, 10), + (0x3e1, 10), + (0x7d4, 11), + (0x7d9, 11), + (0xfd3, 12), + (0xfde, 12), + (0x1fdd, 13), + (0x1fd9, 13), + (0x1fe2, 13), + (0x1fea, 13), + (0x1ff1, 13), + (0x1ff6, 13), + (0x7d2, 11), + (0x3d4, 10), + (0x3da, 10), + (0x7c7, 11), + (0x7d7, 11), + (0x7e2, 11), + (0xfce, 12), + (0xfdb, 12), + (0x1fd8, 13), + (0x1fee, 13), + (0x3ff0, 14), + (0x1ff4, 13), + (0x3ff2, 14), + (0x7e1, 11), + (0x3df, 10), + (0x7c9, 11), + (0x7d6, 11), + (0xfca, 12), + (0xfd0, 12), + (0xfe5, 12), + (0xfe6, 12), + (0x1feb, 13), + (0x1fef, 13), + (0x3ff3, 14), + (0x3ff4, 14), + (0x3ff5, 14), + (0xfe0, 12), + (0x7ce, 11), + (0x7d5, 11), + (0xfc6, 12), + (0xfd1, 12), + (0xfe1, 12), + (0x1fe0, 13), + (0x1fe8, 13), + (0x1ff0, 13), + (0x3ff1, 14), + (0x3ff8, 14), + (0x3ff6, 14), + (0x7ffc, 15), + (0xfe8, 12), + (0x7df, 11), + (0xfc9, 12), + (0xfd7, 12), + (0xfdc, 12), + (0x1fdc, 13), + (0x1fdf, 13), + (0x1fed, 13), + (0x1ff5, 13), + (0x3ff9, 14), + (0x3ffb, 14), + (0x7ffd, 15), + (0x7ffe, 15), + (0x1fe7, 13), + (0xfcc, 12), + (0xfd6, 12), + (0xfdf, 12), + (0x1fde, 13), + (0x1fda, 13), + (0x1fe5, 13), + (0x1ff2, 13), + (0x3ffa, 14), + (0x3ff7, 14), + (0x3ffc, 14), + (0x3ffd, 14), + (0x7fff, 15), +]; + +pub(crate) static SPECTRUM_10: [(u32, u8); 169] = [ + (0x22, 6), + (0x8, 5), + (0x1d, 6), + (0x26, 6), + (0x5f, 7), + (0xd3, 8), + (0x1cf, 9), + (0x3d0, 10), + (0x3d7, 10), + (0x3ed, 10), + (0x7f0, 11), + (0x7f6, 11), + (0xffd, 12), + (0x7, 5), + (0x0, 4), + (0x1, 4), + (0x9, 5), + (0x20, 6), + (0x54, 7), + (0x60, 7), + (0xd5, 8), + (0xdc, 8), + (0x1d4, 9), + (0x3cd, 10), + (0x3de, 10), + (0x7e7, 11), + (0x1c, 6), + (0x2, 4), + (0x6, 5), + (0xc, 5), + (0x1e, 6), + (0x28, 6), + (0x5b, 7), + (0xcd, 8), + (0xd9, 8), + (0x1ce, 9), + (0x1dc, 9), + (0x3d9, 10), + (0x3f1, 10), + (0x25, 6), + (0xb, 5), + (0xa, 5), + (0xd, 5), + (0x24, 6), + (0x57, 7), + (0x61, 7), + (0xcc, 8), + (0xdd, 8), + (0x1cc, 9), + (0x1de, 9), + (0x3d3, 10), + (0x3e7, 10), + (0x5d, 7), + (0x21, 6), + (0x1f, 6), + (0x23, 6), + (0x27, 6), + (0x59, 7), + (0x64, 7), + (0xd8, 8), + (0xdf, 8), + (0x1d2, 9), + (0x1e2, 9), + (0x3dd, 10), + (0x3ee, 10), + (0xd1, 8), + (0x55, 7), + (0x29, 6), + (0x56, 7), + (0x58, 7), + (0x62, 7), + (0xce, 8), + (0xe0, 8), + (0xe2, 8), + (0x1da, 9), + (0x3d4, 10), + (0x3e3, 10), + (0x7eb, 11), + (0x1c9, 9), + (0x5e, 7), + (0x5a, 7), + (0x5c, 7), + (0x63, 7), + (0xca, 8), + (0xda, 8), + (0x1c7, 9), + (0x1ca, 9), + (0x1e0, 9), + (0x3db, 10), + (0x3e8, 10), + (0x7ec, 11), + (0x1e3, 9), + (0xd2, 8), + (0xcb, 8), + (0xd0, 8), + (0xd7, 8), + (0xdb, 8), + (0x1c6, 9), + (0x1d5, 9), + (0x1d8, 9), + (0x3ca, 10), + (0x3da, 10), + (0x7ea, 11), + (0x7f1, 11), + (0x1e1, 9), + (0xd4, 8), + (0xcf, 8), + (0xd6, 8), + (0xde, 8), + (0xe1, 8), + (0x1d0, 9), + (0x1d6, 9), + (0x3d1, 10), + (0x3d5, 10), + (0x3f2, 10), + (0x7ee, 11), + (0x7fb, 11), + (0x3e9, 10), + (0x1cd, 9), + (0x1c8, 9), + (0x1cb, 9), + (0x1d1, 9), + (0x1d7, 9), + (0x1df, 9), + (0x3cf, 10), + (0x3e0, 10), + (0x3ef, 10), + (0x7e6, 11), + (0x7f8, 11), + (0xffa, 12), + (0x3eb, 10), + (0x1dd, 9), + (0x1d3, 9), + (0x1d9, 9), + (0x1db, 9), + (0x3d2, 10), + (0x3cc, 10), + (0x3dc, 10), + (0x3ea, 10), + (0x7ed, 11), + (0x7f3, 11), + (0x7f9, 11), + (0xff9, 12), + (0x7f2, 11), + (0x3ce, 10), + (0x1e4, 9), + (0x3cb, 10), + (0x3d8, 10), + (0x3d6, 10), + (0x3e2, 10), + (0x3e5, 10), + (0x7e8, 11), + (0x7f4, 11), + (0x7f5, 11), + (0x7f7, 11), + (0xffb, 12), + (0x7fa, 11), + (0x3ec, 10), + (0x3df, 10), + (0x3e1, 10), + (0x3e4, 10), + (0x3e6, 10), + (0x3f0, 10), + (0x7e9, 11), + (0x7ef, 11), + (0xff8, 12), + (0xffe, 12), + (0xffc, 12), + (0xfff, 12), +]; + +pub(crate) static SPECTRUM_11: [(u32, u8); 289] = [ + (0x0, 4), + (0x6, 5), + (0x19, 6), + (0x3d, 7), + (0x9c, 8), + (0xc6, 8), + (0x1a7, 9), + (0x390, 10), + (0x3c2, 10), + (0x3df, 10), + (0x7e6, 11), + (0x7f3, 11), + (0xffb, 12), + (0x7ec, 11), + (0xffa, 12), + (0xffe, 12), + (0x38e, 10), + (0x5, 5), + (0x1, 4), + (0x8, 5), + (0x14, 6), + (0x37, 7), + (0x42, 7), + (0x92, 8), + (0xaf, 8), + (0x191, 9), + (0x1a5, 9), + (0x1b5, 9), + (0x39e, 10), + (0x3c0, 10), + (0x3a2, 10), + (0x3cd, 10), + (0x7d6, 11), + (0xae, 8), + (0x17, 6), + (0x7, 5), + (0x9, 5), + (0x18, 6), + (0x39, 7), + (0x40, 7), + (0x8e, 8), + (0xa3, 8), + (0xb8, 8), + (0x199, 9), + (0x1ac, 9), + (0x1c1, 9), + (0x3b1, 10), + (0x396, 10), + (0x3be, 10), + (0x3ca, 10), + (0x9d, 8), + (0x3c, 7), + (0x15, 6), + (0x16, 6), + (0x1a, 6), + (0x3b, 7), + (0x44, 7), + (0x91, 8), + (0xa5, 8), + (0xbe, 8), + (0x196, 9), + (0x1ae, 9), + (0x1b9, 9), + (0x3a1, 10), + (0x391, 10), + (0x3a5, 10), + (0x3d5, 10), + (0x94, 8), + (0x9a, 8), + (0x36, 7), + (0x38, 7), + (0x3a, 7), + (0x41, 7), + (0x8c, 8), + (0x9b, 8), + (0xb0, 8), + (0xc3, 8), + (0x19e, 9), + (0x1ab, 9), + (0x1bc, 9), + (0x39f, 10), + (0x38f, 10), + (0x3a9, 10), + (0x3cf, 10), + (0x93, 8), + (0xbf, 8), + (0x3e, 7), + (0x3f, 7), + (0x43, 7), + (0x45, 7), + (0x9e, 8), + (0xa7, 8), + (0xb9, 8), + (0x194, 9), + (0x1a2, 9), + (0x1ba, 9), + (0x1c3, 9), + (0x3a6, 10), + (0x3a7, 10), + (0x3bb, 10), + (0x3d4, 10), + (0x9f, 8), + (0x1a0, 9), + (0x8f, 8), + (0x8d, 8), + (0x90, 8), + (0x98, 8), + (0xa6, 8), + (0xb6, 8), + (0xc4, 8), + (0x19f, 9), + (0x1af, 9), + (0x1bf, 9), + (0x399, 10), + (0x3bf, 10), + (0x3b4, 10), + (0x3c9, 10), + (0x3e7, 10), + (0xa8, 8), + (0x1b6, 9), + (0xab, 8), + (0xa4, 8), + (0xaa, 8), + (0xb2, 8), + (0xc2, 8), + (0xc5, 8), + (0x198, 9), + (0x1a4, 9), + (0x1b8, 9), + (0x38c, 10), + (0x3a4, 10), + (0x3c4, 10), + (0x3c6, 10), + (0x3dd, 10), + (0x3e8, 10), + (0xad, 8), + (0x3af, 10), + (0x192, 9), + (0xbd, 8), + (0xbc, 8), + (0x18e, 9), + (0x197, 9), + (0x19a, 9), + (0x1a3, 9), + (0x1b1, 9), + (0x38d, 10), + (0x398, 10), + (0x3b7, 10), + (0x3d3, 10), + (0x3d1, 10), + (0x3db, 10), + (0x7dd, 11), + (0xb4, 8), + (0x3de, 10), + (0x1a9, 9), + (0x19b, 9), + (0x19c, 9), + (0x1a1, 9), + (0x1aa, 9), + (0x1ad, 9), + (0x1b3, 9), + (0x38b, 10), + (0x3b2, 10), + (0x3b8, 10), + (0x3ce, 10), + (0x3e1, 10), + (0x3e0, 10), + (0x7d2, 11), + (0x7e5, 11), + (0xb7, 8), + (0x7e3, 11), + (0x1bb, 9), + (0x1a8, 9), + (0x1a6, 9), + (0x1b0, 9), + (0x1b2, 9), + (0x1b7, 9), + (0x39b, 10), + (0x39a, 10), + (0x3ba, 10), + (0x3b5, 10), + (0x3d6, 10), + (0x7d7, 11), + (0x3e4, 10), + (0x7d8, 11), + (0x7ea, 11), + (0xba, 8), + (0x7e8, 11), + (0x3a0, 10), + (0x1bd, 9), + (0x1b4, 9), + (0x38a, 10), + (0x1c4, 9), + (0x392, 10), + (0x3aa, 10), + (0x3b0, 10), + (0x3bc, 10), + (0x3d7, 10), + (0x7d4, 11), + (0x7dc, 11), + (0x7db, 11), + (0x7d5, 11), + (0x7f0, 11), + (0xc1, 8), + (0x7fb, 11), + (0x3c8, 10), + (0x3a3, 10), + (0x395, 10), + (0x39d, 10), + (0x3ac, 10), + (0x3ae, 10), + (0x3c5, 10), + (0x3d8, 10), + (0x3e2, 10), + (0x3e6, 10), + (0x7e4, 11), + (0x7e7, 11), + (0x7e0, 11), + (0x7e9, 11), + (0x7f7, 11), + (0x190, 9), + (0x7f2, 11), + (0x393, 10), + (0x1be, 9), + (0x1c0, 9), + (0x394, 10), + (0x397, 10), + (0x3ad, 10), + (0x3c3, 10), + (0x3c1, 10), + (0x3d2, 10), + (0x7da, 11), + (0x7d9, 11), + (0x7df, 11), + (0x7eb, 11), + (0x7f4, 11), + (0x7fa, 11), + (0x195, 9), + (0x7f8, 11), + (0x3bd, 10), + (0x39c, 10), + (0x3ab, 10), + (0x3a8, 10), + (0x3b3, 10), + (0x3b9, 10), + (0x3d0, 10), + (0x3e3, 10), + (0x3e5, 10), + (0x7e2, 11), + (0x7de, 11), + (0x7ed, 11), + (0x7f1, 11), + (0x7f9, 11), + (0x7fc, 11), + (0x193, 9), + (0xffd, 12), + (0x3dc, 10), + (0x3b6, 10), + (0x3c7, 10), + (0x3cc, 10), + (0x3cb, 10), + (0x3d9, 10), + (0x3da, 10), + (0x7d3, 11), + (0x7e1, 11), + (0x7ee, 11), + (0x7ef, 11), + (0x7f5, 11), + (0x7f6, 11), + (0xffc, 12), + (0xfff, 12), + (0x19d, 9), + (0x1c2, 9), + (0xb5, 8), + (0xa1, 8), + (0x96, 8), + (0x97, 8), + (0x95, 8), + (0x99, 8), + (0xa0, 8), + (0xa2, 8), + (0xac, 8), + (0xa9, 8), + (0xb1, 8), + (0xb3, 8), + (0xbb, 8), + (0xc0, 8), + (0x18f, 9), + (0x4, 5), +]; diff --git a/crates/aac/src/ics.rs b/crates/aac/src/ics.rs new file mode 100644 index 0000000000..d70c31e9f9 --- /dev/null +++ b/crates/aac/src/ics.rs @@ -0,0 +1,151 @@ +//! `ics_info` and helpers shared by the encoder and decoder (ISO/IEC 14496-3 §4.4.2.1, §4.5.2.3). + +use deckcraft_bitstream::{BitReader, BitWriter}; + +use crate::mdct::WindowShape; +use crate::tables::{swb_offsets_long, swb_offsets_short}; +use crate::{Error, Result}; + +/// `window_sequence`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum WindowSequence { + #[default] + OnlyLong = 0, + LongStart = 1, + EightShort = 2, + LongStop = 3, +} + +impl WindowSequence { + pub fn from_bits(v: u32) -> Self { + match v & 3 { + 0 => WindowSequence::OnlyLong, + 1 => WindowSequence::LongStart, + 2 => WindowSequence::EightShort, + _ => WindowSequence::LongStop, + } + } +} + +/// Decoded/encoded `ics_info`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IcsInfo { + pub window_sequence: WindowSequence, + pub window_shape: WindowShape, + pub max_sfb: usize, + pub num_groups: usize, + /// Windows per group (only the first `num_groups` entries are used; long windows: `[1]`). + pub group_len: [u8; 8], +} + +impl Default for IcsInfo { + fn default() -> Self { + IcsInfo { + window_sequence: WindowSequence::OnlyLong, + window_shape: WindowShape::Sine, + max_sfb: 0, + num_groups: 1, + group_len: [1, 0, 0, 0, 0, 0, 0, 0], + } + } +} + +impl IcsInfo { + #[inline] + pub fn is_short(&self) -> bool { + self.window_sequence == WindowSequence::EightShort + } + #[inline] + pub fn num_windows(&self) -> usize { + if self.is_short() { 8 } else { 1 } + } + /// Band offsets of one window. + pub fn swb_offsets(&self, sf_index: u8) -> &'static [u16] { + if self.is_short() { swb_offsets_short(sf_index) } else { swb_offsets_long(sf_index) } + } + /// Number of bands for this window length (`num_swb`). + pub fn num_swb(&self, sf_index: u8) -> usize { + self.swb_offsets(sf_index).len() - 1 + } + /// First window of group `g`. + pub fn group_start(&self, g: usize) -> usize { + self.group_len[..g].iter().map(|&l| l as usize).sum() + } + + pub fn parse(br: &mut BitReader, sf_index: u8) -> Result { + if br.read_bits(1)? != 0 { + // ics_reserved_bit; tolerated by most decoders + } + let window_sequence = WindowSequence::from_bits(br.read_bits(2)?); + let window_shape = WindowShape::from_bit(br.read_bits(1)? == 1); + let mut info = IcsInfo { window_sequence, window_shape, ..Default::default() }; + if info.is_short() { + info.max_sfb = br.read_bits(4)? as usize; + let grouping = br.read_bits(7)?; + info.num_groups = 1; + info.group_len = [1, 0, 0, 0, 0, 0, 0, 0]; + for w in 1..8 { + if grouping & (1 << (6 - (w - 1))) != 0 { + info.group_len[info.num_groups - 1] += 1; + } else { + info.num_groups += 1; + info.group_len[info.num_groups - 1] = 1; + } + } + } else { + info.max_sfb = br.read_bits(6)? as usize; + if br.read_bits(1)? == 1 { + return Err(Error::Unsupported("predictor data (AAC Main / LTP)")); + } + } + if info.max_sfb > info.num_swb(sf_index) { + return Err(Error::Bitstream("max_sfb exceeds the number of scalefactor bands")); + } + Ok(info) + } + + pub fn write(&self, bw: &mut BitWriter) { + bw.write_bits(0, 1); + bw.write_bits(self.window_sequence as u32, 2); + bw.write_bits(self.window_shape.bit(), 1); + if self.is_short() { + bw.write_bits(self.max_sfb as u32, 4); + let mut grouping = 0u32; + let mut w = 0; + for g in 0..self.num_groups { + for i in 0..self.group_len[g] as usize { + if w > 0 && i > 0 { + grouping |= 1 << (6 - (w - 1)); + } + w += 1; + } + } + bw.write_bits(grouping, 7); + } else { + bw.write_bits(self.max_sfb as u32, 6); + bw.write_bits(0, 1); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn grouping_roundtrip() { + let info = IcsInfo { + window_sequence: WindowSequence::EightShort, + window_shape: WindowShape::Kbd, + max_sfb: 12, + num_groups: 3, + group_len: [3, 1, 4, 0, 0, 0, 0, 0], + }; + let mut bw = BitWriter::new(); + info.write(&mut bw); + let d = bw.finish(); + let p = IcsInfo::parse(&mut BitReader::new(&d), 4).unwrap(); + assert_eq!(p, info); + assert_eq!(p.group_start(2), 4); + } +} diff --git a/crates/aac/src/lib.rs b/crates/aac/src/lib.rs new file mode 100644 index 0000000000..da9c89af30 --- /dev/null +++ b/crates/aac/src/lib.rs @@ -0,0 +1,79 @@ +//! Ported from FilmCraft (`crates/aac`, commit cee1feb), same authors and licence. +//! +//! Clean-room AAC-LC encoder and decoder (ISO/IEC 14496-3, MPEG-4 Audio, Low Complexity profile). +//! +//! - [`Encoder`]: planar f32 → raw access units (+ [`Encoder::audio_specific_config`] for MP4 `esds`, +//! optional ADTS framing). Psychoacoustic model, window switching, M/S, TNS, bit reservoir, CBR/VBR, +//! 1–8 channels. +//! - [`Decoder`]: `AudioSpecificConfig` + raw access units → planar f32. Supports M/S, intensity +//! stereo, PNS, TNS, pulse data, both window shapes and every window sequence. +//! +//! HE-AAC (SBR/PS), Main/SSR/LTP profiles, coupling channels and 960-sample frames are not supported; +//! for explicitly signalled HE-AAC only the AAC-LC core is decoded (at the core rate). +//! +//! Layer L0: depends only on `deckcraft-bitstream` (+ `thiserror`, optional `rayon`); no `unsafe`; +//! builds for `wasm32-unknown-unknown`. +//! +//! ``` +//! use deckcraft_aac::{Decoder, Encoder, EncoderConfig}; +//! +//! let mut enc = Encoder::new(EncoderConfig::cbr(48_000, 2, 128_000))?; +//! let left: Vec = (0..4800).map(|i| (i as f32 * 0.05).sin() * 0.5).collect(); +//! let right = left.clone(); +//! let mut aus = enc.encode(&[&left, &right]); // 0..n access units +//! aus.extend(enc.flush()); +//! let asc = enc.audio_specific_config(); // for the MP4 `esds` +//! assert_eq!(enc.priming_samples(), 1024); // MP4 edit list media_time +//! +//! let mut dec = Decoder::new(&asc)?; +//! for au in &aus { +//! let pcm = dec.decode(au)?; // planar, 1024 samples per channel +//! assert_eq!(pcm.len(), 2); +//! } +//! # Ok::<(), deckcraft_aac::Error>(()) +//! ``` + +// Index loops over parallel per-band arrays mirror the spec text and stay as in FilmCraft so +// the port can be re-synced. +#![allow(clippy::needless_range_loop)] +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] + +mod config; +mod decoder; +mod encoder; +mod huffman; +mod huffman_tables; +mod ics; +mod mdct; +mod tables; +mod tns; + +pub use config::{AdtsHeader, AudioSpecificConfig, ElementType, ProgramConfig, config_layout, split_adts}; +pub use decoder::Decoder; +pub use encoder::{BitrateMode, Encoder, EncoderConfig, EncoderStats}; +pub use mdct::WindowShape; +pub use tables::{SAMPLE_RATES, sample_rate_index}; + +/// Samples per AAC-LC frame (per channel). +pub const FRAME_LEN: usize = 1024; + +/// Errors from configuration parsing, encoding and decoding. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("invalid configuration: {0}")] + InvalidConfig(&'static str), + #[error("unsupported feature: {0}")] + Unsupported(&'static str), + #[error("invalid bitstream: {0}")] + Bitstream(&'static str), + #[error("unexpected end of access unit")] + Eof, +} + +impl From for Error { + fn from(_: deckcraft_bitstream::BitError) -> Self { + Error::Eof + } +} + +pub type Result = std::result::Result; diff --git a/crates/aac/src/mdct.rs b/crates/aac/src/mdct.rs new file mode 100644 index 0000000000..74d9f2958c --- /dev/null +++ b/crates/aac/src/mdct.rs @@ -0,0 +1,274 @@ +//! MDCT / IMDCT (ISO/IEC 14496-3 §4.6.18) via a DCT-IV computed with an N/4-point complex FFT, +//! plus the sine and Kaiser–Bessel-derived windows. + +use std::f64::consts::PI; +use std::sync::OnceLock; + +/// Radix-2 complex FFT (forward, `e^{-2πi nk/P}`). +struct Fft { + n: usize, + twiddle: Vec<[f32; 2]>, + rev: Vec, +} + +impl Fft { + fn new(n: usize) -> Fft { + assert!(n.is_power_of_two()); + let bits = n.trailing_zeros(); + let rev = (0..n as u32).map(|i| if bits == 0 { 0 } else { i.reverse_bits() >> (32 - bits) }).collect(); + let twiddle = (0..n / 2) + .map(|k| { + let a = -2.0 * PI * k as f64 / n as f64; + [a.cos() as f32, a.sin() as f32] + }) + .collect(); + Fft { n, twiddle, rev } + } + + fn run(&self, re: &mut [f32], im: &mut [f32]) { + let n = self.n; + for i in 0..n { + let j = self.rev[i] as usize; + if j > i { + re.swap(i, j); + im.swap(i, j); + } + } + let mut len = 2; + while len <= n { + let half = len / 2; + let step = n / len; + for start in (0..n).step_by(len) { + for k in 0..half { + let [wr, wi] = self.twiddle[k * step]; + let a = start + k; + let b = a + half; + let xr = re[b] * wr - im[b] * wi; + let xi = re[b] * wi + im[b] * wr; + re[b] = re[a] - xr; + im[b] = im[a] - xi; + re[a] += xr; + im[a] += xi; + } + } + len <<= 1; + } + } +} + +/// MDCT of `2M` inputs to `M` outputs (unscaled sums): +/// `X[k] = Σ x[n] cos(π/M (n + ½ + M/2)(k + ½))`, and the matching unscaled inverse. +pub struct Mdct { + m: usize, + fft: Fft, + pre: Vec<[f32; 2]>, + post: Vec<[f32; 2]>, +} + +impl Mdct { + pub fn new(m: usize) -> Mdct { + let q = m / 2; + let pre = (0..q) + .map(|n| { + let a = -PI * n as f64 / m as f64; + [a.cos() as f32, a.sin() as f32] + }) + .collect(); + let post = (0..q) + .map(|k| { + let a = -PI * (k as f64 + 0.25) / m as f64; + [a.cos() as f32, a.sin() as f32] + }) + .collect(); + Mdct { m, fft: Fft::new(q), pre, post } + } + + /// DCT-IV: `out[k] = Σ u[n] cos(π/M (n+½)(k+½))`. + fn dct4(&self, u: &[f32], out: &mut [f32]) { + let m = self.m; + let q = m / 2; + let mut re = vec![0f32; q]; + let mut im = vec![0f32; q]; + for n in 0..q { + let (a, b) = (u[2 * n], u[m - 1 - 2 * n]); + let [c, s] = self.pre[n]; + re[n] = a * c - b * s; + im[n] = a * s + b * c; + } + self.fft.run(&mut re, &mut im); + for k in 0..q { + let [c, s] = self.post[k]; + let wr = re[k] * c - im[k] * s; + let wi = re[k] * s + im[k] * c; + out[2 * k] = wr; + out[m - 1 - 2 * k] = -wi; + } + } + + /// Forward MDCT of `x` (length 2M, already windowed) into `out` (length M). + pub fn forward(&self, x: &[f32], out: &mut [f32]) { + let m = self.m; + let h = m / 2; + let mut u = vec![0f32; m]; + for n in 0..h { + // quarters a, b, c, d of length M/2 + u[n] = -x[m + h - 1 - n] - x[m + h + n]; + u[h + n] = x[n] - x[m - 1 - n]; + } + self.dct4(&u, out); + } + + /// Inverse MDCT (unscaled): `y[n] = Σ X[k] cos(π/M (n + ½ + M/2)(k + ½))`, `y` has length 2M. + pub fn inverse(&self, spec: &[f32], y: &mut [f32]) { + let m = self.m; + let h = m / 2; + let mut u = vec![0f32; m]; + self.dct4(spec, &mut u); + for n in 0..h { + y[n] = u[h + n]; + y[m - 1 - n] = -u[h + n]; + y[m + h - 1 - n] = -u[n]; + y[m + h + n] = -u[n]; + } + } +} + +/// Cached transforms for 1024 and 128 lines. +pub fn mdct_long() -> &'static Mdct { + static M: OnceLock = OnceLock::new(); + M.get_or_init(|| Mdct::new(1024)) +} +pub fn mdct_short() -> &'static Mdct { + static M: OnceLock = OnceLock::new(); + M.get_or_init(|| Mdct::new(128)) +} + +/// Window shape (`window_shape` bit): 0 = sine, 1 = Kaiser–Bessel derived. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum WindowShape { + #[default] + Sine, + Kbd, +} + +impl WindowShape { + pub fn from_bit(b: bool) -> Self { + if b { WindowShape::Kbd } else { WindowShape::Sine } + } + pub fn bit(self) -> u32 { + (self == WindowShape::Kbd) as u32 + } +} + +fn sine_half(m: usize) -> Vec { + let n = 2 * m; + (0..m).map(|i| (PI / n as f64 * (i as f64 + 0.5)).sin() as f32).collect() +} + +fn bessel_i0(x: f64) -> f64 { + let mut sum = 1.0; + let mut term = 1.0; + let y = x * x / 4.0; + for k in 1..60 { + term *= y / (k as f64 * k as f64); + sum += term; + if term < sum * 1e-17 { + break; + } + } + sum +} + +fn kbd_half(m: usize, alpha: f64) -> Vec { + // w[n] = sqrt(Σ_{j<=n} W(j) / Σ_{j<=M} W(j)), W the Kaiser kernel of length M+1. + let kernel: Vec = (0..=m) + .map(|j| { + let r = (j as f64 - m as f64 / 2.0) / (m as f64 / 2.0); + bessel_i0(PI * alpha * (1.0 - r * r).max(0.0).sqrt()) + }) + .collect(); + let total: f64 = kernel.iter().sum(); + let mut acc = 0.0; + (0..m) + .map(|n| { + acc += kernel[n]; + (acc / total).sqrt() as f32 + }) + .collect() +} + +/// Rising half (length N/2) of the window of the given shape; `long` selects N = 2048, else N = 256. +pub fn window(shape: WindowShape, long: bool) -> &'static [f32] { + static W: OnceLock<[Vec; 4]> = OnceLock::new(); + let w = W.get_or_init(|| [sine_half(1024), kbd_half(1024, 4.0), sine_half(128), kbd_half(128, 6.0)]); + let i = match (long, shape) { + (true, WindowShape::Sine) => 0, + (true, WindowShape::Kbd) => 1, + (false, WindowShape::Sine) => 2, + (false, WindowShape::Kbd) => 3, + }; + &w[i] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn direct(x: &[f32]) -> Vec { + let m = x.len() / 2; + (0..m).map(|k| (0..2 * m).map(|n| x[n] as f64 * (PI / m as f64 * (n as f64 + 0.5 + m as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum()).collect() + } + + #[test] + fn forward_matches_definition() { + for m in [128usize, 1024] { + let x: Vec = (0..2 * m).map(|i| ((i * 7919) % 101) as f32 / 50.0 - 1.0).collect(); + let mut out = vec![0f32; m]; + Mdct::new(m).forward(&x, &mut out); + let d = direct(&x); + for k in 0..m { + assert!((out[k] as f64 - d[k]).abs() < 1e-3 * (m as f64).sqrt(), "m={m} k={k} {} {}", out[k], d[k]); + } + } + } + + #[test] + fn inverse_matches_definition() { + let m = 128; + let spec: Vec = (0..m).map(|i| ((i * 31) % 17) as f32 - 8.0).collect(); + let mut y = vec![0f32; 2 * m]; + Mdct::new(m).inverse(&spec, &mut y); + for n in 0..2 * m { + let d: f64 = (0..m).map(|k| spec[k] as f64 * (PI / m as f64 * (n as f64 + 0.5 + m as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum(); + assert!((y[n] as f64 - d).abs() < 1e-3, "n={n}"); + } + } + + #[test] + fn tdac_perfect_reconstruction() { + let m = 1024; + let t = Mdct::new(m); + for shape in [WindowShape::Sine, WindowShape::Kbd] { + let w = window(shape, true); + let win = |n: usize| if n < m { w[n] } else { w[2 * m - 1 - n] }; + let x: Vec = (0..4 * m).map(|i| ((i as f32) * 0.013).sin() + ((i * 13) % 7) as f32 * 0.05).collect(); + let mut out = vec![0f32; 4 * m]; + for f in 0..3 { + let seg: Vec = (0..2 * m).map(|n| x[f * m + n] * win(n)).collect(); + let mut spec = vec![0f32; m]; + t.forward(&seg, &mut spec); + for v in spec.iter_mut() { + *v *= 2.0; + } + let mut y = vec![0f32; 2 * m]; + t.inverse(&spec, &mut y); + for n in 0..2 * m { + out[f * m + n] += y[n] * win(n) / m as f32; + } + } + for n in m..3 * m { + assert!((out[n] - x[n]).abs() < 1e-4, "{shape:?} n={n} {} {}", out[n], x[n]); + } + } + } +} diff --git a/crates/aac/src/tables.rs b/crates/aac/src/tables.rs new file mode 100644 index 0000000000..7b17e183ca --- /dev/null +++ b/crates/aac/src/tables.rs @@ -0,0 +1,143 @@ +//! Sampling-rate dependent tables from ISO/IEC 14496-3 §4.5.4 (scalefactor bands) and §4.6.9 (TNS). + +/// Sampling frequencies indexed by `samplingFrequencyIndex`. +pub const SAMPLE_RATES: [u32; 13] = [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350]; + +/// Map an arbitrary sample rate to its `samplingFrequencyIndex`, if it is one of the standard rates. +pub fn sample_rate_index(rate: u32) -> Option { + SAMPLE_RATES.iter().position(|&r| r == rate).map(|i| i as u8) +} + +/// The table index used for scalefactor-band tables when a non-standard rate is signalled explicitly +/// (§4.5.4: rates are mapped to the nearest table by frequency ranges). +pub fn table_index_for_rate(rate: u32) -> u8 { + match rate { + 92017.. => 0, + 75132.. => 1, + 55426.. => 2, + 46009.. => 3, + 37566.. => 4, + 27713.. => 5, + 23004.. => 6, + 18783.. => 7, + 13856.. => 8, + 11502.. => 9, + 9391.. => 10, + _ => 11, + } +} + +const SWB_LONG_96: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 64, 72, 80, 88, 96, 108, 120, 132, 144, 156, 172, 188, 212, 240, 276, 320, 384, 448, + 512, 576, 640, 704, 768, 832, 896, 960, 1024, +]; +const SWB_LONG_64: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 64, 72, 80, 88, 100, 112, 124, 140, 156, 172, 192, 216, 240, 268, 304, 344, 384, 424, + 464, 504, 544, 584, 624, 664, 704, 744, 784, 824, 864, 904, 944, 984, 1024, +]; +const SWB_LONG_48: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 48, 56, 64, 72, 80, 88, 96, 108, 120, 132, 144, 160, 176, 196, 216, 240, 264, 292, 320, 352, 384, 416, + 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800, 832, 864, 896, 928, 1024, +]; +const SWB_LONG_32: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 48, 56, 64, 72, 80, 88, 96, 108, 120, 132, 144, 160, 176, 196, 216, 240, 264, 292, 320, 352, 384, 416, + 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800, 832, 864, 896, 928, 960, 992, 1024, +]; +const SWB_LONG_24: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 52, 60, 68, 76, 84, 92, 100, 108, 116, 124, 136, 148, 160, 172, 188, 204, 220, 240, 260, 284, 308, + 336, 364, 396, 432, 468, 508, 552, 600, 652, 704, 768, 832, 896, 960, 1024, +]; +const SWB_LONG_16: &[u16] = &[ + 0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 100, 112, 124, 136, 148, 160, 172, 184, 196, 212, 228, 244, 260, 280, 300, 320, 344, 368, 396, 424, + 456, 492, 532, 572, 616, 664, 716, 772, 832, 896, 960, 1024, +]; +const SWB_LONG_8: &[u16] = &[ + 0, 12, 24, 36, 48, 60, 72, 84, 96, 108, 120, 132, 144, 156, 172, 188, 204, 220, 236, 252, 268, 288, 308, 328, 348, 372, 396, 420, 448, 476, 508, + 544, 580, 620, 664, 712, 764, 820, 880, 944, 1024, +]; + +const SWB_SHORT_96: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 32, 40, 48, 64, 92, 128]; +const SWB_SHORT_48: &[u16] = &[0, 4, 8, 12, 16, 20, 28, 36, 44, 56, 68, 80, 96, 112, 128]; +const SWB_SHORT_24: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 28, 36, 44, 52, 64, 76, 92, 108, 128]; +const SWB_SHORT_16: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 60, 72, 88, 108, 128]; +const SWB_SHORT_8: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 28, 36, 44, 52, 60, 72, 88, 108, 128]; + +/// Scalefactor band offsets (`swb_offset_long_window`, 1024-line frames); `len() == num_swb + 1`. +pub fn swb_offsets_long(sf_index: u8) -> &'static [u16] { + match sf_index { + 0 | 1 => SWB_LONG_96, + 2 => SWB_LONG_64, + 3 | 4 => SWB_LONG_48, + 5 => SWB_LONG_32, + 6 | 7 => SWB_LONG_24, + 8..=10 => SWB_LONG_16, + _ => SWB_LONG_8, + } +} + +/// Scalefactor band offsets for one short window (128 lines). +pub fn swb_offsets_short(sf_index: u8) -> &'static [u16] { + match sf_index { + 0..=2 => SWB_SHORT_96, + 3..=5 => SWB_SHORT_48, + 6 | 7 => SWB_SHORT_24, + 8..=10 => SWB_SHORT_16, + _ => SWB_SHORT_8, + } +} + +/// `TNS_MAX_BANDS` for AAC-LC (long, short). +pub fn tns_max_bands(sf_index: u8, short: bool) -> usize { + const LONG: [u8; 13] = [31, 31, 34, 40, 42, 51, 46, 46, 42, 42, 42, 39, 39]; + const SHORT: [u8; 13] = [9, 9, 10, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14]; + let i = (sf_index as usize).min(12); + if short { SHORT[i] as usize } else { LONG[i] as usize } +} + +/// `TNS_MAX_ORDER` for AAC-LC. +pub fn tns_max_order(short: bool) -> usize { + if short { 7 } else { 12 } +} + +/// `x^(4/3)` for the integer quantised magnitudes 0..=8191. +pub fn pow43(q: u32) -> f32 { + use std::sync::OnceLock; + static TABLE: OnceLock> = OnceLock::new(); + let t = TABLE.get_or_init(|| (0..8192u32).map(|i| (i as f64).powf(4.0 / 3.0) as f32).collect()); + match t.get(q as usize) { + Some(v) => *v, + None => (q as f64).powf(4.0 / 3.0) as f32, + } +} + +/// `2^(0.25 * (sf - 100))`, the dequantisation gain of scalefactor `sf`. +#[inline] +pub fn sf_gain(sf: i32) -> f32 { + (0.25 * (sf as f64 - 100.0)).exp2() as f32 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn band_tables_are_monotonic_and_complete() { + for i in 0..13u8 { + let l = swb_offsets_long(i); + assert_eq!(*l.last().unwrap(), 1024); + assert!(l.windows(2).all(|w| w[0] < w[1] && (w[1] - w[0]) % 4 == 0)); + let s = swb_offsets_short(i); + assert_eq!(*s.last().unwrap(), 128); + assert!(s.windows(2).all(|w| w[0] < w[1] && (w[1] - w[0]) % 4 == 0)); + assert!(tns_max_bands(i, false) < l.len()); + assert!(tns_max_bands(i, true) < s.len()); + } + assert_eq!(swb_offsets_long(3).len() - 1, 49); + assert_eq!(swb_offsets_long(5).len() - 1, 51); + assert_eq!(swb_offsets_long(0).len() - 1, 41); + assert_eq!(swb_offsets_long(2).len() - 1, 47); + assert_eq!(swb_offsets_long(6).len() - 1, 47); + assert_eq!(swb_offsets_long(8).len() - 1, 43); + assert_eq!(swb_offsets_long(11).len() - 1, 40); + } +} diff --git a/crates/aac/src/tns.rs b/crates/aac/src/tns.rs new file mode 100644 index 0000000000..b0c86a0de5 --- /dev/null +++ b/crates/aac/src/tns.rs @@ -0,0 +1,313 @@ +//! Temporal noise shaping (ISO/IEC 14496-3 §4.6.9): syntax, coefficient conversion, and the +//! decoder (all-pole) and encoder (all-zero) filters. + +use std::f64::consts::FRAC_PI_2; + +use deckcraft_bitstream::{BitReader, BitWriter}; + +use crate::ics::IcsInfo; +use crate::tables::{tns_max_bands, tns_max_order}; +use crate::{Error, Result}; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TnsFilter { + /// Length in scalefactor bands. + pub length: u8, + pub order: u8, + /// false = upward, true = downward. + pub direction: bool, + pub coef_compress: bool, + /// Quantised reflection coefficients (signed, `coef_res + 3 - coef_compress` bits each). + pub coef: [i8; 12], +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TnsWindow { + pub n_filt: u8, + /// false = 3-bit, true = 4-bit coefficient resolution. + pub coef_res: bool, + pub filt: [TnsFilter; 4], +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TnsData { + pub windows: [TnsWindow; 8], +} + +impl TnsData { + pub fn parse(br: &mut BitReader, info: &IcsInfo) -> Result { + let short = info.is_short(); + let mut t = TnsData::default(); + for w in 0..info.num_windows() { + let win = &mut t.windows[w]; + win.n_filt = br.read_bits(if short { 1 } else { 2 })? as u8; + if win.n_filt == 0 { + continue; + } + win.coef_res = br.read_bits(1)? == 1; + for f in 0..win.n_filt as usize { + let filt = &mut win.filt[f]; + filt.length = br.read_bits(if short { 4 } else { 6 })? as u8; + filt.order = br.read_bits(if short { 3 } else { 5 })? as u8; + if filt.order as usize > tns_max_order(short) { + return Err(Error::Bitstream("TNS order too large")); + } + if filt.order > 0 { + filt.direction = br.read_bits(1)? == 1; + filt.coef_compress = br.read_bits(1)? == 1; + let bits = 3 + win.coef_res as u32 - filt.coef_compress as u32; + for i in 0..filt.order as usize { + let v = br.read_bits(bits)? as i32; + let v = if v & (1 << (bits - 1)) != 0 { v - (1 << bits) } else { v }; + filt.coef[i] = v as i8; + } + } + } + } + Ok(t) + } + + pub fn write(&self, bw: &mut BitWriter, info: &IcsInfo) { + let short = info.is_short(); + for w in 0..info.num_windows() { + let win = &self.windows[w]; + bw.write_bits(win.n_filt as u32, if short { 1 } else { 2 }); + if win.n_filt == 0 { + continue; + } + bw.write_bits(win.coef_res as u32, 1); + for f in 0..win.n_filt as usize { + let filt = &win.filt[f]; + bw.write_bits(filt.length as u32, if short { 4 } else { 6 }); + bw.write_bits(filt.order as u32, if short { 3 } else { 5 }); + if filt.order > 0 { + bw.write_bits(filt.direction as u32, 1); + bw.write_bits(filt.coef_compress as u32, 1); + let bits = 3 + win.coef_res as u32 - filt.coef_compress as u32; + for i in 0..filt.order as usize { + bw.write_bits(filt.coef[i] as i32 as u32 & ((1 << bits) - 1), bits); + } + } + } + } + } + + /// Bits `write` would produce. + pub fn bit_count(&self, info: &IcsInfo) -> u32 { + let short = info.is_short(); + let mut n = 0; + for w in 0..info.num_windows() { + let win = &self.windows[w]; + n += if short { 1 } else { 2 }; + if win.n_filt == 0 { + continue; + } + n += 1; + for f in 0..win.n_filt as usize { + let filt = &win.filt[f]; + n += if short { 7 } else { 11 }; + if filt.order > 0 { + n += 2 + filt.order as u32 * (3 + win.coef_res as u32 - filt.coef_compress as u32); + } + } + } + n + } +} + +/// Dequantise one reflection coefficient. +pub fn dequant_coef(c: i8, coef_res: bool) -> f64 { + let res = 3 + coef_res as i32; + let iqfac = ((1 << (res - 1)) as f64 - 0.5) / FRAC_PI_2; + let iqfac_m = ((1 << (res - 1)) as f64 + 0.5) / FRAC_PI_2; + let c = c as f64; + (c / if c >= 0.0 { iqfac } else { iqfac_m }).sin() +} + +/// Quantise a reflection coefficient (inverse of [`dequant_coef`], nearest index). +pub fn quant_coef(k: f64, coef_res: bool) -> i8 { + let res = 3 + coef_res as i32; + let iqfac = ((1 << (res - 1)) as f64 - 0.5) / FRAC_PI_2; + let iqfac_m = ((1 << (res - 1)) as f64 + 0.5) / FRAC_PI_2; + let a = k.clamp(-1.0, 1.0).asin(); + let lo = -(1 << (res - 1)); + let hi = (1 << (res - 1)) - 1; + let q = if a >= 0.0 { (a * iqfac).round() } else { (a * iqfac_m).round() }; + (q as i32).clamp(lo, hi) as i8 +} + +/// Reflection coefficients → direct-form LPC `a[0..=order]` (`a[0] = 1`). +pub fn parcor_to_lpc(k: &[f64]) -> [f64; 13] { + let mut a = [0f64; 13]; + a[0] = 1.0; + let mut b = [0f64; 13]; + for m in 1..=k.len() { + for i in 1..m { + b[i] = a[i] + k[m - 1] * a[m - i]; + } + a[1..m].copy_from_slice(&b[1..m]); + a[m] = k[m - 1]; + } + a +} + +/// Filter region `[start, end)` of every TNS filter of window `w`, in spectrum lines of that window. +fn regions(info: &IcsInfo, sf_index: u8, win: &TnsWindow) -> Vec<(usize, usize, usize)> { + let short = info.is_short(); + let swb = info.swb_offsets(sf_index); + let num_swb = swb.len() - 1; + let lim = tns_max_bands(sf_index, short).min(info.max_sfb); + let mut out = Vec::new(); + let mut bottom = num_swb; + for f in 0..win.n_filt as usize { + let filt = &win.filt[f]; + let top = bottom; + bottom = top.saturating_sub(filt.length as usize); + if filt.order == 0 { + continue; + } + let start = swb[bottom.min(lim)] as usize; + let end = swb[top.min(lim)] as usize; + if end > start { + out.push((f, start, end)); + } + } + out +} + +fn lpc_of(filt: &TnsFilter, coef_res: bool) -> [f64; 13] { + let k: Vec = filt.coef[..filt.order as usize].iter().map(|&c| dequant_coef(c, coef_res)).collect(); + parcor_to_lpc(&k) +} + +/// Decoder: apply the all-pole TNS synthesis filters to `spec` (1024 lines; short windows at 128 stride). +pub fn apply_decoder(spec: &mut [f32], info: &IcsInfo, sf_index: u8, tns: &TnsData) { + let wlen = if info.is_short() { 128 } else { 1024 }; + for w in 0..info.num_windows() { + let win = &tns.windows[w]; + for (f, start, end) in regions(info, sf_index, win) { + let filt = &win.filt[f]; + let a = lpc_of(filt, win.coef_res); + let order = filt.order as usize; + let s = &mut spec[w * wlen..(w + 1) * wlen]; + let size = end - start; + let mut state = [0f64; 12]; + for m in 0..size { + let idx = if filt.direction { end - 1 - m } else { start + m }; + let mut y = s[idx] as f64; + for i in 0..order { + y -= a[i + 1] * state[i]; + } + for i in (1..order).rev() { + state[i] = state[i - 1]; + } + if order > 0 { + state[0] = y; + } + s[idx] = y as f32; + } + } + } +} + +/// Encoder: apply the all-zero TNS analysis filters (exact inverse of [`apply_decoder`]). +pub fn apply_encoder(spec: &mut [f32], info: &IcsInfo, sf_index: u8, tns: &TnsData) { + let wlen = if info.is_short() { 128 } else { 1024 }; + for w in 0..info.num_windows() { + let win = &tns.windows[w]; + for (f, start, end) in regions(info, sf_index, win) { + let filt = &win.filt[f]; + let a = lpc_of(filt, win.coef_res); + let order = filt.order as usize; + let s = &mut spec[w * wlen..(w + 1) * wlen]; + let size = end - start; + let mut state = [0f64; 12]; + for m in 0..size { + let idx = if filt.direction { end - 1 - m } else { start + m }; + let x = s[idx] as f64; + let mut y = x; + for i in 0..order { + y += a[i + 1] * state[i]; + } + for i in (1..order).rev() { + state[i] = state[i - 1]; + } + if order > 0 { + state[0] = x; + } + s[idx] = y as f32; + } + } + } +} + +/// Levinson–Durbin on autocorrelation `r[0..=order]`: returns reflection coefficients and the +/// prediction gain `r[0] / error`. +pub fn levinson(r: &[f64], order: usize) -> (Vec, f64) { + let mut a = vec![0f64; order + 1]; + a[0] = 1.0; + let mut err = r[0]; + let mut ks = Vec::with_capacity(order); + if err <= 0.0 { + return (vec![0.0; order], 1.0); + } + for m in 1..=order { + let mut acc = r[m]; + for i in 1..m { + acc += a[i] * r[m - i]; + } + let k = -acc / err; + let prev = a.clone(); + for i in 1..m { + a[i] = prev[i] + k * prev[m - i]; + } + a[m] = k; + err *= 1.0 - k * k; + ks.push(k); + if err <= r[0] * 1e-9 { + ks.resize(order, 0.0); + break; + } + } + (ks, r[0] / err.max(r[0] * 1e-9)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ics::WindowSequence; + use crate::mdct::WindowShape; + + #[test] + fn coef_quant_roundtrip() { + for res in [false, true] { + let n = if res { 8 } else { 4 }; + for c in -n..n { + let k = dequant_coef(c as i8, res); + assert_eq!(quant_coef(k, res), c as i8); + } + } + } + + #[test] + fn encoder_decoder_filters_are_inverse() { + let info = IcsInfo { window_sequence: WindowSequence::OnlyLong, window_shape: WindowShape::Sine, max_sfb: 49, ..Default::default() }; + let mut t = TnsData::default(); + t.windows[0].n_filt = 1; + t.windows[0].coef_res = true; + t.windows[0].filt[0] = TnsFilter { length: 40, order: 4, direction: true, coef_compress: false, coef: [3, -2, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0] }; + let orig: Vec = (0..1024).map(|i| ((i * 37 % 101) as f32 - 50.0) * 0.1).collect(); + let mut s = orig.clone(); + apply_encoder(&mut s, &info, 4, &t); + assert_ne!(s, orig); + apply_decoder(&mut s, &info, 4, &t); + for i in 0..1024 { + assert!((s[i] - orig[i]).abs() < 1e-3); + } + let mut bw = BitWriter::new(); + t.write(&mut bw, &info); + assert_eq!(bw.bit_len() as u32, t.bit_count(&info)); + let d = bw.finish(); + assert_eq!(TnsData::parse(&mut BitReader::new(&d), &info).unwrap(), t); + } +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 222ed6159f..ff7cc52f6c 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -21,6 +21,8 @@ deckcraft-pptx = { workspace = true } deckcraft-media = { workspace = true } deckcraft-h264 = { workspace = true } deckcraft-isobmff = { workspace = true } +deckcraft-aac = { workspace = true } +deckcraft-anim = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } i_overlay = { workspace = true } diff --git a/crates/engine/src/cmd/file.rs b/crates/engine/src/cmd/file.rs index 6e13320bfa..7edaece865 100644 --- a/crates/engine/src/cmd/file.rs +++ b/crates/engine/src/cmd/file.rs @@ -33,7 +33,7 @@ pub fn specs() -> Vec { cmd!(noundo "file.saveAs", "Save As…", ["File"], Some("Cmd+Shift+S"), "{path, format?}", has_doc, save_as), cmd!(noundo "file.saveTemplate", "Save as Template…", ["File"], None, "{path}", has_doc, save_as), cmd!(query "file.saveBytes", "Save to Data", [], None, "{format?: deckcraft|pptx} → {data: base64}", has_doc, save_bytes_cmd), - cmd!(noundo "file.export", "Export…", ["File"], None, "{path, format?: png|jpeg|pptx|deckcraft|outline|pdf|gif|mp4|mov (else by path), slide?: index, all?: bool, scale?: px per pt, layout?: slides|notes|handouts (pdf), perPage?: 1|2|3|4|6|9, paper?: letter|a4|legal|a3 (pdf), orientation?: portrait|landscape (pdf), dpi?, slides?: [index]|range (\"1-3,5\"), includeHidden?, textLayer?, frame?, width?: px 16-3840|small|medium|large|extraLarge (gif), secondsPerSlide?: 0.1-600 (gif; 5), loop?: bool (gif; true: forever), height?: 480|720|1080|2160 (mp4/mov), fps? (mp4/mov)} → {path, bytes} or {files} (png, jpeg); gif adds frames, width, height, secondsPerSlide", has_doc, export), + cmd!(noundo "file.export", "Export…", ["File"], None, "{path, format?: png|jpeg|pptx|deckcraft|outline|pdf|gif|mp4|mov (else by path), slide?: index, all?: bool, scale?: px per pt, layout?: slides|notes|handouts (pdf), perPage?: 1|2|3|4|6|9, paper?: letter|a4|legal|a3 (pdf), orientation?: portrait|landscape (pdf), dpi?, slides?: [index]|range (\"1-3,5\"), includeHidden?, textLayer?, frame?, width?: px 16-3840|small|medium|large|extraLarge (gif), secondsPerSlide?: 0.1-600 (gif; 5; also mp4/mov), loop?: bool (gif; true: forever), height?: 480|720|1080|2160 (mp4/mov), fps? (mp4/mov), secondsPerClick? (mp4/mov), audio?: aac|pcm|none (mp4/mov; pcm in .mov)} → {path, bytes} or {files} (png, jpeg); gif adds frames, width, height, secondsPerSlide", has_doc, export), cmd!( query "file.info", "Info", @@ -552,17 +552,28 @@ fn export(s: &mut Session, p: &Value) -> Result { /// File › Export › Create a Video. fn video_export(doc: &deckcraft_model::Presentation, path: &str, format: &str, p: &Value) -> Result { + use crate::video::AudioFormat; let d = crate::video::VideoOpts::default(); + let audio = match str_param(p, "audio") { + None | Some("aac") => AudioFormat::Aac, + Some("pcm") => AudioFormat::Pcm, + Some("none") => AudioFormat::None, + Some(other) => return Err(bad("file.export", format!("audio “{other}”: use aac, pcm (.mov) or none"))), + }; let opts = crate::video::VideoOpts { height: usize_param(p, "height").map(|h| h.clamp(2, 4320) as u32).unwrap_or(d.height), fps: usize_param(p, "fps").map(|f| f.clamp(1, 60) as u32).unwrap_or(d.fps), seconds_per_slide: f64_or(p, "secondsPerSlide", d.seconds_per_slide).clamp(0.1, 3600.0), + seconds_per_click: f64_or(p, "secondsPerClick", d.seconds_per_click).clamp(0.0, 3600.0), include_hidden: bool_or(p, "includeHidden", false), + audio, }; - let (bytes, frames, (w, h)) = crate::video::encode(doc, format, &opts).map_err(|e| bad("file.export", e))?; - write_file(path, &bytes)?; + let v = crate::video::encode(doc, format, &opts).map_err(|e| bad("file.export", e))?; + write_file(path, &v.bytes)?; let fps = opts.fps.clamp(1, 60); - Ok(json!({"path": path, "bytes": bytes.len(), "frames": frames, "width": w, "height": h, "fps": fps, "seconds": frames as f64 / fps as f64})) + Ok( + json!({"path": path, "bytes": v.bytes.len(), "frames": v.frames, "width": v.size.0, "height": v.size.1, "fps": fps, "seconds": v.frames as f64 / fps as f64, "audioSeconds": v.audio_seconds, "warnings": v.warnings}), + ) } /// File › Info: what the Info page shows about the active presentation. diff --git a/crates/engine/src/video.rs b/crates/engine/src/video.rs index dbb30f705e..c03fec10d9 100644 --- a/crates/engine/src/video.rs +++ b/crates/engine/src/video.rs @@ -1,14 +1,33 @@ -//! Video export (File › Export › Create a Video): the deck as an MP4 or MOV, one frame stream -//! of each slide's transition then the slide held for its timing. Frames come from the slide -//! renderer and are encoded with DeckCraft's own H.264 encoder ([`deckcraft_h264::Encoder`]: -//! intra-coded changes, skipped unchanged macroblocks) into an MP4 or QuickTime file -//! ([`deckcraft_isobmff::Mp4Writer`]); no external tool is needed. Animations within a slide are -//! shown in their final state, and there is no audio track. +//! Video export (File › Export › Create a Video): the deck as an MP4 or MOV. Each slide plays +//! its transition, then its animations on the slide-show timeline (click steps one after another, +//! a pause before each), then stays up for its timing. Frames come from the slide renderer and are +//! encoded with DeckCraft's own H.264 encoder ([`deckcraft_h264::Encoder`]: intra-coded changes, +//! skipped unchanged macroblocks); the sound of the slides' audio and video clips (narration is +//! recorded as such clips) and of transitions is mixed and encoded as AAC ([`deckcraft_aac`]), or +//! kept as PCM in a MOV. Everything is muxed by [`deckcraft_isobmff::Mp4Writer`]; no external +//! tool is needed. Video clips show their poster frame. -use deckcraft_model::Presentation; +use std::collections::HashMap; +use std::sync::Arc; + +use deckcraft_anim::{Sequence, ShowState, Timeline}; +use deckcraft_model::anim::AnimClass; +use deckcraft_model::{MediaId, Presentation, ShapeId, ShapeKind}; +use deckcraft_render::{ParaState, ShapeState}; /// The most frames one export writes (two hours at 30 fps), against runaway timings. pub const MAX_FRAMES: u64 = 2 * 60 * 60 * 30; +/// Audio sample rate of the exported track. +pub const AUDIO_RATE: u32 = 48_000; + +/// How the sound is written. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AudioFormat { + Aac, + /// 16-bit PCM (MOV only). + Pcm, + None, +} /// Export options, as `file.export` takes them. #[derive(Clone, Debug, PartialEq)] @@ -17,25 +36,67 @@ pub struct VideoOpts { /// slide's aspect ratio. Both are rounded to even numbers, as H.264 needs. pub height: u32, pub fps: u32, - /// How long a slide stays up when it has no "advance after" timing of its own. + /// How long a slide stays up after its animations when it has no "advance after" timing. pub seconds_per_slide: f64, + /// The pause before each click step of a slide's animations. + pub seconds_per_click: f64, pub include_hidden: bool, + pub audio: AudioFormat, } impl Default for VideoOpts { fn default() -> Self { - VideoOpts { height: 1080, fps: 30, seconds_per_slide: 5.0, include_hidden: false } + VideoOpts { height: 1080, fps: 30, seconds_per_slide: 5.0, seconds_per_click: 1.0, include_hidden: false, audio: AudioFormat::Aac } } } -/// One step of the timeline: a transition frame at progress `t`, or slide `index` held for -/// `frames` frames. +/// One step of the timeline: a transition frame at progress `t`; a frame of slide `index`'s +/// animations at main step `step`, `t` seconds in; the slide still before main step `step` for +/// `frames` frames; or the slide with every animation done, held for `frames` frames. #[derive(Clone, Debug, PartialEq)] pub enum Step { Transition { index: usize, t: f64 }, + Anim { index: usize, step: usize, t: f64 }, + Still { index: usize, step: usize, frames: u64 }, Hold { index: usize, frames: u64 }, } +impl Step { + pub fn index(&self) -> usize { + match self { + Step::Transition { index, .. } | Step::Anim { index, .. } | Step::Still { index, .. } | Step::Hold { index, .. } => *index, + } + } + fn frames(&self) -> u64 { + match self { + Step::Still { frames, .. } | Step::Hold { frames, .. } => *frames, + _ => 1, + } + } +} + +/// A sound that starts at `start` seconds into the video and stops by `until`. +#[derive(Clone, Debug, PartialEq)] +pub struct AudioCue { + pub media: MediaId, + pub start: f64, + pub until: f64, + pub trim_start: f64, + pub trim_end: f64, + pub volume: f64, + pub fade_in: f64, + pub fade_out: f64, + pub looped: bool, +} + +/// The video's frames and sounds. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Plan { + pub steps: Vec, + pub cues: Vec, + pub frames: u64, +} + /// The frame size for `opts` on this deck's slide size: even, at least 2×2. pub fn frame_size(pres: &Presentation, opts: &VideoOpts) -> (u32, u32) { let h = opts.height.clamp(2, 4320) & !1; @@ -51,36 +112,158 @@ fn frames_for(seconds: f64, fps: u32) -> u64 { ((s * fps as f64).round() as u64).max(1) } +fn secs(ms: u32) -> f64 { + f64::from(ms) / 1000.0 +} + +/// The clip a cue plays, from a media shape. +fn clip_cue(c: &deckcraft_model::MediaClip, start: f64, until: f64) -> AudioCue { + AudioCue { + media: c.media, + start, + until, + trim_start: secs(c.trim_start_ms), + trim_end: secs(c.trim_end_ms), + volume: if c.volume.is_finite() { c.volume.clamp(0.0, 1.0) } else { 1.0 }, + fade_in: secs(c.fade_in_ms), + fade_out: secs(c.fade_out_ms), + looped: c.loop_play, + } +} + +/// Pushes steps while counting frames against [`MAX_FRAMES`]. +struct Layout { + steps: Vec, + frames: u64, + fps: u32, +} + +impl Layout { + /// Add `step`; `false` once the cap is reached. + fn push(&mut self, step: Step) -> bool { + let room = MAX_FRAMES.saturating_sub(self.frames); + if room == 0 { + return false; + } + let step = match step { + Step::Still { index, step, frames } => Step::Still { index, step, frames: frames.min(room) }, + Step::Hold { index, frames } => Step::Hold { index, frames: frames.min(room) }, + s => s, + }; + self.frames = self.frames.saturating_add(step.frames()); + self.steps.push(step); + true + } + fn now(&self) -> f64 { + self.frames as f64 / f64::from(self.fps) + } +} + /// The timeline: for each slide shown, its transition's frames (none for "none", a cut or a -/// zero duration) then the slide held for its "advance after" time, else `seconds_per_slide`. +/// zero duration), its main animation sequence (each click step after a `seconds_per_click` +/// pause; steps that start by themselves follow on), then the slide held for its "advance +/// after" time (counted from the slide's start, as recorded timings are), else +/// `seconds_per_slide`. Sounds: transition sounds, clips that play automatically or from a +/// media effect, until their end or the slide's (the deck's when they play across slides). /// The total is capped at [`MAX_FRAMES`]. -pub fn timeline(pres: &Presentation, opts: &VideoOpts) -> Vec { +pub fn plan(pres: &Presentation, opts: &VideoOpts) -> Plan { let fps = opts.fps.clamp(1, 60); - let mut out = vec![]; - let mut total: u64 = 0; - for (index, slide) in pres.slides.iter().enumerate() { + let mut lay = Layout { steps: vec![], frames: 0, fps }; + let mut cues: Vec<(AudioCue, bool)> = vec![]; // (cue, ends with its slide) + let animate = !pres.show.without_animation; + 'slides: for (index, slide) in pres.slides.iter().enumerate() { if slide.hidden && !opts.include_hidden { continue; } - if let Some(tr) = slide.transition.as_ref().filter(|t| !matches!(t.kind.as_str(), "none" | "cut" | "") && t.duration_ms > 0) { - let n = frames_for(tr.duration_ms as f64 / 1000.0, fps); - for k in 0..n { - if total >= MAX_FRAMES { - return out; + let slide_start = lay.now(); + if let Some(tr) = slide.transition.as_ref() { + if let Some(m) = tr.sound { + cues.push(( + AudioCue { + media: m, + start: slide_start, + until: f64::MAX, + trim_start: 0.0, + trim_end: 0.0, + volume: 1.0, + fade_in: 0.0, + fade_out: 0.0, + looped: false, + }, + false, + )); + } + if !matches!(tr.kind.as_str(), "none" | "cut" | "") && tr.duration_ms > 0 { + let n = frames_for(secs(tr.duration_ms), fps); + for k in 0..n { + if !lay.push(Step::Transition { index, t: k as f64 / n as f64 }) { + break 'slides; + } } - out.push(Step::Transition { index, t: k as f64 / n as f64 }); - total = total.saturating_add(1); } } - let secs = slide.transition.as_ref().and_then(|t| t.advance_after_ms).map(|ms| ms as f64 / 1000.0).unwrap_or(opts.seconds_per_slide); - let frames = frames_for(secs, fps).min(MAX_FRAMES.saturating_sub(total)); - if frames == 0 { - return out; + let content_start = lay.now(); + let first_cue = cues.len(); + let mut started: Vec = vec![]; + for sh in &slide.shapes { + if let ShapeKind::Media(c) = &sh.kind + && c.autoplay + { + cues.push((clip_cue(c, content_start, f64::MAX), !c.play_across_slides)); + started.push(sh.id); + } + } + let tl = if animate { ShowState::timeline(pres, index) } else { Timeline::default() }; + let effects = tl.effects(); + for step in 0..tl.steps() { + if !tl.step_is_auto(step) && !lay.push(Step::Still { index, step, frames: frames_for(opts.seconds_per_click, fps) }) { + break 'slides; + } + let step_start = lay.now(); + for e in effects.iter().filter(|e| e.sequence == Sequence::Main && e.step == step && e.class == AnimClass::Media) { + if started.contains(&e.shape) { + continue; + } + if let Some(ShapeKind::Media(c)) = slide.shape(e.shape).map(|s| &s.kind) { + cues.push((clip_cue(c, step_start + e.start.max(0.0), f64::MAX), !c.play_across_slides)); + started.push(e.shape); + } + } + let dur = tl.step_duration(step); + if dur > 0.0 { + let n = frames_for(dur, fps); + for k in 0..n { + if !lay.push(Step::Anim { index, step, t: k as f64 * dur / n as f64 }) { + break 'slides; + } + } + } + } + let hold = match slide.transition.as_ref().and_then(|t| t.advance_after_ms) { + Some(ms) => (secs(ms) - (lay.now() - slide_start)).max(0.5), + None => opts.seconds_per_slide, + }; + if !lay.push(Step::Hold { index, frames: frames_for(hold, fps) }) { + break; + } + let end = lay.now(); + for (c, ends) in cues.iter_mut().skip(first_cue) { + if *ends { + c.until = end; + } } - out.push(Step::Hold { index, frames }); - total = total.saturating_add(frames); } - out + let total = lay.now(); + let cues = cues.into_iter().map(|(mut c, _)| { + c.until = c.until.min(total); + c + }); + Plan { steps: lay.steps, cues: cues.filter(|c| c.until > c.start).collect(), frames: lay.frames } +} + +/// The steps of [`plan`]. +pub fn timeline(pres: &Presentation, opts: &VideoOpts) -> Vec { + plan(pres, opts).steps } /// An image fitted to exactly `w`×`h` RGBA (the renderer's size can differ by a pixel): copied @@ -102,45 +285,169 @@ fn fit(img: &deckcraft_render::Image, w: u32, h: u32) -> Vec { } /// The renderer options for a frame `h` pixels high. -fn render_opts(pres: &Presentation, h: u32) -> deckcraft_render::RenderOpts<'static> { +fn render_opts<'a>(pres: &Presentation, h: u32, state: Option<&'a dyn Fn(ShapeId) -> ShapeState>) -> deckcraft_render::RenderOpts<'a> { let scale = h as f64 / pres.slide_size.height.max(1.0); - deckcraft_render::RenderOpts { scale: if scale.is_finite() { scale.clamp(0.01, 16.0) } else { 1.0 }, ..Default::default() } + deckcraft_render::RenderOpts { scale: if scale.is_finite() { scale.clamp(0.01, 16.0) } else { 1.0 }, state, ..Default::default() } +} + +/// A shape's state at main step `step`, `t` seconds in (as the slide show draws it). +fn state_for(tl: &Timeline, id: ShapeId, step: usize, t: f64) -> ShapeState { + let a = tl.state(id, None, step, t); + ShapeState { + visible: a.visible, + opacity: a.opacity, + offset: deckcraft_geom::Vec2::new(a.offset_x, a.offset_y), + scale: (a.scale_x, a.scale_y), + rotate: a.rotate, + clip: a.clip, + tint: a.tint, + paras: tl + .paragraph_targets(id) + .into_iter() + .map(|p| { + let a = tl.state(id, Some(p), step, t); + (p, ParaState { visible: a.visible, opacity: a.opacity, offset: deckcraft_geom::Vec2::new(a.offset_x, a.offset_y) }) + }) + .collect(), + } } -/// Every frame of `steps`, in order, handed to `sink` (RGBA, `w`×`h`). A held slide is rendered -/// once. Stops at the first error `sink` returns. +/// Every frame of `steps`, in order, handed to `sink` (RGBA, `w`×`h`). A still or held slide +/// is rendered once. Stops at the first error `sink` returns. pub fn frames(pres: &Presentation, steps: &[Step], (w, h): (u32, u32), sink: &mut dyn FnMut(&[u8]) -> Result<(), String>) -> Result { - let opts = render_opts(pres, h); + let animate = !pres.show.without_animation; + let mut timelines: HashMap> = HashMap::new(); + let mut tl_of = + |i: usize| timelines.entry(i).or_insert_with(|| Arc::new(if animate { ShowState::timeline(pres, i) } else { Timeline::default() })).clone(); let mut n: u64 = 0; for step in steps { - match step { - Step::Transition { index, t } => { - let img = deckcraft_render::render_transition(pres, *index, *t, &opts); - sink(&fit(&img, w, h))?; - n = n.saturating_add(1); + let tl = tl_of(step.index()); + let (st, t) = match step { + Step::Transition { .. } => (0, 0.0), + Step::Anim { step, t, .. } => (*step, *t), + Step::Still { step, .. } => (*step, 0.0), + Step::Hold { .. } => (tl.steps(), 0.0), + }; + // During a transition only the incoming slide's shapes take its starting state. + let incoming = pres.slides.get(step.index()).cloned(); + let f = move |id: ShapeId| { + if incoming.as_ref().is_some_and(|s| s.shape(id).is_some()) { state_for(&tl, id, st, t) } else { ShapeState::default() } + }; + let opts = render_opts(pres, h, Some(&f)); + let img = match step { + Step::Transition { index, t } => deckcraft_render::render_transition(pres, *index, *t, &opts), + _ => deckcraft_render::render_slide(pres, step.index(), &opts), + }; + let px = fit(&img, w, h); + for _ in 0..step.frames() { + sink(&px)?; + n = n.saturating_add(1); + } + } + Ok(n) +} + +/// Decoded clips at [`AUDIO_RATE`], interleaved stereo, by media id (`None`: undecodable). +fn decode_clips(pres: &Presentation, cues: &[AudioCue], warnings: &mut Vec) -> HashMap>>> { + let mut out = HashMap::new(); + for c in cues { + if out.contains_key(&c.media) { + continue; + } + let pcm = pres.media(c.media).filter(|m| !m.data.is_empty()).and_then(|m| match deckcraft_media::audio::decode(&m.data) { + Ok(p) => Some(p), + Err(e) => { + warnings.push(format!("{}: no sound in the video ({e})", m.name)); + None } - Step::Hold { index, frames } => { - let img = fit(&deckcraft_render::render_slide(pres, *index, &opts), w, h); - for _ in 0..*frames { - sink(&img)?; - n = n.saturating_add(1); - } + }); + let stereo = pcm.map(|p| { + let p = deckcraft_media::audio::resample(&p, AUDIO_RATE); + let ch = usize::from(p.channels.max(1)); + let mut s = Vec::with_capacity(p.frames().saturating_mul(2)); + for f in 0..p.frames() { + let l = p.samples.get(f * ch).copied().unwrap_or(0.0); + let r = if ch > 1 { p.samples.get(f * ch + 1).copied().unwrap_or(0.0) } else { l }; + s.push(l); + s.push(r); + } + Arc::new(s) + }); + out.insert(c.media, stereo); + } + out +} + +/// Mix every cue into the stereo block starting at sample frame `at` (`out`: interleaved). +fn mix_block(cues: &[AudioCue], clips: &HashMap>>>, at: u64, out: &mut [f32]) { + out.iter_mut().for_each(|s| *s = 0.0); + let rate = f64::from(AUDIO_RATE); + let len = (out.len() / 2) as u64; + for c in cues { + let Some(Some(clip)) = clips.get(&c.media) else { continue }; + let frames = (clip.len() / 2) as u64; + let first = (c.trim_start * rate) as u64; + let last = frames.saturating_sub((c.trim_end * rate) as u64); + if last <= first { + continue; + } + let (start, until) = ((c.start * rate) as u64, (c.until * rate) as u64); + if until <= at || start >= at + len { + continue; + } + let span = (last - first) as f64 / rate; + for k in 0..len { + let g = at + k; + if g < start || g >= until { + continue; + } + let local = g - start; + if !c.looped && first + local >= last { + break; + } + let pos = first + if c.looped { local % (last - first) } else { local }; + let tl = local as f64 / rate; + let mut gain = c.volume; + if c.fade_in > 0.0 && tl < c.fade_in { + gain *= tl / c.fade_in; + } + let left = if c.looped { f64::MAX } else { span - tl }; + if c.fade_out > 0.0 && left < c.fade_out { + gain *= (left / c.fade_out).max(0.0); + } + let (i, o) = (pos as usize * 2, k as usize * 2); + if let (Some(l), Some(r)) = (clip.get(i), clip.get(i + 1)) { + out[o] += l * gain as f32; + out[o + 1] += r * gain as f32; } } } - Ok(n) + for s in out.iter_mut() { + *s = s.clamp(-1.0, 1.0); + } } /// A key frame at least this often (seconds), so players can seek. const KEY_EVERY_S: u32 = 2; -/// Encode the deck to `path` as `mp4` or `mov`, at quality `qp` (H.264 QP, lower is better). -/// The file is assembled in memory and written through a temporary sibling, so a failed export -/// leaves no half file. Returns the frame count and size. -pub fn encode(pres: &Presentation, format: &str, opts: &VideoOpts) -> Result<(Vec, u64, (u32, u32)), String> { - use deckcraft_isobmff::{AvcConfig, Brand, CodecConfig, FourCc, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; - let steps = timeline(pres, opts); - if steps.is_empty() { +/// The encoded file. +pub struct Encoded { + pub bytes: Vec, + pub frames: u64, + pub size: (u32, u32), + /// Seconds of sound (0: no audio track). + pub audio_seconds: f64, + pub warnings: Vec, +} + +/// Encode the deck as `mp4` or `mov`. The file is assembled in memory. +pub fn encode(pres: &Presentation, format: &str, opts: &VideoOpts) -> Result { + use deckcraft_isobmff::{AvcConfig, Brand, CodecConfig, FourCc, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + if opts.audio == AudioFormat::Pcm && format != "mov" { + return Err("PCM sound needs a .mov file; use AAC for .mp4".into()); + } + let plan = plan(pres, opts); + if plan.steps.is_empty() { return Err("there are no slides to put in the video".into()); } let (w, h) = frame_size(pres, opts); @@ -154,10 +461,9 @@ pub fn encode(pres: &Presentation, format: &str, opts: &VideoOpts) -> Result<(Ve let key_every = u64::from(fps * KEY_EVERY_S); let mut since_key: u64 = 0; let mut total: u64 = 0; - for (i, step) in steps.iter().enumerate() { - // A key frame starts each slide and each transition, so the file seeks by slide. - let continues = matches!((i.checked_sub(1).and_then(|p| steps.get(p)), step), (Some(Step::Transition { index: a, .. }), Step::Transition { index: b, .. }) if a == b); - let mut start = !continues; + for (i, step) in plan.steps.iter().enumerate() { + // A key frame starts each slide (at its transition), so the file seeks by slide. + let mut start = i.checked_sub(1).and_then(|p| plan.steps.get(p)).is_none_or(|p| p.index() != step.index()); let mut sink = |px: &[u8]| -> Result<(), String> { let pic = enc.encode_rgba(px, start || since_key >= key_every); start = false; @@ -167,9 +473,64 @@ pub fn encode(pres: &Presentation, format: &str, opts: &VideoOpts) -> Result<(Ve }; total = total.saturating_add(frames(pres, std::slice::from_ref(step), (w, h), &mut sink)?); } - let n = total; + // Sound. + let mut warnings = vec![]; + let mut audio_seconds = 0.0; + if opts.audio != AudioFormat::None && !plan.cues.is_empty() { + let clips = decode_clips(pres, &plan.cues, &mut warnings); + if clips.values().any(Option::is_some) { + let samples = total.saturating_mul(u64::from(AUDIO_RATE)) / u64::from(fps); + audio_seconds = samples as f64 / f64::from(AUDIO_RATE); + const BLOCK: usize = deckcraft_aac::FRAME_LEN; + let mut buf = vec![0f32; BLOCK * 2]; + match opts.audio { + AudioFormat::Pcm => { + let pcm = PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: 2, sample_rate: f64::from(AUDIO_RATE) }; + let at = mux.add_track(TrackConfig::new(SampleEntry::pcm(pcm), AUDIO_RATE)).map_err(|e| e.to_string())?; + let mut pos = 0u64; + while pos < samples { + let n = (samples - pos).min(BLOCK as u64) as usize; + mix_block(&plan.cues, &clips, pos, &mut buf[..n * 2]); + let data: Vec = buf[..n * 2].iter().flat_map(|s| ((s * 32767.0).round() as i16).to_le_bytes()).collect(); + mux.write_sample(at, WriteSample { data: &data, duration: n as u32, composition_offset: 0, is_sync: true }) + .map_err(|e| e.to_string())?; + pos += n as u64; + } + } + _ => { + let mut aac = + deckcraft_aac::Encoder::new(deckcraft_aac::EncoderConfig::cbr(AUDIO_RATE, 2, 160_000)).map_err(|e| e.to_string())?; + let mut cfg = TrackConfig::new(SampleEntry::aac(aac.audio_specific_config(), 2, AUDIO_RATE), AUDIO_RATE); + cfg.media_start = Some(i64::from(aac.priming_samples())); + let at = mux.add_track(cfg).map_err(|e| e.to_string())?; + let write = |units: Vec>, mux: &mut Mp4Writer>>| -> Result<(), String> { + for u in units { + mux.write_sample(at, WriteSample { data: &u, duration: BLOCK as u32, composition_offset: 0, is_sync: true }) + .map_err(|e| e.to_string())?; + } + Ok(()) + }; + let mut pos = 0u64; + let (mut l, mut r) = (vec![0f32; BLOCK], vec![0f32; BLOCK]); + while pos < samples { + let n = (samples - pos).min(BLOCK as u64) as usize; + mix_block(&plan.cues, &clips, pos, &mut buf[..n * 2]); + l.clear(); + r.clear(); + for k in 0..n { + l.push(buf[k * 2]); + r.push(buf[k * 2 + 1]); + } + write(aac.encode(&[&l, &r]), &mut mux)?; + pos += n as u64; + } + write(aac.flush(), &mut mux)?; + } + } + } + } let bytes = mux.finish_faststart().map_err(|e| e.to_string())?.into_inner(); - Ok((bytes, n, (w, h))) + Ok(Encoded { bytes, frames: total, size: (w, h), audio_seconds, warnings }) } #[cfg(test)] @@ -197,7 +558,8 @@ mod tests { let t = timeline(&p, &o); assert_eq!(t.first(), Some(&Step::Hold { index: 0, frames: 30 })); assert_eq!(t.iter().filter(|s| matches!(s, Step::Transition { index: 1, .. })).count(), 5); - assert_eq!(t.last(), Some(&Step::Hold { index: 1, frames: 20 }), "the hidden slide is left out"); + // "Advance after" counts from the slide's start, its transition included. + assert_eq!(t.last(), Some(&Step::Hold { index: 1, frames: 15 }), "the hidden slide is left out"); let with_hidden = timeline(&p, &VideoOpts { include_hidden: true, ..o }); assert_eq!(with_hidden.last(), Some(&Step::Hold { index: 2, frames: 30 })); } @@ -208,13 +570,7 @@ mod tests { for s in &mut p.slides { std::sync::Arc::make_mut(s).transition = Some(Transition { advance_after_ms: Some(u32::MAX), ..Default::default() }); } - let total: u64 = timeline(&p, &VideoOpts { fps: 60, ..Default::default() }) - .iter() - .map(|s| match s { - Step::Hold { frames, .. } => *frames, - Step::Transition { .. } => 1, - }) - .sum(); + let total: u64 = timeline(&p, &VideoOpts { fps: 60, ..Default::default() }).iter().map(Step::frames).sum(); assert!(total <= MAX_FRAMES); let (w, h) = frame_size(&p, &VideoOpts { height: 721, ..Default::default() }); assert_eq!((w % 2, h), (0, 720)); @@ -237,15 +593,16 @@ mod tests { fn encodes_a_playable_mp4() { let mut p = deck(2); std::sync::Arc::make_mut(&mut p.slides[1]).transition = Some(Transition { kind: "fade".into(), duration_ms: 200, ..Default::default() }); - let o = VideoOpts { height: 72, fps: 10, seconds_per_slide: 0.5, include_hidden: false }; - let (bytes, n, (w, h)) = encode(&p, "mp4", &o).unwrap(); + let o = VideoOpts { height: 72, fps: 10, seconds_per_slide: 0.5, ..Default::default() }; + let v = encode(&p, "mp4", &o).unwrap(); + let (bytes, n, (w, h)) = (v.bytes, v.frames, v.size); assert_eq!(n, 5 + 2 + 5); let f = deckcraft_isobmff::open(bytes.as_slice()).unwrap(); let t = &f.tracks[0]; assert_eq!(t.samples.len(), n as usize); - // Key frames start each slide and each transition. + // Key frames start each slide (at its transition). let keys: Vec = t.samples.iter().enumerate().filter(|(_, s)| s.is_sync).map(|(i, _)| i).collect(); - assert_eq!(keys, vec![0, 5, 7]); + assert_eq!(keys, vec![0, 5]); let mut dec = deckcraft_h264::Decoder::new(); // Rebuild an Annex B stream from the length-prefixed samples and the avcC parameter sets. let Some(deckcraft_isobmff::CodecConfig::Avc(avc)) = t.entries.first().map(|e| &e.codec) else { panic!("not AVC") }; @@ -269,4 +626,100 @@ mod tests { assert_eq!(pics.len(), n as usize); assert_eq!((pics[0].width, pics[0].height), (w, h)); } + + /// A mono 16-bit WAV of a sine tone. + fn wav(rate: u32, secs: f64, hz: f64) -> Vec { + let n = (f64::from(rate) * secs) as u32; + let mut b = vec![]; + b.extend(b"RIFF"); + b.extend((36 + n * 2).to_le_bytes()); + b.extend(b"WAVEfmt "); + b.extend(16u32.to_le_bytes()); + b.extend(1u16.to_le_bytes()); + b.extend(1u16.to_le_bytes()); + b.extend(rate.to_le_bytes()); + b.extend((rate * 2).to_le_bytes()); + b.extend(2u16.to_le_bytes()); + b.extend(16u16.to_le_bytes()); + b.extend(b"data"); + b.extend((n * 2).to_le_bytes()); + for k in 0..n { + let v = ((k as f64 / f64::from(rate) * hz * std::f64::consts::TAU).sin() * 12000.0) as i16; + b.extend(v.to_le_bytes()); + } + b + } + + /// A slide with a text box flying in on click and a tone that plays by itself. + fn animated_deck() -> Presentation { + use serde_json::json; + let mut s = crate::Session::with_new(); + s.execute("slide.new", &json!({"layout": "blank"})).unwrap(); + s.execute("insert.textBox", &json!({"rect": [100, 100, 400, 120], "text": "Hello"})).unwrap(); + s.execute("animation.add", &json!({"effect": "fade", "start": "onClick", "duration": 1000})).unwrap(); + let data = crate::cmd::base64_encode(&wav(22_050, 1.0, 440.0)); + s.execute("insert.audio", &json!({"data": data, "name": "tone.wav"})).unwrap(); + s.execute("media.options", &json!({"autoplay": true})).unwrap(); + let mut p = (*s.doc().unwrap().doc).clone(); + p.slides.retain(|sl| sl.shapes.iter().any(|sh| matches!(sh.kind, ShapeKind::Media(_)))); + p + } + + #[test] + fn animations_play_on_the_show_timeline() { + let p = animated_deck(); + let o = VideoOpts { fps: 10, seconds_per_slide: 1.0, seconds_per_click: 0.5, height: 90, ..Default::default() }; + let plan = plan(&p, &o); + assert_eq!(plan.steps.first(), Some(&Step::Still { index: 0, step: 0, frames: 5 }), "{:?}", plan.steps); + assert_eq!(plan.steps.iter().filter(|s| matches!(s, Step::Anim { .. })).count(), 10); + assert_eq!(plan.steps.last(), Some(&Step::Hold { index: 0, frames: 10 })); + // The tone plays from the slide's start, for the slide. + assert_eq!(plan.cues.len(), 1); + assert_eq!((plan.cues[0].start, plan.cues[0].until), (0.0, 2.5)); + // Before the click the text is hidden; halfway through the fade it shows faintly; at + // the end fully. + let size = frame_size(&p, &o); + let ink = |step: Step| { + let mut dark = 0usize; + frames(&p, &[step], size, &mut |px| { + dark = px.as_chunks::<4>().0.iter().filter(|q| q[0] < 200).count(); + Ok(()) + }) + .unwrap(); + dark + }; + let (before, mid, after) = ( + ink(Step::Still { index: 0, step: 0, frames: 1 }), + ink(Step::Anim { index: 0, step: 0, t: 0.5 }), + ink(Step::Hold { index: 0, frames: 1 }), + ); + assert!(before < after && after > 0, "{before} {mid} {after}"); + } + + #[test] + fn sound_is_mixed_and_encoded_as_aac() { + let p = animated_deck(); + let o = VideoOpts { fps: 10, seconds_per_slide: 1.0, seconds_per_click: 0.5, height: 72, ..Default::default() }; + let v = encode(&p, "mp4", &o).unwrap(); + assert!((v.audio_seconds - 2.5).abs() < 0.01, "{}", v.audio_seconds); + let f = deckcraft_isobmff::open(v.bytes.as_slice()).unwrap(); + assert_eq!(f.tracks.len(), 2); + let a = &f.tracks[1]; + let Some(deckcraft_isobmff::CodecConfig::Aac(cfg)) = a.entries.first().map(|e| &e.codec) else { panic!("not AAC") }; + let mut dec = deckcraft_aac::Decoder::new(&cfg.asc).unwrap(); + let mut energy = vec![]; + for i in 0..a.samples.len() { + let pcm = dec.decode(&f.read_sample(v.bytes.as_slice(), 1, i).unwrap()).unwrap(); + energy.push(pcm[0].iter().map(|x| x * x).sum::() / pcm[0].len().max(1) as f32); + } + // The one-second tone, then silence. + let loud = energy.iter().filter(|e| **e > 0.01).count(); + let quiet = energy.iter().filter(|e| **e < 1e-5).count(); + assert!(loud >= 40 && quiet >= 50, "loud {loud} quiet {quiet} of {}", energy.len()); + // PCM in a MOV. + let v = encode(&p, "mov", &VideoOpts { audio: AudioFormat::Pcm, ..o.clone() }).unwrap(); + let f = deckcraft_isobmff::open(v.bytes.as_slice()).unwrap(); + assert!(matches!(f.tracks[1].entries.first().map(|e| &e.codec), Some(deckcraft_isobmff::CodecConfig::Pcm(_)))); + assert!(encode(&p, "mp4", &VideoOpts { audio: AudioFormat::Pcm, ..o }).is_err()); + } } diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 8258237762..f4257e88c9 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -29,7 +29,7 @@ PowerPoint opens and saves; Viewer = opens only). | PNG / JPEG slide images | export | n/a | yes (`file.export`, `deckcraft-cli render`) | Matches the editor's renderer. PNG `pHYs` and JPEG JFIF density are `scale` × 72 dpi (default scale 2 is 144 dpi), so Preview shows the slide's real size | render and engine tests | | TIFF / BMP / GIF slide images | export | n/a | no | Easy (the `image` crate writes them) | none | | Animated GIF of the deck | export | n/a | yes (`file.export` format `gif`, Export…, `deckcraft-cli`, MCP `export`) | One frame per slide from the slide renderer; size small/medium/large/extra large or 16–3840 px wide, 0.1–600 s per slide, loop forever or play once, slide range, hidden slides skipped; a 256-colour palette per frame (exact, or NeuQuant above 256 colours). Transitions and build animations are not animated | `crates/engine/src/gif_export.rs` tests (decoded frames, size, delay, loop, hidden slides, hostile params), MCP, CLI and export dialog tests | -| MP4 / MOV video of the deck | export | n/a | yes | `file.export` mp4/mov: transitions then each slide for its timing, 480p–4K, own H.264 encoder (intra + skip); no in-slide animations, no audio | none | +| MP4 / MOV video of the deck | export | n/a | yes | `file.export` mp4/mov: transitions then each slide for its timing, 480p–4K, own H.264 encoder (intra + skip); in-slide animations on the show timeline; sound (media, narration, transition sounds) as AAC or PCM in .mov; video clips show their poster frame | none | | Print | yes | n/a | no | PR #66 (browser) open | none | ## Pictures and media inside decks diff --git a/docs/gaps.md b/docs/gaps.md index e24ba23a92..2c33e70c16 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -138,14 +138,13 @@ Detail in [file-format-parity.md](file-format-parity.md). - **Impact:** handouts and speaker notes are printed in many workplaces. - **Estimate:** 5–8 h (print through the PDF path and each OS's print dialog). **Beta blocker.** -### Video has no animations or audio; the animated GIF has no transitions or builds -- **Evidence:** `file.export` writes MP4/MOV: each slide's transition, then the slide held for its - "advance after" time (else a default), at 480p–4K, encoded by DeckCraft's own H.264 encoder - (`crates/h264/src/encoder.rs`). Animations inside a slide show in their final state and the video - has no audio track (narration, media sound). The animated GIF (#171) has one frame per slide. -- **Impact:** sharing a talk as a video is a common request; builds and narration are part of it. -- **Estimate:** 4–8 h (animation timeline frames, an AAC track from the media and narration, an - Export dialog page). +### Video clips are stills in exported video +- **Evidence:** `file.export` writes MP4/MOV with each transition, the in-slide animations played on + the slide-show timeline (`secondsPerClick` pause before each click), and a sound track (embedded + audio, narration clips, transition sounds) as AAC (`crates/aac`, ported from FilmCraft) or 16-bit + PCM in .mov. Video clips show only their poster frame. +- **Impact:** decks with embedded video lose that motion in the exported video. +- **Estimate:** 3–5 h (decode clip frames into the render, an Export dialog page). ### Embedded MicroType Express font subsets aren't drawn - **Evidence:** `p:embeddedFontLst` is read and written. A raw TrueType/OpenType face, or one inside an Embedded OpenType wrapper, is loaded and used to draw. A MicroType Express subset (`application/x-fontdata` with neither form) is kept and written back, and the text still uses a stand-in. diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index b9bb409b25..8d739832da 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -41,6 +41,7 @@ pub const TABLE: &[(&str, Class)] = &[ ("ogg", Class::Standalone), ("opus", Class::Standalone), ("h264", Class::Layer(1)), + ("aac", Class::Layer(1)), ("hevc", Class::Layer(1)), ("vp9", Class::Layer(1)), ("av1", Class::Layer(1)), From 98e422159481dc1e3bc3533afbd52ec2e28774dc Mon Sep 17 00:00:00 2001 From: Nikita Rakitin Date: Wed, 14 Oct 2026 02:24:32 +0300 Subject: [PATCH 3/4] Video: embedded video clips play in MP4/MOV export A video clip that plays (autoplay or a Media effect) now shows its pictures in the exported video instead of its poster frame. Pictures come from deckcraft-media's VideoDecoder (the slide show's own decoder), decoded in order as the export's clock reaches them, with trim, loop and rewind honoured; after the end the last picture stays unless the clip rewinds. A still or held slide is rendered per frame only while a clip on it plays. RenderOpts gains video_frames (straight RGBA per media shape), which the renderer draws over the clip's frame in place of the poster. Clips that can't be decoded keep the poster and are listed in the export's warnings. Co-authored-by: Cursor --- crates/engine/src/video.rs | 243 +++++++++++++++++++++++++++++--- crates/render/src/lib.rs | 17 ++- crates/render/src/paint.rs | 24 ++++ crates/render/src/transition.rs | 1 + docs/file-format-parity.md | 2 +- docs/gaps.md | 14 +- 6 files changed, 269 insertions(+), 32 deletions(-) diff --git a/crates/engine/src/video.rs b/crates/engine/src/video.rs index c03fec10d9..9987c82812 100644 --- a/crates/engine/src/video.rs +++ b/crates/engine/src/video.rs @@ -4,8 +4,8 @@ //! encoded with DeckCraft's own H.264 encoder ([`deckcraft_h264::Encoder`]: intra-coded changes, //! skipped unchanged macroblocks); the sound of the slides' audio and video clips (narration is //! recorded as such clips) and of transitions is mixed and encoded as AAC ([`deckcraft_aac`]), or -//! kept as PCM in a MOV. Everything is muxed by [`deckcraft_isobmff::Mp4Writer`]; no external -//! tool is needed. Video clips show their poster frame. +//! kept as PCM in a MOV. Video clips play their pictures (decoded by [`deckcraft_media`]) while +//! they play. Everything is muxed by [`deckcraft_isobmff::Mp4Writer`]; no external tool is needed. use std::collections::HashMap; use std::sync::Arc; @@ -87,6 +87,12 @@ pub struct AudioCue { pub fade_in: f64, pub fade_out: f64, pub looped: bool, + /// The video clip whose pictures play with this cue: (slide index, shape). + pub picture: Option<(usize, ShapeId)>, + /// Back to the poster frame when the clip ends. + pub rewind: bool, + /// The clip's length (0: unknown, play until the pictures run out). + pub length: f64, } /// The video's frames and sounds. @@ -117,7 +123,7 @@ fn secs(ms: u32) -> f64 { } /// The clip a cue plays, from a media shape. -fn clip_cue(c: &deckcraft_model::MediaClip, start: f64, until: f64) -> AudioCue { +fn clip_cue(c: &deckcraft_model::MediaClip, index: usize, shape: ShapeId, start: f64, until: f64) -> AudioCue { AudioCue { media: c.media, start, @@ -128,6 +134,9 @@ fn clip_cue(c: &deckcraft_model::MediaClip, start: f64, until: f64) -> AudioCue fade_in: secs(c.fade_in_ms), fade_out: secs(c.fade_out_ms), looped: c.loop_play, + picture: c.video.then_some((index, shape)), + rewind: c.rewind, + length: secs(c.duration_ms), } } @@ -189,6 +198,9 @@ pub fn plan(pres: &Presentation, opts: &VideoOpts) -> Plan { fade_in: 0.0, fade_out: 0.0, looped: false, + picture: None, + rewind: false, + length: 0.0, }, false, )); @@ -209,7 +221,7 @@ pub fn plan(pres: &Presentation, opts: &VideoOpts) -> Plan { if let ShapeKind::Media(c) = &sh.kind && c.autoplay { - cues.push((clip_cue(c, content_start, f64::MAX), !c.play_across_slides)); + cues.push((clip_cue(c, index, sh.id, content_start, f64::MAX), !c.play_across_slides)); started.push(sh.id); } } @@ -225,7 +237,7 @@ pub fn plan(pres: &Presentation, opts: &VideoOpts) -> Plan { continue; } if let Some(ShapeKind::Media(c)) = slide.shape(e.shape).map(|s| &s.kind) { - cues.push((clip_cue(c, step_start + e.start.max(0.0), f64::MAX), !c.play_across_slides)); + cues.push((clip_cue(c, index, e.shape, step_start + e.start.max(0.0), f64::MAX), !c.play_across_slides)); started.push(e.shape); } } @@ -313,15 +325,136 @@ fn state_for(tl: &Timeline, id: ShapeId, step: usize, t: f64) -> ShapeState { } /// Every frame of `steps`, in order, handed to `sink` (RGBA, `w`×`h`). A still or held slide -/// is rendered once. Stops at the first error `sink` returns. +/// is rendered once. Stops at the first error `sink` returns. Video clips show their poster. pub fn frames(pres: &Presentation, steps: &[Step], (w, h): (u32, u32), sink: &mut dyn FnMut(&[u8]) -> Result<(), String>) -> Result { - let animate = !pres.show.without_animation; - let mut timelines: HashMap> = HashMap::new(); - let mut tl_of = - |i: usize| timelines.entry(i).or_insert_with(|| Arc::new(if animate { ShowState::timeline(pres, i) } else { Timeline::default() })).clone(); + let mut r = Renderer::new(pres, &[], 30, (w, h)); let mut n: u64 = 0; for step in steps { - let tl = tl_of(step.index()); + n = n.saturating_add(r.step(step, sink)?); + } + Ok(n) +} + +/// A video clip's pictures, decoded in order as the export reaches them. +struct ClipPictures { + dec: deckcraft_media::VideoDecoder, + cur: Option, + next: Option, + done: bool, +} + +impl ClipPictures { + /// The picture shown `t` seconds into the media (the first one before it starts). + fn at(&mut self, t: f64) -> Option<&deckcraft_media::Frame> { + const EPS: f64 = 1e-6; + if self.cur.as_ref().is_some_and(|c| c.time > t + EPS) { + self.dec.seek(t); + self.cur = None; + self.next = None; + self.done = false; + } + loop { + if self.next.is_none() && !self.done { + match self.dec.next_frame() { + Ok(Some(f)) => self.next = Some(f), + // The end, or a broken stream: keep the last picture. + Ok(None) | Err(_) => self.done = true, + } + } + match &self.next { + Some(n) if n.time <= t + EPS || self.cur.is_none() => { + let late = n.time > t + EPS; + self.cur = self.next.take(); + if late { + break; + } + } + _ => break, + } + } + self.cur.as_ref() + } +} + +/// Renders steps in order, knowing where on the video's clock each frame falls so playing +/// video clips show the right picture. +struct Renderer<'p> { + pres: &'p Presentation, + cues: Vec, + fps: u32, + size: (u32, u32), + /// Frames rendered so far: the clock. + at: u64, + timelines: HashMap>, + pictures: HashMap>, + warnings: Vec, +} + +impl<'p> Renderer<'p> { + fn new(pres: &'p Presentation, cues: &[AudioCue], fps: u32, size: (u32, u32)) -> Self { + let cues = cues.iter().filter(|c| c.picture.is_some()).cloned().collect(); + Renderer { pres, cues, fps: fps.max(1), size, at: 0, timelines: HashMap::new(), pictures: HashMap::new(), warnings: vec![] } + } + + /// Where in its media each video clip of slide `index` is at `t` seconds of the video. + fn clip_times(&self, index: usize, t: f64) -> Vec<(ShapeId, MediaId, f64)> { + let mut out = vec![]; + for c in &self.cues { + let Some((i, shape)) = c.picture else { continue }; + if i != index || t < c.start || t >= c.until { + continue; + } + let local = t - c.start; + let span = (c.length - c.trim_start - c.trim_end).max(0.0); + let pos = if c.length <= 0.0 { + local + } else if c.looped && span > 0.0 { + local % span + } else if local >= span && c.rewind { + continue; + } else { + local.min(span) + }; + out.push((shape, c.media, c.trim_start + pos)); + } + out + } + + /// Whether any clip of slide `index` plays during `n` frames from the clock. + fn plays_during(&self, index: usize, n: u64) -> bool { + let fps = f64::from(self.fps); + let (t0, t1) = (self.at as f64 / fps, self.at.saturating_add(n) as f64 / fps); + self.cues.iter().any(|c| c.picture.is_some_and(|p| p.0 == index) && c.start < t1 && c.until > t0) + } + + fn pictures_for(&mut self, media: MediaId) -> Option<&mut ClipPictures> { + let pres = self.pres; + let warnings = &mut self.warnings; + self.pictures + .entry(media) + .or_insert_with(|| { + let m = pres.media(media)?; + match deckcraft_media::VideoDecoder::open(m.data.clone()) { + Ok(dec) => Some(ClipPictures { dec, cur: None, next: None, done: false }), + Err(e) => { + warnings.push(format!("{}: the video shows its poster frame ({e})", m.name)); + None + } + } + }) + .as_mut() + } + + /// Render one step's frames into `sink`. + fn step(&mut self, step: &Step, sink: &mut dyn FnMut(&[u8]) -> Result<(), String>) -> Result { + let pres = self.pres; + let animate = !pres.show.without_animation; + let index = step.index(); + let tl = self + .timelines + .entry(index) + .or_insert_with(|| Arc::new(if animate { ShowState::timeline(pres, index) } else { Timeline::default() })) + .clone(); let (st, t) = match step { Step::Transition { .. } => (0, 0.0), Step::Anim { step, t, .. } => (*step, *t), @@ -329,22 +462,39 @@ pub fn frames(pres: &Presentation, steps: &[Step], (w, h): (u32, u32), sink: &mu Step::Hold { .. } => (tl.steps(), 0.0), }; // During a transition only the incoming slide's shapes take its starting state. - let incoming = pres.slides.get(step.index()).cloned(); + let incoming = pres.slides.get(index).cloned(); let f = move |id: ShapeId| { if incoming.as_ref().is_some_and(|s| s.shape(id).is_some()) { state_for(&tl, id, st, t) } else { ShapeState::default() } }; - let opts = render_opts(pres, h, Some(&f)); - let img = match step { - Step::Transition { index, t } => deckcraft_render::render_transition(pres, *index, *t, &opts), - _ => deckcraft_render::render_slide(pres, step.index(), &opts), - }; - let px = fit(&img, w, h); - for _ in 0..step.frames() { + let total = step.frames(); + // A still slide is drawn once, unless a video clip on it is playing. + let each = total > 1 && self.plays_during(index, total); + let mut px = vec![]; + for k in 0..total { + if k == 0 || each { + let now = self.at as f64 / f64::from(self.fps); + let times = self.clip_times(index, now); + let mut held: Vec<(ShapeId, deckcraft_media::Frame)> = vec![]; + for (shape, media, ct) in times { + if let Some(p) = self.pictures_for(media).and_then(|p| p.at(ct)) { + held.push((shape, p.clone())); + } + } + let vf: Vec<(ShapeId, deckcraft_render::VideoFrame)> = + held.iter().map(|(s, f)| (*s, deckcraft_render::VideoFrame { width: f.width, height: f.height, rgba: &f.rgba })).collect(); + let mut opts = render_opts(pres, self.size.1, Some(&f)); + opts.video_frames = &vf; + let img = match step { + Step::Transition { index, t } => deckcraft_render::render_transition(pres, *index, *t, &opts), + _ => deckcraft_render::render_slide(pres, index, &opts), + }; + px = fit(&img, self.size.0, self.size.1); + } sink(&px)?; - n = n.saturating_add(1); + self.at = self.at.saturating_add(1); } + Ok(total) } - Ok(n) } /// Decoded clips at [`AUDIO_RATE`], interleaved stereo, by media id (`None`: undecodable). @@ -461,6 +611,7 @@ pub fn encode(pres: &Presentation, format: &str, opts: &VideoOpts) -> Result Result Vec { + use deckcraft_isobmff::{AvcConfig, Brand, CodecConfig, FourCc, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + let mut enc = deckcraft_h264::Encoder::new(deckcraft_h264::EncoderConfig { width: 64, height: 64, fps: 10, qp: 20 }).unwrap(); + let avc = AvcConfig::new(vec![enc.sps()], vec![enc.pps()], 4); + let entry = SampleEntry::video(FourCc::new(b"avc1"), CodecConfig::Avc(avc), 64, 64); + let mut mux = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap(); + let t = mux.add_track(TrackConfig::new(entry, 10)).unwrap(); + for k in 0..20 { + let c = if k < 10 { [230u8, 20, 20, 255] } else { [20, 20, 230, 255] }; + let px: Vec = std::iter::repeat_n(c, 64 * 64).flatten().collect(); + let pic = enc.encode_rgba(&px, k == 0 || k == 10); + mux.write_sample(t, WriteSample { data: &pic.length_prefixed(), duration: 1, composition_offset: 0, is_sync: pic.key }).unwrap(); + } + mux.finish().unwrap().into_inner() + } + + #[test] + fn video_clips_play_their_pictures() { + use serde_json::json; + let mut s = crate::Session::with_new(); + s.execute("slide.new", &json!({"layout": "blank"})).unwrap(); + let data = crate::cmd::base64_encode(&red_then_blue()); + s.execute("insert.video", &json!({"data": data, "name": "clip.mp4", "rect": [0, 0, 960, 540]})).unwrap(); + s.execute("media.options", &json!({"autoplay": true})).unwrap(); + let mut p = (*s.doc().unwrap().doc).clone(); + p.slides.retain(|sl| sl.shapes.iter().any(|sh| matches!(sh.kind, ShapeKind::Media(_)))); + for sl in &mut p.slides { + Arc::make_mut(sl).transition = None; + } + let o = VideoOpts { fps: 10, seconds_per_slide: 3.0, height: 64, audio: AudioFormat::None, ..Default::default() }; + let v = encode(&p, "mp4", &o).unwrap(); + assert_eq!(v.frames, 30); + let mut dec = deckcraft_media::VideoDecoder::open(Arc::new(v.bytes)).unwrap(); + let mut centre = vec![]; + while let Some(f) = dec.next_frame().unwrap() { + let i = ((f.height / 2 * f.width + f.width / 2) * 4) as usize; + centre.push([f.rgba[i], f.rgba[i + 1], f.rgba[i + 2]]); + } + let red = |c: &[u8; 3]| c[0] > 150 && c[2] < 100; + let blue = |c: &[u8; 3]| c[2] > 150 && c[0] < 100; + // Red for the first second, blue for the second, then the last picture held. + assert!(centre[..9].iter().all(red), "{centre:?}"); + assert!(centre[11..].iter().all(blue), "{centre:?}"); + } } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 93b0dc955e..e3825bddb5 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -227,6 +227,8 @@ pub struct RenderOpts<'a> { pub editing_text: Option, /// Animated GIF pictures and how far into their play they are (seconds); others show frame 1. pub gif_times: &'a [(ShapeId, f64)], + /// Video clips showing a decoded frame instead of their poster (video export). + pub video_frames: &'a [(ShapeId, VideoFrame<'a>)], /// Worker threads (0 = this thread; required when effects use filters). pub threads: u16, /// Translate the slide within the output (pixels) and output size override. @@ -240,6 +242,14 @@ pub struct RenderOpts<'a> { pub now: Option, } +/// A decoded video picture: straight RGBA, `width`×`height`. +#[derive(Clone, Copy, Debug)] +pub struct VideoFrame<'a> { + pub width: u32, + pub height: u32, + pub rgba: &'a [u8], +} + impl Default for RenderOpts<'_> { fn default() -> Self { RenderOpts { @@ -251,6 +261,7 @@ impl Default for RenderOpts<'_> { skip_text: None, editing_text: None, gif_times: &[], + video_frames: &[], threads: 0, offset: (0.0, 0.0), size: None, @@ -1084,7 +1095,11 @@ impl Renderer { ShapeKind::Media(mc) => { ctx.set_transform(m); let bounds = Rect::new(0.0, 0.0, w, h); - if let Some(p) = mc.poster { + let playing = if mc.video { f.opts.video_frames.iter().find(|v| v.0 == s.id).map(|v| v.1) } else { None }; + if let Some(vf) = playing + && paint::video_frame(ctx, vf, bounds, m) + { + } else if let Some(p) = mc.poster { let pf = deckcraft_model::style::PictureFill { media: p, ..Default::default() }; paint::picture(ctx, f.pres, &pf, bounds, m, None); } else if mc.video { diff --git a/crates/render/src/paint.rs b/crates/render/src/paint.rs index ee572fd989..8cf6b45833 100644 --- a/crates/render/src/paint.rs +++ b/crates/render/src/paint.rs @@ -279,6 +279,30 @@ pub fn tile_origin(bounds: Rect, align: &str, tile: (f64, f64), offset: (f64, f6 Point::new(bounds.x0 + ax * (bounds.width() - tile.0) + ox, bounds.y0 + ay * (bounds.height() - tile.1) + oy) } +/// A decoded video frame stretched over `bounds`; `false` when it has no usable pixels. +pub fn video_frame(ctx: &mut RenderContext, f: crate::VideoFrame, bounds: Rect, m: Affine) -> bool { + let (w, h) = (f.width, f.height); + let n = (w as usize).saturating_mul(h as usize); + if w == 0 || h == 0 || w > u32::from(u16::MAX) || h > u32::from(u16::MAX) || f.rgba.len() < n.saturating_mul(4) { + return false; + } + let data: Vec = f.rgba.as_chunks::<4>().0[..n] + .iter() + .map(|p| { + let a = u16::from(p[3]); + let k = |c: u8| ((u16::from(c) * a + 127) / 255) as u8; + vello_cpu::color::PremulRgba8 { r: k(p[0]), g: k(p[1]), b: k(p[2]), a: p[3] } + }) + .collect(); + let pm = vello_cpu::Pixmap::from_parts(data, w as u16, h as u16); + let image = vello_cpu::Image { + image: vello_cpu::ImageSource::Pixmap(std::sync::Arc::new(pm)), + sampler: peniko::ImageSampler { quality: peniko::ImageQuality::High, ..Default::default() }, + }; + draw_image(ctx, &image, f64::from(w), f64::from(h), bounds, m, 1.0); + true +} + fn draw_image(ctx: &mut RenderContext, image: &vello_cpu::Image, iw: f64, ih: f64, dest: Rect, m: Affine, alpha: f64) { let t = m * Affine::translate(dest.origin().to_vec2()) * Affine::scale_non_uniform(dest.width() / iw, dest.height() / ih); ctx.set_transform(t); diff --git a/crates/render/src/transition.rs b/crates/render/src/transition.rs index bd6e331756..b16af7c758 100644 --- a/crates/render/src/transition.rs +++ b/crates/render/src/transition.rs @@ -107,6 +107,7 @@ fn copy_opts<'a>(o: &RenderOpts<'a>) -> RenderOpts<'a> { skip_text: o.skip_text, editing_text: o.editing_text, gif_times: o.gif_times, + video_frames: o.video_frames, threads: o.threads, offset: o.offset, size: o.size, diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index f4257e88c9..d3633243d2 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -29,7 +29,7 @@ PowerPoint opens and saves; Viewer = opens only). | PNG / JPEG slide images | export | n/a | yes (`file.export`, `deckcraft-cli render`) | Matches the editor's renderer. PNG `pHYs` and JPEG JFIF density are `scale` × 72 dpi (default scale 2 is 144 dpi), so Preview shows the slide's real size | render and engine tests | | TIFF / BMP / GIF slide images | export | n/a | no | Easy (the `image` crate writes them) | none | | Animated GIF of the deck | export | n/a | yes (`file.export` format `gif`, Export…, `deckcraft-cli`, MCP `export`) | One frame per slide from the slide renderer; size small/medium/large/extra large or 16–3840 px wide, 0.1–600 s per slide, loop forever or play once, slide range, hidden slides skipped; a 256-colour palette per frame (exact, or NeuQuant above 256 colours). Transitions and build animations are not animated | `crates/engine/src/gif_export.rs` tests (decoded frames, size, delay, loop, hidden slides, hostile params), MCP, CLI and export dialog tests | -| MP4 / MOV video of the deck | export | n/a | yes | `file.export` mp4/mov: transitions then each slide for its timing, 480p–4K, own H.264 encoder (intra + skip); in-slide animations on the show timeline; sound (media, narration, transition sounds) as AAC or PCM in .mov; video clips show their poster frame | none | +| MP4 / MOV video of the deck | export | n/a | yes | `file.export` mp4/mov: transitions then each slide for its timing, 480p–4K, own H.264 encoder (intra + skip); in-slide animations on the show timeline; sound (media, narration, transition sounds) as AAC or PCM in .mov; video clips play | none | | Print | yes | n/a | no | PR #66 (browser) open | none | ## Pictures and media inside decks diff --git a/docs/gaps.md b/docs/gaps.md index 2c33e70c16..2d34dcd76e 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -138,13 +138,13 @@ Detail in [file-format-parity.md](file-format-parity.md). - **Impact:** handouts and speaker notes are printed in many workplaces. - **Estimate:** 5–8 h (print through the PDF path and each OS's print dialog). **Beta blocker.** -### Video clips are stills in exported video -- **Evidence:** `file.export` writes MP4/MOV with each transition, the in-slide animations played on - the slide-show timeline (`secondsPerClick` pause before each click), and a sound track (embedded - audio, narration clips, transition sounds) as AAC (`crates/aac`, ported from FilmCraft) or 16-bit - PCM in .mov. Video clips show only their poster frame. -- **Impact:** decks with embedded video lose that motion in the exported video. -- **Estimate:** 3–5 h (decode clip frames into the render, an Export dialog page). +### The animated GIF has no transitions or builds +- **Evidence:** `file.export` writes MP4/MOV with transitions, in-slide animations on the + slide-show timeline, video clips playing their pictures (decoded by `deckcraft-media`), and a + sound track (AAC or PCM in .mov). The animated GIF (#171) still has one frame per slide, and + there is no Export dialog page for video yet. +- **Impact:** small; GIF is a niche export. +- **Estimate:** 2–3 h (an Export dialog page; the GIF can reuse the video's timeline frames). ### Embedded MicroType Express font subsets aren't drawn - **Evidence:** `p:embeddedFontLst` is read and written. A raw TrueType/OpenType face, or one inside an Embedded OpenType wrapper, is loaded and used to draw. A MicroType Express subset (`application/x-fontdata` with neither form) is kept and written back, and the text still uses a stand-in. From 9f7ede5b704e8d9e3303fc156f711449a5215b56 Mon Sep 17 00:00:00 2001 From: Nikita Rakitin Date: Wed, 14 Oct 2026 02:36:23 +0300 Subject: [PATCH 4/4] Export: Create a Video page with progress and Cancel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit File › Export › Create a Video sets the resolution (4K, 1080p, 720p, 480p), frames per second, seconds on each slide and before each click, the sound (AAC in MP4, uncompressed in MOV, none) and hidden slides, then exports in the background: the page shows a progress bar counting frames and a Cancel button, and the status bar reports the result. Commands: app.exportVideo (asks where unless given a path; .mov for PCM) and app.cancelVideoExport. The engine gains video::encode_with (progress callback that can stop the export) and cmd::file::video_opts, so the page and file.export read the same options. On the web the video is encoded at once and downloaded. Co-authored-by: Cursor --- crates/engine/src/cmd/file.rs | 14 ++- crates/engine/src/video.rs | 31 ++++++- crates/ui-egui/src/backstage.rs | 127 ++++++++++++++++++++++++++ crates/ui-egui/src/backstage_tests.rs | 99 ++++++++++++++++++++ crates/ui-egui/src/lib.rs | 74 +++++++++++++++ crates/ui-egui/src/video_job.rs | 74 +++++++++++++++ docs/gaps.md | 7 +- docs/parity-checklist.md | 2 +- docs/ui-parity.md | 4 +- 9 files changed, 421 insertions(+), 11 deletions(-) create mode 100644 crates/ui-egui/src/video_job.rs diff --git a/crates/engine/src/cmd/file.rs b/crates/engine/src/cmd/file.rs index 7edaece865..bc74666943 100644 --- a/crates/engine/src/cmd/file.rs +++ b/crates/engine/src/cmd/file.rs @@ -550,8 +550,9 @@ fn export(s: &mut Session, p: &Value) -> Result { } } -/// File › Export › Create a Video. -fn video_export(doc: &deckcraft_model::Presentation, path: &str, format: &str, p: &Value) -> Result { +/// The `file.export` video options (`height`, `fps`, `secondsPerSlide`, `secondsPerClick`, +/// `includeHidden`, `audio`: aac|pcm|none) as [`crate::video::VideoOpts`]. +pub fn video_opts(p: &Value) -> Result { use crate::video::AudioFormat; let d = crate::video::VideoOpts::default(); let audio = match str_param(p, "audio") { @@ -560,14 +561,19 @@ fn video_export(doc: &deckcraft_model::Presentation, path: &str, format: &str, p Some("none") => AudioFormat::None, Some(other) => return Err(bad("file.export", format!("audio “{other}”: use aac, pcm (.mov) or none"))), }; - let opts = crate::video::VideoOpts { + Ok(crate::video::VideoOpts { height: usize_param(p, "height").map(|h| h.clamp(2, 4320) as u32).unwrap_or(d.height), fps: usize_param(p, "fps").map(|f| f.clamp(1, 60) as u32).unwrap_or(d.fps), seconds_per_slide: f64_or(p, "secondsPerSlide", d.seconds_per_slide).clamp(0.1, 3600.0), seconds_per_click: f64_or(p, "secondsPerClick", d.seconds_per_click).clamp(0.0, 3600.0), include_hidden: bool_or(p, "includeHidden", false), audio, - }; + }) +} + +/// File › Export › Create a Video. +fn video_export(doc: &deckcraft_model::Presentation, path: &str, format: &str, p: &Value) -> Result { + let opts = video_opts(p)?; let v = crate::video::encode(doc, format, &opts).map_err(|e| bad("file.export", e))?; write_file(path, &v.bytes)?; let fps = opts.fps.clamp(1, 60); diff --git a/crates/engine/src/video.rs b/crates/engine/src/video.rs index 9987c82812..c7684b8c1b 100644 --- a/crates/engine/src/video.rs +++ b/crates/engine/src/video.rs @@ -592,6 +592,15 @@ pub struct Encoded { /// Encode the deck as `mp4` or `mov`. The file is assembled in memory. pub fn encode(pres: &Presentation, format: &str, opts: &VideoOpts) -> Result { + encode_with(pres, format, opts, &mut |_, _| true) +} + +/// What [`encode_with`] fails with when its progress callback asks it to stop. +pub const CANCELLED: &str = "the video export was cancelled"; + +/// [`encode`], calling `progress(frames done, frames in all)` after each frame; when it returns +/// `false` the export stops with [`CANCELLED`]. +pub fn encode_with(pres: &Presentation, format: &str, opts: &VideoOpts, progress: &mut dyn FnMut(u64, u64) -> bool) -> Result { use deckcraft_isobmff::{AvcConfig, Brand, CodecConfig, FourCc, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions}; if opts.audio == AudioFormat::Pcm && format != "mov" { return Err("PCM sound needs a .mov file; use AAC for .mp4".into()); @@ -611,16 +620,20 @@ pub fn encode(pres: &Presentation, format: &str, opts: &VideoOpts) -> Result Result<(), String> { let pic = enc.encode_rgba(px, start || since_key >= key_every); start = false; since_key = if pic.key { 1 } else { since_key.saturating_add(1) }; let data = pic.length_prefixed(); - mux.write_sample(track, WriteSample { data: &data, duration: 1, composition_offset: 0, is_sync: pic.key }).map_err(|e| e.to_string()) + mux.write_sample(track, WriteSample { data: &data, duration: 1, composition_offset: 0, is_sync: pic.key }).map_err(|e| e.to_string())?; + done = done.saturating_add(1); + if progress(done, all) { Ok(()) } else { Err(CANCELLED.to_string()) } }; total = total.saturating_add(renderer.step(step, &mut sink)?); } @@ -919,4 +932,20 @@ mod tests { assert!(centre[..9].iter().all(red), "{centre:?}"); assert!(centre[11..].iter().all(blue), "{centre:?}"); } + + #[test] + fn progress_counts_frames_and_can_cancel() { + let p = animated_deck(); + let o = VideoOpts { fps: 10, seconds_per_slide: 1.0, seconds_per_click: 0.5, height: 64, audio: AudioFormat::None, ..Default::default() }; + let mut seen = vec![]; + let v = encode_with(&p, "mp4", &o, &mut |d, n| { + seen.push((d, n)); + true + }) + .unwrap(); + assert_eq!(seen.len() as u64, v.frames); + assert_eq!(seen.last(), Some(&(v.frames, v.frames))); + let r = encode_with(&p, "mp4", &o, &mut |d, _| d < 3); + assert_eq!(r.err().as_deref(), Some(CANCELLED)); + } } diff --git a/crates/ui-egui/src/backstage.rs b/crates/ui-egui/src/backstage.rs index f7bfe25300..ce31ab92d8 100644 --- a/crates/ui-egui/src/backstage.rs +++ b/crates/ui-egui/src/backstage.rs @@ -145,6 +145,7 @@ pub enum PrintScope { pub enum ExportTask { Pdf, Handouts, + Video, FileType, } @@ -175,6 +176,14 @@ pub struct Backstage { pub file_type: &'static str, pub handouts_per_page: u8, pub pdf_hidden: bool, + /// Export › Create a Video: frame height, frames a second, seconds a slide and before each + /// click, the sound (`aac`, `pcm`, `none`), and hidden slides. + pub video_height: u32, + pub video_fps: u32, + pub video_slide_s: f64, + pub video_click_s: f64, + pub video_audio: &'static str, + pub video_hidden: bool, } impl Backstage { @@ -197,6 +206,12 @@ impl Backstage { file_type: "pptx", handouts_per_page: 3, pdf_hidden: false, + video_height: 1080, + video_fps: 30, + video_slide_s: 5.0, + video_click_s: 1.0, + video_audio: "aac", + video_hidden: false, } } @@ -1247,6 +1262,7 @@ fn export(app: &mut SlideApp, ui: &mut Ui, bs: &mut Backstage) { for (task, icon, label) in [ (ExportTask::Pdf, Icon::File, "Create PDF Document"), (ExportTask::Handouts, Icon::Notes, "Create Handouts"), + (ExportTask::Video, Icon::Video, "Create a Video"), (ExportTask::FileType, Icon::Export, "Change File Type"), ] { if place_row(ui, icon, label, bs.export == task, 244.0).clicked() { @@ -1290,6 +1306,7 @@ fn export(app: &mut SlideApp, ui: &mut Ui, bs: &mut Backstage) { app.export_as("pdf", json!({"layout": "handouts", "perPage": bs.handouts_per_page})); } } + ExportTask::Video => video_page(app, ui, bs), ExportTask::FileType => { ui.label(RichText::new("Change File Type").font(theme::font(16.0)).color(t.text)); ui.add_space(8.0); @@ -1319,6 +1336,116 @@ fn export(app: &mut SlideApp, ui: &mut Ui, bs: &mut Backstage) { }); } +/// Video resolutions as PowerPoint names them: (frame height, label). +pub const VIDEO_SIZES: &[(u32, &str)] = &[ + (2160, "Ultra HD (4K) · 3840 × 2160"), + (1080, "Full HD (1080p) · 1920 × 1080"), + (720, "HD (720p) · 1280 × 720"), + (480, "Standard (480p) · 852 × 480"), +]; +/// The sound choices: (`audio` param, label). +pub const VIDEO_AUDIO: &[(&str, &str)] = &[("aac", "AAC sound (MP4)"), ("pcm", "Uncompressed sound (MOV)"), ("none", "No sound")]; + +/// Export › Create a Video: the options, then the export with its progress and Cancel. +fn video_page(app: &mut SlideApp, ui: &mut Ui, bs: &mut Backstage) { + let t = Tokens::get(ui.ctx()); + ui.label(RichText::new("Create a Video").font(theme::font(16.0)).color(t.text)); + bullets( + ui, + &[ + "Plays the transitions, animations, narration and media", + "MP4 plays almost anywhere; MOV can keep the sound uncompressed", + "The presentation keeps its own file", + ], + ); + let running = app.video_job.is_some(); + ui.add_enabled_ui(!running, |ui| { + let label = |ui: &mut Ui, s: &str| { + ui.label(RichText::new(s).font(theme::font(12.5)).color(t.text_dim)); + }; + label(ui, "Resolution"); + let size = VIDEO_SIZES.iter().find(|v| v.0 == bs.video_height).map(|v| v.1).unwrap_or("Custom"); + egui::ComboBox::from_id_salt("backstage_video_size").width(300.0).selected_text(size).show_ui(ui, |ui| { + for (h, l) in VIDEO_SIZES { + if ui.selectable_label(bs.video_height == *h, *l).clicked() { + bs.video_height = *h; + } + } + }); + ui.add_space(6.0); + label(ui, "Frames per second"); + egui::ComboBox::from_id_salt("backstage_video_fps").width(300.0).selected_text(format!("{} fps", bs.video_fps)).show_ui(ui, |ui| { + for f in [24u32, 25, 30, 60] { + if ui.selectable_label(bs.video_fps == f, format!("{f} fps")).clicked() { + bs.video_fps = f; + } + } + }); + ui.add_space(6.0); + ui.horizontal(|ui| { + ui.add(egui::DragValue::new(&mut bs.video_slide_s).range(0.1..=3600.0).speed(0.1).max_decimals(2).suffix(" s")); + ui.label("on each slide without a timing"); + }); + ui.horizontal(|ui| { + ui.add(egui::DragValue::new(&mut bs.video_click_s).range(0.0..=3600.0).speed(0.1).max_decimals(2).suffix(" s")); + ui.label("before each animation click"); + }); + ui.add_space(6.0); + label(ui, "Sound"); + let audio = VIDEO_AUDIO.iter().find(|a| a.0 == bs.video_audio).map(|a| a.1).unwrap_or("AAC sound (MP4)"); + egui::ComboBox::from_id_salt("backstage_video_audio").width(300.0).selected_text(audio).show_ui(ui, |ui| { + for (id, l) in VIDEO_AUDIO { + if ui.selectable_label(bs.video_audio == *id, *l).clicked() { + bs.video_audio = id; + } + } + }); + ui.checkbox(&mut bs.video_hidden, "Include hidden slides"); + }); + ui.add_space(10.0); + match app.video_job.as_ref().map(|j| (j.progress(), j.cancelling())) { + None => { + if square_button(ui, Icon::Video, "Create Video", true).clicked() { + let params = json!({ + "height": bs.video_height, + "fps": bs.video_fps, + "secondsPerSlide": bs.video_slide_s, + "secondsPerClick": bs.video_click_s, + "audio": bs.video_audio, + "includeHidden": bs.video_hidden, + }); + if let Err(e) = app.run("app.exportVideo", params) { + app.set_status(e); + } + } + } + Some(((done, total), cancelling)) => { + let frac = if total > 0 { done as f32 / total as f32 } else { 0.0 }; + let text = if cancelling { + "Cancelling…".to_string() + } else if total > 0 { + format!("Frame {done} of {total}") + } else { + "Starting…".into() + }; + // A flat bar in the accent colour, the frame count beside it. + ui.horizontal(|ui| { + let (r, resp) = ui.allocate_exact_size(vec2(300.0, 8.0), Sense::hover()); + resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::ProgressIndicator, true, &text)); + let p = ui.painter(); + p.rect_filled(r, CornerRadius::same(4), t.border); + let fill = egui::Rect::from_min_size(r.min, vec2((r.width() * frac.clamp(0.0, 1.0)).max(8.0), r.height())); + p.rect_filled(fill, CornerRadius::same(4), t.accent); + ui.label(RichText::new(&text).font(theme::font(12.0)).color(t.text_dim)); + }); + ui.add_space(6.0); + if ui.add_enabled(!cancelling, egui::Button::new("Cancel")).clicked() { + let _ = app.run("app.cancelVideoExport", json!({})); + } + } + } +} + fn bullets(ui: &mut Ui, items: &[&str]) { ui.add_space(4.0); for b in items { diff --git a/crates/ui-egui/src/backstage_tests.rs b/crates/ui-egui/src/backstage_tests.rs index 9b3906a2a3..4a304b768b 100644 --- a/crates/ui-egui/src/backstage_tests.rs +++ b/crates/ui-egui/src/backstage_tests.rs @@ -535,3 +535,102 @@ fn print_slides_follow_the_settings() { bs.print_layout = "slides"; assert_eq!(print_page_size(&bs, &doc), (doc.slide_size.width as f32, doc.slide_size.height as f32), "slides print slide-shaped"); } + +/// Steps the window until the background video export has finished (at most ~60 s). +fn wait_for_video(h: &mut Harness<'_, SlideApp>) { + for _ in 0..3000 { + if h.state().video_job.is_none() { + return; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + h.run_steps(1); + } + panic!("the video export didn't finish"); +} + +/// The window as PNG into `$DECKCRAFT_UI_SHOTS/.png`, to look at (when a GPU renderer +/// is available and the variable is set). +fn shot(h: &mut Harness<'_, SlideApp>, name: &str) { + if let (Ok(dir), Ok(img)) = (std::env::var("DECKCRAFT_UI_SHOTS"), h.render()) { + let _ = img.save(std::path::Path::new(&dir).join(format!("{name}.png"))); + } +} + +#[test] +fn export_creates_a_video_with_the_chosen_options() { + let dir = temp_dir("video"); + let (services, asked) = saving_into(&dir); + let mut h = harness(SlideApp::new(Session::with_new(), services)); + h.state_mut().run("app.backstage", json!({"page": "export"})).unwrap(); + h.run_steps(3); + click_on_page(&h, "Create a Video"); + h.run_steps(3); + assert_eq!(h.state().backstage.as_ref().map(|b| b.export), Some(ExportTask::Video)); + for label in ["Resolution", "Frames per second", "Sound", "Include hidden slides", "Create Video"] { + assert!(h.query_all_by_label(label).next().is_some(), "no “{label}” on the page"); + } + // Pick Standard (480p) in the resolution list, then keep the rest small for the test. + h.query_all_by_role(Role::ComboBox).find(|n| !in_nav(n.rect())).expect("the resolution list").click(); + h.run_steps(3); + h.query_all_by_label_contains("Standard (480p)").last().expect("480p in the list").click(); + h.run_steps(3); + assert_eq!(h.state().backstage.as_ref().map(|b| b.video_height), Some(480)); + shot(&mut h, "video-export-page"); + if let Some(b) = h.state_mut().backstage.as_mut() { + b.video_height = 90; + b.video_fps = 10; + b.video_slide_s = 0.3; + b.video_click_s = 0.1; + b.video_audio = "pcm"; + } + h.run_steps(2); + click_on_page(&h, "Create Video"); + h.run_steps(2); + wait_for_video(&mut h); + let title = h.state().session.active().unwrap().title(); + // PCM sound means a MOV. + assert_eq!(asked.borrow().last(), Some(&format!("{title}.mov"))); + let bytes = std::fs::read(dir.join(format!("{title}.mov"))).unwrap(); + assert_eq!(&bytes[4..8], b"ftyp"); + let status = h.state().status.as_ref().map(|s| s.0.clone()).unwrap_or_default(); + assert!(status.starts_with("Exported"), "{status}"); + assert!(h.state().session.active().unwrap().path.is_none(), "exports are copies"); +} + +#[test] +fn a_video_export_shows_progress_and_can_be_cancelled() { + let dir = temp_dir("video-cancel"); + let (services, _) = saving_into(&dir); + let mut h = harness(SlideApp::new(Session::with_new(), services)); + h.state_mut().run("app.backstage", json!({"page": "export"})).unwrap(); + h.run_steps(3); + click_on_page(&h, "Create a Video"); + h.run_steps(3); + if let Some(b) = h.state_mut().backstage.as_mut() { + b.video_height = 360; + b.video_slide_s = 600.0; + } + click_on_page(&h, "Create Video"); + h.run_steps(2); + assert!(h.state().video_job.is_some()); + assert!(h.state_mut().run("app.exportVideo", json!({})).is_err(), "one export at a time"); + // The progress bar counts frames. + let mut seen = false; + for _ in 0..500 { + h.run_steps(1); + if h.query_all_by_label_contains("Frame ").next().is_some() { + seen = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert!(seen, "no progress shown"); + shot(&mut h, "video-export-progress"); + click_on_page(&h, "Cancel"); + h.run_steps(2); + wait_for_video(&mut h); + assert_eq!(h.state().status.as_ref().map(|s| s.0.as_str()), Some("Video export cancelled")); + let title = h.state().session.active().unwrap().title(); + assert!(!dir.join(format!("{title}.mp4")).exists(), "nothing written"); + assert_eq!(h.state_mut().run("app.cancelVideoExport", json!({})).unwrap(), json!({"cancelled": false})); +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 8867407062..ca5341d53f 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -35,6 +35,7 @@ pub mod theme; pub mod thumbs; pub mod toast; pub mod uifonts; +pub mod video_job; pub mod widgets; use std::collections::HashMap; @@ -406,6 +407,13 @@ pub const UI_COMMANDS: &[(&str, &str, Option<&str>, &str)] = &[ ("app.exportDialog", "Export…", None, "{}"), ("app.browseThemeDialog", "Browse for Themes…", None, "{}"), ("app.compareDialog", "Compare…", None, "{}"), + ( + "app.exportVideo", + "Create a Video", + None, + "{path?, height?, fps?, secondsPerSlide?, secondsPerClick?, audio?: aac|pcm|none, includeHidden?} → {started}", + ), + ("app.cancelVideoExport", "Cancel the Video Export", None, "{} → {cancelled}"), ("app.dialog", "Open Dialog", None, "{id}"), ("app.pasteSpecial", "Paste Special…", Some("Cmd+Alt+V"), "{}"), ("app.qat", "Quick Access Toolbar", None, "{add: command id, params?, label?, icon?} | {remove: id, params?} | {reset: true} → [buttons]"), @@ -453,6 +461,8 @@ pub struct SlideApp { pub status: Option<(String, f64)>, /// A presentation being read in the background, with its progress window ([`opening`]). pub opening: Option, + /// File › Export › Create a Video while it runs (desktop). + pub video_job: Option, /// A brief notice over the bottom of the window (missing fonts on opening a file). pub toast: Option, /// Documents (by uid) whose missing fonts have been mentioned. @@ -545,6 +555,7 @@ impl SlideApp { media: media::MediaHost::new(audio_out), status: None, opening: None, + video_job: None, toast: None, fonts_noted: Default::default(), perf: Perf { frame_ms: 0.0, render_ms: 0.0, fps: 0.0 }, @@ -781,6 +792,63 @@ impl SlideApp { self.download(&suggested, bytes) } + /// File › Export › Create a Video: asks where (unless `path` is given; `.mov` when the sound + /// is PCM) and exports in the background on the desktop, or at once as a download on the + /// web. `{started}` is false when the save dialog was dismissed. + fn export_video(&mut self, p: &Value) -> Result { + if self.video_job.is_some() { + return Err("a video is already being exported".into()); + } + let opts = deckcraft_engine::cmd::file::video_opts(p).map_err(|e| e.to_string())?; + let Some(doc) = self.session.active().map(|d| d.doc.clone()) else { return Err("no presentation is open".into()) }; + let ext = if opts.audio == deckcraft_engine::video::AudioFormat::Pcm { "mov" } else { "mp4" }; + let suggested = self.suggested_name(ext); + let path = match p.get("path").and_then(Value::as_str) { + Some(path) => Some(path.to_string()), + None => match self.services.pick_save.as_mut() { + Some(pick) => match pick(&suggested) { + Some(path) => Some(path), + None => return Ok(json!({"started": false})), + }, + None => None, + }, + }; + match path { + #[cfg(not(target_arch = "wasm32"))] + Some(path) => { + let format = if path.to_ascii_lowercase().ends_with(".mov") { "mov" } else { "mp4" }; + self.video_job = Some(video_job::start(doc, path.clone(), format.into(), opts)?); + self.set_status(format!("Exporting the video to {path}…")); + Ok(json!({"started": true, "path": path})) + } + #[cfg(target_arch = "wasm32")] + Some(_) => Err("this window can't write files; the video is a download instead".into()), + None => { + // The web: no threads, so encode now and hand the file over. + let bytes = deckcraft_engine::video::encode(&doc, ext, &opts) + .map(|v| v.bytes) + .map_err(|e| deckcraft_engine::EngineError::BadParams { cmd: "app.exportVideo".into(), msg: e }); + Ok(json!({"started": self.download(&suggested, bytes)})) + } + } + } + + /// The background video export: its outcome in the status bar once it's done. + fn tick_video_job(&mut self, ctx: &egui::Context) { + let Some(job) = &self.video_job else { return }; + match job.poll() { + Some(r) => { + self.video_job = None; + match r { + Ok(msg) => self.set_status(msg), + Err(e) if e == deckcraft_engine::video::CANCELLED => self.set_status("Video export cancelled"), + Err(e) => self.set_status(format!("Couldn't export the video: {e}")), + } + } + None => ctx.request_repaint_after(std::time::Duration::from_millis(100)), + } + } + /// Hands `bytes` to the browser as a download (web); false (with the reason in the status /// bar) when there are no bytes or no way to download. fn download(&mut self, name: &str, bytes: deckcraft_engine::Result>) -> bool { @@ -1044,6 +1112,11 @@ impl SlideApp { self.set_status("Compare needs the desktop app for now".to_string()); } } + "app.exportVideo" => return self.export_video(p), + "app.cancelVideoExport" => { + let running = self.video_job.as_ref().inspect(|j| j.cancel()).is_some(); + return Ok(json!({"cancelled": running})); + } "app.dialog" => { let d = p.get("id").and_then(Value::as_str).ok_or("missing `id`")?; self.dialog = Some(dialogs::Dialog::new(d)); @@ -1782,6 +1855,7 @@ impl SlideApp { self.tick_media(&ctx); eyedropper::tick(self, &ctx); self.opening_ui(&ctx); + self.tick_video_job(&ctx); if self.show.is_some() { self.anim_preview = None; show::ui(self, ui); diff --git a/crates/ui-egui/src/video_job.rs b/crates/ui-egui/src/video_job.rs new file mode 100644 index 0000000000..c9e10fbe41 --- /dev/null +++ b/crates/ui-egui/src/video_job.rs @@ -0,0 +1,74 @@ +//! File › Export › Create a Video in the background: the export runs on a thread of its own +//! (on the desktop) while the window shows its progress and can cancel it. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::mpsc::Receiver; + +use deckcraft_engine::video::VideoOpts; +use deckcraft_model::Presentation; + +/// A video export under way. +pub struct VideoJob { + pub path: String, + done: Arc, + total: Arc, + cancel: Arc, + rx: Receiver>, +} + +impl VideoJob { + /// Frames written and frames in all (0 until the plan is known). + pub fn progress(&self) -> (u64, u64) { + (self.done.load(Ordering::Relaxed), self.total.load(Ordering::Relaxed)) + } + /// Ask the export to stop; it does after the frame it is on. + pub fn cancel(&self) { + self.cancel.store(true, Ordering::Relaxed); + } + pub fn cancelling(&self) -> bool { + self.cancel.load(Ordering::Relaxed) + } + /// The outcome once the export has finished: a status line, or why it failed. + pub fn poll(&self) -> Option> { + match self.rx.try_recv() { + Ok(r) => Some(r), + Err(std::sync::mpsc::TryRecvError::Empty) => None, + Err(std::sync::mpsc::TryRecvError::Disconnected) => Some(Err("the video export stopped unexpectedly".into())), + } + } +} + +/// Encode `doc` as `format` (`mp4`/`mov`) with `opts` and write it to `path`, calling `progress` +/// after each frame (it returns `false` to stop). The status line for the window. +#[cfg(not(target_arch = "wasm32"))] +pub fn export(doc: &Presentation, path: &str, format: &str, opts: &VideoOpts, progress: &mut dyn FnMut(u64, u64) -> bool) -> Result { + let v = deckcraft_engine::video::encode_with(doc, format, opts, progress)?; + std::fs::write(path, &v.bytes).map_err(|e| format!("couldn't write {path}: {e}"))?; + let secs = v.frames as f64 / f64::from(opts.fps.clamp(1, 60)); + let mut msg = format!("Exported {path} ({secs:.1} s)"); + if let Some(w) = v.warnings.first() { + msg.push_str(&format!(" — {w}")); + } + Ok(msg) +} + +/// Start exporting on a thread of its own. +#[cfg(not(target_arch = "wasm32"))] +pub fn start(doc: Arc, path: String, format: String, opts: VideoOpts) -> Result { + let (done, total, cancel) = (Arc::new(AtomicU64::new(0)), Arc::new(AtomicU64::new(0)), Arc::new(AtomicBool::new(false))); + let (tx, rx) = std::sync::mpsc::channel(); + let (d, t, c, p) = (done.clone(), total.clone(), cancel.clone(), path.clone()); + std::thread::Builder::new() + .name("video-export".into()) + .spawn(move || { + let r = export(&doc, &p, &format, &opts, &mut |n, all| { + d.store(n, Ordering::Relaxed); + t.store(all, Ordering::Relaxed); + !c.load(Ordering::Relaxed) + }); + let _ = tx.send(r); + }) + .map_err(|e| format!("couldn't start the video export: {e}"))?; + Ok(VideoJob { path, done, total, cancel, rx }) +} diff --git a/docs/gaps.md b/docs/gaps.md index 2d34dcd76e..c289f931e8 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -141,10 +141,11 @@ Detail in [file-format-parity.md](file-format-parity.md). ### The animated GIF has no transitions or builds - **Evidence:** `file.export` writes MP4/MOV with transitions, in-slide animations on the slide-show timeline, video clips playing their pictures (decoded by `deckcraft-media`), and a - sound track (AAC or PCM in .mov). The animated GIF (#171) still has one frame per slide, and - there is no Export dialog page for video yet. + sound track (AAC or PCM in .mov); File › Export › Create a Video sets the options and runs it + in the background with progress and Cancel. The animated GIF (#171) still has one frame per + slide. - **Impact:** small; GIF is a niche export. -- **Estimate:** 2–3 h (an Export dialog page; the GIF can reuse the video's timeline frames). +- **Estimate:** 2–4 h (reuse the video's timeline frames for the GIF). ### Embedded MicroType Express font subsets aren't drawn - **Evidence:** `p:embeddedFontLst` is read and written. A raw TrueType/OpenType face, or one inside an Embedded OpenType wrapper, is loaded and used to draw. A MicroType Express subset (`application/x-fontdata` with neither form) is kept and written back, and the text still uses a stand-in. diff --git a/docs/parity-checklist.md b/docs/parity-checklist.md index 94928d8c07..3b00d66a41 100644 --- a/docs/parity-checklist.md +++ b/docs/parity-checklist.md @@ -65,7 +65,7 @@ Open P0 items: Contextual tabs (Shape Format, Picture Format, Table Design/Layou | Files | Export to PDF (slides, notes pages, handouts; text layer, links, bookmarks; vector artwork pending) | P0 | P | | Files | Export slides as PNG/JPEG | P0 | D | | Files | Export outline / RTF | P2 | P | -| Files | Export video (MP4) / animated GIF (GIF: one frame per slide, no transitions; no video) | P2 | P | +| Files | Export video (MP4/MOV with animations, media and sound; File › Export › Create a Video) / animated GIF (one frame per slide, no transitions) | P2 | P | | Files | Print (slides, handouts, notes, outline) | P1 | M | | Files | Password protection | P2 | M | | Files | Reuse slides from another file | P1 | P | diff --git a/docs/ui-parity.md b/docs/ui-parity.md index cc97dae477..e3e6228cc3 100644 --- a/docs/ui-parity.md +++ b/docs/ui-parity.md @@ -24,7 +24,7 @@ ends and are omitted from thumbnails, slide shows and exports. | Ribbon tabs: Home, Insert, Draw, Design, Transitions, Animations, Slide Show, Record, Review, View | 10 | 10 | Record tab is a placeholder | | Colour menus: theme and standard colours, hex field, Eyedropper | yes (plus a More Colors dialog) | yes: theme and standard colours, hex field and an Eyedropper that takes the colour drawn under the pointer on the slide (#367) | More Colors dialog; Eyedropper only takes colours from the slide, not the rest of the screen | | Contextual tabs: Shape Format, Picture Format, Table Design, Table Layout, Chart Design, Chart Format, Video/Audio Format and Playback | yes | most; checklist P0 row partial | Chart Format, media tabs thinner | -| File tab (Backstage): Home, New, Open, Info, Save, Save As, Print, Share, Export, Close, Account, Options | yes | yes (#76), see [File tab](#file-tab-backstage); no Share (cloud) | Video/GIF/CD exports, templates beyond the built-in themes | +| File tab (Backstage): Home, New, Open, Info, Save, Save As, Print, Share, Export, Close, Account, Options | yes | yes (#76), see [File tab](#file-tab-backstage); no Share (cloud) | GIF/CD exports, templates beyond the built-in themes | | Start screen: new, themes, templates, recent, open | yes, the same page as File › Home | yes: File › Home with Home, New, Open, Account and Options (#76) | | | Status bar: slide counter, language, notes, comments, views, zoom slider | yes | yes | | | Command search (Tell Me) | yes | command palette | | @@ -55,7 +55,7 @@ menu colours taken from the theme tokens (menu `chrome`, page `card`). | Save As | Places, folder, file name, type, Save, More options | Type (PowerPoint, DeckCraft, PDF, Outline, PNG, JPEG) and Save, which opens the system Save dialog (web: Download). .pptx/.deckcraft take the new file; the others are copies (`file.saveAs`, `file.export`) | | Print | Print and Copies, Printer, Print All / Selection / Current / Custom Range, Slides box, layout (Full Page Slides, Notes Pages, Outline, Handouts 1–9), sides, collation, colour, Edit Header & Footer; page preview with pager and zoom | Print, Print All Slides / Current Slide / Custom Range, Slides box (`1-3, 5`), Full Page Slides / Notes Pages / Handouts 1–9, Print Hidden Slides, Frame Slides, Edit Header & Footer; preview with pager. DeckCraft prints through a PDF (`file.export`) opened in the system viewer, which has the printers (web: downloaded). No printer list, copies, sides or colour | | Share | Cloud sharing | Not offered | -| Export | Create PDF/XPS Document, Create a Video, Create an Animated GIF, Package Presentation for CD, Create Handouts, Change File Type | Create PDF Document, Create Handouts (a PDF), Change File Type. No video, GIF or CD package | +| Export | Create PDF/XPS Document, Create a Video, Create an Animated GIF, Package Presentation for CD, Create Handouts, Change File Type | Create PDF Document, Create Handouts (a PDF), Create a Video (resolution, fps, seconds per slide and click, sound; progress and Cancel), Change File Type. No GIF or CD package | | Close | Closes; asks Save / Don't Save / Cancel about unsaved changes | Same (`file.close` after the prompt); Ctrl+W asks too | | Account | User information, sign-in, Office Background and Office Theme, product information, About | Appearance (light/dark, `view.dark`), product information, About DeckCraft. No accounts to sign in to | | Options | Opens the PowerPoint Options dialog | Opens Preferences |