diff --git a/Cargo.lock b/Cargo.lock index e2bc79659f..167cfbd5f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -909,6 +909,15 @@ dependencies = [ "winresource", ] +[[package]] +name = "deckcraft-aac" +version = "0.7.0" +dependencies = [ + "deckcraft-bitstream", + "rayon", + "thiserror 2.0.21", +] + [[package]] name = "deckcraft-anim" version = "0.7.0" @@ -954,11 +963,14 @@ dependencies = [ name = "deckcraft-engine" version = "0.7.0" dependencies = [ + "deckcraft-aac", "deckcraft-anim", "deckcraft-color", "deckcraft-fonts", "deckcraft-format", "deckcraft-geom", + "deckcraft-h264", + "deckcraft-isobmff", "deckcraft-media", "deckcraft-model", "deckcraft-pdf", diff --git a/Cargo.toml b/Cargo.toml index c9cb818236..b815999fb8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ deckcraft-ui-egui = { path = "crates/ui-egui" } deckcraft-media = { path = "crates/media" } # codecs (ported from FilmCraft, same owner and licence) deckcraft-bitstream = { path = "crates/bitstream" } +deckcraft-aac = { path = "crates/aac" } deckcraft-h264 = { path = "crates/h264" } deckcraft-hevc = { path = "crates/hevc" } deckcraft-vp9 = { path = "crates/vp9" } diff --git a/crates/aac/Cargo.toml b/crates/aac/Cargo.toml new file mode 100644 index 0000000000..1df55220c7 --- /dev/null +++ b/crates/aac/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "deckcraft-aac" +description = "Clean-room AAC-LC encoder and decoder (ISO/IEC 14496-3), ported from FilmCraft" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[features] +default = [] +rayon = ["dep:rayon"] + +[dependencies] +deckcraft-bitstream = { workspace = true } +thiserror = { workspace = true } +rayon = { workspace = true, optional = true } diff --git a/crates/aac/README.md b/crates/aac/README.md new file mode 100644 index 0000000000..9132995baf --- /dev/null +++ b/crates/aac/README.md @@ -0,0 +1,191 @@ +# deckcraft-aac + +A clean-room AAC-LC encoder and decoder (ISO/IEC 14496-3, MPEG-4 Audio, Low Complexity profile), +written from the standard and the public psychoacoustics literature. Layer L0: it depends only on +`deckcraft-bitstream` and `thiserror` (plus optional `rayon`), has no `unsafe`, and builds for +`wasm32-unknown-unknown`. + +No FFmpeg/FAAC/FDK-AAC/libfaad or other GPL/LGPL code was read or ported. ffmpeg is used only as an +external **test oracle** (decoding our streams, producing reference streams). + +## API + +```rust +use deckcraft_aac::{Decoder, Encoder, EncoderConfig, BitrateMode, AdtsHeader, split_adts}; + +// Encoder: planar f32 (±1.0) → raw access units +let mut enc = Encoder::new(EncoderConfig::cbr(48_000, 2, 128_000))?; // or EncoderConfig::vbr(48_000, 2, 3.0) +let mut aus: Vec> = enc.encode(&[&left, &right]); // 0..n AUs per call, any chunk size +aus.extend(enc.flush()); // zero-pads and drains the look-ahead +let asc = enc.audio_specific_config(); // bytes for the MP4 `esds` +let delay = enc.priming_samples(); // 1024 → edit list media_time +let stats = enc.stats(); // frames, bytes, bitrate(), short blocks, M/S, TNS + +// Decoder: AudioSpecificConfig + raw AUs → planar f32, 1024 samples per channel per AU +let mut dec = Decoder::new(&asc)?; +let pcm: Vec> = dec.decode(&aus[0])?; + +// ADTS +let frames = split_adts(&adts_bytes)?; // [(AdtsHeader, raw payload)] +let mut dec = Decoder::from_adts(&frames[0].0)?; // channel config 0 → layout from the in-band PCE +``` + +`EncoderConfig` fields: `sample_rate` (the 13 AAC rates, 7.35–96 kHz), `channels` (1–8), `mode` +(`BitrateMode::Cbr(bps)` / `BitrateMode::Vbr(quality 1.0..=5.0)`), `bandwidth` (Hz, `None` = automatic +from bitrate/quality), `tns`, `ms_stereo`, `window_shape` (sine/KBD), `adts` (prefix each AU with a +7-byte ADTS header). + +**Channel order** is AAC element order, for both encoder input and decoder output: + +| channels | configuration | order | +|---|---|---| +| 1 | 1 | C | +| 2 | 2 | L R | +| 3 | 3 | C L R | +| 4 | 4 | C L R Cs | +| 5 | 5 | C L R Ls Rs | +| 6 | 6 | C L R Ls Rs LFE | +| 7 | 0 + PCE | C L R Ls Rs Cs LFE (PCE in the ASC; repeated in every AU when ADTS-framed) | +| 8 | 7 | C Lc Rc L R Ls Rs LFE | + +The decoder also reads configurations 11 (6.1: C L R Ls Rs Cs LFE) and 12 (7.1: C L R Ls Rs Lsr Rsr +LFE) from ISO/IEC 14496-3 Amd 4; ffmpeg 9 writes 7.1 AAC as configuration 12. + +`Decoder::layout()` reports the element layout (it follows a PCE for configuration 0). + +## Encoder + +Per 1024-sample frame and channel element: + +1. **Transient detection**: energy of the first-difference (high-passed) signal in 128-sample + sub-blocks against the mean of the previous eight; one frame of look-ahead selects + `ONLY_LONG` / `LONG_START` / `EIGHT_SHORT` / `LONG_STOP`. Short windows are grouped by energy + similarity (an attack starts a new group). LFE is always long. +2. **MDCT** (2048/256 points) via a DCT-IV on an N/4-point radix-2 complex FFT; sine or KBD + (α = 4 long, 6 short) windows. +3. **Psychoacoustic model** in the MDCT domain: band energies, tonality from the spectral flatness of + a ≥16-line neighbourhood, Schroeder's spreading function in the Bark domain, tonality-dependent + masking offset (30 dB tonal … 6 dB noise), Terhardt threshold in quiet (capped at 40 dB SPL) as a + floor, pre-echo control (threshold ≤ 2× previous frame), perceptual entropy. +4. **TNS** on long windows: Levinson–Durbin (order 8) over the spectrum above 1.4 kHz; applied when the + prediction gain exceeds 1.4, with 4-bit reflection coefficients. +5. **M/S** per band (and group) for channel pairs (always common window): chosen when the estimated + perceptual cost of M and S (with the stricter L/R threshold, halved) is below that of L and R. +6. **Rate–distortion loop**: one multiplier λ scales every band's allowed noise. Each band gets the + coarsest scalefactor whose *measured* quantisation error meets its target (binary search, + `0.4054` rounding), scalefactor differences are kept within ±60, codebooks and sections are chosen + by dynamic programming over all 12 codebook states, and λ is bracketed and bisected until the frame + fits its budget (CBR) or fixed from the quality (VBR). +7. **Bit reservoir** (CBR): frame budgets follow the perceptual entropy (short-window frames get extra), + bounded by the 6144-bit-per-channel decoder buffer; overflow is padded with fill elements and the + final frame drains the reservoir, so the stream is exactly `frames × bitrate × 1024 / rate`. + +Bandwidth: automatic from bitrate per full-bandwidth channel (e.g. 64 kbps/ch → 16 kHz, 96 → 19 kHz, +≥144 → 22 kHz) or from quality in VBR. + +Delay: 1024 priming samples (`priming_samples()`); after `flush()` the decoded stream minus the priming +covers every input sample (`ceil(n / 1024) + 1` access units). + +Performance (Apple Silicon, release, single thread): 10 s of 44.1 kHz stereo encodes at ~85× realtime +and decodes at ~1000× realtime. With the `rayon` feature channels are quantised in parallel inside the +rate loop. + +## Decoder + +Full AAC-LC syntax: SCE/CPE/LFE, DSE, FIL, PCE (reconfigures a configuration-0 stream), all window +sequences and both window shapes (shape of the previous frame for the left half), section data, +scalefactors, pulse data, TNS (all-pole synthesis, long and short), M/S (per band / all bands), +intensity stereo (in/out of phase, M/S-inverted), PNS (with correlated noise for `ms_used` noise bands +as specified), IMDCT and overlap-add. Output is planar f32 at ±1.0 full scale. All reads are bounds +checked: malformed access units return `Err`, never panic. + +## Test results + +All produced by `cargo test -p deckcraft-aac -- --nocapture` (ffmpeg 9.0 at `/opt/homebrew/bin` as +oracle; tests skip when it is absent). Fixtures go to `target/fixtures/aac/`. + +### Encoder quality / bitrate (2 s synthetic signals, decoded by ffmpeg) + +SNR is full-band; bwSNR is measured after low-passing both signals at the coded bandwidth; +segSNR averages 1024-sample segments above −50 dBFS (clamped to −10…60 dB). "us-vs-ff" is the +largest difference between our decoder and ffmpeg's on the same stream. Every stream decodes without +ffmpeg errors, no output clips (peak < 1.0), and CBR is within ±0.1% of target (test limit ±5%). + +| signal | kbps | actual | bw Hz | SNR | bwSNR | segSNR | short frames | us-vs-ff | +|---|---|---|---|---|---|---|---|---| +| sweep 2ch 44.1k | 64 | 64.00 | 11000 | 12.1 | 36.1 | 43.6 | 1 | 5e-7 | +| sweep 2ch 44.1k | 128 | 127.92 | 16000 | 26.1 | 27.3 | 54.2 | 1 | 3e-7 | +| sweep 2ch 44.1k | 320 | 319.90 | 21609 | 46.6 | 46.6 | 59.6 | 1 | 3e-7 | +| white noise 2ch 44.1k | 64 | 64.00 | 11000 | 1.3 | 3.3 | 1.3 | 1 | 5e-7 | +| white noise 2ch 44.1k | 320 | 319.91 | 21609 | 16.4 | 16.5 | 16.6 | 1 | 1.5e-6 | +| speech-like 2ch 48k | 64 | 64.00 | 11000 | 13.6 | 14.0 | 15.2 | 13 | 2e-7 | +| speech-like 2ch 48k | 128 | 127.98 | 16000 | 21.3 | 23.0 | 20.7 | 13 | 2e-7 | +| speech-like 2ch 48k | 320 | 319.96 | 22000 | 31.2 | 31.5 | 31.3 | 13 | 2e-7 | +| chords 2ch 44.1k | 64 | 64.01 | 11000 | 12.4 | 12.4 | 20.6 | 4 | 2e-7 | +| chords 2ch 44.1k | 128 | 128.00 | 16000 | 20.1 | 20.1 | 38.5 | 4 | 2e-7 | +| chords 2ch 44.1k | 320 | 319.99 | 21609 | 35.9 | 35.9 | 55.5 | 4 | 2e-7 | +| clicks 2ch 48k | 64 | 64.00 | 11000 | 10.4 | 11.2 | 48.6 | 11 | 9e-8 | +| clicks 2ch 48k | 128 | 128.00 | 16000 | 20.2 | 25.1 | 53.9 | 11 | 2e-7 | +| clicks 2ch 48k | 320 | 319.95 | 22000 | 39.6 | 39.6 | 58.0 | 11 | 1e-7 | +| speech-like 1ch 44.1k | 64 | 64.00 | 16000 | 18.4 | 19.2 | 18.0 | 15 | 2e-7 | +| chords 5.1 48k | 320 | 320.00 | 16000 | 27.2 | 27.2 | 37.9 | 9 | 2e-7 | +| chords 3/4/5/7/8ch 48k | 192–448 | exact | 16000 | 26–28 | 26–28 | 38 | 7–13 | 2e-7 | +| chords 2ch 8 k … 96 k (10 rates) | 24–256 | exact | 3.9–21 k | 13–38 | | 17–47 | | ≤3e-7 | + +Reference points from ffmpeg's own AAC encoder on the same signals (stereo 48 kHz, segSNR): +sweep 128k 39.7, speech 128k 21.2, chords 128k 32.8, clicks 128k 43.3 (ours: 52/20/39/54). Pure SNR +is not a perceptual measure: the model deliberately leaves noise under the masking threshold, and +full-band white noise is the worst case by design. + +VBR (`Vbr(q)`, 44.1 kHz stereo): chords q1 62 kbps / q3 139 / q5 225 (segSNR 18 / 39 / 51); +speech-like q1 42 / q3 166 / q5 272 kbps (segSNR 13 / 29 / 41). Bitrate and quality grow +monotonically with q. + +### Decoder accuracy vs ffmpeg (streams from `ffmpeg -c:a aac`, PNS off) + +| stream | max abs error | +|---|---| +| stereo 44.1k 128k / 48k 256k / 96k 256k | 1.2e-7 / 1.2e-7 / 1.3e-7 | +| mono 48k 64k / 22.05k 32k / 16k 24k / 8k 16k | 1.5e-7 / 1.2e-7 / 1.2e-7 / 8.9e-8 | +| stereo 44.1k 96k with intensity stereo + M/S + TNS | 1.2e-7 | +| stereo 32k 48k with intensity stereo | 1.2e-7 | +| 3.0, 5.1 (320k), 7.1 wide with PCE (448k) | 1.2e-7, 1.5e-7, 1.3e-7 | +| stereo 44.1k 160k `-aac_coder fast` | 1.2e-7 | + +Our own encoder's streams: ≤1.6e-6 against ffmpeg for every case above. PNS (ffmpeg's encoder does not +emit it, so streams are hand-built): total energy within 0.15 dB of ffmpeg; per-frame energy within +0.8 dB (random vectors differ). For noise bands flagged `ms_used` we produce identical L/R noise as +the standard specifies; ffmpeg produces independent noise there. + +### Robustness + +12 000 mutated access units (bit flips, byte overwrites, truncation, extension, splices, random +buffers) over mono/stereo/5.1/7-channel streams, 20 000 random `AudioSpecificConfig`s and 2 000 random +ADTS streams: no panics. The encoder accepts NaN/±inf/over-range input, any chunk size, and missing +channels. + +## Clean-room notes + +- Syntax, tables (scalefactor bands, TNS limits), filterbank, TNS, PNS, M/S and intensity stereo are + implemented from ISO/IEC 14496-3. +- The Huffman codebooks (`src/huffman_tables.rs`) are the normative values of the standard's Annex + 4.A, recovered by black-box probing of the ffmpeg *binary* as an oracle: `tools/` contains the + Python (stdlib-only) scripts that craft single-frame streams repeating a candidate bit string, fit the + decoded output to the IMDCT basis, and walk the code tree. Every table is verified to be a complete + prefix code (Kraft sum exactly 1) covering each index once (unit test), and the decoder matches + ffmpeg to ~1e-7 on real streams. +- Psychoacoustics follow textbook models (Bark scale, Schroeder spreading, Terhardt ATH, Johnston-style + tonality/SFM offsets, perceptual entropy). + +## Gaps / not implemented + +- HE-AAC v1/v2 (SBR/PS): explicit signalling is parsed and only the AAC-LC core is decoded (half-rate, + no SBR); implicit SBR fill elements are skipped. +- AAC Main (prediction), SSR (gain control), LTP, ER/LD/ELD object types, coupling channel elements + (CCE → `Err(Unsupported)`), 960-sample frames. +- Encoder: no PNS or intensity stereo (decoder supports both), no TNS on short windows, no pulse data; + window shape is fixed per stream (sine or KBD). Short blocks at onsets cost segmental SNR on + speech-like material relative to long windows + TNS; perceptually they avoid pre-echo. +- No error concealment: a corrupt AU returns `Err`; the caller may substitute silence. +- ADTS with CRC is parsed (header skipped, CRC not verified); multi-block ADTS frames are split per + frame only. diff --git a/crates/aac/src/config.rs b/crates/aac/src/config.rs new file mode 100644 index 0000000000..58126beaea --- /dev/null +++ b/crates/aac/src/config.rs @@ -0,0 +1,447 @@ +//! `AudioSpecificConfig` (ISO/IEC 14496-3 §1.6.2.1), `program_config_element` (§4.4.1.1) and ADTS +//! framing (§1.A.2.2). + +use deckcraft_bitstream::{BitReader, BitWriter}; + +use crate::tables::{SAMPLE_RATES, table_index_for_rate}; +use crate::{Error, Result}; + +/// Syntactic element types (`id_syn_ele`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ElementType { + /// single_channel_element + Sce, + /// channel_pair_element + Cpe, + /// lfe_channel_element + Lfe, +} + +impl ElementType { + pub fn channels(self) -> usize { + if self == ElementType::Cpe { 2 } else { 1 } + } +} + +/// A `program_config_element` (only the parts that affect channel mapping are interpreted). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ProgramConfig { + pub element_instance_tag: u8, + pub object_type: u8, + pub sf_index: u8, + /// (is_cpe, tag) for front, side and back channel elements. + pub front: Vec<(bool, u8)>, + pub side: Vec<(bool, u8)>, + pub back: Vec<(bool, u8)>, + pub lfe: Vec, + pub assoc_data: Vec, + /// (is_ind_sw, tag) + pub cc: Vec<(bool, u8)>, + pub comment: Vec, +} + +impl ProgramConfig { + pub fn parse(br: &mut BitReader) -> Result { + let mut p = ProgramConfig { + element_instance_tag: br.read_bits(4)? as u8, + object_type: br.read_bits(2)? as u8, + sf_index: br.read_bits(4)? as u8, + ..Default::default() + }; + let nf = br.read_bits(4)?; + let ns = br.read_bits(4)?; + let nb = br.read_bits(4)?; + let nl = br.read_bits(2)?; + let na = br.read_bits(3)?; + let nc = br.read_bits(4)?; + if br.read_bits(1)? == 1 { + br.skip(4)?; + } + if br.read_bits(1)? == 1 { + br.skip(4)?; + } + if br.read_bits(1)? == 1 { + br.skip(3)?; + } + for (n, v) in [(nf, &mut p.front), (ns, &mut p.side), (nb, &mut p.back)] { + for _ in 0..n { + let cpe = br.read_bits(1)? == 1; + v.push((cpe, br.read_bits(4)? as u8)); + } + } + for _ in 0..nl { + p.lfe.push(br.read_bits(4)? as u8); + } + for _ in 0..na { + p.assoc_data.push(br.read_bits(4)? as u8); + } + for _ in 0..nc { + let ind = br.read_bits(1)? == 1; + p.cc.push((ind, br.read_bits(4)? as u8)); + } + br.byte_align(); + let len = br.read_bits(8)?; + for _ in 0..len { + p.comment.push(br.read_bits(8)? as u8); + } + Ok(p) + } + + pub fn write(&self, bw: &mut BitWriter) { + bw.write_bits(self.element_instance_tag as u32, 4); + bw.write_bits(self.object_type as u32, 2); + bw.write_bits(self.sf_index as u32, 4); + bw.write_bits(self.front.len() as u32, 4); + bw.write_bits(self.side.len() as u32, 4); + bw.write_bits(self.back.len() as u32, 4); + bw.write_bits(self.lfe.len() as u32, 2); + bw.write_bits(self.assoc_data.len() as u32, 3); + bw.write_bits(self.cc.len() as u32, 4); + bw.write_bits(0, 3); // no mixdowns + for v in [&self.front, &self.side, &self.back] { + for &(cpe, tag) in v { + bw.write_bits(cpe as u32, 1); + bw.write_bits(tag as u32, 4); + } + } + for &t in &self.lfe { + bw.write_bits(t as u32, 4); + } + for &t in &self.assoc_data { + bw.write_bits(t as u32, 4); + } + for &(ind, t) in &self.cc { + bw.write_bits(ind as u32, 1); + bw.write_bits(t as u32, 4); + } + bw.align_zero(); + bw.write_bits(self.comment.len().min(255) as u32, 8); + for &b in self.comment.iter().take(255) { + bw.write_bits(b as u32, 8); + } + } + + /// Elements in output channel order (front, side, back, LFE). + pub fn layout(&self) -> Vec<(ElementType, u8)> { + let mut out = Vec::new(); + for v in [&self.front, &self.side, &self.back] { + for &(cpe, tag) in v { + out.push((if cpe { ElementType::Cpe } else { ElementType::Sce }, tag)); + } + } + for &t in &self.lfe { + out.push((ElementType::Lfe, t)); + } + out + } +} + +/// Element layout of `channelConfiguration` 1..=7, 11 and 12 (tags as conventionally assigned; 11 and +/// 12 from ISO/IEC 14496-3 Amd 4: 6.1 with a back centre, 7.1 with back surrounds). +pub fn config_layout(channel_config: u8) -> Option> { + use ElementType::*; + Some(match channel_config { + 1 => vec![(Sce, 0)], + 2 => vec![(Cpe, 0)], + 3 => vec![(Sce, 0), (Cpe, 0)], + 4 => vec![(Sce, 0), (Cpe, 0), (Sce, 1)], + 5 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1)], + 6 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Lfe, 0)], + 7 | 12 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Cpe, 2), (Lfe, 0)], + 11 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Sce, 1), (Lfe, 0)], + _ => return None, + }) +} + +/// Parsed `AudioSpecificConfig`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AudioSpecificConfig { + /// Audio object type of the core (2 = AAC LC). + pub object_type: u8, + /// Core sampling rate. + pub sample_rate: u32, + /// Index into the scalefactor-band tables (the signalled index, or derived from an explicit rate). + pub sf_index: u8, + pub channel_config: u8, + pub frame_length_flag: bool, + /// Explicitly signalled SBR/PS (HE-AAC v1/v2); only the AAC-LC core is decoded. + pub sbr: bool, + pub ps: bool, + /// Output (SBR) rate when `sbr` is set. + pub extension_sample_rate: Option, + pub pce: Option, +} + +fn read_aot(br: &mut BitReader) -> Result { + let a = br.read_bits(5)? as u8; + Ok(if a == 31 { 32 + br.read_bits(6)? as u8 } else { a }) +} + +fn read_rate(br: &mut BitReader) -> Result<(u8, u32)> { + let i = br.read_bits(4)? as u8; + if i == 15 { + let r = br.read_bits(24)?; + return Ok((table_index_for_rate(r), r)); + } + let r = *SAMPLE_RATES.get(i as usize).ok_or(Error::InvalidConfig("reserved sampling frequency index"))?; + Ok((i, r)) +} + +impl AudioSpecificConfig { + /// A plain AAC-LC configuration. + pub fn lc(sf_index: u8, channel_config: u8) -> AudioSpecificConfig { + AudioSpecificConfig { + object_type: 2, + sample_rate: SAMPLE_RATES[sf_index as usize], + sf_index, + channel_config, + frame_length_flag: false, + sbr: false, + ps: false, + extension_sample_rate: None, + pce: None, + } + } + + pub fn parse(data: &[u8]) -> Result { + let mut br = BitReader::new(data); + let mut aot = read_aot(&mut br)?; + let (sf_index, sample_rate) = read_rate(&mut br)?; + let channel_config = br.read_bits(4)? as u8; + let (mut sbr, mut ps, mut extension_sample_rate) = (false, false, None); + if aot == 5 || aot == 29 { + sbr = true; + ps = aot == 29; + let (_, ext) = read_rate(&mut br)?; + extension_sample_rate = Some(ext); + aot = read_aot(&mut br)?; + } + if aot != 2 { + return Err(Error::Unsupported(match aot { + 1 => "AAC Main profile", + 3 => "AAC SSR profile", + 4 => "AAC LTP profile", + 23 => "AAC-LD", + 39 => "AAC-ELD", + _ => "audio object type other than AAC LC", + })); + } + let frame_length_flag = br.read_bits(1)? == 1; + if br.read_bits(1)? == 1 { + br.skip(14)?; // coreCoderDelay + } + let _extension_flag = br.read_bits(1)?; + let pce = if channel_config == 0 { Some(ProgramConfig::parse(&mut br)?) } else { None }; + if frame_length_flag { + return Err(Error::Unsupported("960-sample frames")); + } + let cfg = + AudioSpecificConfig { object_type: aot, sample_rate, sf_index, channel_config, frame_length_flag, sbr, ps, extension_sample_rate, pce }; + if cfg.channels() == 0 { + return Err(Error::InvalidConfig("no channels")); + } + Ok(cfg) + } + + /// Serialise (AAC-LC, no SBR signalling). + pub fn to_bytes(&self) -> Vec { + let mut bw = BitWriter::new(); + bw.write_bits(self.object_type as u32, 5); + match crate::tables::sample_rate_index(self.sample_rate) { + Some(i) => bw.write_bits(i as u32, 4), + None => { + bw.write_bits(15, 4); + bw.write_bits(self.sample_rate, 24); + } + } + bw.write_bits(self.channel_config as u32, 4); + bw.write_bits(0, 3); // frameLengthFlag, dependsOnCoreCoder, extensionFlag + if self.channel_config == 0 + && let Some(p) = &self.pce + { + p.write(&mut bw); + } + bw.finish() + } + + /// Elements in output channel order. + pub fn layout(&self) -> Vec<(ElementType, u8)> { + match &self.pce { + Some(p) if self.channel_config == 0 => p.layout(), + _ => config_layout(self.channel_config).unwrap_or_default(), + } + } + + pub fn channels(&self) -> usize { + self.layout().iter().map(|e| e.0.channels()).sum() + } +} + +/// A parsed ADTS header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AdtsHeader { + /// Audio object type (`profile + 1`). + pub object_type: u8, + pub sf_index: u8, + pub channel_config: u8, + /// Whole frame length including the header. + pub frame_length: usize, + pub buffer_fullness: u16, + /// Header length (7, or 9 with CRC). + pub header_length: usize, + pub raw_data_blocks: u8, +} + +impl AdtsHeader { + pub fn parse(data: &[u8]) -> Result { + let mut br = BitReader::new(data); + if br.read_bits(12)? != 0xFFF { + return Err(Error::Bitstream("missing ADTS syncword")); + } + br.skip(1)?; // ID + if br.read_bits(2)? != 0 { + return Err(Error::Bitstream("ADTS layer must be 0")); + } + let protection_absent = br.read_bits(1)? == 1; + let object_type = br.read_bits(2)? as u8 + 1; + let sf_index = br.read_bits(4)? as u8; + if sf_index as usize >= SAMPLE_RATES.len() { + return Err(Error::Bitstream("invalid ADTS sampling frequency index")); + } + br.skip(1)?; + let channel_config = br.read_bits(3)? as u8; + br.skip(4)?; + let frame_length = br.read_bits(13)? as usize; + let buffer_fullness = br.read_bits(11)? as u16; + let raw_data_blocks = br.read_bits(2)? as u8 + 1; + let header_length = if protection_absent { 7 } else { 9 }; + if frame_length < header_length { + return Err(Error::Bitstream("ADTS frame shorter than its header")); + } + Ok(AdtsHeader { object_type, sf_index, channel_config, frame_length, buffer_fullness, header_length, raw_data_blocks }) + } + + /// The equivalent `AudioSpecificConfig`. Channel configuration 0 (PCE in-band) is resolved by the + /// decoder from the first frame; this returns stereo-less layout in that case. + pub fn to_asc(&self) -> AudioSpecificConfig { + AudioSpecificConfig { object_type: self.object_type, ..AudioSpecificConfig::lc(self.sf_index, self.channel_config) } + } + + /// Write a 7-byte header (no CRC) for a frame whose raw payload is `payload_len` bytes. + pub fn write(object_type: u8, sf_index: u8, channel_config: u8, payload_len: usize, buffer_fullness: u16) -> [u8; 7] { + let len = (payload_len + 7) as u32; + let mut bw = BitWriter::new(); + bw.write_bits(0xFFF, 12); + bw.write_bits(0, 1); // MPEG-4 + bw.write_bits(0, 2); + bw.write_bits(1, 1); // protection absent + bw.write_bits((object_type.max(1) - 1) as u32 & 3, 2); + bw.write_bits(sf_index as u32, 4); + bw.write_bits(0, 1); + bw.write_bits(channel_config as u32 & 7, 3); + bw.write_bits(0, 4); + bw.write_bits(len & 0x1FFF, 13); + bw.write_bits(buffer_fullness as u32 & 0x7FF, 11); + bw.write_bits(0, 2); + let v = bw.finish(); + let mut out = [0u8; 7]; + out.copy_from_slice(&v); + out + } +} + +/// Split an ADTS stream into `(header, raw payload)` frames. Stops at the first malformed or +/// truncated frame (returning what was parsed so far); errors only if nothing parses. +pub fn split_adts(mut data: &[u8]) -> Result> { + let mut out = Vec::new(); + while data.len() >= 7 { + let h = match AdtsHeader::parse(data) { + Ok(h) => h, + Err(e) => { + if out.is_empty() { + return Err(e); + } + break; + } + }; + if h.frame_length > data.len() { + break; + } + out.push((h, &data[h.header_length..h.frame_length])); + data = &data[h.frame_length..]; + } + if out.is_empty() { + return Err(Error::Bitstream("no ADTS frames")); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn asc_roundtrip() { + let a = AudioSpecificConfig::lc(4, 2); + let b = a.to_bytes(); + assert_eq!(b, vec![0x12, 0x10]); + assert_eq!(AudioSpecificConfig::parse(&b).unwrap(), a); + // 48 kHz 5.1 + let a = AudioSpecificConfig::lc(3, 6); + assert_eq!(AudioSpecificConfig::parse(&a.to_bytes()).unwrap().channels(), 6); + } + + #[test] + fn amd4_channel_configurations() { + // 6.1 and 7.1 back (#922); 22.2 and 7.1 with heights stay unsupported + let channels = |c| AudioSpecificConfig::parse(&AudioSpecificConfig::lc(3, c).to_bytes()).map(|a| a.channels()).ok(); + assert_eq!([7, 11, 12].map(channels), [Some(8), Some(7), Some(8)]); + assert_eq!([13, 14].map(config_layout), [None, None]); + } + + #[test] + fn asc_with_pce() { + let pce = ProgramConfig { + sf_index: 3, + object_type: 1, + front: vec![(false, 0), (true, 0)], + side: vec![(true, 1)], + back: vec![(false, 1)], + lfe: vec![0], + ..Default::default() + }; + let a = AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) }; + let p = AudioSpecificConfig::parse(&a.to_bytes()).unwrap(); + assert_eq!(p.channels(), 7); + assert_eq!(p, a); + } + + #[test] + fn he_aac_asc_parses_core() { + // AOT 5, 24 kHz core, stereo, ext 48 kHz, core AOT 2 + let mut bw = BitWriter::new(); + bw.write_bits(5, 5); + bw.write_bits(6, 4); + bw.write_bits(2, 4); + bw.write_bits(3, 4); + bw.write_bits(2, 5); + bw.write_bits(0, 3); + let a = AudioSpecificConfig::parse(&bw.finish()).unwrap(); + assert!(a.sbr); + assert_eq!(a.sample_rate, 24000); + assert_eq!(a.extension_sample_rate, Some(48000)); + } + + #[test] + fn adts_roundtrip() { + let h = AdtsHeader::write(2, 4, 2, 100, 0x7FF); + let mut frame = h.to_vec(); + frame.extend(std::iter::repeat_n(0u8, 100)); + let p = AdtsHeader::parse(&frame).unwrap(); + assert_eq!(p.frame_length, 107); + assert_eq!((p.object_type, p.sf_index, p.channel_config), (2, 4, 2)); + let frames = split_adts(&frame).unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].1.len(), 100); + } +} diff --git a/crates/aac/src/decoder.rs b/crates/aac/src/decoder.rs new file mode 100644 index 0000000000..7b3f5ec99f --- /dev/null +++ b/crates/aac/src/decoder.rs @@ -0,0 +1,584 @@ +//! AAC-LC decoder: raw access units (`raw_data_block`, ISO/IEC 14496-3 §4.4.2.1) → planar f32. + +use deckcraft_bitstream::BitReader; + +use crate::config::{AudioSpecificConfig, ElementType, ProgramConfig}; +use crate::huffman::{self, INTENSITY_HCB, INTENSITY_HCB2, NOISE_HCB, ZERO_HCB}; +use crate::ics::{IcsInfo, WindowSequence}; +use crate::mdct::{WindowShape, mdct_long, mdct_short, window}; +use crate::tables::{pow43, sf_gain}; +use crate::tns::{self, TnsData}; +use crate::{Error, Result}; + +const ID_SCE: u32 = 0; +const ID_CPE: u32 = 1; +const ID_CCE: u32 = 2; +const ID_LFE: u32 = 3; +const ID_DSE: u32 = 4; +const ID_PCE: u32 = 5; +const ID_FIL: u32 = 6; +const ID_END: u32 = 7; + +/// `extension_type` of a fill element carrying SBR data (ISO/IEC 14496-3 §4.5.2.8.2, Table 4.121). +const EXT_SBR_DATA: u32 = 13; +const EXT_SBR_DATA_CRC: u32 = 14; + +/// One decoded `individual_channel_stream` before stereo processing and synthesis. +struct Ics { + info: IcsInfo, + /// Band type per (group, sfb). + band_type: [[u8; 64]; 8], + /// Scalefactor / noise energy / intensity position per (group, sfb). + sf: [[i32; 64]; 8], + tns: Option, + /// Dequantised spectrum (short windows at stride 128). + spec: Vec, +} + +#[derive(Clone)] +struct ChannelState { + overlap: Vec, + prev_shape: WindowShape, +} + +/// AAC-LC decoder. +pub struct Decoder { + asc: AudioSpecificConfig, + layout: Vec<(ElementType, u8)>, + /// First output channel of each layout entry. + offsets: Vec, + channels: Vec, + noise_state: u32, + /// A fill element carried SBR data (implicitly signalled HE-AAC). + sbr_seen: bool, +} + +impl Decoder { + /// Create a decoder from an `AudioSpecificConfig`. + pub fn new(asc: &[u8]) -> Result { + Self::from_config(AudioSpecificConfig::parse(asc)?) + } + + /// Create a decoder for an ADTS stream from its first header. With channel configuration 0 the + /// channel layout is taken from the program config element carried in the first frame. + pub fn from_adts(header: &crate::config::AdtsHeader) -> Result { + Self::from_config(header.to_asc()) + } + + pub fn from_config(asc: AudioSpecificConfig) -> Result { + if asc.object_type != 2 { + return Err(Error::Unsupported("audio object type other than AAC LC")); + } + let layout = asc.layout(); + if layout.is_empty() && asc.channel_config != 0 { + return Err(Error::InvalidConfig("unsupported channel configuration")); + } + let mut d = Decoder { asc, layout: Vec::new(), offsets: Vec::new(), channels: Vec::new(), noise_state: 0x2545_F491, sbr_seen: false }; + d.set_layout(layout); + Ok(d) + } + + fn set_layout(&mut self, layout: Vec<(ElementType, u8)>) { + let mut offsets = Vec::with_capacity(layout.len()); + let mut n = 0; + for e in &layout { + offsets.push(n); + n += e.0.channels(); + } + self.layout = layout; + self.offsets = offsets; + self.channels = vec![ChannelState { overlap: vec![0.0; 1024], prev_shape: WindowShape::Sine }; n]; + } + + pub fn config(&self) -> &AudioSpecificConfig { + &self.asc + } + pub fn sample_rate(&self) -> u32 { + self.asc.sample_rate + } + /// Whether the stream is HE-AAC: SBR signalled in the `AudioSpecificConfig` (explicit) or SBR + /// data seen in a fill element of a decoded access unit (implicit). Only the AAC-LC core is + /// decoded, at [`Self::sample_rate`]; the SBR high band is not reconstructed. + pub fn sbr(&self) -> bool { + self.asc.sbr || self.sbr_seen + } + pub fn channels(&self) -> usize { + self.channels.len() + } + /// Element layout in output channel order (C, L/R, surround pairs, LFE for configurations 1–7). + pub fn layout(&self) -> &[(ElementType, u8)] { + &self.layout + } + + /// Clear the overlap state (e.g. after a seek). + pub fn reset(&mut self) { + for c in &mut self.channels { + c.overlap.fill(0.0); + c.prev_shape = WindowShape::Sine; + } + } + + fn sf_index(&self) -> u8 { + self.asc.sf_index + } + + /// Decode one access unit (one `raw_data_block`) into 1024 samples per channel. + pub fn decode(&mut self, au: &[u8]) -> Result>> { + self.decode_block(au, true) + } + + fn decode_block(&mut self, au: &[u8], allow_reconfig: bool) -> Result>> { + let mut br = BitReader::new(au); + let nch = self.channels.len(); + let mut saw_element = false; + let mut out: Vec>> = vec![None; nch]; + let mut used = vec![false; self.layout.len()]; + loop { + let id = br.read_bits(3)?; + saw_element |= matches!(id, ID_SCE | ID_CPE | ID_LFE); + match id { + ID_SCE | ID_LFE => { + let tag = br.read_bits(4)? as u8; + let ty = if id == ID_SCE { ElementType::Sce } else { ElementType::Lfe }; + let slot = self.find_slot(ty, tag, &used)?; + used[slot] = true; + let mut ics = self.decode_ics(&mut br, None)?; + let ch = self.offsets[slot]; + self.fill_noise(&mut ics, None); + out[ch] = Some(self.finish_channel(ch, &mut ics)); + } + ID_CPE => { + let tag = br.read_bits(4)? as u8; + let slot = self.find_slot(ElementType::Cpe, tag, &used)?; + used[slot] = true; + let ch = self.offsets[slot]; + let [a, b] = self.decode_cpe(&mut br, ch)?; + out[ch] = Some(a); + out[ch + 1] = Some(b); + } + ID_CCE => return Err(Error::Unsupported("coupling channel element")), + ID_DSE => { + br.skip(4)?; + let align = br.read_bits(1)? == 1; + let mut count = br.read_bits(8)? as usize; + if count == 255 { + count += br.read_bits(8)? as usize; + } + if align { + br.byte_align(); + } + br.skip(count * 8)?; + } + ID_PCE => { + let pce = ProgramConfig::parse(&mut br)?; + if allow_reconfig && !saw_element && self.asc.channel_config == 0 && self.asc.pce.as_ref() != Some(&pce) { + let layout = pce.layout(); + if !layout.is_empty() && layout.iter().map(|e| e.0.channels()).sum::() <= 64 { + self.asc.pce = Some(pce); + self.set_layout(layout); + return self.decode_block(au, false); + } + } + } + ID_FIL => { + let mut count = br.read_bits(4)? as usize; + if count == 15 { + count += br.read_bits(8)? as usize; + count -= 1; + } + if count > 0 { + let ext = br.read_bits(4)?; + self.sbr_seen |= matches!(ext, EXT_SBR_DATA | EXT_SBR_DATA_CRC); + br.skip(count * 8 - 4)?; + } + } + _ => break, // ID_END + } + if id == ID_END { + break; + } + } + if nch == 0 && !saw_element { + return Err(Error::InvalidConfig("channel configuration 0 without a program config element")); + } + let mut res = Vec::with_capacity(nch); + for o in out { + res.push(o.ok_or(Error::Bitstream("access unit is missing a channel element"))?); + } + Ok(res) + } + + fn find_slot(&self, ty: ElementType, tag: u8, used: &[bool]) -> Result { + let compatible = |t: ElementType| t == ty || (ty != ElementType::Cpe && t != ElementType::Cpe); + if let Some(i) = self.layout.iter().enumerate().position(|(i, &(t, g))| t == ty && g == tag && !used[i]) { + return Ok(i); + } + if let Some(i) = self.layout.iter().enumerate().position(|(i, &(t, _))| t == ty && !used[i]) { + return Ok(i); + } + self.layout + .iter() + .enumerate() + .position(|(i, &(t, _))| compatible(t) && !used[i]) + .ok_or(Error::Bitstream("unexpected channel element for the channel configuration")) + } + + fn decode_cpe(&mut self, br: &mut BitReader, ch: usize) -> Result<[Vec; 2]> { + let common_window = br.read_bits(1)? == 1; + let mut ms_mask_present = 0; + let mut ms_used = [[false; 64]; 8]; + let mut info = None; + if common_window { + let i = IcsInfo::parse(br, self.sf_index())?; + ms_mask_present = br.read_bits(2)?; + if ms_mask_present == 3 { + return Err(Error::Bitstream("reserved ms_mask_present")); + } + if ms_mask_present == 1 { + for g in 0..i.num_groups { + for sfb in 0..i.max_sfb { + ms_used[g][sfb] = br.read_bits(1)? == 1; + } + } + } else if ms_mask_present == 2 { + ms_used = [[true; 64]; 8]; + } + info = Some(i); + } + let mut l = self.decode_ics(br, info)?; + let mut r = self.decode_ics(br, info)?; + let ms = if ms_mask_present > 0 { Some(&ms_used) } else { None }; + self.fill_noise(&mut l, None); + self.fill_noise(&mut r, if common_window { Some((&l, ms)) } else { None }); + if common_window { + let swb = l.info.swb_offsets(self.sf_index()); + for g in 0..l.info.num_groups { + let w0 = l.info.group_start(g); + for sfb in 0..l.info.max_sfb { + let (bl, br_) = (l.band_type[g][sfb], r.band_type[g][sfb]); + let range = swb[sfb] as usize..swb[sfb + 1] as usize; + if br_ == INTENSITY_HCB || br_ == INTENSITY_HCB2 { + let mut c: f32 = if br_ == INTENSITY_HCB { 1.0 } else { -1.0 }; + if ms_mask_present > 0 && ms_used[g][sfb] { + c = -c; + } + let scale = c * (-0.25 * r.sf[g][sfb].clamp(-400, 400) as f64).exp2() as f32; + for w in w0..w0 + l.info.group_len[g] as usize { + for k in range.clone() { + r.spec[w * 128 + k] = l.spec[w * 128 + k] * scale; + } + } + } else if ms_mask_present > 0 && ms_used[g][sfb] && bl < NOISE_HCB && br_ < NOISE_HCB { + for w in w0..w0 + l.info.group_len[g] as usize { + for k in range.clone() { + let (m, s) = (l.spec[w * 128 + k], r.spec[w * 128 + k]); + l.spec[w * 128 + k] = m + s; + r.spec[w * 128 + k] = m - s; + } + } + } + } + } + } + Ok([self.finish_channel(ch, &mut l), self.finish_channel(ch + 1, &mut r)]) + } + + fn decode_ics(&mut self, br: &mut BitReader, common: Option) -> Result { + let sfi = self.sf_index(); + let global_gain = br.read_bits(8)? as i32; + let info = match common { + Some(i) => i, + None => IcsInfo::parse(br, sfi)?, + }; + let swb = info.swb_offsets(sfi); + let num_swb = swb.len() - 1; + let mut ics = Ics { info, band_type: [[0; 64]; 8], sf: [[0; 64]; 8], tns: None, spec: vec![0.0; 1024] }; + + // section_data + let (sect_bits, esc) = if info.is_short() { (3, 7) } else { (5, 31) }; + for g in 0..info.num_groups { + let mut k = 0; + while k < info.max_sfb { + let cb = br.read_bits(4)? as u8; + if cb == 12 { + return Err(Error::Bitstream("reserved codebook 12")); + } + let mut len = 0usize; + loop { + let inc = br.read_bits(sect_bits)?; + len += inc as usize; + if inc != esc { + break; + } + } + if k + len > info.max_sfb { + return Err(Error::Bitstream("section exceeds max_sfb")); + } + if len == 0 { + // zero-length sections are legal but useless; guard against endless loops + if br.bits_left() == 0 { + return Err(Error::Eof); + } + } + for sfb in k..k + len { + ics.band_type[g][sfb] = cb; + } + k += len; + } + } + + // scale_factor_data + let mut sf = global_gain; + let mut noise = global_gain - 90; + let mut is_pos = 0i32; + let mut noise_first = true; + for g in 0..info.num_groups { + for sfb in 0..info.max_sfb { + match ics.band_type[g][sfb] { + ZERO_HCB => ics.sf[g][sfb] = 0, + INTENSITY_HCB | INTENSITY_HCB2 => { + is_pos += huffman::decode_sf(br)?; + ics.sf[g][sfb] = is_pos; + } + NOISE_HCB => { + if noise_first { + noise += br.read_bits(9)? as i32 - 256; + noise_first = false; + } else { + noise += huffman::decode_sf(br)?; + } + ics.sf[g][sfb] = noise; + } + _ => { + sf += huffman::decode_sf(br)?; + if !(0..=255).contains(&sf) { + return Err(Error::Bitstream("scalefactor out of range")); + } + ics.sf[g][sfb] = sf; + } + } + } + } + + // pulse_data + let mut pulses: Vec<(usize, i32)> = Vec::new(); + if br.read_bits(1)? == 1 { + if info.is_short() { + return Err(Error::Bitstream("pulse data in short window")); + } + let n = br.read_bits(2)? as usize + 1; + let start = br.read_bits(6)? as usize; + if start >= num_swb { + return Err(Error::Bitstream("pulse_start_sfb out of range")); + } + let mut k = swb[start] as usize; + for _ in 0..n { + k += br.read_bits(5)? as usize; + let amp = br.read_bits(4)? as i32; + if k >= 1024 { + return Err(Error::Bitstream("pulse position out of range")); + } + pulses.push((k, amp)); + } + } + if br.read_bits(1)? == 1 { + ics.tns = Some(TnsData::parse(br, &info)?); + } + if br.read_bits(1)? == 1 { + return Err(Error::Unsupported("gain control data (AAC SSR)")); + } + + // spectral_data + let mut q = vec![0i32; 1024]; + let mut vals = [0i32; 4]; + for g in 0..info.num_groups { + let w0 = info.group_start(g); + let glen = info.group_len[g] as usize; + for sfb in 0..info.max_sfb { + let cb = ics.band_type[g][sfb]; + if cb == ZERO_HCB || cb >= NOISE_HCB { + continue; + } + let dim = huffman::book(cb).dim; + let (lo, hi) = (swb[sfb] as usize, swb[sfb + 1] as usize); + for w in w0..w0 + glen { + let base = w * 128; + let mut k = lo; + while k < hi { + huffman::decode_spectral(br, cb, &mut vals)?; + q[base + k..base + k + dim].copy_from_slice(&vals[..dim]); + k += dim; + } + } + } + } + for (k, amp) in pulses { + if q[k] > 0 { + q[k] += amp; + } else { + q[k] -= amp; + } + } + + // dequantise + for g in 0..info.num_groups { + let w0 = info.group_start(g); + for sfb in 0..info.max_sfb { + let cb = ics.band_type[g][sfb]; + if cb == ZERO_HCB || cb >= NOISE_HCB { + continue; + } + let gain = sf_gain(ics.sf[g][sfb]); + for w in w0..w0 + info.group_len[g] as usize { + for k in swb[sfb] as usize..swb[sfb + 1] as usize { + let v = q[w * 128 + k]; + let m = pow43(v.unsigned_abs()) * gain; + ics.spec[w * 128 + k] = if v < 0 { -m } else { m }; + } + } + } + } + Ok(ics) + } + + fn next_random(&mut self) -> f32 { + self.noise_state = self.noise_state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (self.noise_state as i32) as f32 + } + + /// Perceptual noise substitution. `pair` = (left channel, ms mask) for the right channel of a CPE: + /// bands that are noise in both channels with `ms_used` reuse the left channel's noise vector. + fn fill_noise(&mut self, ics: &mut Ics, pair: Option<(&Ics, Option<&[[bool; 64]; 8]>)>) { + let info = ics.info; + let swb = info.swb_offsets(self.sf_index()); + for g in 0..info.num_groups { + let w0 = info.group_start(g); + for sfb in 0..info.max_sfb { + if ics.band_type[g][sfb] != NOISE_HCB { + continue; + } + let (lo, hi) = (swb[sfb] as usize, swb[sfb + 1] as usize); + let target = (0.25 * ics.sf[g][sfb].clamp(-400, 400) as f64).exp2(); + let correlated = match pair { + Some((l, Some(ms))) => ms[g][sfb] && l.band_type[g][sfb] == NOISE_HCB, + _ => false, + }; + for w in w0..w0 + info.group_len[g] as usize { + let band = w * 128 + lo..w * 128 + hi; + if correlated && let Some((l, _)) = pair { + ics.spec[band.clone()].copy_from_slice(&l.spec[band.clone()]); + } else { + for k in band.clone() { + ics.spec[k] = self.next_random(); + } + } + let energy: f64 = ics.spec[band.clone()].iter().map(|&v| v as f64 * v as f64).sum(); + let scale = if energy > 0.0 { (target / energy.sqrt()) as f32 } else { 0.0 }; + for k in band { + ics.spec[k] *= scale; + } + } + } + } + } + + /// TNS + filterbank for a single channel element; returns output samples. + fn finish_channel(&mut self, ch: usize, ics: &mut Ics) -> Vec { + if let Some(t) = &ics.tns { + tns::apply_decoder(&mut ics.spec, &ics.info, self.sf_index(), t); + } + synth(&mut self.channels[ch], &ics.info, &ics.spec) + } +} + +/// Inverse MDCT, windowing and overlap-add (§4.6.18), output scaled to ±1.0 full scale. +fn synth(state: &mut ChannelState, info: &IcsInfo, spec: &[f32]) -> Vec { + let mut buf = vec![0f32; 2048]; + let prev = state.prev_shape; + let cur = info.window_shape; + let norm = 1.0 / 32768.0; + match info.window_sequence { + WindowSequence::EightShort => { + let t = mdct_short(); + let mut y = vec![0f32; 256]; + let wc = window(cur, false); + for w in 0..8 { + t.inverse(&spec[w * 128..(w + 1) * 128], &mut y); + let wl = window(if w == 0 { prev } else { cur }, false); + let off = 448 + 128 * w; + let s = norm / 128.0; + for n in 0..128 { + buf[off + n] += y[n] * wl[n] * s; + buf[off + 128 + n] += y[128 + n] * wc[127 - n] * s; + } + } + } + seq => { + let mut y = vec![0f32; 2048]; + mdct_long().inverse(&spec[..1024], &mut y); + let s = norm / 1024.0; + match seq { + WindowSequence::LongStop => { + let ws = window(prev, false); + for n in 448..576 { + buf[n] = y[n] * ws[n - 448] * s; + } + for n in 576..1024 { + buf[n] = y[n] * s; + } + } + _ => { + let wl = window(prev, true); + for n in 0..1024 { + buf[n] = y[n] * wl[n] * s; + } + } + } + match seq { + WindowSequence::LongStart => { + let ws = window(cur, false); + for n in 1024..1472 { + buf[n] = y[n] * s; + } + for n in 1472..1600 { + buf[n] = y[n] * ws[127 - (n - 1472)] * s; + } + } + _ => { + let wl = window(cur, true); + for n in 1024..2048 { + buf[n] = y[n] * wl[2047 - n] * s; + } + } + } + } + } + let out: Vec = (0..1024).map(|n| state.overlap[n] + buf[n]).collect(); + state.overlap.copy_from_slice(&buf[1024..]); + state.prev_shape = cur; + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Implicitly signalled HE-AAC (an AAC-LC config) is recognised from the SBR fill element. + #[test] + fn sbr_fill_element_marks_implicit_he_aac() { + // AAC-LC, 22.05 kHz, stereo + let mut dec = Decoder::new(&[0x13, 0x90]).unwrap(); + assert!(!dec.sbr()); + // ID_FIL, count 1, extension_type EXT_SBR_DATA (+ 4 bits of payload), ID_END + let _ = dec.decode(&[0b1100_0011, 0b1010_0001, 0b1100_0000]); + assert!(dec.sbr()); + // other fill payloads (EXT_FILL) don't + let mut dec = Decoder::new(&[0x13, 0x90]).unwrap(); + let _ = dec.decode(&[0b1100_0010, 0b0000_0001, 0b1100_0000]); + assert!(!dec.sbr()); + // explicit signalling: AOT 5, core 22.05 kHz, stereo, extension 44.1 kHz, core AOT 2 + let dec = Decoder::new(&[0x2B, 0x92, 0x08, 0x00]).unwrap(); + assert!(dec.sbr()); + assert_eq!(dec.sample_rate(), 22_050); + } +} diff --git a/crates/aac/src/encoder/mod.rs b/crates/aac/src/encoder/mod.rs new file mode 100644 index 0000000000..17b0e27433 --- /dev/null +++ b/crates/aac/src/encoder/mod.rs @@ -0,0 +1,833 @@ +//! AAC-LC encoder. +//! +//! Pipeline per 1024-sample frame and channel element: +//! 1. transient detection on high-passed 128-sample sub-blocks (one frame of look-ahead) → +//! window sequence (`ONLY_LONG` / `LONG_START` / `EIGHT_SHORT` / `LONG_STOP`) and short-window grouping; +//! 2. windowed MDCT (sine or KBD); +//! 3. psychoacoustic model → per-band masking thresholds and perceptual entropy; +//! 4. TNS analysis (long windows; Levinson–Durbin, applied when the prediction gain is high); +//! 5. per-band M/S decision for channel pairs (common window); +//! 6. rate–distortion loop: a single multiplier λ scales every band's allowed noise; each band gets the +//! coarsest scalefactor meeting its noise target, sections/codebooks are chosen by dynamic +//! programming, and λ is bisected until the frame fits its bit budget (CBR) or fixed (VBR); +//! 7. bit reservoir (CBR): budget follows the frame's PE, bounded by the decoder buffer +//! (6144 bits/channel); overflow is padded with fill elements so the rate stays constant. + +mod psy; +mod quant; + +use deckcraft_bitstream::BitWriter; + +use crate::config::{AdtsHeader, AudioSpecificConfig, ElementType, ProgramConfig, config_layout}; +use crate::ics::{IcsInfo, WindowSequence}; +use crate::mdct::{WindowShape, mdct_long, mdct_short, window}; +use crate::tables::{sample_rate_index, swb_offsets_long, swb_offsets_short, tns_max_bands}; +use crate::tns::{self, TnsData, TnsFilter}; +use crate::{Error, FRAME_LEN, Result}; +use psy::Psy; +use quant::{ChannelInput, QuantChannel, quantise_channel, write_ics}; + +/// Rate control mode. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum BitrateMode { + /// Constant bitrate in bits per second (bit reservoir + fill elements). + Cbr(u32), + /// Constant quality, 1.0 (small) ..= 5.0 (transparent-ish). 3.0 targets the masking threshold. + Vbr(f32), +} + +/// Encoder settings. +#[derive(Debug, Clone, PartialEq)] +pub struct EncoderConfig { + /// One of the 13 AAC sampling rates (8–96 kHz). + pub sample_rate: u32, + /// 1–8. Channel order is the AAC element order: C; L R; C L R; C L R Cs; C L R Ls Rs; + /// C L R Ls Rs LFE; (7) C L R Ls Rs Cs LFE; (8) C Lc Rc L R Ls Rs LFE. + pub channels: usize, + pub mode: BitrateMode, + /// Audio bandwidth in Hz (`None` = chosen from the bitrate / quality). + pub bandwidth: Option, + /// Temporal noise shaping on long windows. + pub tns: bool, + /// Per-band mid/side stereo for channel pairs. + pub ms_stereo: bool, + pub window_shape: WindowShape, + /// Prefix every access unit with an ADTS header. + pub adts: bool, +} + +impl EncoderConfig { + pub fn cbr(sample_rate: u32, channels: usize, bitrate_bps: u32) -> EncoderConfig { + EncoderConfig { + sample_rate, + channels, + mode: BitrateMode::Cbr(bitrate_bps), + bandwidth: None, + tns: true, + ms_stereo: true, + window_shape: WindowShape::Sine, + adts: false, + } + } + pub fn vbr(sample_rate: u32, channels: usize, quality: f32) -> EncoderConfig { + EncoderConfig { mode: BitrateMode::Vbr(quality), ..Self::cbr(sample_rate, channels, 0) } + } +} + +/// Running statistics. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct EncoderStats { + pub frames: u64, + /// Raw access-unit bytes (without ADTS headers). + pub bytes: u64, + /// Element-frames coded with eight short windows. + pub short_blocks: u64, + pub ms_bands: u64, + pub stereo_bands: u64, + pub tns_filters: u64, + /// Bits spent on fill elements (CBR padding). + pub fill_bits: u64, +} + +impl EncoderStats { + /// Average bitrate of the raw access units. + pub fn bitrate(&self, sample_rate: u32) -> f64 { + if self.frames == 0 { 0.0 } else { self.bytes as f64 * 8.0 * sample_rate as f64 / (self.frames as f64 * FRAME_LEN as f64) } + } +} + +struct ElementState { + ty: ElementType, + tag: u8, + first_ch: usize, + prev_seq: WindowSequence, + short_next: Option, + /// Previous long-window thresholds per channel (pre-echo control). + prev_thr: Vec>, +} + +/// Per-frame analysis of one element. +struct ElementFrame { + info: IcsInfo, + chans: Vec, + ms_mask: Option<(u32, Vec)>, +} + +const HISTORY: usize = 2048; + +/// AAC-LC encoder. +pub struct Encoder { + cfg: EncoderConfig, + sf_index: u8, + asc: AudioSpecificConfig, + psy: Psy, + elements: Vec, + /// Input history per channel; `hist[c][0]` is absolute sample `origin`. + hist: Vec>, + origin: i64, + total_in: u64, + frames_done: u64, + flushed: bool, + bandwidth: u32, + // rate control + mean_acc: u64, + level: i64, + res_max: i64, + max_frame_bits: i64, + pe_avg: f32, + lambda: f32, + stats: EncoderStats, +} + +impl Encoder { + pub fn new(cfg: EncoderConfig) -> Result { + let sf_index = + sample_rate_index(cfg.sample_rate).ok_or(Error::InvalidConfig("sample rate must be one of the 13 AAC rates (7350–96000 Hz)"))?; + if !(1..=8).contains(&cfg.channels) { + return Err(Error::InvalidConfig("channels must be 1..=8")); + } + use ElementType::*; + let (channel_config, layout) = match cfg.channels { + 7 => (0u8, vec![(Sce, 0u8), (Cpe, 0), (Cpe, 1), (Sce, 1), (Lfe, 0)]), + 8 => (7, config_layout(7).unwrap_or_default()), + n => (n as u8, config_layout(n as u8).unwrap_or_default()), + }; + let mut asc = AudioSpecificConfig::lc(sf_index, channel_config); + if channel_config == 0 { + asc.pce = Some(ProgramConfig { + element_instance_tag: 0, + object_type: 1, + sf_index, + front: vec![(false, 0), (true, 0)], + side: vec![(true, 1)], + back: vec![(false, 1)], + lfe: vec![0], + ..Default::default() + }); + } + let mut elements = Vec::new(); + let mut ch = 0; + for (ty, tag) in layout { + elements.push(ElementState { + ty, + tag, + first_ch: ch, + prev_seq: WindowSequence::OnlyLong, + short_next: None, + prev_thr: vec![Vec::new(); ty.channels()], + }); + ch += ty.channels(); + } + let nfull = cfg.channels - elements.iter().filter(|e| e.ty == Lfe).count(); + let nyq = cfg.sample_rate / 2; + let auto_bw = match cfg.mode { + BitrateMode::Cbr(b) => { + let per = b as f32 / nfull.max(1) as f32 / 1000.0; + let pts = [ + (12.0f32, 4000.0f32), + (16.0, 5500.0), + (24.0, 8000.0), + (32.0, 11000.0), + (48.0, 14000.0), + (64.0, 16000.0), + (80.0, 17500.0), + (96.0, 19000.0), + (112.0, 20000.0), + (144.0, 22000.0), + ]; + interp(&pts, per) as u32 + } + BitrateMode::Vbr(q) => interp(&[(1.0, 13000.0), (2.0, 15000.0), (3.0, 17000.0), (4.0, 19000.0), (5.0, 20000.0)], q) as u32, + }; + let bandwidth = cfg.bandwidth.unwrap_or(auto_bw).clamp(1000, nyq.saturating_sub(nyq / 50).max(1000)); + let max_frame_bits = 6144 * cfg.channels as i64; + let mean = match cfg.mode { + BitrateMode::Cbr(b) => b as i64 * FRAME_LEN as i64 / cfg.sample_rate as i64, + BitrateMode::Vbr(_) => 0, + }; + if let BitrateMode::Cbr(b) = cfg.mode + && (b < 1000 * cfg.channels as u32 || mean > max_frame_bits) + { + return Err(Error::InvalidConfig("bitrate out of range for the channel count and sample rate")); + } + let res_max = (max_frame_bits - mean).max(0); + let hist = vec![vec![0f32; HISTORY]; cfg.channels]; + Ok(Encoder { + psy: Psy::new(cfg.sample_rate, sf_index), + sf_index, + asc, + elements, + hist, + origin: -(HISTORY as i64), + total_in: 0, + frames_done: 0, + flushed: false, + bandwidth, + mean_acc: 0, + level: 0, + res_max, + max_frame_bits, + pe_avg: 0.0, + lambda: 1.0, + stats: EncoderStats::default(), + cfg, + }) + } + + pub fn config(&self) -> &EncoderConfig { + &self.cfg + } + + /// `AudioSpecificConfig` bytes for the MP4 `esds` decoder-specific info. + pub fn audio_specific_config(&self) -> Vec { + self.asc.to_bytes() + } + + /// Encoder delay: the first 1024 decoded samples are priming and should be skipped (MP4 edit + /// list `media_time`, or iTunSMPB). + pub fn priming_samples(&self) -> u32 { + FRAME_LEN as u32 + } + + /// Audio bandwidth in Hz. + pub fn bandwidth(&self) -> u32 { + self.bandwidth + } + + pub fn stats(&self) -> EncoderStats { + self.stats + } + + /// Feed planar samples (±1.0 full scale); returns any complete access units. Missing channels are + /// treated as silence; all channels are truncated to the shortest slice. + pub fn encode(&mut self, planar: &[&[f32]]) -> Vec> { + if self.flushed { + return Vec::new(); + } + let n = planar.iter().take(self.cfg.channels).map(|p| p.len()).min().unwrap_or(0); + for c in 0..self.cfg.channels { + match planar.get(c) { + Some(p) => self.hist[c].extend(p[..n].iter().map(|v| if v.is_finite() { v.clamp(-8.0, 8.0) } else { 0.0 })), + None => self.hist[c].extend(std::iter::repeat_n(0.0, n)), + } + } + self.total_in += n as u64; + let mut out = Vec::new(); + while self.available() >= self.needed_for(self.frames_done) { + out.push(self.encode_frame(false)); + } + out + } + + /// Encode the remaining input (zero-padded). Afterwards the decoded stream, minus + /// [`priming_samples`](Self::priming_samples), covers every input sample. + pub fn flush(&mut self) -> Vec> { + if self.flushed { + return Vec::new(); + } + let frames = if self.total_in == 0 { 0 } else { self.total_in.div_ceil(FRAME_LEN as u64) + 1 }; + let mut out = Vec::new(); + while self.frames_done < frames { + let need = self.needed_for(self.frames_done); + let have = self.available(); + if have < need { + for h in &mut self.hist { + h.extend(std::iter::repeat_n(0.0, (need - have) as usize)); + } + } + let last = self.frames_done + 1 == frames; + out.push(self.encode_frame(last)); + } + self.flushed = true; + out + } + + /// Absolute sample index one past the buffered input. + fn available(&self) -> i64 { + self.origin + self.hist[0].len() as i64 + } + /// Input needed (absolute end) before frame `t` can be encoded (window + transient look-ahead). + fn needed_for(&self, t: u64) -> i64 { + (t as i64 + 1) * FRAME_LEN as i64 + 640 + } + + fn sample(&self, ch: usize, abs: i64) -> f32 { + let i = abs - self.origin; + if i < 0 { 0.0 } else { self.hist[ch].get(i as usize).copied().unwrap_or(0.0) } + } + + fn subblock_energy(&self, ch: usize, j: i64) -> f32 { + let s = j * 128; + let mut e = 0f32; + let mut prev = self.sample(ch, s - 1); + for n in s..s + 128 { + let x = self.sample(ch, n); + let d = x - prev; + e += d * d; + prev = x; + } + e + } + + /// Transient in the region covered by frame `t`'s short windows. + fn transient(&self, first_ch: usize, nch: usize, t: i64) -> bool { + for ch in first_ch..first_ch + nch { + let base = 8 * t; + let mut hist: Vec = (base - 13..base - 5).map(|j| self.subblock_energy(ch, j)).collect(); + for j in base - 5..=base + 4 { + let e = self.subblock_energy(ch, j); + let mean = hist.iter().sum::() / hist.len() as f32; + if e > 1.5e-4 && e > 10.0 * mean { + return true; + } + hist.remove(0); + hist.push(e); + } + } + false + } + + fn max_sfb(&self, short: bool, lfe: bool) -> usize { + let bw = if lfe { 240.min(self.bandwidth) } else { self.bandwidth }; + let (swb, m) = if short { (swb_offsets_short(self.sf_index), 128f64) } else { (swb_offsets_long(self.sf_index), 1024f64) }; + let bin = (bw as f64 / (self.cfg.sample_rate as f64 / 2.0) * m).ceil() as u16; + swb.iter().take(swb.len() - 1).filter(|&&o| o < bin).count().max(1) + } + + fn mdct(&self, ch: usize, seq: WindowSequence) -> Vec { + let shape = self.cfg.window_shape; + let seg = &self.hist[ch][HISTORY - FRAME_LEN..HISTORY + FRAME_LEN]; + let mut spec = vec![0f32; 1024]; + let scale = 32768.0 * 2.0; + if seq == WindowSequence::EightShort { + let ws = window(shape, false); + let mut buf = vec![0f32; 256]; + for w in 0..8 { + let off = 448 + 128 * w; + for n in 0..128 { + buf[n] = seg[off + n] * ws[n] * scale; + buf[128 + n] = seg[off + 128 + n] * ws[127 - n] * scale; + } + mdct_short().forward(&buf, &mut spec[w * 128..(w + 1) * 128]); + } + return spec; + } + let wl = window(shape, true); + let ws = window(shape, false); + let mut buf = vec![0f32; 2048]; + for n in 0..1024 { + let w = match seq { + WindowSequence::LongStop => { + if n < 448 { + 0.0 + } else if n < 576 { + ws[n - 448] + } else { + 1.0 + } + } + _ => wl[n], + }; + buf[n] = seg[n] * w * scale; + } + for n in 1024..2048 { + let w = match seq { + WindowSequence::LongStart => { + if n < 1472 { + 1.0 + } else if n < 1600 { + ws[127 - (n - 1472)] + } else { + 0.0 + } + } + _ => wl[2047 - n], + }; + buf[n] = seg[n] * w * scale; + } + mdct_long().forward(&buf, &mut spec); + spec + } + + fn tns_analyse(&self, spec: &mut [f32], info: &IcsInfo) -> Option { + let swb = swb_offsets_long(self.sf_index); + let lim = tns_max_bands(self.sf_index, false).min(info.max_sfb); + let hz = self.cfg.sample_rate as f64 / 2048.0; + let start = swb.iter().position(|&o| o as f64 * hz >= 1400.0).unwrap_or(swb.len() - 1); + if lim < start + 4 { + return None; + } + let (lo, hi) = (swb[start] as usize, swb[lim] as usize); + let x = &spec[lo..hi]; + const ORDER: usize = 8; + let mut r = [0f64; ORDER + 1]; + for (lag, rv) in r.iter_mut().enumerate() { + let mut acc = 0f64; + for n in lag..x.len() { + acc += x[n] as f64 * x[n - lag] as f64; + } + // Gaussian lag window for robustness + *rv = acc * (-0.5 * (0.02 * lag as f64).powi(2)).exp(); + } + if r[0] <= 1e-3 { + return None; + } + let (ks, gain) = tns::levinson(&r, ORDER); + if gain < 1.4 { + return None; + } + let mut coef = [0i8; 12]; + let mut order = 0; + for (i, &k) in ks.iter().enumerate() { + coef[i] = tns::quant_coef(k, true); + if coef[i] != 0 { + order = i + 1; + } + } + if order == 0 { + return None; + } + let mut t = TnsData::default(); + t.windows[0].n_filt = 1; + t.windows[0].coef_res = true; + t.windows[0].filt[0] = TnsFilter { length: (swb.len() - 1 - start) as u8, order: order as u8, direction: false, coef_compress: false, coef }; + tns::apply_encoder(spec, info, self.sf_index, &t); + Some(t) + } + + fn analyse_element(&mut self, ei: usize, t: i64) -> (ElementFrame, f32) { + let (ty, first, nch, prev_seq, cached) = { + let e = &self.elements[ei]; + (e.ty, e.first_ch, e.ty.channels(), e.prev_seq, e.short_next) + }; + let lfe = ty == ElementType::Lfe; + let (short_now, short_next) = + if lfe { (false, false) } else { (cached.unwrap_or_else(|| self.transient(first, nch, t)), self.transient(first, nch, t + 1)) }; + use WindowSequence::*; + let seq = if short_now { + EightShort + } else if prev_seq == EightShort || prev_seq == LongStart { + if short_next { EightShort } else { LongStop } + } else if short_next { + LongStart + } else { + OnlyLong + }; + let specs: Vec> = (first..first + nch).map(|c| self.mdct(c, seq)).collect(); + let short = seq == EightShort; + let mut info = IcsInfo { window_sequence: seq, window_shape: self.cfg.window_shape, max_sfb: self.max_sfb(short, lfe), ..Default::default() }; + if short { + // group windows of similar energy; an attack starts a new group + let en: Vec = + (0..8).map(|w| specs.iter().map(|s| s[w * 128..(w + 1) * 128].iter().map(|v| v * v).sum::()).sum::() + 1.0).collect(); + info.num_groups = 1; + info.group_len = [1, 0, 0, 0, 0, 0, 0, 0]; + for w in 1..8 { + let r = en[w] / en[w - 1]; + if !(1.0 / 6.0..=3.0).contains(&r) { + info.num_groups += 1; + info.group_len[info.num_groups - 1] = 1; + } else { + info.group_len[info.num_groups - 1] += 1; + } + } + } + let mut pe = 0f32; + let mut specs = specs; + let mut thrs: Vec>> = Vec::with_capacity(nch); + let mut tnss = Vec::with_capacity(nch); + for (i, spec) in specs.iter_mut().enumerate() { + let mut r = self.psy.analyse(spec, short); + pe += r.pe; + let e = &mut self.elements[ei]; + if !short { + if prev_seq != EightShort && e.prev_thr[i].len() == r.thr[0].len() { + for (t, p) in r.thr[0].iter_mut().zip(&e.prev_thr[i]) { + *t = t.min(2.0 * p.max(1e-9)); + } + } + e.prev_thr[i] = r.thr[0].clone(); + } else { + e.prev_thr[i].clear(); + } + // group thresholds: the most sensitive window times the group length + let mut thr_g = vec![vec![0f32; 64]; info.num_groups]; + for g in 0..info.num_groups { + let w0 = info.group_start(g); + let gl = info.group_len[g] as usize; + for sfb in 0..info.max_sfb { + let m = (w0..w0 + gl).map(|w| r.thr[w][sfb]).fold(f32::MAX, f32::min); + thr_g[g][sfb] = m * gl as f32; + } + } + thrs.push(thr_g); + let tns = if self.cfg.tns && !short && !lfe { self.tns_analyse(spec, &info) } else { None }; + if let Some(t) = &tns { + self.stats.tns_filters += t.windows[0].n_filt as u64; + } + tnss.push(tns); + } + let swb = info.swb_offsets(self.sf_index); + let mut ms_mask = None; + if ty == ElementType::Cpe { + let mut used = Vec::with_capacity(info.num_groups * info.max_sfb); + let (l, r) = specs.split_at_mut(1); + let (l, r) = (&mut l[0], &mut r[0]); + for g in 0..info.num_groups { + let w0 = info.group_start(g); + let gl = info.group_len[g] as usize; + for sfb in 0..info.max_sfb { + let (mut el, mut er, mut em, mut es) = (0f32, 0f32, 0f32, 0f32); + for w in w0..w0 + gl { + for k in w * 128 + swb[sfb] as usize..w * 128 + swb[sfb + 1] as usize { + let (a, b) = (l[k], r[k]); + el += a * a; + er += b * b; + em += 0.25 * (a + b) * (a + b); + es += 0.25 * (a - b) * (a - b); + } + } + let (tl, tr) = (thrs[0][g][sfb].max(1e-9), thrs[1][g][sfb].max(1e-9)); + let tm = 0.5 * tl.min(tr); + let c = |e: f32, t: f32| (e / t).max(1.0).log2(); + let use_ms = self.cfg.ms_stereo && c(em, tm) + c(es, tm) < c(el, tl) + c(er, tr); + used.push(use_ms); + self.stats.stereo_bands += 1; + if use_ms { + self.stats.ms_bands += 1; + thrs[0][g][sfb] = tm; + thrs[1][g][sfb] = tm; + for w in w0..w0 + gl { + for k in w * 128 + swb[sfb] as usize..w * 128 + swb[sfb + 1] as usize { + let (a, b) = (l[k], r[k]); + l[k] = 0.5 * (a + b); + r[k] = 0.5 * (a - b); + } + } + } + } + } + let n_ms = used.iter().filter(|&&u| u).count(); + let present = if n_ms == 0 { + 0 + } else if n_ms == used.len() { + 2 + } else { + 1 + }; + ms_mask = Some((present, used)); + } + let chans = specs.iter().zip(thrs.iter()).zip(tnss).map(|((s, th), tn)| ChannelInput::new(info, swb, s, th, tn)).collect(); + let e = &mut self.elements[ei]; + e.prev_seq = seq; + e.short_next = Some(short_next); + if short { + self.stats.short_blocks += 1; + } + (ElementFrame { info, chans, ms_mask }, pe) + } + + fn element_overhead(&self, ei: usize, f: &ElementFrame) -> u32 { + let info_bits = if f.info.is_short() { 15 } else { 11 }; + match self.elements[ei].ty { + ElementType::Cpe => { + let ms = match &f.ms_mask { + Some((1, used)) => used.len() as u32, + _ => 0, + }; + 3 + 4 + 1 + info_bits + 2 + ms + } + _ => 3 + 4 + info_bits, + } + } + + fn quantise_all(frames: &[ElementFrame], lambda: f32) -> Vec> { + #[cfg(feature = "rayon")] + { + use rayon::prelude::*; + frames.par_iter().map(|f| f.chans.par_iter().map(|c| quantise_channel(c, lambda)).collect()).collect() + } + #[cfg(not(feature = "rayon"))] + { + frames.iter().map(|f| f.chans.iter().map(|c| quantise_channel(c, lambda)).collect()).collect() + } + } + + /// `last`: final frame of the stream — a CBR stream spends (or pads away) the whole reservoir so + /// the total size is exactly `frames × bitrate × 1024 / rate`. + fn encode_frame(&mut self, last: bool) -> Vec { + let t = self.frames_done as i64; + let mut frames = Vec::with_capacity(self.elements.len()); + let mut pe = 0f32; + for ei in 0..self.elements.len() { + let (f, p) = self.analyse_element(ei, t); + pe += p; + frames.push(f); + } + let pce_bits = if self.cfg.adts && self.asc.channel_config == 0 { + let mut bw = BitWriter::new(); + if let Some(p) = &self.asc.pce { + p.write(&mut bw); + } + 3 + bw.bit_len() as u32 + 8 + } else { + 0 + }; + let overhead: u32 = (0..frames.len()).map(|i| self.element_overhead(i, &frames[i])).sum::() + 3 + 7 + pce_bits; + let total_bits = |q: &Vec>| overhead + q.iter().flatten().map(|c| c.bits).sum::(); + + let any_short = frames.iter().any(|f| f.info.is_short()); + let (mean, target) = match self.cfg.mode { + BitrateMode::Cbr(b) => { + self.mean_acc += b as u64 * FRAME_LEN as u64; + let mean = (self.mean_acc / self.cfg.sample_rate as u64) as i64; + self.mean_acc %= self.cfg.sample_rate as u64; + self.pe_avg = if self.pe_avg == 0.0 { pe.max(1.0) } else { 0.95 * self.pe_avg + 0.05 * pe.max(1.0) }; + let mut ratio = (pe.max(1.0) / self.pe_avg).clamp(0.7, 1.6); + if any_short { + ratio = (ratio * 1.3).min(2.0); + } + let mut desired = mean as f32 * ratio + (self.level as f32 - 0.5 * self.res_max as f32) * 0.15; + desired = desired.min((mean + self.level) as f32).max((mean - (self.res_max - self.level)) as f32); + desired = desired.min(self.max_frame_bits as f32); + if last { + desired = (mean + self.level) as f32; + } + (mean, desired.max(0.0) as u32) + } + BitrateMode::Vbr(_) => (0, self.max_frame_bits as u32), + }; + + let quant = match self.cfg.mode { + BitrateMode::Vbr(q) => { + let d = q.clamp(0.5, 6.0) - 1.0; + let mut lambda = 10f32.powf(-0.2 - 1.2 * d + 0.09 * d * d); + let mut res = Self::quantise_all(&frames, lambda); + while total_bits(&res) > target && lambda < 1e9 { + lambda *= 2.0; + res = Self::quantise_all(&frames, lambda); + } + res + } + BitrateMode::Cbr(_) => { + let mut lam = self.lambda.clamp(1e-6, 1e9); + let mut cur = Self::quantise_all(&frames, lam); + let (mut lo, mut hi, mut best); + if total_bits(&cur) <= target { + hi = lam; + best = cur; + lo = lam; + loop { + lo /= 4.0; + if lo < 1e-6 { + break; + } + cur = Self::quantise_all(&frames, lo); + if total_bits(&cur) > target { + break; + } + hi = lo; + best = cur; + } + } else { + lo = lam; + loop { + lam *= 4.0; + cur = Self::quantise_all(&frames, lam); + if total_bits(&cur) <= target || lam > 1e9 { + break; + } + lo = lam; + } + hi = lam; + best = cur; + } + for _ in 0..6 { + if hi / lo < 1.03 { + break; + } + let mid = (lo * hi).sqrt(); + let r = Self::quantise_all(&frames, mid); + if total_bits(&r) <= target { + hi = mid; + best = r; + } else { + lo = mid; + } + } + self.lambda = hi; + best + } + }; + + // assemble the raw_data_block + let mut bw = BitWriter::new(); + if pce_bits > 0 + && let Some(p) = &self.asc.pce + { + bw.write_bits(5, 3); + p.write(&mut bw); + } + for (ei, (f, q)) in frames.iter().zip(&quant).enumerate() { + let e = &self.elements[ei]; + match e.ty { + ElementType::Cpe => { + bw.write_bits(1, 3); + bw.write_bits(e.tag as u32, 4); + bw.write_bits(1, 1); // common_window + f.info.write(&mut bw); + let (present, used) = f.ms_mask.clone().unwrap_or((0, Vec::new())); + bw.write_bits(present, 2); + if present == 1 { + for &u in &used { + bw.write_bits(u as u32, 1); + } + } + write_ics(&mut bw, &f.chans[0], &q[0], false); + write_ics(&mut bw, &f.chans[1], &q[1], false); + } + ty => { + bw.write_bits(if ty == ElementType::Lfe { 3 } else { 0 }, 3); + bw.write_bits(e.tag as u32, 4); + write_ics(&mut bw, &f.chans[0], &q[0], true); + } + } + } + if let BitrateMode::Cbr(_) = self.cfg.mode { + let projected = ((bw.bit_len() as i64 + 3 + 7) / 8) * 8; + let excess = self.level + mean - projected - if last { 0 } else { self.res_max }; + if excess > 0 { + self.stats.fill_bits += write_fill(&mut bw, excess as u64); + } + } + bw.write_bits(7, 3); // ID_END + let au = bw.finish(); + let used = au.len() as i64 * 8; + if let BitrateMode::Cbr(_) = self.cfg.mode { + self.level = (self.level + mean - used).clamp(0, self.res_max); + } + self.stats.frames += 1; + self.stats.bytes += au.len() as u64; + + // slide the input window + for h in &mut self.hist { + h.drain(..FRAME_LEN); + } + self.origin += FRAME_LEN as i64; + self.frames_done += 1; + + if self.cfg.adts { + let fullness = match self.cfg.mode { + BitrateMode::Cbr(_) => ((self.level / (32 * self.cfg.channels as i64)) as u16).min(0x7FE), + BitrateMode::Vbr(_) => 0x7FF, + }; + let mut out = AdtsHeader::write(2, self.sf_index, self.asc.channel_config, au.len(), fullness).to_vec(); + out.extend_from_slice(&au); + out + } else { + au + } + } +} + +/// Write fill elements totalling at least `bits - 7` bits (at most `bits + 8`); returns bits written. +fn write_fill(bw: &mut BitWriter, bits: u64) -> u64 { + let start = bw.bit_len() as u64; + let mut left = bits as i64; + while left >= 7 { + let n = (((left - 7) / 8) as usize).min(269); + let n = if n >= 15 && (left - 15) / 8 < 15 { 14 } else { n }; + bw.write_bits(6, 3); + if n >= 15 { + bw.write_bits(15, 4); + bw.write_bits((n - 14) as u32, 8); + } else { + bw.write_bits(n as u32, 4); + } + if n > 0 { + bw.write_bits(0, 8); // extension_type EXT_FILL + fill_nibble + for _ in 1..n { + bw.write_bits(0xA5, 8); + } + } + let el = 7 + if n >= 15 { 8 } else { 0 } + 8 * n as i64; + left -= el; + if n == 0 { + break; + } + } + bw.bit_len() as u64 - start +} + +fn interp(pts: &[(f32, f32)], x: f32) -> f32 { + if x <= pts[0].0 { + return pts[0].1; + } + for w in pts.windows(2) { + if x <= w[1].0 { + let t = (x - w[0].0) / (w[1].0 - w[0].0); + return w[0].1 + t * (w[1].1 - w[0].1); + } + } + pts[pts.len() - 1].1 +} diff --git a/crates/aac/src/encoder/psy.rs b/crates/aac/src/encoder/psy.rs new file mode 100644 index 0000000000..1101a76234 --- /dev/null +++ b/crates/aac/src/encoder/psy.rs @@ -0,0 +1,176 @@ +//! Psychoacoustic model operating on MDCT coefficients. +//! +//! Per scalefactor band: energy; tonality from the spectral flatness measure (SFM) of a +//! neighbourhood of ≥ 16 lines; masking-threshold PSD by spreading the band PSDs with Schroeder's +//! spreading function in the Bark domain; a tonality-dependent masking offset (≈18 dB for tonal, +//! ≈6 dB for noise-like maskers); the absolute threshold of hearing (Terhardt) as a floor; and +//! perceptual entropy (PE) as an estimate of the bits the frame needs. + +use crate::tables::{swb_offsets_long, swb_offsets_short}; + +/// Precomputed per-band data for one window length. +pub struct BandModel { + pub offsets: &'static [u16], + /// Linear spreading factor `[maskee][masker]` (power). + spread: Vec>, + /// ATH as band energy in encoder spectrum units. + ath: Vec, + /// Neighbourhood `[lo, hi)` in lines used for the tonality estimate. + tonal_span: Vec<(usize, usize)>, +} + +fn bark(f: f64) -> f64 { + 13.0 * (0.00076 * f).atan() + 3.5 * (f / 7500.0).powi(2).atan() +} + +/// Terhardt's threshold in quiet (dB SPL), capped at 40 dB so content near the band edge is not +/// discarded wholesale (the bandwidth setting limits the coded range instead). +fn ath_db(f: f64) -> f64 { + let f = (f / 1000.0).max(0.02); + (3.64 * f.powf(-0.8) - 6.5 * (-0.6 * (f - 3.3).powi(2)).exp() + 1e-3 * f.powi(4)).min(40.0) +} + +impl BandModel { + fn new(sample_rate: u32, offsets: &'static [u16], m: usize) -> BandModel { + let nb = offsets.len() - 1; + let hz_per_line = sample_rate as f64 / (2.0 * m as f64); + let centre: Vec = (0..nb).map(|b| bark((offsets[b] as f64 + offsets[b + 1] as f64) * 0.5 * hz_per_line)).collect(); + let spread = (0..nb) + .map(|i| { + (0..nb) + .map(|j| { + let dz = centre[i] - centre[j] + 0.474; + let db = 15.81 + 7.5 * dz - 17.5 * (1.0 + dz * dz).sqrt(); + if db < -60.0 { 0.0 } else { 10f64.powf(db / 10.0) as f32 } + }) + .collect() + }) + .collect(); + // A full-scale (16-bit) sine of power P_ref = 32768²/2 has MDCT energy ≈ 2·M²·P per frame + // with the encoder's scaling; it is taken as 96 dB SPL. 3 dB of safety margin. + let p_ref = 32768f64 * 32768.0 / 2.0; + let ath = (0..nb) + .map(|b| { + let mut lo = f64::MAX; + for k in offsets[b]..offsets[b + 1] { + lo = lo.min(ath_db((k as f64 + 0.5) * hz_per_line)); + } + (2.0 * (m * m) as f64 * p_ref * 10f64.powf((lo - 96.0 - 3.0) / 10.0)) as f32 + }) + .collect(); + let tonal_span = (0..nb) + .map(|b| { + let (lo, hi) = (offsets[b] as usize, offsets[b + 1] as usize); + if hi - lo >= 16 { + (lo, hi) + } else { + let c = (lo + hi) / 2; + let lo = c.saturating_sub(8); + let hi = (lo + 16).min(m); + (hi.saturating_sub(16), hi) + } + }) + .collect(); + BandModel { offsets, spread, ath, tonal_span } + } + + pub fn num_bands(&self) -> usize { + self.offsets.len() - 1 + } +} + +pub struct Psy { + pub long: BandModel, + pub short: BandModel, +} + +/// Per-window, per-band analysis of one channel. +#[derive(Clone, Default)] +pub struct PsyResult { + /// `[window][band]` energy. + pub en: Vec>, + /// `[window][band]` masking threshold (allowed noise energy). + pub thr: Vec>, + pub pe: f32, +} + +impl Psy { + pub fn new(sample_rate: u32, sf_index: u8) -> Psy { + Psy { + long: BandModel::new(sample_rate, swb_offsets_long(sf_index), 1024), + short: BandModel::new(sample_rate, swb_offsets_short(sf_index), 128), + } + } + + /// Analyse a spectrum: 1024 lines (long) or 8×128 (short). + pub fn analyse(&self, spec: &[f32], short: bool) -> PsyResult { + let (model, nwin, wlen) = if short { (&self.short, 8, 128) } else { (&self.long, 1, 1024) }; + let nb = model.num_bands(); + let mut res = PsyResult { en: Vec::with_capacity(nwin), thr: Vec::with_capacity(nwin), pe: 0.0 }; + let mut psd = vec![0f32; nb]; + let mut tonality = vec![0f32; nb]; + for w in 0..nwin { + let s = &spec[w * wlen..(w + 1) * wlen]; + let mut en = vec![0f32; nb]; + for b in 0..nb { + let (lo, hi) = (model.offsets[b] as usize, model.offsets[b + 1] as usize); + en[b] = s[lo..hi].iter().map(|v| v * v).sum(); + psd[b] = en[b] / (hi - lo) as f32; + // spectral flatness over the neighbourhood + let (tlo, thi) = model.tonal_span[b]; + let n = (thi - tlo) as f64; + let mut sum = 0f64; + let mut log_sum = 0f64; + for &v in &s[tlo..thi] { + let p = v as f64 * v as f64 + 1e-3; + sum += p; + log_sum += p.ln(); + } + let sfm_db = 10.0 * ((log_sum / n).exp() / (sum / n)).log10(); + // real-valued Gaussian noise gives ≈ -5.5 dB; a pure tone ≤ -25 dB + tonality[b] = ((-sfm_db - 6.0) / 19.0).clamp(0.0, 1.0) as f32; + } + let mut thr = vec![0f32; nb]; + for b in 0..nb { + let row = &model.spread[b]; + let mut sp = 0f32; + for j in 0..nb { + sp += psd[j] * row[j]; + } + let width = (model.offsets[b + 1] - model.offsets[b]) as f32; + let offset_db = tonality[b] * 30.0 + (1.0 - tonality[b]) * 6.0; + let t = sp * width * 10f32.powf(-offset_db / 10.0); + thr[b] = t.max(model.ath[b]); + if en[b] > thr[b] { + res.pe += width * 0.5 * (en[b] / thr[b]).log2(); + } + } + res.en.push(en); + res.thr.push(thr); + } + res + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tone_is_masked_less_than_noise() { + let psy = Psy::new(44100, 4); + let tone: Vec = (0..1024).map(|k| if k == 100 { 1e6 } else { 0.0 }).collect(); + let mut seed = 1u32; + let noise: Vec = (0..1024) + .map(|_| { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + ((seed >> 8) as f32 / (1 << 24) as f32 - 0.5) * 1e4 + }) + .collect(); + let rt = psy.analyse(&tone, false); + let rn = psy.analyse(&noise, false); + let b = psy.long.offsets.iter().position(|&o| o > 100).unwrap() - 1; + assert!(rt.thr[0][b] < rt.en[0][b] / 30.0); + assert!(rn.thr[0][b] > rn.en[0][b] / 10.0); + } +} diff --git a/crates/aac/src/encoder/quant.rs b/crates/aac/src/encoder/quant.rs new file mode 100644 index 0000000000..e07ac67d72 --- /dev/null +++ b/crates/aac/src/encoder/quant.rs @@ -0,0 +1,382 @@ +//! Quantisation, scalefactor selection, codebook/section selection, bit counting and writing of one +//! `individual_channel_stream`. +//! +//! Coefficients are kept in bitstream order ("interleaved": group → band → window → line), so every +//! (group, band) is a contiguous slice and short-window grouping needs no special casing. + +use deckcraft_bitstream::BitWriter; + +use crate::huffman::{self, ZERO_HCB}; +use crate::ics::IcsInfo; +use crate::tables::pow43; +use crate::tns::TnsData; + +/// Magic rounding offset for AAC quantisation (`0.5 - 0.0946`), minimising the error for the +/// `x^(3/4)` companding. +const ROUND: f32 = 0.4054; +const MAX_Q: i32 = 8191; + +/// One (group, band) of a channel, in bitstream order. +#[derive(Clone, Copy, Debug)] +pub struct Band { + /// Range in the interleaved coefficient array. + pub start: usize, + pub end: usize, + /// Allowed noise energy at λ = 1. + pub thr: f32, + /// Coefficient energy. + pub en: f32, + /// Largest `|x|^(3/4)`. + pub max34: f32, +} + +/// Everything the rate loop needs for one channel. +pub struct ChannelInput { + pub info: IcsInfo, + /// Interleaved (bitstream-order) coefficients. + pub x: Vec, + pub xabs: Vec, + pub x34: Vec, + pub bands: Vec, + pub tns: Option, +} + +impl ChannelInput { + /// `spec` is window-major (short windows at stride 128); `thr[g][sfb]` and the grouping come from + /// `info`. + pub fn new(info: IcsInfo, swb: &[u16], spec: &[f32], thr: &[Vec], tns: Option) -> ChannelInput { + let mut x = Vec::with_capacity(1024); + let mut bands = Vec::new(); + for g in 0..info.num_groups { + let w0 = info.group_start(g); + for sfb in 0..info.max_sfb { + let start = x.len(); + for w in w0..w0 + info.group_len[g] as usize { + x.extend_from_slice(&spec[w * 128 + swb[sfb] as usize..w * 128 + swb[sfb + 1] as usize]); + } + bands.push(Band { start, end: x.len(), thr: thr[g][sfb], en: 0.0, max34: 0.0 }); + } + } + let xabs: Vec = x.iter().map(|v| v.abs()).collect(); + let x34: Vec = xabs.iter().map(|v| v.powf(0.75)).collect(); + for b in &mut bands { + b.en = x[b.start..b.end].iter().map(|v| v * v).sum(); + b.max34 = x34[b.start..b.end].iter().cloned().fold(0.0, f32::max); + } + ChannelInput { info, x, xabs, x34, bands, tns } + } +} + +/// A quantised channel ready to be written. +#[derive(Clone, Default)] +pub struct QuantChannel { + pub q: Vec, + /// Scalefactor per band (only meaningful where `cb != 0`). + pub sf: Vec, + pub cb: Vec, + /// Per group: (codebook, number of bands). + pub sections: Vec>, + pub global_gain: i32, + /// Bits of the ICS excluding `ics_info`. + pub bits: u32, +} + +#[inline] +fn gain34(sf: i32) -> f32 { + (-0.1875 * (sf as f32 - 100.0)).exp2() +} +#[inline] +fn gain(sf: i32) -> f32 { + (0.25 * (sf as f32 - 100.0)).exp2() +} + +/// Quantise a slice at `sf`, returning the distortion; writes the (signed) values to `out` if given. +fn quantise(x: &[f32], xabs: &[f32], x34: &[f32], sf: i32, out: Option<&mut [i32]>) -> f32 { + let g34 = gain34(sf); + let g = gain(sf); + let mut d = 0f32; + match out { + Some(out) => { + for i in 0..x.len() { + let q = ((x34[i] * g34 + ROUND) as i32).min(MAX_Q); + let e = xabs[i] - pow43(q as u32) * g; + d += e * e; + out[i] = if x[i] < 0.0 { -q } else { q }; + } + } + None => { + for i in 0..x.len() { + let q = ((x34[i] * g34 + ROUND) as i32).min(MAX_Q); + let e = xabs[i] - pow43(q as u32) * g; + d += e * e; + } + } + } + d +} + +/// Largest scalefactor whose distortion stays within `allowed`. +fn search_sf(ch: &ChannelInput, b: &Band, allowed: f32) -> i32 { + let r = b.start..b.end; + let (x, xa, x34) = (&ch.x[r.clone()], &ch.xabs[r.clone()], &ch.x34[r]); + let l2 = b.max34.log2(); + let lo = ((100.0 - ((MAX_Q as f32 + 0.5).log2() - l2) / 0.1875).ceil() as i32).clamp(0, 255); + let zero = ((100.0 - ((1.0 - ROUND).log2() - l2) / 0.1875).floor() as i32 + 1).clamp(0, 256); + let mut a = lo; + let mut hi = (zero - 1).clamp(lo, 255); + if quantise(x, xa, x34, a, None) > allowed { + return a; + } + while a < hi { + let mid = (a + hi + 1) / 2; + if quantise(x, xa, x34, mid, None) <= allowed { + a = mid; + } else { + hi = mid - 1; + } + } + a +} + +/// Candidate codebooks for a band whose largest magnitude is `m` (0 for all-zero bands). +fn candidates(m: i32) -> &'static [u8] { + match m { + 0 => &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + 1 => &[1, 2, 3, 4, 5, 6], + 2 => &[3, 4, 5, 6, 7, 8], + 3..=4 => &[5, 6, 7, 8, 9, 10], + 5..=7 => &[7, 8, 9, 10, 11], + 8..=12 => &[9, 10, 11], + _ => &[11], + } +} + +/// Quantise one channel at rate-distortion multiplier `lambda` and select sections. +pub fn quantise_channel(ch: &ChannelInput, lambda: f32) -> QuantChannel { + let nb = ch.bands.len(); + let mut sf = vec![-1i32; nb]; + for (i, b) in ch.bands.iter().enumerate() { + let allowed = lambda * b.thr; + if b.max34 <= 0.0 || b.en <= allowed { + continue; + } + sf[i] = search_sf(ch, b, allowed); + } + // Scalefactor differences between coded bands must stay within ±60: lower (refine) offenders, + // then quantise; repeat if a band unexpectedly quantised to all zeros (it leaves the chain). + let mut q = vec![0i32; ch.x.len()]; + let mut maxq = vec![0i32; nb]; + loop { + for _ in 0..8 { + let mut changed = false; + let coded: Vec = (0..nb).filter(|&i| sf[i] >= 0).collect(); + for w in coded.windows(2) { + if sf[w[1]] > sf[w[0]] + 60 { + sf[w[1]] = sf[w[0]] + 60; + changed = true; + } + } + for w in coded.windows(2).rev() { + if sf[w[0]] > sf[w[1]] + 60 { + sf[w[0]] = sf[w[1]] + 60; + changed = true; + } + } + if !changed { + break; + } + } + let mut removed = false; + for (i, b) in ch.bands.iter().enumerate() { + let r = b.start..b.end; + if sf[i] < 0 { + q[r].fill(0); + maxq[i] = 0; + continue; + } + quantise(&ch.x[r.clone()], &ch.xabs[r.clone()], &ch.x34[r.clone()], sf[i], Some(&mut q[r.clone()])); + maxq[i] = q[r].iter().map(|v| v.abs()).max().unwrap_or(0); + if maxq[i] == 0 { + sf[i] = -1; + removed = true; + } + } + if !removed { + break; + } + } + + // Section selection: dynamic programming per group over codebooks. + let info = &ch.info; + let (sect_bits, esc) = if info.is_short() { (3u32, 7usize) } else { (5u32, 31usize) }; + let mut cb = vec![ZERO_HCB; nb]; + let mut sections = Vec::with_capacity(info.num_groups); + let mut spectral_bits = 0u32; + let mut section_bits = 0u32; + let per_group = info.max_sfb; + for g in 0..info.num_groups { + let base = g * per_group; + let n = per_group; + if n == 0 { + sections.push(Vec::new()); + continue; + } + // cost[i][c] + const INF: u32 = u32::MAX / 4; + let mut cost = vec![[INF; 12]; n]; + for i in 0..n { + let bi = base + i; + let b = &ch.bands[bi]; + let vals = &q[b.start..b.end]; + for &c in candidates(maxq[bi]) { + cost[i][c as usize] = if c == 0 { + 0 + } else { + // all-zero band inside a coded section also needs a (zero) scalefactor difference + huffman::count_bits(c, vals) + if maxq[bi] == 0 { 1 } else { 0 } + }; + } + } + let new_sec = 4 + sect_bits; + let mut best = [INF; 12]; + let mut back = vec![[0u8; 12]; n]; + for c in 0..12 { + if cost[0][c] < INF { + best[c] = new_sec + cost[0][c]; + } + } + for i in 1..n { + let (mut min_c, mut min_v) = (0usize, INF); + for c in 0..12 { + if best[c] < min_v { + min_v = best[c]; + min_c = c; + } + } + let mut nb_ = [INF; 12]; + for c in 0..12 { + if cost[i][c] >= INF { + continue; + } + let stay = best[c]; + let switch = min_v + new_sec; + if stay <= switch { + nb_[c] = stay + cost[i][c]; + back[i][c] = c as u8; + } else { + nb_[c] = switch + cost[i][c]; + back[i][c] = min_c as u8; + } + } + best = nb_; + } + let mut c = (0..12).min_by_key(|&c| best[c]).unwrap_or(0); + for i in (0..n).rev() { + cb[base + i] = c as u8; + c = back[i][c] as usize; + } + // run-length sections and exact costs + let mut secs: Vec<(u8, usize)> = Vec::new(); + for i in 0..n { + let c = cb[base + i]; + match secs.last_mut() { + Some((lc, len)) if *lc == c => *len += 1, + _ => secs.push((c, 1)), + } + if c != ZERO_HCB { + let b = &ch.bands[base + i]; + spectral_bits += huffman::count_bits(c, &q[b.start..b.end]); + } + } + for &(_, len) in &secs { + section_bits += 4 + sect_bits * (len / esc + 1) as u32; + } + sections.push(secs); + } + + // Scalefactors: global gain is the first coded band's value; bands coded with a non-zero + // codebook but all-zero values reuse the previous value (difference 0). + let first = (0..nb).find(|&i| sf[i] >= 0).map(|i| sf[i]).unwrap_or(100); + let mut prev = first; + let mut sf_bits = 0u32; + for i in 0..nb { + if cb[i] == ZERO_HCB { + continue; + } + if sf[i] < 0 { + sf[i] = prev; + } + sf_bits += huffman::sf_bits(sf[i] - prev); + prev = sf[i]; + } + let tns_bits = 1 + ch.tns.as_ref().map(|t| t.bit_count(info)).unwrap_or(0); + let bits = 8 + section_bits + sf_bits + 1 + tns_bits + 1 + spectral_bits; + QuantChannel { q, sf, cb, sections, global_gain: first, bits } +} + +/// Write an `individual_channel_stream`; `write_info` is false inside a common-window CPE. +pub fn write_ics(bw: &mut BitWriter, ch: &ChannelInput, qc: &QuantChannel, write_info: bool) { + let info = &ch.info; + bw.write_bits(qc.global_gain as u32, 8); + if write_info { + info.write(bw); + } + let (sect_bits, esc) = if info.is_short() { (3u32, 7usize) } else { (5u32, 31usize) }; + for secs in &qc.sections { + for &(c, len) in secs { + bw.write_bits(c as u32, 4); + let mut l = len; + while l >= esc { + bw.write_bits(esc as u32, sect_bits); + l -= esc; + } + bw.write_bits(l as u32, sect_bits); + } + } + let mut prev = qc.global_gain; + for i in 0..ch.bands.len() { + if qc.cb[i] == ZERO_HCB { + continue; + } + huffman::write_sf(bw, qc.sf[i] - prev); + prev = qc.sf[i]; + } + bw.write_bits(0, 1); // pulse_data_present + match &ch.tns { + Some(t) => { + bw.write_bits(1, 1); + t.write(bw, info); + } + None => bw.write_bits(0, 1), + } + bw.write_bits(0, 1); // gain_control_data_present + for (i, b) in ch.bands.iter().enumerate() { + if qc.cb[i] != ZERO_HCB { + huffman::write_spectral(bw, qc.cb[i], &qc.q[b.start..b.end]); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ics::WindowSequence; + use crate::mdct::WindowShape; + use crate::tables::swb_offsets_long; + use deckcraft_bitstream::BitWriter; + + #[test] + fn bit_count_matches_written_bits() { + let swb = swb_offsets_long(4); + let info = IcsInfo { window_sequence: WindowSequence::OnlyLong, window_shape: WindowShape::Sine, max_sfb: 40, ..Default::default() }; + let spec: Vec = (0..1024).map(|i| ((i * 7919 % 1000) as f32 - 500.0) * (1000.0 / (i as f32 + 10.0))).collect(); + let thr = vec![vec![1000.0f32; 64]]; + let ch = ChannelInput::new(info, swb, &spec, &thr, None); + for lambda in [0.01f32, 1.0, 100.0, 1e5] { + let qc = quantise_channel(&ch, lambda); + let mut bw = BitWriter::new(); + write_ics(&mut bw, &ch, &qc, false); + assert_eq!(bw.bit_len() as u32, qc.bits, "lambda {lambda}"); + } + } +} diff --git a/crates/aac/src/huffman.rs b/crates/aac/src/huffman.rs new file mode 100644 index 0000000000..2e116ed283 --- /dev/null +++ b/crates/aac/src/huffman.rs @@ -0,0 +1,306 @@ +//! Huffman coding of spectral values and scalefactors (ISO/IEC 14496-3 §4.6.3, Annex 4.A). + +use std::sync::OnceLock; + +use deckcraft_bitstream::{BitReader, BitWriter}; + +use crate::huffman_tables::{ + SCALEFACTOR, SPECTRUM_1, SPECTRUM_2, SPECTRUM_3, SPECTRUM_4, SPECTRUM_5, SPECTRUM_6, SPECTRUM_7, SPECTRUM_8, SPECTRUM_9, SPECTRUM_10, SPECTRUM_11, +}; +use crate::{Error, Result}; + +/// `ZERO_HCB`: band quantised to zero, nothing transmitted. +pub const ZERO_HCB: u8 = 0; +/// `ESC_HCB`: codebook 11 with escape sequences. +pub const ESC_HCB: u8 = 11; +/// `NOISE_HCB`: perceptual noise substitution. +pub const NOISE_HCB: u8 = 13; +/// `INTENSITY_HCB2`: intensity stereo, out of phase. +pub const INTENSITY_HCB2: u8 = 14; +/// `INTENSITY_HCB`: intensity stereo, in phase. +pub const INTENSITY_HCB: u8 = 15; + +/// Parameters of one spectrum codebook. +pub struct Book { + /// Values per codeword (4 = quad, 2 = pair). + pub dim: usize, + /// Signed values in the codeword (no separate sign bits). + pub signed: bool, + /// Largest absolute value (16 = escape for codebook 11). + pub lav: u32, + /// Radix of the index (`2*lav+1` signed, `lav+1` unsigned). + pub modulo: u32, + pub codes: &'static [(u32, u8)], +} + +static BOOKS: [Book; 11] = [ + Book { dim: 4, signed: true, lav: 1, modulo: 3, codes: &SPECTRUM_1 }, + Book { dim: 4, signed: true, lav: 1, modulo: 3, codes: &SPECTRUM_2 }, + Book { dim: 4, signed: false, lav: 2, modulo: 3, codes: &SPECTRUM_3 }, + Book { dim: 4, signed: false, lav: 2, modulo: 3, codes: &SPECTRUM_4 }, + Book { dim: 2, signed: true, lav: 4, modulo: 9, codes: &SPECTRUM_5 }, + Book { dim: 2, signed: true, lav: 4, modulo: 9, codes: &SPECTRUM_6 }, + Book { dim: 2, signed: false, lav: 7, modulo: 8, codes: &SPECTRUM_7 }, + Book { dim: 2, signed: false, lav: 7, modulo: 8, codes: &SPECTRUM_8 }, + Book { dim: 2, signed: false, lav: 12, modulo: 13, codes: &SPECTRUM_9 }, + Book { dim: 2, signed: false, lav: 12, modulo: 13, codes: &SPECTRUM_10 }, + Book { dim: 2, signed: false, lav: 16, modulo: 17, codes: &SPECTRUM_11 }, +]; + +/// Spectrum codebook 1..=11. +#[inline] +pub fn book(cb: u8) -> &'static Book { + &BOOKS[(cb as usize).clamp(1, 11) - 1] +} + +const LEAF: u32 = 1 << 31; + +/// Binary decoding tree for one codebook. +struct Tree { + nodes: Vec<[u32; 2]>, +} + +impl Tree { + fn build(codes: &[(u32, u8)]) -> Tree { + let mut nodes: Vec<[u32; 2]> = vec![[0, 0]]; + for (sym, &(code, len)) in codes.iter().enumerate() { + let mut node = 0usize; + for i in (0..len).rev() { + let bit = ((code >> i) & 1) as usize; + if i == 0 { + nodes[node][bit] = LEAF | sym as u32; + } else { + if nodes[node][bit] == 0 { + nodes.push([0, 0]); + nodes[node][bit] = (nodes.len() - 1) as u32; + } + node = nodes[node][bit] as usize; + } + } + } + Tree { nodes } + } + + #[inline] + fn decode(&self, br: &mut BitReader) -> Result { + let mut node = 0usize; + loop { + let bit = br.read_bits(1)? as usize; + let next = self.nodes[node][bit]; + if next & LEAF != 0 { + return Ok(next & !LEAF); + } + if next == 0 { + return Err(Error::Bitstream("invalid Huffman codeword")); + } + node = next as usize; + } + } +} + +fn trees() -> &'static [Tree; 12] { + static TREES: OnceLock<[Tree; 12]> = OnceLock::new(); + TREES.get_or_init(|| std::array::from_fn(|i| if i == 0 { Tree::build(&SCALEFACTOR) } else { Tree::build(BOOKS[i - 1].codes) })) +} + +/// Decode one scalefactor difference (`-60..=60`). +#[inline] +pub fn decode_sf(br: &mut BitReader) -> Result { + Ok(trees()[0].decode(br)? as i32 - 60) +} + +/// Write one scalefactor difference (`-60..=60`). +#[inline] +pub fn write_sf(bw: &mut BitWriter, diff: i32) { + let (code, len) = SCALEFACTOR[(diff + 60).clamp(0, 120) as usize]; + bw.write_bits(code, len as u32); +} + +/// Bits of one scalefactor difference. +#[inline] +pub fn sf_bits(diff: i32) -> u32 { + SCALEFACTOR[(diff + 60).clamp(0, 120) as usize].1 as u32 +} + +/// Decode one codeword of codebook `cb` (1..=11) into `out[..dim]`, including sign bits and escapes. +pub fn decode_spectral(br: &mut BitReader, cb: u8, out: &mut [i32]) -> Result<()> { + let b = book(cb); + let mut idx = trees()[cb as usize].decode(br)?; + for i in (0..b.dim).rev() { + out[i] = (idx % b.modulo) as i32; + idx /= b.modulo; + } + if b.signed { + for v in out[..b.dim].iter_mut() { + *v -= b.lav as i32; + } + return Ok(()); + } + for v in out[..b.dim].iter_mut() { + if *v != 0 && br.read_bits(1)? == 1 { + *v = -*v; + } + } + if cb == ESC_HCB { + for v in out[..b.dim].iter_mut() { + if v.abs() == 16 { + let mut n = 0u32; + while br.read_bits(1)? == 1 { + n += 1; + if n > 8 { + return Err(Error::Bitstream("escape sequence too long")); + } + } + let mag = (1i32 << (n + 4)) + br.read_bits(n + 4)? as i32; + *v = if *v < 0 { -mag } else { mag }; + } + } + } + Ok(()) +} + +/// Codebook index of the (sign-stripped for unsigned books) values. +#[inline] +fn index_of(b: &Book, vals: &[i32]) -> usize { + let mut idx = 0u32; + for &v in &vals[..b.dim] { + let d = if b.signed { (v + b.lav as i32) as u32 } else { v.unsigned_abs().min(b.lav) }; + idx = idx * b.modulo + d; + } + idx as usize +} + +#[inline] +fn escape_bits(mag: u32) -> u32 { + // mag >= 16: N ones, a zero, N+4 bits where 2^(N+4) <= mag < 2^(N+5). + let n = 31 - mag.leading_zeros() - 4; + 2 * n + 5 +} + +/// Bits needed to code `vals` (a multiple of the codebook dimension) with codebook `cb` (1..=11). +/// The values must fit the codebook (`|v| <= lav`, or `<= 8191` for codebook 11). +pub fn count_bits(cb: u8, vals: &[i32]) -> u32 { + let b = book(cb); + let mut bits = 0u32; + for chunk in vals.chunks_exact(b.dim) { + bits += b.codes[index_of(b, chunk)].1 as u32; + if !b.signed { + for &v in chunk { + if v != 0 { + bits += 1; + if cb == ESC_HCB && v.unsigned_abs() >= 16 { + bits += escape_bits(v.unsigned_abs()); + } + } + } + } + } + bits +} + +/// Write `vals` with codebook `cb` (1..=11). +pub fn write_spectral(bw: &mut BitWriter, cb: u8, vals: &[i32]) { + let b = book(cb); + for chunk in vals.chunks_exact(b.dim) { + let (code, len) = b.codes[index_of(b, chunk)]; + bw.write_bits(code, len as u32); + if b.signed { + continue; + } + for &v in chunk { + if v != 0 { + bw.write_bits((v < 0) as u32, 1); + } + } + if cb == ESC_HCB { + for &v in chunk { + let mag = v.unsigned_abs(); + if mag >= 16 { + let n = 31 - mag.leading_zeros() - 4; + for _ in 0..n { + bw.write_bits(1, 1); + } + bw.write_bits(0, 1); + bw.write_bits(mag - (1 << (n + 4)), n + 4); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn check_complete(codes: &[(u32, u8)]) { + // Kraft sum exactly one and prefix-free. + let max = codes.iter().map(|c| c.1).max().unwrap() as u32; + let mut kraft = 0u64; + for &(_, l) in codes { + kraft += 1u64 << (max - l as u32); + } + assert_eq!(kraft, 1u64 << max); + for (i, &(ci, li)) in codes.iter().enumerate() { + for (j, &(cj, lj)) in codes.iter().enumerate() { + if i != j && li <= lj { + assert_ne!(cj >> (lj - li), ci, "codeword {i} is a prefix of {j}"); + } + } + } + } + + #[test] + fn codebooks_are_complete_prefix_codes() { + check_complete(&SCALEFACTOR); + assert_eq!(SCALEFACTOR[60], (0, 1)); + for b in &BOOKS { + assert_eq!(b.codes.len() as u32, b.modulo.pow(b.dim as u32)); + check_complete(b.codes); + } + } + + #[test] + fn spectral_roundtrip_every_book() { + for cb in 1..=11u8 { + let b = book(cb); + let lim = if cb == 11 { 8191 } else { b.lav as i32 }; + let mut vals = Vec::new(); + let mut seed = 12345u32; + for _ in 0..400 { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + let r = (seed >> 8) as i32; + let v = if cb == 11 && r % 3 == 0 { r % (lim + 1) } else { r % (b.lav.min(16) as i32 + 1) }; + let v = v.min(lim); + vals.push(if b.signed || r & 1 == 0 { v } else { -v }); + } + if b.signed { + for v in vals.iter_mut() { + *v = (*v).clamp(-(b.lav as i32), b.lav as i32); + } + } + let mut bw = BitWriter::new(); + write_spectral(&mut bw, cb, &vals); + assert_eq!(bw.bit_len() as u32, count_bits(cb, &vals)); + let data = bw.finish(); + let mut br = BitReader::new(&data); + let mut out = vec![0i32; vals.len()]; + for c in out.chunks_mut(b.dim) { + decode_spectral(&mut br, cb, c).unwrap(); + } + assert_eq!(out, vals, "codebook {cb}"); + } + } + + #[test] + fn scalefactor_roundtrip() { + let mut bw = BitWriter::new(); + for d in -60..=60 { + write_sf(&mut bw, d); + } + let data = bw.finish(); + let mut br = BitReader::new(&data); + for d in -60..=60 { + assert_eq!(decode_sf(&mut br).unwrap(), d); + } + } +} diff --git a/crates/aac/src/huffman_tables.rs b/crates/aac/src/huffman_tables.rs new file mode 100644 index 0000000000..c65c7df609 --- /dev/null +++ b/crates/aac/src/huffman_tables.rs @@ -0,0 +1,1406 @@ +//! AAC Huffman codebooks (ISO/IEC 14496-3 §4.A.1, spectrum codebooks 1–11 and the scalefactor codebook). +//! +//! Each table is indexed by the codebook index defined in the standard and stores `(codeword, length)`. +//! Generated data: derived by black-box probing of an external decoder used purely as a test oracle +//! (every table is a complete prefix code — Kraft sum exactly 1 — covering every index once; see +//! `tests::codebooks_are_complete_prefix_codes`). + +#![allow(clippy::unreadable_literal)] + +pub(crate) static SCALEFACTOR: [(u32, u8); 121] = [ + (0x3ffe8, 18), + (0x3ffe6, 18), + (0x3ffe7, 18), + (0x3ffe5, 18), + (0x7fff5, 19), + (0x7fff1, 19), + (0x7ffed, 19), + (0x7fff6, 19), + (0x7ffee, 19), + (0x7ffef, 19), + (0x7fff0, 19), + (0x7fffc, 19), + (0x7fffd, 19), + (0x7ffff, 19), + (0x7fffe, 19), + (0x7fff7, 19), + (0x7fff8, 19), + (0x7fffb, 19), + (0x7fff9, 19), + (0x3ffe4, 18), + (0x7fffa, 19), + (0x3ffe3, 18), + (0x1ffef, 17), + (0x1fff0, 17), + (0xfff5, 16), + (0x1ffee, 17), + (0xfff2, 16), + (0xfff3, 16), + (0xfff4, 16), + (0xfff1, 16), + (0x7ff6, 15), + (0x7ff7, 15), + (0x3ff9, 14), + (0x3ff5, 14), + (0x3ff7, 14), + (0x3ff3, 14), + (0x3ff6, 14), + (0x3ff2, 14), + (0x1ff7, 13), + (0x1ff5, 13), + (0xff9, 12), + (0xff7, 12), + (0xff6, 12), + (0x7f9, 11), + (0xff4, 12), + (0x7f8, 11), + (0x3f9, 10), + (0x3f7, 10), + (0x3f5, 10), + (0x1f8, 9), + (0x1f7, 9), + (0xfa, 8), + (0xf8, 8), + (0xf6, 8), + (0x79, 7), + (0x3a, 6), + (0x38, 6), + (0x1a, 5), + (0xb, 4), + (0x4, 3), + (0x0, 1), + (0xa, 4), + (0xc, 4), + (0x1b, 5), + (0x39, 6), + (0x3b, 6), + (0x78, 7), + (0x7a, 7), + (0xf7, 8), + (0xf9, 8), + (0x1f6, 9), + (0x1f9, 9), + (0x3f4, 10), + (0x3f6, 10), + (0x3f8, 10), + (0x7f5, 11), + (0x7f4, 11), + (0x7f6, 11), + (0x7f7, 11), + (0xff5, 12), + (0xff8, 12), + (0x1ff4, 13), + (0x1ff6, 13), + (0x1ff8, 13), + (0x3ff8, 14), + (0x3ff4, 14), + (0xfff0, 16), + (0x7ff4, 15), + (0xfff6, 16), + (0x7ff5, 15), + (0x3ffe2, 18), + (0x7ffd9, 19), + (0x7ffda, 19), + (0x7ffdb, 19), + (0x7ffdc, 19), + (0x7ffdd, 19), + (0x7ffde, 19), + (0x7ffd8, 19), + (0x7ffd2, 19), + (0x7ffd3, 19), + (0x7ffd4, 19), + (0x7ffd5, 19), + (0x7ffd6, 19), + (0x7fff2, 19), + (0x7ffdf, 19), + (0x7ffe7, 19), + (0x7ffe8, 19), + (0x7ffe9, 19), + (0x7ffea, 19), + (0x7ffeb, 19), + (0x7ffe6, 19), + (0x7ffe0, 19), + (0x7ffe1, 19), + (0x7ffe2, 19), + (0x7ffe3, 19), + (0x7ffe4, 19), + (0x7ffe5, 19), + (0x7ffd7, 19), + (0x7ffec, 19), + (0x7fff4, 19), + (0x7fff3, 19), +]; + +pub(crate) static SPECTRUM_1: [(u32, u8); 81] = [ + (0x7f8, 11), + (0x1f1, 9), + (0x7fd, 11), + (0x3f5, 10), + (0x68, 7), + (0x3f0, 10), + (0x7f7, 11), + (0x1ec, 9), + (0x7f5, 11), + (0x3f1, 10), + (0x72, 7), + (0x3f4, 10), + (0x74, 7), + (0x11, 5), + (0x76, 7), + (0x1eb, 9), + (0x6c, 7), + (0x3f6, 10), + (0x7fc, 11), + (0x1e1, 9), + (0x7f1, 11), + (0x1f0, 9), + (0x61, 7), + (0x1f6, 9), + (0x7f2, 11), + (0x1ea, 9), + (0x7fb, 11), + (0x1f2, 9), + (0x69, 7), + (0x1ed, 9), + (0x77, 7), + (0x17, 5), + (0x6f, 7), + (0x1e6, 9), + (0x64, 7), + (0x1e5, 9), + (0x67, 7), + (0x15, 5), + (0x62, 7), + (0x12, 5), + (0x0, 1), + (0x14, 5), + (0x65, 7), + (0x16, 5), + (0x6d, 7), + (0x1e9, 9), + (0x63, 7), + (0x1e4, 9), + (0x6b, 7), + (0x13, 5), + (0x71, 7), + (0x1e3, 9), + (0x70, 7), + (0x1f3, 9), + (0x7fe, 11), + (0x1e7, 9), + (0x7f3, 11), + (0x1ef, 9), + (0x60, 7), + (0x1ee, 9), + (0x7f0, 11), + (0x1e2, 9), + (0x7fa, 11), + (0x3f3, 10), + (0x6a, 7), + (0x1e8, 9), + (0x75, 7), + (0x10, 5), + (0x73, 7), + (0x1f4, 9), + (0x6e, 7), + (0x3f7, 10), + (0x7f6, 11), + (0x1e0, 9), + (0x7f9, 11), + (0x3f2, 10), + (0x66, 7), + (0x1f5, 9), + (0x7ff, 11), + (0x1f7, 9), + (0x7f4, 11), +]; + +pub(crate) static SPECTRUM_2: [(u32, u8); 81] = [ + (0x1f3, 9), + (0x6f, 7), + (0x1fd, 9), + (0xeb, 8), + (0x23, 6), + (0xea, 8), + (0x1f7, 9), + (0xe8, 8), + (0x1fa, 9), + (0xf2, 8), + (0x2d, 6), + (0x70, 7), + (0x20, 6), + (0x6, 5), + (0x2b, 6), + (0x6e, 7), + (0x28, 6), + (0xe9, 8), + (0x1f9, 9), + (0x66, 7), + (0xf8, 8), + (0xe7, 8), + (0x1b, 6), + (0xf1, 8), + (0x1f4, 9), + (0x6b, 7), + (0x1f5, 9), + (0xec, 8), + (0x2a, 6), + (0x6c, 7), + (0x2c, 6), + (0xa, 5), + (0x27, 6), + (0x67, 7), + (0x1a, 6), + (0xf5, 8), + (0x24, 6), + (0x8, 5), + (0x1f, 6), + (0x9, 5), + (0x0, 3), + (0x7, 5), + (0x1d, 6), + (0xb, 5), + (0x30, 6), + (0xef, 8), + (0x1c, 6), + (0x64, 7), + (0x1e, 6), + (0xc, 5), + (0x29, 6), + (0xf3, 8), + (0x2f, 6), + (0xf0, 8), + (0x1fc, 9), + (0x71, 7), + (0x1f2, 9), + (0xf4, 8), + (0x21, 6), + (0xe6, 8), + (0xf7, 8), + (0x68, 7), + (0x1f8, 9), + (0xee, 8), + (0x22, 6), + (0x65, 7), + (0x31, 6), + (0x2, 4), + (0x26, 6), + (0xed, 8), + (0x25, 6), + (0x6a, 7), + (0x1fb, 9), + (0x72, 7), + (0x1fe, 9), + (0x69, 7), + (0x2e, 6), + (0xf6, 8), + (0x1ff, 9), + (0x6d, 7), + (0x1f6, 9), +]; + +pub(crate) static SPECTRUM_3: [(u32, u8); 81] = [ + (0x0, 1), + (0x9, 4), + (0xef, 8), + (0xb, 4), + (0x19, 5), + (0xf0, 8), + (0x1eb, 9), + (0x1e6, 9), + (0x3f2, 10), + (0xa, 4), + (0x35, 6), + (0x1ef, 9), + (0x34, 6), + (0x37, 6), + (0x1e9, 9), + (0x1ed, 9), + (0x1e7, 9), + (0x3f3, 10), + (0x1ee, 9), + (0x3ed, 10), + (0x1ffa, 13), + (0x1ec, 9), + (0x1f2, 9), + (0x7f9, 11), + (0x7f8, 11), + (0x3f8, 10), + (0xff8, 12), + (0x8, 4), + (0x38, 6), + (0x3f6, 10), + (0x36, 6), + (0x75, 7), + (0x3f1, 10), + (0x3eb, 10), + (0x3ec, 10), + (0xff4, 12), + (0x18, 5), + (0x76, 7), + (0x7f4, 11), + (0x39, 6), + (0x74, 7), + (0x3ef, 10), + (0x1f3, 9), + (0x1f4, 9), + (0x7f6, 11), + (0x1e8, 9), + (0x3ea, 10), + (0x1ffc, 13), + (0xf2, 8), + (0x1f1, 9), + (0xffb, 12), + (0x3f5, 10), + (0x7f3, 11), + (0xffc, 12), + (0xee, 8), + (0x3f7, 10), + (0x7ffe, 15), + (0x1f0, 9), + (0x7f5, 11), + (0x7ffd, 15), + (0x1ffb, 13), + (0x3ffa, 14), + (0xffff, 16), + (0xf1, 8), + (0x3f0, 10), + (0x3ffc, 14), + (0x1ea, 9), + (0x3ee, 10), + (0x3ffb, 14), + (0xff6, 12), + (0xffa, 12), + (0x7ffc, 15), + (0x7f2, 11), + (0xff5, 12), + (0xfffe, 16), + (0x3f4, 10), + (0x7f7, 11), + (0x7ffb, 15), + (0xff7, 12), + (0xff9, 12), + (0x7ffa, 15), +]; + +pub(crate) static SPECTRUM_4: [(u32, u8); 81] = [ + (0x7, 4), + (0x16, 5), + (0xf6, 8), + (0x18, 5), + (0x8, 4), + (0xef, 8), + (0x1ef, 9), + (0xf3, 8), + (0x7f8, 11), + (0x19, 5), + (0x17, 5), + (0xed, 8), + (0x15, 5), + (0x1, 4), + (0xe2, 8), + (0xf0, 8), + (0x70, 7), + (0x3f0, 10), + (0x1ee, 9), + (0xf1, 8), + (0x7fa, 11), + (0xee, 8), + (0xe4, 8), + (0x3f2, 10), + (0x7f6, 11), + (0x3ef, 10), + (0x7fd, 11), + (0x5, 4), + (0x14, 5), + (0xf2, 8), + (0x9, 4), + (0x4, 4), + (0xe5, 8), + (0xf4, 8), + (0xe8, 8), + (0x3f4, 10), + (0x6, 4), + (0x2, 4), + (0xe7, 8), + (0x3, 4), + (0x0, 4), + (0x6b, 7), + (0xe3, 8), + (0x69, 7), + (0x1f3, 9), + (0xeb, 8), + (0xe6, 8), + (0x3f6, 10), + (0x6e, 7), + (0x6a, 7), + (0x1f4, 9), + (0x3ec, 10), + (0x1f0, 9), + (0x3f9, 10), + (0xf5, 8), + (0xec, 8), + (0x7fb, 11), + (0xea, 8), + (0x6f, 7), + (0x3f7, 10), + (0x7f9, 11), + (0x3f3, 10), + (0xfff, 12), + (0xe9, 8), + (0x6d, 7), + (0x3f8, 10), + (0x6c, 7), + (0x68, 7), + (0x1f5, 9), + (0x3ee, 10), + (0x1f2, 9), + (0x7f4, 11), + (0x7f7, 11), + (0x3f1, 10), + (0xffe, 12), + (0x3ed, 10), + (0x1f1, 9), + (0x7f5, 11), + (0x7fe, 11), + (0x3f5, 10), + (0x7fc, 11), +]; + +pub(crate) static SPECTRUM_5: [(u32, u8); 81] = [ + (0x1fff, 13), + (0xff7, 12), + (0x7f4, 11), + (0x7e8, 11), + (0x3f1, 10), + (0x7ee, 11), + (0x7f9, 11), + (0xff8, 12), + (0x1ffd, 13), + (0xffd, 12), + (0x7f1, 11), + (0x3e8, 10), + (0x1e8, 9), + (0xf0, 8), + (0x1ec, 9), + (0x3ee, 10), + (0x7f2, 11), + (0xffa, 12), + (0xff4, 12), + (0x3ef, 10), + (0x1f2, 9), + (0xe8, 8), + (0x70, 7), + (0xec, 8), + (0x1f0, 9), + (0x3ea, 10), + (0x7f3, 11), + (0x7eb, 11), + (0x1eb, 9), + (0xea, 8), + (0x1a, 5), + (0x8, 4), + (0x19, 5), + (0xee, 8), + (0x1ef, 9), + (0x7ed, 11), + (0x3f0, 10), + (0xf2, 8), + (0x73, 7), + (0xb, 4), + (0x0, 1), + (0xa, 4), + (0x71, 7), + (0xf3, 8), + (0x7e9, 11), + (0x7ef, 11), + (0x1ee, 9), + (0xef, 8), + (0x18, 5), + (0x9, 4), + (0x1b, 5), + (0xeb, 8), + (0x1e9, 9), + (0x7ec, 11), + (0x7f6, 11), + (0x3eb, 10), + (0x1f3, 9), + (0xed, 8), + (0x72, 7), + (0xe9, 8), + (0x1f1, 9), + (0x3ed, 10), + (0x7f7, 11), + (0xff6, 12), + (0x7f0, 11), + (0x3e9, 10), + (0x1ed, 9), + (0xf1, 8), + (0x1ea, 9), + (0x3ec, 10), + (0x7f8, 11), + (0xff9, 12), + (0x1ffc, 13), + (0xffc, 12), + (0xff5, 12), + (0x7ea, 11), + (0x3f3, 10), + (0x3f2, 10), + (0x7f5, 11), + (0xffb, 12), + (0x1ffe, 13), +]; + +pub(crate) static SPECTRUM_6: [(u32, u8); 81] = [ + (0x7fe, 11), + (0x3fd, 10), + (0x1f1, 9), + (0x1eb, 9), + (0x1f4, 9), + (0x1ea, 9), + (0x1f0, 9), + (0x3fc, 10), + (0x7fd, 11), + (0x3f6, 10), + (0x1e5, 9), + (0xea, 8), + (0x6c, 7), + (0x71, 7), + (0x68, 7), + (0xf0, 8), + (0x1e6, 9), + (0x3f7, 10), + (0x1f3, 9), + (0xef, 8), + (0x32, 6), + (0x27, 6), + (0x28, 6), + (0x26, 6), + (0x31, 6), + (0xeb, 8), + (0x1f7, 9), + (0x1e8, 9), + (0x6f, 7), + (0x2e, 6), + (0x8, 4), + (0x4, 4), + (0x6, 4), + (0x29, 6), + (0x6b, 7), + (0x1ee, 9), + (0x1ef, 9), + (0x72, 7), + (0x2d, 6), + (0x2, 4), + (0x0, 4), + (0x3, 4), + (0x2f, 6), + (0x73, 7), + (0x1fa, 9), + (0x1e7, 9), + (0x6e, 7), + (0x2b, 6), + (0x7, 4), + (0x1, 4), + (0x5, 4), + (0x2c, 6), + (0x6d, 7), + (0x1ec, 9), + (0x1f9, 9), + (0xee, 8), + (0x30, 6), + (0x24, 6), + (0x2a, 6), + (0x25, 6), + (0x33, 6), + (0xec, 8), + (0x1f2, 9), + (0x3f8, 10), + (0x1e4, 9), + (0xed, 8), + (0x6a, 7), + (0x70, 7), + (0x69, 7), + (0x74, 7), + (0xf1, 8), + (0x3fa, 10), + (0x7ff, 11), + (0x3f9, 10), + (0x1f6, 9), + (0x1ed, 9), + (0x1f8, 9), + (0x1e9, 9), + (0x1f5, 9), + (0x3fb, 10), + (0x7fc, 11), +]; + +pub(crate) static SPECTRUM_7: [(u32, u8); 64] = [ + (0x0, 1), + (0x5, 3), + (0x37, 6), + (0x74, 7), + (0xf2, 8), + (0x1eb, 9), + (0x3ed, 10), + (0x7f7, 11), + (0x4, 3), + (0xc, 4), + (0x35, 6), + (0x71, 7), + (0xec, 8), + (0xee, 8), + (0x1ee, 9), + (0x1f5, 9), + (0x36, 6), + (0x34, 6), + (0x72, 7), + (0xea, 8), + (0xf1, 8), + (0x1e9, 9), + (0x1f3, 9), + (0x3f5, 10), + (0x73, 7), + (0x70, 7), + (0xeb, 8), + (0xf0, 8), + (0x1f1, 9), + (0x1f0, 9), + (0x3ec, 10), + (0x3fa, 10), + (0xf3, 8), + (0xed, 8), + (0x1e8, 9), + (0x1ef, 9), + (0x3ef, 10), + (0x3f1, 10), + (0x3f9, 10), + (0x7fb, 11), + (0x1ed, 9), + (0xef, 8), + (0x1ea, 9), + (0x1f2, 9), + (0x3f3, 10), + (0x3f8, 10), + (0x7f9, 11), + (0x7fc, 11), + (0x3ee, 10), + (0x1ec, 9), + (0x1f4, 9), + (0x3f4, 10), + (0x3f7, 10), + (0x7f8, 11), + (0xffd, 12), + (0xffe, 12), + (0x7f6, 11), + (0x3f0, 10), + (0x3f2, 10), + (0x3f6, 10), + (0x7fa, 11), + (0x7fd, 11), + (0xffc, 12), + (0xfff, 12), +]; + +pub(crate) static SPECTRUM_8: [(u32, u8); 64] = [ + (0xe, 5), + (0x5, 4), + (0x10, 5), + (0x30, 6), + (0x6f, 7), + (0xf1, 8), + (0x1fa, 9), + (0x3fe, 10), + (0x3, 4), + (0x0, 3), + (0x4, 4), + (0x12, 5), + (0x2c, 6), + (0x6a, 7), + (0x75, 7), + (0xf8, 8), + (0xf, 5), + (0x2, 4), + (0x6, 4), + (0x14, 5), + (0x2e, 6), + (0x69, 7), + (0x72, 7), + (0xf5, 8), + (0x2f, 6), + (0x11, 5), + (0x13, 5), + (0x2a, 6), + (0x32, 6), + (0x6c, 7), + (0xec, 8), + (0xfa, 8), + (0x71, 7), + (0x2b, 6), + (0x2d, 6), + (0x31, 6), + (0x6d, 7), + (0x70, 7), + (0xf2, 8), + (0x1f9, 9), + (0xef, 8), + (0x68, 7), + (0x33, 6), + (0x6b, 7), + (0x6e, 7), + (0xee, 8), + (0xf9, 8), + (0x3fc, 10), + (0x1f8, 9), + (0x74, 7), + (0x73, 7), + (0xed, 8), + (0xf0, 8), + (0xf6, 8), + (0x1f6, 9), + (0x1fd, 9), + (0x3fd, 10), + (0xf3, 8), + (0xf4, 8), + (0xf7, 8), + (0x1f7, 9), + (0x1fb, 9), + (0x1fc, 9), + (0x3ff, 10), +]; + +pub(crate) static SPECTRUM_9: [(u32, u8); 169] = [ + (0x0, 1), + (0x5, 3), + (0x37, 6), + (0xe7, 8), + (0x1de, 9), + (0x3ce, 10), + (0x3d9, 10), + (0x7c8, 11), + (0x7cd, 11), + (0xfc8, 12), + (0xfdd, 12), + (0x1fe4, 13), + (0x1fec, 13), + (0x4, 3), + (0xc, 4), + (0x35, 6), + (0x72, 7), + (0xea, 8), + (0xed, 8), + (0x1e2, 9), + (0x3d1, 10), + (0x3d3, 10), + (0x3e0, 10), + (0x7d8, 11), + (0xfcf, 12), + (0xfd5, 12), + (0x36, 6), + (0x34, 6), + (0x71, 7), + (0xe8, 8), + (0xec, 8), + (0x1e1, 9), + (0x3cf, 10), + (0x3dd, 10), + (0x3db, 10), + (0x7d0, 11), + (0xfc7, 12), + (0xfd4, 12), + (0xfe4, 12), + (0xe6, 8), + (0x70, 7), + (0xe9, 8), + (0x1dd, 9), + (0x1e3, 9), + (0x3d2, 10), + (0x3dc, 10), + (0x7cc, 11), + (0x7ca, 11), + (0x7de, 11), + (0xfd8, 12), + (0xfea, 12), + (0x1fdb, 13), + (0x1df, 9), + (0xeb, 8), + (0x1dc, 9), + (0x1e6, 9), + (0x3d5, 10), + (0x3de, 10), + (0x7cb, 11), + (0x7dd, 11), + (0x7dc, 11), + (0xfcd, 12), + (0xfe2, 12), + (0xfe7, 12), + (0x1fe1, 13), + (0x3d0, 10), + (0x1e0, 9), + (0x1e4, 9), + (0x3d6, 10), + (0x7c5, 11), + (0x7d1, 11), + (0x7db, 11), + (0xfd2, 12), + (0x7e0, 11), + (0xfd9, 12), + (0xfeb, 12), + (0x1fe3, 13), + (0x1fe9, 13), + (0x7c4, 11), + (0x1e5, 9), + (0x3d7, 10), + (0x7c6, 11), + (0x7cf, 11), + (0x7da, 11), + (0xfcb, 12), + (0xfda, 12), + (0xfe3, 12), + (0xfe9, 12), + (0x1fe6, 13), + (0x1ff3, 13), + (0x1ff7, 13), + (0x7d3, 11), + (0x3d8, 10), + (0x3e1, 10), + (0x7d4, 11), + (0x7d9, 11), + (0xfd3, 12), + (0xfde, 12), + (0x1fdd, 13), + (0x1fd9, 13), + (0x1fe2, 13), + (0x1fea, 13), + (0x1ff1, 13), + (0x1ff6, 13), + (0x7d2, 11), + (0x3d4, 10), + (0x3da, 10), + (0x7c7, 11), + (0x7d7, 11), + (0x7e2, 11), + (0xfce, 12), + (0xfdb, 12), + (0x1fd8, 13), + (0x1fee, 13), + (0x3ff0, 14), + (0x1ff4, 13), + (0x3ff2, 14), + (0x7e1, 11), + (0x3df, 10), + (0x7c9, 11), + (0x7d6, 11), + (0xfca, 12), + (0xfd0, 12), + (0xfe5, 12), + (0xfe6, 12), + (0x1feb, 13), + (0x1fef, 13), + (0x3ff3, 14), + (0x3ff4, 14), + (0x3ff5, 14), + (0xfe0, 12), + (0x7ce, 11), + (0x7d5, 11), + (0xfc6, 12), + (0xfd1, 12), + (0xfe1, 12), + (0x1fe0, 13), + (0x1fe8, 13), + (0x1ff0, 13), + (0x3ff1, 14), + (0x3ff8, 14), + (0x3ff6, 14), + (0x7ffc, 15), + (0xfe8, 12), + (0x7df, 11), + (0xfc9, 12), + (0xfd7, 12), + (0xfdc, 12), + (0x1fdc, 13), + (0x1fdf, 13), + (0x1fed, 13), + (0x1ff5, 13), + (0x3ff9, 14), + (0x3ffb, 14), + (0x7ffd, 15), + (0x7ffe, 15), + (0x1fe7, 13), + (0xfcc, 12), + (0xfd6, 12), + (0xfdf, 12), + (0x1fde, 13), + (0x1fda, 13), + (0x1fe5, 13), + (0x1ff2, 13), + (0x3ffa, 14), + (0x3ff7, 14), + (0x3ffc, 14), + (0x3ffd, 14), + (0x7fff, 15), +]; + +pub(crate) static SPECTRUM_10: [(u32, u8); 169] = [ + (0x22, 6), + (0x8, 5), + (0x1d, 6), + (0x26, 6), + (0x5f, 7), + (0xd3, 8), + (0x1cf, 9), + (0x3d0, 10), + (0x3d7, 10), + (0x3ed, 10), + (0x7f0, 11), + (0x7f6, 11), + (0xffd, 12), + (0x7, 5), + (0x0, 4), + (0x1, 4), + (0x9, 5), + (0x20, 6), + (0x54, 7), + (0x60, 7), + (0xd5, 8), + (0xdc, 8), + (0x1d4, 9), + (0x3cd, 10), + (0x3de, 10), + (0x7e7, 11), + (0x1c, 6), + (0x2, 4), + (0x6, 5), + (0xc, 5), + (0x1e, 6), + (0x28, 6), + (0x5b, 7), + (0xcd, 8), + (0xd9, 8), + (0x1ce, 9), + (0x1dc, 9), + (0x3d9, 10), + (0x3f1, 10), + (0x25, 6), + (0xb, 5), + (0xa, 5), + (0xd, 5), + (0x24, 6), + (0x57, 7), + (0x61, 7), + (0xcc, 8), + (0xdd, 8), + (0x1cc, 9), + (0x1de, 9), + (0x3d3, 10), + (0x3e7, 10), + (0x5d, 7), + (0x21, 6), + (0x1f, 6), + (0x23, 6), + (0x27, 6), + (0x59, 7), + (0x64, 7), + (0xd8, 8), + (0xdf, 8), + (0x1d2, 9), + (0x1e2, 9), + (0x3dd, 10), + (0x3ee, 10), + (0xd1, 8), + (0x55, 7), + (0x29, 6), + (0x56, 7), + (0x58, 7), + (0x62, 7), + (0xce, 8), + (0xe0, 8), + (0xe2, 8), + (0x1da, 9), + (0x3d4, 10), + (0x3e3, 10), + (0x7eb, 11), + (0x1c9, 9), + (0x5e, 7), + (0x5a, 7), + (0x5c, 7), + (0x63, 7), + (0xca, 8), + (0xda, 8), + (0x1c7, 9), + (0x1ca, 9), + (0x1e0, 9), + (0x3db, 10), + (0x3e8, 10), + (0x7ec, 11), + (0x1e3, 9), + (0xd2, 8), + (0xcb, 8), + (0xd0, 8), + (0xd7, 8), + (0xdb, 8), + (0x1c6, 9), + (0x1d5, 9), + (0x1d8, 9), + (0x3ca, 10), + (0x3da, 10), + (0x7ea, 11), + (0x7f1, 11), + (0x1e1, 9), + (0xd4, 8), + (0xcf, 8), + (0xd6, 8), + (0xde, 8), + (0xe1, 8), + (0x1d0, 9), + (0x1d6, 9), + (0x3d1, 10), + (0x3d5, 10), + (0x3f2, 10), + (0x7ee, 11), + (0x7fb, 11), + (0x3e9, 10), + (0x1cd, 9), + (0x1c8, 9), + (0x1cb, 9), + (0x1d1, 9), + (0x1d7, 9), + (0x1df, 9), + (0x3cf, 10), + (0x3e0, 10), + (0x3ef, 10), + (0x7e6, 11), + (0x7f8, 11), + (0xffa, 12), + (0x3eb, 10), + (0x1dd, 9), + (0x1d3, 9), + (0x1d9, 9), + (0x1db, 9), + (0x3d2, 10), + (0x3cc, 10), + (0x3dc, 10), + (0x3ea, 10), + (0x7ed, 11), + (0x7f3, 11), + (0x7f9, 11), + (0xff9, 12), + (0x7f2, 11), + (0x3ce, 10), + (0x1e4, 9), + (0x3cb, 10), + (0x3d8, 10), + (0x3d6, 10), + (0x3e2, 10), + (0x3e5, 10), + (0x7e8, 11), + (0x7f4, 11), + (0x7f5, 11), + (0x7f7, 11), + (0xffb, 12), + (0x7fa, 11), + (0x3ec, 10), + (0x3df, 10), + (0x3e1, 10), + (0x3e4, 10), + (0x3e6, 10), + (0x3f0, 10), + (0x7e9, 11), + (0x7ef, 11), + (0xff8, 12), + (0xffe, 12), + (0xffc, 12), + (0xfff, 12), +]; + +pub(crate) static SPECTRUM_11: [(u32, u8); 289] = [ + (0x0, 4), + (0x6, 5), + (0x19, 6), + (0x3d, 7), + (0x9c, 8), + (0xc6, 8), + (0x1a7, 9), + (0x390, 10), + (0x3c2, 10), + (0x3df, 10), + (0x7e6, 11), + (0x7f3, 11), + (0xffb, 12), + (0x7ec, 11), + (0xffa, 12), + (0xffe, 12), + (0x38e, 10), + (0x5, 5), + (0x1, 4), + (0x8, 5), + (0x14, 6), + (0x37, 7), + (0x42, 7), + (0x92, 8), + (0xaf, 8), + (0x191, 9), + (0x1a5, 9), + (0x1b5, 9), + (0x39e, 10), + (0x3c0, 10), + (0x3a2, 10), + (0x3cd, 10), + (0x7d6, 11), + (0xae, 8), + (0x17, 6), + (0x7, 5), + (0x9, 5), + (0x18, 6), + (0x39, 7), + (0x40, 7), + (0x8e, 8), + (0xa3, 8), + (0xb8, 8), + (0x199, 9), + (0x1ac, 9), + (0x1c1, 9), + (0x3b1, 10), + (0x396, 10), + (0x3be, 10), + (0x3ca, 10), + (0x9d, 8), + (0x3c, 7), + (0x15, 6), + (0x16, 6), + (0x1a, 6), + (0x3b, 7), + (0x44, 7), + (0x91, 8), + (0xa5, 8), + (0xbe, 8), + (0x196, 9), + (0x1ae, 9), + (0x1b9, 9), + (0x3a1, 10), + (0x391, 10), + (0x3a5, 10), + (0x3d5, 10), + (0x94, 8), + (0x9a, 8), + (0x36, 7), + (0x38, 7), + (0x3a, 7), + (0x41, 7), + (0x8c, 8), + (0x9b, 8), + (0xb0, 8), + (0xc3, 8), + (0x19e, 9), + (0x1ab, 9), + (0x1bc, 9), + (0x39f, 10), + (0x38f, 10), + (0x3a9, 10), + (0x3cf, 10), + (0x93, 8), + (0xbf, 8), + (0x3e, 7), + (0x3f, 7), + (0x43, 7), + (0x45, 7), + (0x9e, 8), + (0xa7, 8), + (0xb9, 8), + (0x194, 9), + (0x1a2, 9), + (0x1ba, 9), + (0x1c3, 9), + (0x3a6, 10), + (0x3a7, 10), + (0x3bb, 10), + (0x3d4, 10), + (0x9f, 8), + (0x1a0, 9), + (0x8f, 8), + (0x8d, 8), + (0x90, 8), + (0x98, 8), + (0xa6, 8), + (0xb6, 8), + (0xc4, 8), + (0x19f, 9), + (0x1af, 9), + (0x1bf, 9), + (0x399, 10), + (0x3bf, 10), + (0x3b4, 10), + (0x3c9, 10), + (0x3e7, 10), + (0xa8, 8), + (0x1b6, 9), + (0xab, 8), + (0xa4, 8), + (0xaa, 8), + (0xb2, 8), + (0xc2, 8), + (0xc5, 8), + (0x198, 9), + (0x1a4, 9), + (0x1b8, 9), + (0x38c, 10), + (0x3a4, 10), + (0x3c4, 10), + (0x3c6, 10), + (0x3dd, 10), + (0x3e8, 10), + (0xad, 8), + (0x3af, 10), + (0x192, 9), + (0xbd, 8), + (0xbc, 8), + (0x18e, 9), + (0x197, 9), + (0x19a, 9), + (0x1a3, 9), + (0x1b1, 9), + (0x38d, 10), + (0x398, 10), + (0x3b7, 10), + (0x3d3, 10), + (0x3d1, 10), + (0x3db, 10), + (0x7dd, 11), + (0xb4, 8), + (0x3de, 10), + (0x1a9, 9), + (0x19b, 9), + (0x19c, 9), + (0x1a1, 9), + (0x1aa, 9), + (0x1ad, 9), + (0x1b3, 9), + (0x38b, 10), + (0x3b2, 10), + (0x3b8, 10), + (0x3ce, 10), + (0x3e1, 10), + (0x3e0, 10), + (0x7d2, 11), + (0x7e5, 11), + (0xb7, 8), + (0x7e3, 11), + (0x1bb, 9), + (0x1a8, 9), + (0x1a6, 9), + (0x1b0, 9), + (0x1b2, 9), + (0x1b7, 9), + (0x39b, 10), + (0x39a, 10), + (0x3ba, 10), + (0x3b5, 10), + (0x3d6, 10), + (0x7d7, 11), + (0x3e4, 10), + (0x7d8, 11), + (0x7ea, 11), + (0xba, 8), + (0x7e8, 11), + (0x3a0, 10), + (0x1bd, 9), + (0x1b4, 9), + (0x38a, 10), + (0x1c4, 9), + (0x392, 10), + (0x3aa, 10), + (0x3b0, 10), + (0x3bc, 10), + (0x3d7, 10), + (0x7d4, 11), + (0x7dc, 11), + (0x7db, 11), + (0x7d5, 11), + (0x7f0, 11), + (0xc1, 8), + (0x7fb, 11), + (0x3c8, 10), + (0x3a3, 10), + (0x395, 10), + (0x39d, 10), + (0x3ac, 10), + (0x3ae, 10), + (0x3c5, 10), + (0x3d8, 10), + (0x3e2, 10), + (0x3e6, 10), + (0x7e4, 11), + (0x7e7, 11), + (0x7e0, 11), + (0x7e9, 11), + (0x7f7, 11), + (0x190, 9), + (0x7f2, 11), + (0x393, 10), + (0x1be, 9), + (0x1c0, 9), + (0x394, 10), + (0x397, 10), + (0x3ad, 10), + (0x3c3, 10), + (0x3c1, 10), + (0x3d2, 10), + (0x7da, 11), + (0x7d9, 11), + (0x7df, 11), + (0x7eb, 11), + (0x7f4, 11), + (0x7fa, 11), + (0x195, 9), + (0x7f8, 11), + (0x3bd, 10), + (0x39c, 10), + (0x3ab, 10), + (0x3a8, 10), + (0x3b3, 10), + (0x3b9, 10), + (0x3d0, 10), + (0x3e3, 10), + (0x3e5, 10), + (0x7e2, 11), + (0x7de, 11), + (0x7ed, 11), + (0x7f1, 11), + (0x7f9, 11), + (0x7fc, 11), + (0x193, 9), + (0xffd, 12), + (0x3dc, 10), + (0x3b6, 10), + (0x3c7, 10), + (0x3cc, 10), + (0x3cb, 10), + (0x3d9, 10), + (0x3da, 10), + (0x7d3, 11), + (0x7e1, 11), + (0x7ee, 11), + (0x7ef, 11), + (0x7f5, 11), + (0x7f6, 11), + (0xffc, 12), + (0xfff, 12), + (0x19d, 9), + (0x1c2, 9), + (0xb5, 8), + (0xa1, 8), + (0x96, 8), + (0x97, 8), + (0x95, 8), + (0x99, 8), + (0xa0, 8), + (0xa2, 8), + (0xac, 8), + (0xa9, 8), + (0xb1, 8), + (0xb3, 8), + (0xbb, 8), + (0xc0, 8), + (0x18f, 9), + (0x4, 5), +]; diff --git a/crates/aac/src/ics.rs b/crates/aac/src/ics.rs new file mode 100644 index 0000000000..d70c31e9f9 --- /dev/null +++ b/crates/aac/src/ics.rs @@ -0,0 +1,151 @@ +//! `ics_info` and helpers shared by the encoder and decoder (ISO/IEC 14496-3 §4.4.2.1, §4.5.2.3). + +use deckcraft_bitstream::{BitReader, BitWriter}; + +use crate::mdct::WindowShape; +use crate::tables::{swb_offsets_long, swb_offsets_short}; +use crate::{Error, Result}; + +/// `window_sequence`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum WindowSequence { + #[default] + OnlyLong = 0, + LongStart = 1, + EightShort = 2, + LongStop = 3, +} + +impl WindowSequence { + pub fn from_bits(v: u32) -> Self { + match v & 3 { + 0 => WindowSequence::OnlyLong, + 1 => WindowSequence::LongStart, + 2 => WindowSequence::EightShort, + _ => WindowSequence::LongStop, + } + } +} + +/// Decoded/encoded `ics_info`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IcsInfo { + pub window_sequence: WindowSequence, + pub window_shape: WindowShape, + pub max_sfb: usize, + pub num_groups: usize, + /// Windows per group (only the first `num_groups` entries are used; long windows: `[1]`). + pub group_len: [u8; 8], +} + +impl Default for IcsInfo { + fn default() -> Self { + IcsInfo { + window_sequence: WindowSequence::OnlyLong, + window_shape: WindowShape::Sine, + max_sfb: 0, + num_groups: 1, + group_len: [1, 0, 0, 0, 0, 0, 0, 0], + } + } +} + +impl IcsInfo { + #[inline] + pub fn is_short(&self) -> bool { + self.window_sequence == WindowSequence::EightShort + } + #[inline] + pub fn num_windows(&self) -> usize { + if self.is_short() { 8 } else { 1 } + } + /// Band offsets of one window. + pub fn swb_offsets(&self, sf_index: u8) -> &'static [u16] { + if self.is_short() { swb_offsets_short(sf_index) } else { swb_offsets_long(sf_index) } + } + /// Number of bands for this window length (`num_swb`). + pub fn num_swb(&self, sf_index: u8) -> usize { + self.swb_offsets(sf_index).len() - 1 + } + /// First window of group `g`. + pub fn group_start(&self, g: usize) -> usize { + self.group_len[..g].iter().map(|&l| l as usize).sum() + } + + pub fn parse(br: &mut BitReader, sf_index: u8) -> Result { + if br.read_bits(1)? != 0 { + // ics_reserved_bit; tolerated by most decoders + } + let window_sequence = WindowSequence::from_bits(br.read_bits(2)?); + let window_shape = WindowShape::from_bit(br.read_bits(1)? == 1); + let mut info = IcsInfo { window_sequence, window_shape, ..Default::default() }; + if info.is_short() { + info.max_sfb = br.read_bits(4)? as usize; + let grouping = br.read_bits(7)?; + info.num_groups = 1; + info.group_len = [1, 0, 0, 0, 0, 0, 0, 0]; + for w in 1..8 { + if grouping & (1 << (6 - (w - 1))) != 0 { + info.group_len[info.num_groups - 1] += 1; + } else { + info.num_groups += 1; + info.group_len[info.num_groups - 1] = 1; + } + } + } else { + info.max_sfb = br.read_bits(6)? as usize; + if br.read_bits(1)? == 1 { + return Err(Error::Unsupported("predictor data (AAC Main / LTP)")); + } + } + if info.max_sfb > info.num_swb(sf_index) { + return Err(Error::Bitstream("max_sfb exceeds the number of scalefactor bands")); + } + Ok(info) + } + + pub fn write(&self, bw: &mut BitWriter) { + bw.write_bits(0, 1); + bw.write_bits(self.window_sequence as u32, 2); + bw.write_bits(self.window_shape.bit(), 1); + if self.is_short() { + bw.write_bits(self.max_sfb as u32, 4); + let mut grouping = 0u32; + let mut w = 0; + for g in 0..self.num_groups { + for i in 0..self.group_len[g] as usize { + if w > 0 && i > 0 { + grouping |= 1 << (6 - (w - 1)); + } + w += 1; + } + } + bw.write_bits(grouping, 7); + } else { + bw.write_bits(self.max_sfb as u32, 6); + bw.write_bits(0, 1); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn grouping_roundtrip() { + let info = IcsInfo { + window_sequence: WindowSequence::EightShort, + window_shape: WindowShape::Kbd, + max_sfb: 12, + num_groups: 3, + group_len: [3, 1, 4, 0, 0, 0, 0, 0], + }; + let mut bw = BitWriter::new(); + info.write(&mut bw); + let d = bw.finish(); + let p = IcsInfo::parse(&mut BitReader::new(&d), 4).unwrap(); + assert_eq!(p, info); + assert_eq!(p.group_start(2), 4); + } +} diff --git a/crates/aac/src/lib.rs b/crates/aac/src/lib.rs new file mode 100644 index 0000000000..da9c89af30 --- /dev/null +++ b/crates/aac/src/lib.rs @@ -0,0 +1,79 @@ +//! Ported from FilmCraft (`crates/aac`, commit cee1feb), same authors and licence. +//! +//! Clean-room AAC-LC encoder and decoder (ISO/IEC 14496-3, MPEG-4 Audio, Low Complexity profile). +//! +//! - [`Encoder`]: planar f32 → raw access units (+ [`Encoder::audio_specific_config`] for MP4 `esds`, +//! optional ADTS framing). Psychoacoustic model, window switching, M/S, TNS, bit reservoir, CBR/VBR, +//! 1–8 channels. +//! - [`Decoder`]: `AudioSpecificConfig` + raw access units → planar f32. Supports M/S, intensity +//! stereo, PNS, TNS, pulse data, both window shapes and every window sequence. +//! +//! HE-AAC (SBR/PS), Main/SSR/LTP profiles, coupling channels and 960-sample frames are not supported; +//! for explicitly signalled HE-AAC only the AAC-LC core is decoded (at the core rate). +//! +//! Layer L0: depends only on `deckcraft-bitstream` (+ `thiserror`, optional `rayon`); no `unsafe`; +//! builds for `wasm32-unknown-unknown`. +//! +//! ``` +//! use deckcraft_aac::{Decoder, Encoder, EncoderConfig}; +//! +//! let mut enc = Encoder::new(EncoderConfig::cbr(48_000, 2, 128_000))?; +//! let left: Vec = (0..4800).map(|i| (i as f32 * 0.05).sin() * 0.5).collect(); +//! let right = left.clone(); +//! let mut aus = enc.encode(&[&left, &right]); // 0..n access units +//! aus.extend(enc.flush()); +//! let asc = enc.audio_specific_config(); // for the MP4 `esds` +//! assert_eq!(enc.priming_samples(), 1024); // MP4 edit list media_time +//! +//! let mut dec = Decoder::new(&asc)?; +//! for au in &aus { +//! let pcm = dec.decode(au)?; // planar, 1024 samples per channel +//! assert_eq!(pcm.len(), 2); +//! } +//! # Ok::<(), deckcraft_aac::Error>(()) +//! ``` + +// Index loops over parallel per-band arrays mirror the spec text and stay as in FilmCraft so +// the port can be re-synced. +#![allow(clippy::needless_range_loop)] +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] + +mod config; +mod decoder; +mod encoder; +mod huffman; +mod huffman_tables; +mod ics; +mod mdct; +mod tables; +mod tns; + +pub use config::{AdtsHeader, AudioSpecificConfig, ElementType, ProgramConfig, config_layout, split_adts}; +pub use decoder::Decoder; +pub use encoder::{BitrateMode, Encoder, EncoderConfig, EncoderStats}; +pub use mdct::WindowShape; +pub use tables::{SAMPLE_RATES, sample_rate_index}; + +/// Samples per AAC-LC frame (per channel). +pub const FRAME_LEN: usize = 1024; + +/// Errors from configuration parsing, encoding and decoding. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("invalid configuration: {0}")] + InvalidConfig(&'static str), + #[error("unsupported feature: {0}")] + Unsupported(&'static str), + #[error("invalid bitstream: {0}")] + Bitstream(&'static str), + #[error("unexpected end of access unit")] + Eof, +} + +impl From for Error { + fn from(_: deckcraft_bitstream::BitError) -> Self { + Error::Eof + } +} + +pub type Result = std::result::Result; diff --git a/crates/aac/src/mdct.rs b/crates/aac/src/mdct.rs new file mode 100644 index 0000000000..74d9f2958c --- /dev/null +++ b/crates/aac/src/mdct.rs @@ -0,0 +1,274 @@ +//! MDCT / IMDCT (ISO/IEC 14496-3 §4.6.18) via a DCT-IV computed with an N/4-point complex FFT, +//! plus the sine and Kaiser–Bessel-derived windows. + +use std::f64::consts::PI; +use std::sync::OnceLock; + +/// Radix-2 complex FFT (forward, `e^{-2πi nk/P}`). +struct Fft { + n: usize, + twiddle: Vec<[f32; 2]>, + rev: Vec, +} + +impl Fft { + fn new(n: usize) -> Fft { + assert!(n.is_power_of_two()); + let bits = n.trailing_zeros(); + let rev = (0..n as u32).map(|i| if bits == 0 { 0 } else { i.reverse_bits() >> (32 - bits) }).collect(); + let twiddle = (0..n / 2) + .map(|k| { + let a = -2.0 * PI * k as f64 / n as f64; + [a.cos() as f32, a.sin() as f32] + }) + .collect(); + Fft { n, twiddle, rev } + } + + fn run(&self, re: &mut [f32], im: &mut [f32]) { + let n = self.n; + for i in 0..n { + let j = self.rev[i] as usize; + if j > i { + re.swap(i, j); + im.swap(i, j); + } + } + let mut len = 2; + while len <= n { + let half = len / 2; + let step = n / len; + for start in (0..n).step_by(len) { + for k in 0..half { + let [wr, wi] = self.twiddle[k * step]; + let a = start + k; + let b = a + half; + let xr = re[b] * wr - im[b] * wi; + let xi = re[b] * wi + im[b] * wr; + re[b] = re[a] - xr; + im[b] = im[a] - xi; + re[a] += xr; + im[a] += xi; + } + } + len <<= 1; + } + } +} + +/// MDCT of `2M` inputs to `M` outputs (unscaled sums): +/// `X[k] = Σ x[n] cos(π/M (n + ½ + M/2)(k + ½))`, and the matching unscaled inverse. +pub struct Mdct { + m: usize, + fft: Fft, + pre: Vec<[f32; 2]>, + post: Vec<[f32; 2]>, +} + +impl Mdct { + pub fn new(m: usize) -> Mdct { + let q = m / 2; + let pre = (0..q) + .map(|n| { + let a = -PI * n as f64 / m as f64; + [a.cos() as f32, a.sin() as f32] + }) + .collect(); + let post = (0..q) + .map(|k| { + let a = -PI * (k as f64 + 0.25) / m as f64; + [a.cos() as f32, a.sin() as f32] + }) + .collect(); + Mdct { m, fft: Fft::new(q), pre, post } + } + + /// DCT-IV: `out[k] = Σ u[n] cos(π/M (n+½)(k+½))`. + fn dct4(&self, u: &[f32], out: &mut [f32]) { + let m = self.m; + let q = m / 2; + let mut re = vec![0f32; q]; + let mut im = vec![0f32; q]; + for n in 0..q { + let (a, b) = (u[2 * n], u[m - 1 - 2 * n]); + let [c, s] = self.pre[n]; + re[n] = a * c - b * s; + im[n] = a * s + b * c; + } + self.fft.run(&mut re, &mut im); + for k in 0..q { + let [c, s] = self.post[k]; + let wr = re[k] * c - im[k] * s; + let wi = re[k] * s + im[k] * c; + out[2 * k] = wr; + out[m - 1 - 2 * k] = -wi; + } + } + + /// Forward MDCT of `x` (length 2M, already windowed) into `out` (length M). + pub fn forward(&self, x: &[f32], out: &mut [f32]) { + let m = self.m; + let h = m / 2; + let mut u = vec![0f32; m]; + for n in 0..h { + // quarters a, b, c, d of length M/2 + u[n] = -x[m + h - 1 - n] - x[m + h + n]; + u[h + n] = x[n] - x[m - 1 - n]; + } + self.dct4(&u, out); + } + + /// Inverse MDCT (unscaled): `y[n] = Σ X[k] cos(π/M (n + ½ + M/2)(k + ½))`, `y` has length 2M. + pub fn inverse(&self, spec: &[f32], y: &mut [f32]) { + let m = self.m; + let h = m / 2; + let mut u = vec![0f32; m]; + self.dct4(spec, &mut u); + for n in 0..h { + y[n] = u[h + n]; + y[m - 1 - n] = -u[h + n]; + y[m + h - 1 - n] = -u[n]; + y[m + h + n] = -u[n]; + } + } +} + +/// Cached transforms for 1024 and 128 lines. +pub fn mdct_long() -> &'static Mdct { + static M: OnceLock = OnceLock::new(); + M.get_or_init(|| Mdct::new(1024)) +} +pub fn mdct_short() -> &'static Mdct { + static M: OnceLock = OnceLock::new(); + M.get_or_init(|| Mdct::new(128)) +} + +/// Window shape (`window_shape` bit): 0 = sine, 1 = Kaiser–Bessel derived. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum WindowShape { + #[default] + Sine, + Kbd, +} + +impl WindowShape { + pub fn from_bit(b: bool) -> Self { + if b { WindowShape::Kbd } else { WindowShape::Sine } + } + pub fn bit(self) -> u32 { + (self == WindowShape::Kbd) as u32 + } +} + +fn sine_half(m: usize) -> Vec { + let n = 2 * m; + (0..m).map(|i| (PI / n as f64 * (i as f64 + 0.5)).sin() as f32).collect() +} + +fn bessel_i0(x: f64) -> f64 { + let mut sum = 1.0; + let mut term = 1.0; + let y = x * x / 4.0; + for k in 1..60 { + term *= y / (k as f64 * k as f64); + sum += term; + if term < sum * 1e-17 { + break; + } + } + sum +} + +fn kbd_half(m: usize, alpha: f64) -> Vec { + // w[n] = sqrt(Σ_{j<=n} W(j) / Σ_{j<=M} W(j)), W the Kaiser kernel of length M+1. + let kernel: Vec = (0..=m) + .map(|j| { + let r = (j as f64 - m as f64 / 2.0) / (m as f64 / 2.0); + bessel_i0(PI * alpha * (1.0 - r * r).max(0.0).sqrt()) + }) + .collect(); + let total: f64 = kernel.iter().sum(); + let mut acc = 0.0; + (0..m) + .map(|n| { + acc += kernel[n]; + (acc / total).sqrt() as f32 + }) + .collect() +} + +/// Rising half (length N/2) of the window of the given shape; `long` selects N = 2048, else N = 256. +pub fn window(shape: WindowShape, long: bool) -> &'static [f32] { + static W: OnceLock<[Vec; 4]> = OnceLock::new(); + let w = W.get_or_init(|| [sine_half(1024), kbd_half(1024, 4.0), sine_half(128), kbd_half(128, 6.0)]); + let i = match (long, shape) { + (true, WindowShape::Sine) => 0, + (true, WindowShape::Kbd) => 1, + (false, WindowShape::Sine) => 2, + (false, WindowShape::Kbd) => 3, + }; + &w[i] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn direct(x: &[f32]) -> Vec { + let m = x.len() / 2; + (0..m).map(|k| (0..2 * m).map(|n| x[n] as f64 * (PI / m as f64 * (n as f64 + 0.5 + m as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum()).collect() + } + + #[test] + fn forward_matches_definition() { + for m in [128usize, 1024] { + let x: Vec = (0..2 * m).map(|i| ((i * 7919) % 101) as f32 / 50.0 - 1.0).collect(); + let mut out = vec![0f32; m]; + Mdct::new(m).forward(&x, &mut out); + let d = direct(&x); + for k in 0..m { + assert!((out[k] as f64 - d[k]).abs() < 1e-3 * (m as f64).sqrt(), "m={m} k={k} {} {}", out[k], d[k]); + } + } + } + + #[test] + fn inverse_matches_definition() { + let m = 128; + let spec: Vec = (0..m).map(|i| ((i * 31) % 17) as f32 - 8.0).collect(); + let mut y = vec![0f32; 2 * m]; + Mdct::new(m).inverse(&spec, &mut y); + for n in 0..2 * m { + let d: f64 = (0..m).map(|k| spec[k] as f64 * (PI / m as f64 * (n as f64 + 0.5 + m as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum(); + assert!((y[n] as f64 - d).abs() < 1e-3, "n={n}"); + } + } + + #[test] + fn tdac_perfect_reconstruction() { + let m = 1024; + let t = Mdct::new(m); + for shape in [WindowShape::Sine, WindowShape::Kbd] { + let w = window(shape, true); + let win = |n: usize| if n < m { w[n] } else { w[2 * m - 1 - n] }; + let x: Vec = (0..4 * m).map(|i| ((i as f32) * 0.013).sin() + ((i * 13) % 7) as f32 * 0.05).collect(); + let mut out = vec![0f32; 4 * m]; + for f in 0..3 { + let seg: Vec = (0..2 * m).map(|n| x[f * m + n] * win(n)).collect(); + let mut spec = vec![0f32; m]; + t.forward(&seg, &mut spec); + for v in spec.iter_mut() { + *v *= 2.0; + } + let mut y = vec![0f32; 2 * m]; + t.inverse(&spec, &mut y); + for n in 0..2 * m { + out[f * m + n] += y[n] * win(n) / m as f32; + } + } + for n in m..3 * m { + assert!((out[n] - x[n]).abs() < 1e-4, "{shape:?} n={n} {} {}", out[n], x[n]); + } + } + } +} diff --git a/crates/aac/src/tables.rs b/crates/aac/src/tables.rs new file mode 100644 index 0000000000..7b17e183ca --- /dev/null +++ b/crates/aac/src/tables.rs @@ -0,0 +1,143 @@ +//! Sampling-rate dependent tables from ISO/IEC 14496-3 §4.5.4 (scalefactor bands) and §4.6.9 (TNS). + +/// Sampling frequencies indexed by `samplingFrequencyIndex`. +pub const SAMPLE_RATES: [u32; 13] = [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350]; + +/// Map an arbitrary sample rate to its `samplingFrequencyIndex`, if it is one of the standard rates. +pub fn sample_rate_index(rate: u32) -> Option { + SAMPLE_RATES.iter().position(|&r| r == rate).map(|i| i as u8) +} + +/// The table index used for scalefactor-band tables when a non-standard rate is signalled explicitly +/// (§4.5.4: rates are mapped to the nearest table by frequency ranges). +pub fn table_index_for_rate(rate: u32) -> u8 { + match rate { + 92017.. => 0, + 75132.. => 1, + 55426.. => 2, + 46009.. => 3, + 37566.. => 4, + 27713.. => 5, + 23004.. => 6, + 18783.. => 7, + 13856.. => 8, + 11502.. => 9, + 9391.. => 10, + _ => 11, + } +} + +const SWB_LONG_96: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 64, 72, 80, 88, 96, 108, 120, 132, 144, 156, 172, 188, 212, 240, 276, 320, 384, 448, + 512, 576, 640, 704, 768, 832, 896, 960, 1024, +]; +const SWB_LONG_64: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 64, 72, 80, 88, 100, 112, 124, 140, 156, 172, 192, 216, 240, 268, 304, 344, 384, 424, + 464, 504, 544, 584, 624, 664, 704, 744, 784, 824, 864, 904, 944, 984, 1024, +]; +const SWB_LONG_48: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 48, 56, 64, 72, 80, 88, 96, 108, 120, 132, 144, 160, 176, 196, 216, 240, 264, 292, 320, 352, 384, 416, + 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800, 832, 864, 896, 928, 1024, +]; +const SWB_LONG_32: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 48, 56, 64, 72, 80, 88, 96, 108, 120, 132, 144, 160, 176, 196, 216, 240, 264, 292, 320, 352, 384, 416, + 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800, 832, 864, 896, 928, 960, 992, 1024, +]; +const SWB_LONG_24: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 52, 60, 68, 76, 84, 92, 100, 108, 116, 124, 136, 148, 160, 172, 188, 204, 220, 240, 260, 284, 308, + 336, 364, 396, 432, 468, 508, 552, 600, 652, 704, 768, 832, 896, 960, 1024, +]; +const SWB_LONG_16: &[u16] = &[ + 0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 100, 112, 124, 136, 148, 160, 172, 184, 196, 212, 228, 244, 260, 280, 300, 320, 344, 368, 396, 424, + 456, 492, 532, 572, 616, 664, 716, 772, 832, 896, 960, 1024, +]; +const SWB_LONG_8: &[u16] = &[ + 0, 12, 24, 36, 48, 60, 72, 84, 96, 108, 120, 132, 144, 156, 172, 188, 204, 220, 236, 252, 268, 288, 308, 328, 348, 372, 396, 420, 448, 476, 508, + 544, 580, 620, 664, 712, 764, 820, 880, 944, 1024, +]; + +const SWB_SHORT_96: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 32, 40, 48, 64, 92, 128]; +const SWB_SHORT_48: &[u16] = &[0, 4, 8, 12, 16, 20, 28, 36, 44, 56, 68, 80, 96, 112, 128]; +const SWB_SHORT_24: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 28, 36, 44, 52, 64, 76, 92, 108, 128]; +const SWB_SHORT_16: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 60, 72, 88, 108, 128]; +const SWB_SHORT_8: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 28, 36, 44, 52, 60, 72, 88, 108, 128]; + +/// Scalefactor band offsets (`swb_offset_long_window`, 1024-line frames); `len() == num_swb + 1`. +pub fn swb_offsets_long(sf_index: u8) -> &'static [u16] { + match sf_index { + 0 | 1 => SWB_LONG_96, + 2 => SWB_LONG_64, + 3 | 4 => SWB_LONG_48, + 5 => SWB_LONG_32, + 6 | 7 => SWB_LONG_24, + 8..=10 => SWB_LONG_16, + _ => SWB_LONG_8, + } +} + +/// Scalefactor band offsets for one short window (128 lines). +pub fn swb_offsets_short(sf_index: u8) -> &'static [u16] { + match sf_index { + 0..=2 => SWB_SHORT_96, + 3..=5 => SWB_SHORT_48, + 6 | 7 => SWB_SHORT_24, + 8..=10 => SWB_SHORT_16, + _ => SWB_SHORT_8, + } +} + +/// `TNS_MAX_BANDS` for AAC-LC (long, short). +pub fn tns_max_bands(sf_index: u8, short: bool) -> usize { + const LONG: [u8; 13] = [31, 31, 34, 40, 42, 51, 46, 46, 42, 42, 42, 39, 39]; + const SHORT: [u8; 13] = [9, 9, 10, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14]; + let i = (sf_index as usize).min(12); + if short { SHORT[i] as usize } else { LONG[i] as usize } +} + +/// `TNS_MAX_ORDER` for AAC-LC. +pub fn tns_max_order(short: bool) -> usize { + if short { 7 } else { 12 } +} + +/// `x^(4/3)` for the integer quantised magnitudes 0..=8191. +pub fn pow43(q: u32) -> f32 { + use std::sync::OnceLock; + static TABLE: OnceLock> = OnceLock::new(); + let t = TABLE.get_or_init(|| (0..8192u32).map(|i| (i as f64).powf(4.0 / 3.0) as f32).collect()); + match t.get(q as usize) { + Some(v) => *v, + None => (q as f64).powf(4.0 / 3.0) as f32, + } +} + +/// `2^(0.25 * (sf - 100))`, the dequantisation gain of scalefactor `sf`. +#[inline] +pub fn sf_gain(sf: i32) -> f32 { + (0.25 * (sf as f64 - 100.0)).exp2() as f32 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn band_tables_are_monotonic_and_complete() { + for i in 0..13u8 { + let l = swb_offsets_long(i); + assert_eq!(*l.last().unwrap(), 1024); + assert!(l.windows(2).all(|w| w[0] < w[1] && (w[1] - w[0]) % 4 == 0)); + let s = swb_offsets_short(i); + assert_eq!(*s.last().unwrap(), 128); + assert!(s.windows(2).all(|w| w[0] < w[1] && (w[1] - w[0]) % 4 == 0)); + assert!(tns_max_bands(i, false) < l.len()); + assert!(tns_max_bands(i, true) < s.len()); + } + assert_eq!(swb_offsets_long(3).len() - 1, 49); + assert_eq!(swb_offsets_long(5).len() - 1, 51); + assert_eq!(swb_offsets_long(0).len() - 1, 41); + assert_eq!(swb_offsets_long(2).len() - 1, 47); + assert_eq!(swb_offsets_long(6).len() - 1, 47); + assert_eq!(swb_offsets_long(8).len() - 1, 43); + assert_eq!(swb_offsets_long(11).len() - 1, 40); + } +} diff --git a/crates/aac/src/tns.rs b/crates/aac/src/tns.rs new file mode 100644 index 0000000000..b0c86a0de5 --- /dev/null +++ b/crates/aac/src/tns.rs @@ -0,0 +1,313 @@ +//! Temporal noise shaping (ISO/IEC 14496-3 §4.6.9): syntax, coefficient conversion, and the +//! decoder (all-pole) and encoder (all-zero) filters. + +use std::f64::consts::FRAC_PI_2; + +use deckcraft_bitstream::{BitReader, BitWriter}; + +use crate::ics::IcsInfo; +use crate::tables::{tns_max_bands, tns_max_order}; +use crate::{Error, Result}; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TnsFilter { + /// Length in scalefactor bands. + pub length: u8, + pub order: u8, + /// false = upward, true = downward. + pub direction: bool, + pub coef_compress: bool, + /// Quantised reflection coefficients (signed, `coef_res + 3 - coef_compress` bits each). + pub coef: [i8; 12], +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TnsWindow { + pub n_filt: u8, + /// false = 3-bit, true = 4-bit coefficient resolution. + pub coef_res: bool, + pub filt: [TnsFilter; 4], +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TnsData { + pub windows: [TnsWindow; 8], +} + +impl TnsData { + pub fn parse(br: &mut BitReader, info: &IcsInfo) -> Result { + let short = info.is_short(); + let mut t = TnsData::default(); + for w in 0..info.num_windows() { + let win = &mut t.windows[w]; + win.n_filt = br.read_bits(if short { 1 } else { 2 })? as u8; + if win.n_filt == 0 { + continue; + } + win.coef_res = br.read_bits(1)? == 1; + for f in 0..win.n_filt as usize { + let filt = &mut win.filt[f]; + filt.length = br.read_bits(if short { 4 } else { 6 })? as u8; + filt.order = br.read_bits(if short { 3 } else { 5 })? as u8; + if filt.order as usize > tns_max_order(short) { + return Err(Error::Bitstream("TNS order too large")); + } + if filt.order > 0 { + filt.direction = br.read_bits(1)? == 1; + filt.coef_compress = br.read_bits(1)? == 1; + let bits = 3 + win.coef_res as u32 - filt.coef_compress as u32; + for i in 0..filt.order as usize { + let v = br.read_bits(bits)? as i32; + let v = if v & (1 << (bits - 1)) != 0 { v - (1 << bits) } else { v }; + filt.coef[i] = v as i8; + } + } + } + } + Ok(t) + } + + pub fn write(&self, bw: &mut BitWriter, info: &IcsInfo) { + let short = info.is_short(); + for w in 0..info.num_windows() { + let win = &self.windows[w]; + bw.write_bits(win.n_filt as u32, if short { 1 } else { 2 }); + if win.n_filt == 0 { + continue; + } + bw.write_bits(win.coef_res as u32, 1); + for f in 0..win.n_filt as usize { + let filt = &win.filt[f]; + bw.write_bits(filt.length as u32, if short { 4 } else { 6 }); + bw.write_bits(filt.order as u32, if short { 3 } else { 5 }); + if filt.order > 0 { + bw.write_bits(filt.direction as u32, 1); + bw.write_bits(filt.coef_compress as u32, 1); + let bits = 3 + win.coef_res as u32 - filt.coef_compress as u32; + for i in 0..filt.order as usize { + bw.write_bits(filt.coef[i] as i32 as u32 & ((1 << bits) - 1), bits); + } + } + } + } + } + + /// Bits `write` would produce. + pub fn bit_count(&self, info: &IcsInfo) -> u32 { + let short = info.is_short(); + let mut n = 0; + for w in 0..info.num_windows() { + let win = &self.windows[w]; + n += if short { 1 } else { 2 }; + if win.n_filt == 0 { + continue; + } + n += 1; + for f in 0..win.n_filt as usize { + let filt = &win.filt[f]; + n += if short { 7 } else { 11 }; + if filt.order > 0 { + n += 2 + filt.order as u32 * (3 + win.coef_res as u32 - filt.coef_compress as u32); + } + } + } + n + } +} + +/// Dequantise one reflection coefficient. +pub fn dequant_coef(c: i8, coef_res: bool) -> f64 { + let res = 3 + coef_res as i32; + let iqfac = ((1 << (res - 1)) as f64 - 0.5) / FRAC_PI_2; + let iqfac_m = ((1 << (res - 1)) as f64 + 0.5) / FRAC_PI_2; + let c = c as f64; + (c / if c >= 0.0 { iqfac } else { iqfac_m }).sin() +} + +/// Quantise a reflection coefficient (inverse of [`dequant_coef`], nearest index). +pub fn quant_coef(k: f64, coef_res: bool) -> i8 { + let res = 3 + coef_res as i32; + let iqfac = ((1 << (res - 1)) as f64 - 0.5) / FRAC_PI_2; + let iqfac_m = ((1 << (res - 1)) as f64 + 0.5) / FRAC_PI_2; + let a = k.clamp(-1.0, 1.0).asin(); + let lo = -(1 << (res - 1)); + let hi = (1 << (res - 1)) - 1; + let q = if a >= 0.0 { (a * iqfac).round() } else { (a * iqfac_m).round() }; + (q as i32).clamp(lo, hi) as i8 +} + +/// Reflection coefficients → direct-form LPC `a[0..=order]` (`a[0] = 1`). +pub fn parcor_to_lpc(k: &[f64]) -> [f64; 13] { + let mut a = [0f64; 13]; + a[0] = 1.0; + let mut b = [0f64; 13]; + for m in 1..=k.len() { + for i in 1..m { + b[i] = a[i] + k[m - 1] * a[m - i]; + } + a[1..m].copy_from_slice(&b[1..m]); + a[m] = k[m - 1]; + } + a +} + +/// Filter region `[start, end)` of every TNS filter of window `w`, in spectrum lines of that window. +fn regions(info: &IcsInfo, sf_index: u8, win: &TnsWindow) -> Vec<(usize, usize, usize)> { + let short = info.is_short(); + let swb = info.swb_offsets(sf_index); + let num_swb = swb.len() - 1; + let lim = tns_max_bands(sf_index, short).min(info.max_sfb); + let mut out = Vec::new(); + let mut bottom = num_swb; + for f in 0..win.n_filt as usize { + let filt = &win.filt[f]; + let top = bottom; + bottom = top.saturating_sub(filt.length as usize); + if filt.order == 0 { + continue; + } + let start = swb[bottom.min(lim)] as usize; + let end = swb[top.min(lim)] as usize; + if end > start { + out.push((f, start, end)); + } + } + out +} + +fn lpc_of(filt: &TnsFilter, coef_res: bool) -> [f64; 13] { + let k: Vec = filt.coef[..filt.order as usize].iter().map(|&c| dequant_coef(c, coef_res)).collect(); + parcor_to_lpc(&k) +} + +/// Decoder: apply the all-pole TNS synthesis filters to `spec` (1024 lines; short windows at 128 stride). +pub fn apply_decoder(spec: &mut [f32], info: &IcsInfo, sf_index: u8, tns: &TnsData) { + let wlen = if info.is_short() { 128 } else { 1024 }; + for w in 0..info.num_windows() { + let win = &tns.windows[w]; + for (f, start, end) in regions(info, sf_index, win) { + let filt = &win.filt[f]; + let a = lpc_of(filt, win.coef_res); + let order = filt.order as usize; + let s = &mut spec[w * wlen..(w + 1) * wlen]; + let size = end - start; + let mut state = [0f64; 12]; + for m in 0..size { + let idx = if filt.direction { end - 1 - m } else { start + m }; + let mut y = s[idx] as f64; + for i in 0..order { + y -= a[i + 1] * state[i]; + } + for i in (1..order).rev() { + state[i] = state[i - 1]; + } + if order > 0 { + state[0] = y; + } + s[idx] = y as f32; + } + } + } +} + +/// Encoder: apply the all-zero TNS analysis filters (exact inverse of [`apply_decoder`]). +pub fn apply_encoder(spec: &mut [f32], info: &IcsInfo, sf_index: u8, tns: &TnsData) { + let wlen = if info.is_short() { 128 } else { 1024 }; + for w in 0..info.num_windows() { + let win = &tns.windows[w]; + for (f, start, end) in regions(info, sf_index, win) { + let filt = &win.filt[f]; + let a = lpc_of(filt, win.coef_res); + let order = filt.order as usize; + let s = &mut spec[w * wlen..(w + 1) * wlen]; + let size = end - start; + let mut state = [0f64; 12]; + for m in 0..size { + let idx = if filt.direction { end - 1 - m } else { start + m }; + let x = s[idx] as f64; + let mut y = x; + for i in 0..order { + y += a[i + 1] * state[i]; + } + for i in (1..order).rev() { + state[i] = state[i - 1]; + } + if order > 0 { + state[0] = x; + } + s[idx] = y as f32; + } + } + } +} + +/// Levinson–Durbin on autocorrelation `r[0..=order]`: returns reflection coefficients and the +/// prediction gain `r[0] / error`. +pub fn levinson(r: &[f64], order: usize) -> (Vec, f64) { + let mut a = vec![0f64; order + 1]; + a[0] = 1.0; + let mut err = r[0]; + let mut ks = Vec::with_capacity(order); + if err <= 0.0 { + return (vec![0.0; order], 1.0); + } + for m in 1..=order { + let mut acc = r[m]; + for i in 1..m { + acc += a[i] * r[m - i]; + } + let k = -acc / err; + let prev = a.clone(); + for i in 1..m { + a[i] = prev[i] + k * prev[m - i]; + } + a[m] = k; + err *= 1.0 - k * k; + ks.push(k); + if err <= r[0] * 1e-9 { + ks.resize(order, 0.0); + break; + } + } + (ks, r[0] / err.max(r[0] * 1e-9)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ics::WindowSequence; + use crate::mdct::WindowShape; + + #[test] + fn coef_quant_roundtrip() { + for res in [false, true] { + let n = if res { 8 } else { 4 }; + for c in -n..n { + let k = dequant_coef(c as i8, res); + assert_eq!(quant_coef(k, res), c as i8); + } + } + } + + #[test] + fn encoder_decoder_filters_are_inverse() { + let info = IcsInfo { window_sequence: WindowSequence::OnlyLong, window_shape: WindowShape::Sine, max_sfb: 49, ..Default::default() }; + let mut t = TnsData::default(); + t.windows[0].n_filt = 1; + t.windows[0].coef_res = true; + t.windows[0].filt[0] = TnsFilter { length: 40, order: 4, direction: true, coef_compress: false, coef: [3, -2, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0] }; + let orig: Vec = (0..1024).map(|i| ((i * 37 % 101) as f32 - 50.0) * 0.1).collect(); + let mut s = orig.clone(); + apply_encoder(&mut s, &info, 4, &t); + assert_ne!(s, orig); + apply_decoder(&mut s, &info, 4, &t); + for i in 0..1024 { + assert!((s[i] - orig[i]).abs() < 1e-3); + } + let mut bw = BitWriter::new(); + t.write(&mut bw, &info); + assert_eq!(bw.bit_len() as u32, t.bit_count(&info)); + let d = bw.finish(); + assert_eq!(TnsData::parse(&mut BitReader::new(&d), &info).unwrap(), t); + } +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 778664a7da..ff7cc52f6c 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -19,6 +19,10 @@ deckcraft-pdf = { workspace = true } deckcraft-format = { workspace = true } deckcraft-pptx = { workspace = true } deckcraft-media = { workspace = true } +deckcraft-h264 = { workspace = true } +deckcraft-isobmff = { workspace = true } +deckcraft-aac = { workspace = true } +deckcraft-anim = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } i_overlay = { workspace = true } diff --git a/crates/engine/src/cmd/file.rs b/crates/engine/src/cmd/file.rs index 4761dc103b..bc74666943 100644 --- a/crates/engine/src/cmd/file.rs +++ b/crates/engine/src/cmd/file.rs @@ -33,7 +33,7 @@ pub fn specs() -> Vec { cmd!(noundo "file.saveAs", "Save As…", ["File"], Some("Cmd+Shift+S"), "{path, format?}", has_doc, save_as), cmd!(noundo "file.saveTemplate", "Save as Template…", ["File"], None, "{path}", has_doc, save_as), cmd!(query "file.saveBytes", "Save to Data", [], None, "{format?: deckcraft|pptx} → {data: base64}", has_doc, save_bytes_cmd), - cmd!(noundo "file.export", "Export…", ["File"], None, "{path, format?: png|jpeg|pptx|deckcraft|outline|pdf|gif (else by path), slide?: index, all?: bool, scale?: px per pt, layout?: slides|notes|handouts (pdf), perPage?: 1|2|3|4|6|9, paper?: letter|a4|legal|a3 (pdf), orientation?: portrait|landscape (pdf), dpi?, slides?: [index]|range (\"1-3,5\"), includeHidden?, textLayer?, frame?, width?: px 16-3840|small|medium|large|extraLarge (gif), secondsPerSlide?: 0.1-600 (gif; 5), loop?: bool (gif; true: forever)} → {path, bytes} or {files} (png, jpeg); gif adds frames, width, height, secondsPerSlide", has_doc, export), + cmd!(noundo "file.export", "Export…", ["File"], None, "{path, format?: png|jpeg|pptx|deckcraft|outline|pdf|gif|mp4|mov (else by path), slide?: index, all?: bool, scale?: px per pt, layout?: slides|notes|handouts (pdf), perPage?: 1|2|3|4|6|9, paper?: letter|a4|legal|a3 (pdf), orientation?: portrait|landscape (pdf), dpi?, slides?: [index]|range (\"1-3,5\"), includeHidden?, textLayer?, frame?, width?: px 16-3840|small|medium|large|extraLarge (gif), secondsPerSlide?: 0.1-600 (gif; 5; also mp4/mov), loop?: bool (gif; true: forever), height?: 480|720|1080|2160 (mp4/mov), fps? (mp4/mov), secondsPerClick? (mp4/mov), audio?: aac|pcm|none (mp4/mov; pcm in .mov)} → {path, bytes} or {files} (png, jpeg); gif adds frames, width, height, secondsPerSlide", has_doc, export), cmd!( query "file.info", "Info", @@ -337,7 +337,7 @@ pub fn pdf_bytes(doc: &Presentation, p: &Value) -> Result> { } /// The output extensions [`format_for_path`] maps to a writer of that format. -pub const WRITABLE_EXTENSIONS: &[&str] = &["deckcraft", "pptx", "potx", "ppsx", "pdf", "png", "jpg", "jpeg", "gif", "txt", "md"]; +pub const WRITABLE_EXTENSIONS: &[&str] = &["deckcraft", "pptx", "potx", "ppsx", "pdf", "png", "jpg", "jpeg", "gif", "mp4", "m4v", "mov", "txt", "md"]; /// The format a file named `path` is written in, or `None` when its extension names no format /// DeckCraft writes (`.odp`, `.svg`, no extension …): [`format_for_path`] would save @@ -368,6 +368,10 @@ pub fn format_for_path(path: &str) -> &'static str { "pdf" } else if l.ends_with(".gif") { "gif" + } else if l.ends_with(".mp4") || l.ends_with(".m4v") { + "mp4" + } else if l.ends_with(".mov") { + "mov" } else if l.ends_with(".txt") || l.ends_with(".md") { "outline" } else { @@ -514,6 +518,7 @@ fn export(s: &mut Session, p: &Value) -> Result { } Ok(json!({"files": written})) } + "mp4" | "mov" => video_export(&st.doc, &path, &format, p), "pdf" => { let bytes = pdf_bytes(&st.doc, p)?; write_file(&path, &bytes)?; @@ -545,6 +550,38 @@ fn export(s: &mut Session, p: &Value) -> Result { } } +/// The `file.export` video options (`height`, `fps`, `secondsPerSlide`, `secondsPerClick`, +/// `includeHidden`, `audio`: aac|pcm|none) as [`crate::video::VideoOpts`]. +pub fn video_opts(p: &Value) -> Result { + use crate::video::AudioFormat; + let d = crate::video::VideoOpts::default(); + let audio = match str_param(p, "audio") { + None | Some("aac") => AudioFormat::Aac, + Some("pcm") => AudioFormat::Pcm, + Some("none") => AudioFormat::None, + Some(other) => return Err(bad("file.export", format!("audio “{other}”: use aac, pcm (.mov) or none"))), + }; + Ok(crate::video::VideoOpts { + height: usize_param(p, "height").map(|h| h.clamp(2, 4320) as u32).unwrap_or(d.height), + fps: usize_param(p, "fps").map(|f| f.clamp(1, 60) as u32).unwrap_or(d.fps), + seconds_per_slide: f64_or(p, "secondsPerSlide", d.seconds_per_slide).clamp(0.1, 3600.0), + seconds_per_click: f64_or(p, "secondsPerClick", d.seconds_per_click).clamp(0.0, 3600.0), + include_hidden: bool_or(p, "includeHidden", false), + audio, + }) +} + +/// File › Export › Create a Video. +fn video_export(doc: &deckcraft_model::Presentation, path: &str, format: &str, p: &Value) -> Result { + let opts = video_opts(p)?; + let v = crate::video::encode(doc, format, &opts).map_err(|e| bad("file.export", e))?; + write_file(path, &v.bytes)?; + let fps = opts.fps.clamp(1, 60); + Ok( + json!({"path": path, "bytes": v.bytes.len(), "frames": v.frames, "width": v.size.0, "height": v.size.1, "fps": fps, "seconds": v.frames as f64 / fps as f64, "audioSeconds": v.audio_seconds, "warnings": v.warnings}), + ) +} + /// File › Info: what the Info page shows about the active presentation. fn info(s: &mut Session, _p: &Value) -> Result { let st = s.doc()?; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ac25d6669a..22c2da3b56 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -28,6 +28,7 @@ pub mod spell; pub mod templates; pub mod thesaurus; pub mod tools; +pub mod video; use std::sync::Arc; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index f7022eab81..934427d707 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -326,6 +326,8 @@ fn writable_format_for_path_rejects_formats_deckcraft_does_not_write() { assert_eq!(f("a.jpg"), Some("jpeg")); assert_eq!(f("a.txt"), Some("outline")); assert_eq!(f("a.gif"), Some("gif")); + assert_eq!(f("a.mp4"), Some("mp4")); + assert_eq!(f("a.mov"), Some("mov")); for path in ["a.odp", "a.bmp", "a.svg", "a.outline", "a", "dir.v2/a"] { assert_eq!(f(path), None, "{path}"); } diff --git a/crates/engine/src/video.rs b/crates/engine/src/video.rs new file mode 100644 index 0000000000..c7684b8c1b --- /dev/null +++ b/crates/engine/src/video.rs @@ -0,0 +1,951 @@ +//! Video export (File › Export › Create a Video): the deck as an MP4 or MOV. Each slide plays +//! its transition, then its animations on the slide-show timeline (click steps one after another, +//! a pause before each), then stays up for its timing. Frames come from the slide renderer and are +//! encoded with DeckCraft's own H.264 encoder ([`deckcraft_h264::Encoder`]: intra-coded changes, +//! skipped unchanged macroblocks); the sound of the slides' audio and video clips (narration is +//! recorded as such clips) and of transitions is mixed and encoded as AAC ([`deckcraft_aac`]), or +//! kept as PCM in a MOV. Video clips play their pictures (decoded by [`deckcraft_media`]) while +//! they play. Everything is muxed by [`deckcraft_isobmff::Mp4Writer`]; no external tool is needed. + +use std::collections::HashMap; +use std::sync::Arc; + +use deckcraft_anim::{Sequence, ShowState, Timeline}; +use deckcraft_model::anim::AnimClass; +use deckcraft_model::{MediaId, Presentation, ShapeId, ShapeKind}; +use deckcraft_render::{ParaState, ShapeState}; + +/// The most frames one export writes (two hours at 30 fps), against runaway timings. +pub const MAX_FRAMES: u64 = 2 * 60 * 60 * 30; +/// Audio sample rate of the exported track. +pub const AUDIO_RATE: u32 = 48_000; + +/// How the sound is written. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AudioFormat { + Aac, + /// 16-bit PCM (MOV only). + Pcm, + None, +} + +/// Export options, as `file.export` takes them. +#[derive(Clone, Debug, PartialEq)] +pub struct VideoOpts { + /// Frame height in pixels (PowerPoint's 480p / 720p / 1080p / 4K); the width follows the + /// slide's aspect ratio. Both are rounded to even numbers, as H.264 needs. + pub height: u32, + pub fps: u32, + /// How long a slide stays up after its animations when it has no "advance after" timing. + pub seconds_per_slide: f64, + /// The pause before each click step of a slide's animations. + pub seconds_per_click: f64, + pub include_hidden: bool, + pub audio: AudioFormat, +} + +impl Default for VideoOpts { + fn default() -> Self { + VideoOpts { height: 1080, fps: 30, seconds_per_slide: 5.0, seconds_per_click: 1.0, include_hidden: false, audio: AudioFormat::Aac } + } +} + +/// One step of the timeline: a transition frame at progress `t`; a frame of slide `index`'s +/// animations at main step `step`, `t` seconds in; the slide still before main step `step` for +/// `frames` frames; or the slide with every animation done, held for `frames` frames. +#[derive(Clone, Debug, PartialEq)] +pub enum Step { + Transition { index: usize, t: f64 }, + Anim { index: usize, step: usize, t: f64 }, + Still { index: usize, step: usize, frames: u64 }, + Hold { index: usize, frames: u64 }, +} + +impl Step { + pub fn index(&self) -> usize { + match self { + Step::Transition { index, .. } | Step::Anim { index, .. } | Step::Still { index, .. } | Step::Hold { index, .. } => *index, + } + } + fn frames(&self) -> u64 { + match self { + Step::Still { frames, .. } | Step::Hold { frames, .. } => *frames, + _ => 1, + } + } +} + +/// A sound that starts at `start` seconds into the video and stops by `until`. +#[derive(Clone, Debug, PartialEq)] +pub struct AudioCue { + pub media: MediaId, + pub start: f64, + pub until: f64, + pub trim_start: f64, + pub trim_end: f64, + pub volume: f64, + pub fade_in: f64, + pub fade_out: f64, + pub looped: bool, + /// The video clip whose pictures play with this cue: (slide index, shape). + pub picture: Option<(usize, ShapeId)>, + /// Back to the poster frame when the clip ends. + pub rewind: bool, + /// The clip's length (0: unknown, play until the pictures run out). + pub length: f64, +} + +/// The video's frames and sounds. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Plan { + pub steps: Vec, + pub cues: Vec, + pub frames: u64, +} + +/// The frame size for `opts` on this deck's slide size: even, at least 2×2. +pub fn frame_size(pres: &Presentation, opts: &VideoOpts) -> (u32, u32) { + let h = opts.height.clamp(2, 4320) & !1; + let aspect = pres.slide_size.width / pres.slide_size.height.max(1.0); + let aspect = if aspect.is_finite() && aspect > 0.0 { aspect.clamp(0.1, 10.0) } else { 16.0 / 9.0 }; + let w = ((h as f64 * aspect).round() as u32).clamp(2, 8192) & !1; + (w, h) +} + +/// Seconds as frames at `fps`, at least one. +fn frames_for(seconds: f64, fps: u32) -> u64 { + let s = if seconds.is_finite() { seconds.clamp(0.0, 3600.0) } else { 0.0 }; + ((s * fps as f64).round() as u64).max(1) +} + +fn secs(ms: u32) -> f64 { + f64::from(ms) / 1000.0 +} + +/// The clip a cue plays, from a media shape. +fn clip_cue(c: &deckcraft_model::MediaClip, index: usize, shape: ShapeId, start: f64, until: f64) -> AudioCue { + AudioCue { + media: c.media, + start, + until, + trim_start: secs(c.trim_start_ms), + trim_end: secs(c.trim_end_ms), + volume: if c.volume.is_finite() { c.volume.clamp(0.0, 1.0) } else { 1.0 }, + fade_in: secs(c.fade_in_ms), + fade_out: secs(c.fade_out_ms), + looped: c.loop_play, + picture: c.video.then_some((index, shape)), + rewind: c.rewind, + length: secs(c.duration_ms), + } +} + +/// Pushes steps while counting frames against [`MAX_FRAMES`]. +struct Layout { + steps: Vec, + frames: u64, + fps: u32, +} + +impl Layout { + /// Add `step`; `false` once the cap is reached. + fn push(&mut self, step: Step) -> bool { + let room = MAX_FRAMES.saturating_sub(self.frames); + if room == 0 { + return false; + } + let step = match step { + Step::Still { index, step, frames } => Step::Still { index, step, frames: frames.min(room) }, + Step::Hold { index, frames } => Step::Hold { index, frames: frames.min(room) }, + s => s, + }; + self.frames = self.frames.saturating_add(step.frames()); + self.steps.push(step); + true + } + fn now(&self) -> f64 { + self.frames as f64 / f64::from(self.fps) + } +} + +/// The timeline: for each slide shown, its transition's frames (none for "none", a cut or a +/// zero duration), its main animation sequence (each click step after a `seconds_per_click` +/// pause; steps that start by themselves follow on), then the slide held for its "advance +/// after" time (counted from the slide's start, as recorded timings are), else +/// `seconds_per_slide`. Sounds: transition sounds, clips that play automatically or from a +/// media effect, until their end or the slide's (the deck's when they play across slides). +/// The total is capped at [`MAX_FRAMES`]. +pub fn plan(pres: &Presentation, opts: &VideoOpts) -> Plan { + let fps = opts.fps.clamp(1, 60); + let mut lay = Layout { steps: vec![], frames: 0, fps }; + let mut cues: Vec<(AudioCue, bool)> = vec![]; // (cue, ends with its slide) + let animate = !pres.show.without_animation; + 'slides: for (index, slide) in pres.slides.iter().enumerate() { + if slide.hidden && !opts.include_hidden { + continue; + } + let slide_start = lay.now(); + if let Some(tr) = slide.transition.as_ref() { + if let Some(m) = tr.sound { + cues.push(( + AudioCue { + media: m, + start: slide_start, + until: f64::MAX, + trim_start: 0.0, + trim_end: 0.0, + volume: 1.0, + fade_in: 0.0, + fade_out: 0.0, + looped: false, + picture: None, + rewind: false, + length: 0.0, + }, + false, + )); + } + if !matches!(tr.kind.as_str(), "none" | "cut" | "") && tr.duration_ms > 0 { + let n = frames_for(secs(tr.duration_ms), fps); + for k in 0..n { + if !lay.push(Step::Transition { index, t: k as f64 / n as f64 }) { + break 'slides; + } + } + } + } + let content_start = lay.now(); + let first_cue = cues.len(); + let mut started: Vec = vec![]; + for sh in &slide.shapes { + if let ShapeKind::Media(c) = &sh.kind + && c.autoplay + { + cues.push((clip_cue(c, index, sh.id, content_start, f64::MAX), !c.play_across_slides)); + started.push(sh.id); + } + } + let tl = if animate { ShowState::timeline(pres, index) } else { Timeline::default() }; + let effects = tl.effects(); + for step in 0..tl.steps() { + if !tl.step_is_auto(step) && !lay.push(Step::Still { index, step, frames: frames_for(opts.seconds_per_click, fps) }) { + break 'slides; + } + let step_start = lay.now(); + for e in effects.iter().filter(|e| e.sequence == Sequence::Main && e.step == step && e.class == AnimClass::Media) { + if started.contains(&e.shape) { + continue; + } + if let Some(ShapeKind::Media(c)) = slide.shape(e.shape).map(|s| &s.kind) { + cues.push((clip_cue(c, index, e.shape, step_start + e.start.max(0.0), f64::MAX), !c.play_across_slides)); + started.push(e.shape); + } + } + let dur = tl.step_duration(step); + if dur > 0.0 { + let n = frames_for(dur, fps); + for k in 0..n { + if !lay.push(Step::Anim { index, step, t: k as f64 * dur / n as f64 }) { + break 'slides; + } + } + } + } + let hold = match slide.transition.as_ref().and_then(|t| t.advance_after_ms) { + Some(ms) => (secs(ms) - (lay.now() - slide_start)).max(0.5), + None => opts.seconds_per_slide, + }; + if !lay.push(Step::Hold { index, frames: frames_for(hold, fps) }) { + break; + } + let end = lay.now(); + for (c, ends) in cues.iter_mut().skip(first_cue) { + if *ends { + c.until = end; + } + } + } + let total = lay.now(); + let cues = cues.into_iter().map(|(mut c, _)| { + c.until = c.until.min(total); + c + }); + Plan { steps: lay.steps, cues: cues.filter(|c| c.until > c.start).collect(), frames: lay.frames } +} + +/// The steps of [`plan`]. +pub fn timeline(pres: &Presentation, opts: &VideoOpts) -> Vec { + plan(pres, opts).steps +} + +/// An image fitted to exactly `w`×`h` RGBA (the renderer's size can differ by a pixel): copied +/// into a black frame, cropped where it overhangs. +fn fit(img: &deckcraft_render::Image, w: u32, h: u32) -> Vec { + let mut out = vec![0u8; (w as usize).saturating_mul(h as usize).saturating_mul(4)]; + for px in out.as_chunks_mut::<4>().0 { + px[3] = 255; + } + let cw = img.width.min(w) as usize; + for y in 0..img.height.min(h) as usize { + let src = y.saturating_mul(img.width as usize).saturating_mul(4); + let dst = y.saturating_mul(w as usize).saturating_mul(4); + if let (Some(s), Some(d)) = (img.pixels.get(src..src + cw * 4), out.get_mut(dst..dst + cw * 4)) { + d.copy_from_slice(s); + } + } + out +} + +/// The renderer options for a frame `h` pixels high. +fn render_opts<'a>(pres: &Presentation, h: u32, state: Option<&'a dyn Fn(ShapeId) -> ShapeState>) -> deckcraft_render::RenderOpts<'a> { + let scale = h as f64 / pres.slide_size.height.max(1.0); + deckcraft_render::RenderOpts { scale: if scale.is_finite() { scale.clamp(0.01, 16.0) } else { 1.0 }, state, ..Default::default() } +} + +/// A shape's state at main step `step`, `t` seconds in (as the slide show draws it). +fn state_for(tl: &Timeline, id: ShapeId, step: usize, t: f64) -> ShapeState { + let a = tl.state(id, None, step, t); + ShapeState { + visible: a.visible, + opacity: a.opacity, + offset: deckcraft_geom::Vec2::new(a.offset_x, a.offset_y), + scale: (a.scale_x, a.scale_y), + rotate: a.rotate, + clip: a.clip, + tint: a.tint, + paras: tl + .paragraph_targets(id) + .into_iter() + .map(|p| { + let a = tl.state(id, Some(p), step, t); + (p, ParaState { visible: a.visible, opacity: a.opacity, offset: deckcraft_geom::Vec2::new(a.offset_x, a.offset_y) }) + }) + .collect(), + } +} + +/// Every frame of `steps`, in order, handed to `sink` (RGBA, `w`×`h`). A still or held slide +/// is rendered once. Stops at the first error `sink` returns. Video clips show their poster. +pub fn frames(pres: &Presentation, steps: &[Step], (w, h): (u32, u32), sink: &mut dyn FnMut(&[u8]) -> Result<(), String>) -> Result { + let mut r = Renderer::new(pres, &[], 30, (w, h)); + let mut n: u64 = 0; + for step in steps { + n = n.saturating_add(r.step(step, sink)?); + } + Ok(n) +} + +/// A video clip's pictures, decoded in order as the export reaches them. +struct ClipPictures { + dec: deckcraft_media::VideoDecoder, + cur: Option, + next: Option, + done: bool, +} + +impl ClipPictures { + /// The picture shown `t` seconds into the media (the first one before it starts). + fn at(&mut self, t: f64) -> Option<&deckcraft_media::Frame> { + const EPS: f64 = 1e-6; + if self.cur.as_ref().is_some_and(|c| c.time > t + EPS) { + self.dec.seek(t); + self.cur = None; + self.next = None; + self.done = false; + } + loop { + if self.next.is_none() && !self.done { + match self.dec.next_frame() { + Ok(Some(f)) => self.next = Some(f), + // The end, or a broken stream: keep the last picture. + Ok(None) | Err(_) => self.done = true, + } + } + match &self.next { + Some(n) if n.time <= t + EPS || self.cur.is_none() => { + let late = n.time > t + EPS; + self.cur = self.next.take(); + if late { + break; + } + } + _ => break, + } + } + self.cur.as_ref() + } +} + +/// Renders steps in order, knowing where on the video's clock each frame falls so playing +/// video clips show the right picture. +struct Renderer<'p> { + pres: &'p Presentation, + cues: Vec, + fps: u32, + size: (u32, u32), + /// Frames rendered so far: the clock. + at: u64, + timelines: HashMap>, + pictures: HashMap>, + warnings: Vec, +} + +impl<'p> Renderer<'p> { + fn new(pres: &'p Presentation, cues: &[AudioCue], fps: u32, size: (u32, u32)) -> Self { + let cues = cues.iter().filter(|c| c.picture.is_some()).cloned().collect(); + Renderer { pres, cues, fps: fps.max(1), size, at: 0, timelines: HashMap::new(), pictures: HashMap::new(), warnings: vec![] } + } + + /// Where in its media each video clip of slide `index` is at `t` seconds of the video. + fn clip_times(&self, index: usize, t: f64) -> Vec<(ShapeId, MediaId, f64)> { + let mut out = vec![]; + for c in &self.cues { + let Some((i, shape)) = c.picture else { continue }; + if i != index || t < c.start || t >= c.until { + continue; + } + let local = t - c.start; + let span = (c.length - c.trim_start - c.trim_end).max(0.0); + let pos = if c.length <= 0.0 { + local + } else if c.looped && span > 0.0 { + local % span + } else if local >= span && c.rewind { + continue; + } else { + local.min(span) + }; + out.push((shape, c.media, c.trim_start + pos)); + } + out + } + + /// Whether any clip of slide `index` plays during `n` frames from the clock. + fn plays_during(&self, index: usize, n: u64) -> bool { + let fps = f64::from(self.fps); + let (t0, t1) = (self.at as f64 / fps, self.at.saturating_add(n) as f64 / fps); + self.cues.iter().any(|c| c.picture.is_some_and(|p| p.0 == index) && c.start < t1 && c.until > t0) + } + + fn pictures_for(&mut self, media: MediaId) -> Option<&mut ClipPictures> { + let pres = self.pres; + let warnings = &mut self.warnings; + self.pictures + .entry(media) + .or_insert_with(|| { + let m = pres.media(media)?; + match deckcraft_media::VideoDecoder::open(m.data.clone()) { + Ok(dec) => Some(ClipPictures { dec, cur: None, next: None, done: false }), + Err(e) => { + warnings.push(format!("{}: the video shows its poster frame ({e})", m.name)); + None + } + } + }) + .as_mut() + } + + /// Render one step's frames into `sink`. + fn step(&mut self, step: &Step, sink: &mut dyn FnMut(&[u8]) -> Result<(), String>) -> Result { + let pres = self.pres; + let animate = !pres.show.without_animation; + let index = step.index(); + let tl = self + .timelines + .entry(index) + .or_insert_with(|| Arc::new(if animate { ShowState::timeline(pres, index) } else { Timeline::default() })) + .clone(); + let (st, t) = match step { + Step::Transition { .. } => (0, 0.0), + Step::Anim { step, t, .. } => (*step, *t), + Step::Still { step, .. } => (*step, 0.0), + Step::Hold { .. } => (tl.steps(), 0.0), + }; + // During a transition only the incoming slide's shapes take its starting state. + let incoming = pres.slides.get(index).cloned(); + let f = move |id: ShapeId| { + if incoming.as_ref().is_some_and(|s| s.shape(id).is_some()) { state_for(&tl, id, st, t) } else { ShapeState::default() } + }; + let total = step.frames(); + // A still slide is drawn once, unless a video clip on it is playing. + let each = total > 1 && self.plays_during(index, total); + let mut px = vec![]; + for k in 0..total { + if k == 0 || each { + let now = self.at as f64 / f64::from(self.fps); + let times = self.clip_times(index, now); + let mut held: Vec<(ShapeId, deckcraft_media::Frame)> = vec![]; + for (shape, media, ct) in times { + if let Some(p) = self.pictures_for(media).and_then(|p| p.at(ct)) { + held.push((shape, p.clone())); + } + } + let vf: Vec<(ShapeId, deckcraft_render::VideoFrame)> = + held.iter().map(|(s, f)| (*s, deckcraft_render::VideoFrame { width: f.width, height: f.height, rgba: &f.rgba })).collect(); + let mut opts = render_opts(pres, self.size.1, Some(&f)); + opts.video_frames = &vf; + let img = match step { + Step::Transition { index, t } => deckcraft_render::render_transition(pres, *index, *t, &opts), + _ => deckcraft_render::render_slide(pres, index, &opts), + }; + px = fit(&img, self.size.0, self.size.1); + } + sink(&px)?; + self.at = self.at.saturating_add(1); + } + Ok(total) + } +} + +/// Decoded clips at [`AUDIO_RATE`], interleaved stereo, by media id (`None`: undecodable). +fn decode_clips(pres: &Presentation, cues: &[AudioCue], warnings: &mut Vec) -> HashMap>>> { + let mut out = HashMap::new(); + for c in cues { + if out.contains_key(&c.media) { + continue; + } + let pcm = pres.media(c.media).filter(|m| !m.data.is_empty()).and_then(|m| match deckcraft_media::audio::decode(&m.data) { + Ok(p) => Some(p), + Err(e) => { + warnings.push(format!("{}: no sound in the video ({e})", m.name)); + None + } + }); + let stereo = pcm.map(|p| { + let p = deckcraft_media::audio::resample(&p, AUDIO_RATE); + let ch = usize::from(p.channels.max(1)); + let mut s = Vec::with_capacity(p.frames().saturating_mul(2)); + for f in 0..p.frames() { + let l = p.samples.get(f * ch).copied().unwrap_or(0.0); + let r = if ch > 1 { p.samples.get(f * ch + 1).copied().unwrap_or(0.0) } else { l }; + s.push(l); + s.push(r); + } + Arc::new(s) + }); + out.insert(c.media, stereo); + } + out +} + +/// Mix every cue into the stereo block starting at sample frame `at` (`out`: interleaved). +fn mix_block(cues: &[AudioCue], clips: &HashMap>>>, at: u64, out: &mut [f32]) { + out.iter_mut().for_each(|s| *s = 0.0); + let rate = f64::from(AUDIO_RATE); + let len = (out.len() / 2) as u64; + for c in cues { + let Some(Some(clip)) = clips.get(&c.media) else { continue }; + let frames = (clip.len() / 2) as u64; + let first = (c.trim_start * rate) as u64; + let last = frames.saturating_sub((c.trim_end * rate) as u64); + if last <= first { + continue; + } + let (start, until) = ((c.start * rate) as u64, (c.until * rate) as u64); + if until <= at || start >= at + len { + continue; + } + let span = (last - first) as f64 / rate; + for k in 0..len { + let g = at + k; + if g < start || g >= until { + continue; + } + let local = g - start; + if !c.looped && first + local >= last { + break; + } + let pos = first + if c.looped { local % (last - first) } else { local }; + let tl = local as f64 / rate; + let mut gain = c.volume; + if c.fade_in > 0.0 && tl < c.fade_in { + gain *= tl / c.fade_in; + } + let left = if c.looped { f64::MAX } else { span - tl }; + if c.fade_out > 0.0 && left < c.fade_out { + gain *= (left / c.fade_out).max(0.0); + } + let (i, o) = (pos as usize * 2, k as usize * 2); + if let (Some(l), Some(r)) = (clip.get(i), clip.get(i + 1)) { + out[o] += l * gain as f32; + out[o + 1] += r * gain as f32; + } + } + } + for s in out.iter_mut() { + *s = s.clamp(-1.0, 1.0); + } +} + +/// A key frame at least this often (seconds), so players can seek. +const KEY_EVERY_S: u32 = 2; + +/// The encoded file. +pub struct Encoded { + pub bytes: Vec, + pub frames: u64, + pub size: (u32, u32), + /// Seconds of sound (0: no audio track). + pub audio_seconds: f64, + pub warnings: Vec, +} + +/// Encode the deck as `mp4` or `mov`. The file is assembled in memory. +pub fn encode(pres: &Presentation, format: &str, opts: &VideoOpts) -> Result { + encode_with(pres, format, opts, &mut |_, _| true) +} + +/// What [`encode_with`] fails with when its progress callback asks it to stop. +pub const CANCELLED: &str = "the video export was cancelled"; + +/// [`encode`], calling `progress(frames done, frames in all)` after each frame; when it returns +/// `false` the export stops with [`CANCELLED`]. +pub fn encode_with(pres: &Presentation, format: &str, opts: &VideoOpts, progress: &mut dyn FnMut(u64, u64) -> bool) -> Result { + use deckcraft_isobmff::{AvcConfig, Brand, CodecConfig, FourCc, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + if opts.audio == AudioFormat::Pcm && format != "mov" { + return Err("PCM sound needs a .mov file; use AAC for .mp4".into()); + } + let plan = plan(pres, opts); + if plan.steps.is_empty() { + return Err("there are no slides to put in the video".into()); + } + let (w, h) = frame_size(pres, opts); + let fps = opts.fps.clamp(1, 60); + let mut enc = deckcraft_h264::Encoder::new(deckcraft_h264::EncoderConfig { width: w, height: h, fps, qp: 24 }).map_err(|e| e.to_string())?; + let avc = AvcConfig::new(vec![enc.sps()], vec![enc.pps()], 4); + let entry = SampleEntry::video(FourCc::new(b"avc1"), CodecConfig::Avc(avc), w.min(u32::from(u16::MAX)) as u16, h.min(u32::from(u16::MAX)) as u16); + let brand = if format == "mov" { Brand::Mov } else { Brand::Mp4 }; + let mut mux = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(brand)).map_err(|e| e.to_string())?; + let track = mux.add_track(TrackConfig::new(entry, fps)).map_err(|e| e.to_string())?; + let key_every = u64::from(fps * KEY_EVERY_S); + let mut since_key: u64 = 0; + let mut total: u64 = 0; + let mut done: u64 = 0; + let mut renderer = Renderer::new(pres, &plan.cues, fps, (w, h)); + for (i, step) in plan.steps.iter().enumerate() { + // A key frame starts each slide (at its transition), so the file seeks by slide. + let mut start = i.checked_sub(1).and_then(|p| plan.steps.get(p)).is_none_or(|p| p.index() != step.index()); + let all = plan.frames; + let mut sink = |px: &[u8]| -> Result<(), String> { + let pic = enc.encode_rgba(px, start || since_key >= key_every); + start = false; + since_key = if pic.key { 1 } else { since_key.saturating_add(1) }; + let data = pic.length_prefixed(); + mux.write_sample(track, WriteSample { data: &data, duration: 1, composition_offset: 0, is_sync: pic.key }).map_err(|e| e.to_string())?; + done = done.saturating_add(1); + if progress(done, all) { Ok(()) } else { Err(CANCELLED.to_string()) } + }; + total = total.saturating_add(renderer.step(step, &mut sink)?); + } + // Sound. + let mut warnings = std::mem::take(&mut renderer.warnings); + let mut audio_seconds = 0.0; + if opts.audio != AudioFormat::None && !plan.cues.is_empty() { + let clips = decode_clips(pres, &plan.cues, &mut warnings); + if clips.values().any(Option::is_some) { + let samples = total.saturating_mul(u64::from(AUDIO_RATE)) / u64::from(fps); + audio_seconds = samples as f64 / f64::from(AUDIO_RATE); + const BLOCK: usize = deckcraft_aac::FRAME_LEN; + let mut buf = vec![0f32; BLOCK * 2]; + match opts.audio { + AudioFormat::Pcm => { + let pcm = PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: 2, sample_rate: f64::from(AUDIO_RATE) }; + let at = mux.add_track(TrackConfig::new(SampleEntry::pcm(pcm), AUDIO_RATE)).map_err(|e| e.to_string())?; + let mut pos = 0u64; + while pos < samples { + let n = (samples - pos).min(BLOCK as u64) as usize; + mix_block(&plan.cues, &clips, pos, &mut buf[..n * 2]); + let data: Vec = buf[..n * 2].iter().flat_map(|s| ((s * 32767.0).round() as i16).to_le_bytes()).collect(); + mux.write_sample(at, WriteSample { data: &data, duration: n as u32, composition_offset: 0, is_sync: true }) + .map_err(|e| e.to_string())?; + pos += n as u64; + } + } + _ => { + let mut aac = + deckcraft_aac::Encoder::new(deckcraft_aac::EncoderConfig::cbr(AUDIO_RATE, 2, 160_000)).map_err(|e| e.to_string())?; + let mut cfg = TrackConfig::new(SampleEntry::aac(aac.audio_specific_config(), 2, AUDIO_RATE), AUDIO_RATE); + cfg.media_start = Some(i64::from(aac.priming_samples())); + let at = mux.add_track(cfg).map_err(|e| e.to_string())?; + let write = |units: Vec>, mux: &mut Mp4Writer>>| -> Result<(), String> { + for u in units { + mux.write_sample(at, WriteSample { data: &u, duration: BLOCK as u32, composition_offset: 0, is_sync: true }) + .map_err(|e| e.to_string())?; + } + Ok(()) + }; + let mut pos = 0u64; + let (mut l, mut r) = (vec![0f32; BLOCK], vec![0f32; BLOCK]); + while pos < samples { + let n = (samples - pos).min(BLOCK as u64) as usize; + mix_block(&plan.cues, &clips, pos, &mut buf[..n * 2]); + l.clear(); + r.clear(); + for k in 0..n { + l.push(buf[k * 2]); + r.push(buf[k * 2 + 1]); + } + write(aac.encode(&[&l, &r]), &mut mux)?; + pos += n as u64; + } + write(aac.flush(), &mut mux)?; + } + } + } + } + let bytes = mux.finish_faststart().map_err(|e| e.to_string())?.into_inner(); + Ok(Encoded { bytes, frames: total, size: (w, h), audio_seconds, warnings }) +} + +#[cfg(test)] +mod tests { + use super::*; + use deckcraft_model::anim::Transition; + + fn deck(n: usize) -> Presentation { + let mut p = deckcraft_model::defaults::new_presentation(None); + while p.slides.len() < n { + let s = p.slides[0].clone(); + p.slides.push(s); + } + p.slides.truncate(n); + p + } + + #[test] + fn timeline_holds_slides_and_plays_transitions() { + let mut p = deck(3); + std::sync::Arc::make_mut(&mut p.slides[1]).transition = + Some(Transition { kind: "fade".into(), duration_ms: 500, advance_after_ms: Some(2000), ..Default::default() }); + std::sync::Arc::make_mut(&mut p.slides[2]).hidden = true; + let o = VideoOpts { fps: 10, seconds_per_slide: 3.0, ..Default::default() }; + let t = timeline(&p, &o); + assert_eq!(t.first(), Some(&Step::Hold { index: 0, frames: 30 })); + assert_eq!(t.iter().filter(|s| matches!(s, Step::Transition { index: 1, .. })).count(), 5); + // "Advance after" counts from the slide's start, its transition included. + assert_eq!(t.last(), Some(&Step::Hold { index: 1, frames: 15 }), "the hidden slide is left out"); + let with_hidden = timeline(&p, &VideoOpts { include_hidden: true, ..o }); + assert_eq!(with_hidden.last(), Some(&Step::Hold { index: 2, frames: 30 })); + } + + #[test] + fn timeline_is_capped_and_sizes_are_even() { + let mut p = deck(2); + for s in &mut p.slides { + std::sync::Arc::make_mut(s).transition = Some(Transition { advance_after_ms: Some(u32::MAX), ..Default::default() }); + } + let total: u64 = timeline(&p, &VideoOpts { fps: 60, ..Default::default() }).iter().map(Step::frames).sum(); + assert!(total <= MAX_FRAMES); + let (w, h) = frame_size(&p, &VideoOpts { height: 721, ..Default::default() }); + assert_eq!((w % 2, h), (0, 720)); + } + + #[test] + fn frames_are_exactly_the_frame_size() { + let p = deck(1); + let size = frame_size(&p, &VideoOpts { height: 90, ..Default::default() }); + let mut seen = vec![]; + let n = frames(&p, &[Step::Hold { index: 0, frames: 2 }], size, &mut |px| { + seen.push(px.len()); + Ok(()) + }); + assert_eq!(n, Ok(2)); + assert!(seen.iter().all(|l| *l == size.0 as usize * size.1 as usize * 4)); + } + + #[test] + fn encodes_a_playable_mp4() { + let mut p = deck(2); + std::sync::Arc::make_mut(&mut p.slides[1]).transition = Some(Transition { kind: "fade".into(), duration_ms: 200, ..Default::default() }); + let o = VideoOpts { height: 72, fps: 10, seconds_per_slide: 0.5, ..Default::default() }; + let v = encode(&p, "mp4", &o).unwrap(); + let (bytes, n, (w, h)) = (v.bytes, v.frames, v.size); + assert_eq!(n, 5 + 2 + 5); + let f = deckcraft_isobmff::open(bytes.as_slice()).unwrap(); + let t = &f.tracks[0]; + assert_eq!(t.samples.len(), n as usize); + // Key frames start each slide (at its transition). + let keys: Vec = t.samples.iter().enumerate().filter(|(_, s)| s.is_sync).map(|(i, _)| i).collect(); + assert_eq!(keys, vec![0, 5]); + let mut dec = deckcraft_h264::Decoder::new(); + // Rebuild an Annex B stream from the length-prefixed samples and the avcC parameter sets. + let Some(deckcraft_isobmff::CodecConfig::Avc(avc)) = t.entries.first().map(|e| &e.codec) else { panic!("not AVC") }; + let mut stream = vec![]; + for ps in avc.sps.iter().chain(&avc.pps) { + stream.extend([0, 0, 0, 1]); + stream.extend(ps); + } + for i in 0..t.samples.len() { + let data = f.read_sample(bytes.as_slice(), 0, i).unwrap(); + let mut k = 0; + while k + 4 <= data.len() { + let len = u32::from_be_bytes([data[k], data[k + 1], data[k + 2], data[k + 3]]) as usize; + stream.extend([0, 0, 0, 1]); + stream.extend(&data[k + 4..k + 4 + len]); + k += 4 + len; + } + } + let mut pics = dec.decode(&stream, 0).unwrap(); + pics.extend(dec.flush()); + assert_eq!(pics.len(), n as usize); + assert_eq!((pics[0].width, pics[0].height), (w, h)); + } + + /// A mono 16-bit WAV of a sine tone. + fn wav(rate: u32, secs: f64, hz: f64) -> Vec { + let n = (f64::from(rate) * secs) as u32; + let mut b = vec![]; + b.extend(b"RIFF"); + b.extend((36 + n * 2).to_le_bytes()); + b.extend(b"WAVEfmt "); + b.extend(16u32.to_le_bytes()); + b.extend(1u16.to_le_bytes()); + b.extend(1u16.to_le_bytes()); + b.extend(rate.to_le_bytes()); + b.extend((rate * 2).to_le_bytes()); + b.extend(2u16.to_le_bytes()); + b.extend(16u16.to_le_bytes()); + b.extend(b"data"); + b.extend((n * 2).to_le_bytes()); + for k in 0..n { + let v = ((k as f64 / f64::from(rate) * hz * std::f64::consts::TAU).sin() * 12000.0) as i16; + b.extend(v.to_le_bytes()); + } + b + } + + /// A slide with a text box flying in on click and a tone that plays by itself. + fn animated_deck() -> Presentation { + use serde_json::json; + let mut s = crate::Session::with_new(); + s.execute("slide.new", &json!({"layout": "blank"})).unwrap(); + s.execute("insert.textBox", &json!({"rect": [100, 100, 400, 120], "text": "Hello"})).unwrap(); + s.execute("animation.add", &json!({"effect": "fade", "start": "onClick", "duration": 1000})).unwrap(); + let data = crate::cmd::base64_encode(&wav(22_050, 1.0, 440.0)); + s.execute("insert.audio", &json!({"data": data, "name": "tone.wav"})).unwrap(); + s.execute("media.options", &json!({"autoplay": true})).unwrap(); + let mut p = (*s.doc().unwrap().doc).clone(); + p.slides.retain(|sl| sl.shapes.iter().any(|sh| matches!(sh.kind, ShapeKind::Media(_)))); + p + } + + #[test] + fn animations_play_on_the_show_timeline() { + let p = animated_deck(); + let o = VideoOpts { fps: 10, seconds_per_slide: 1.0, seconds_per_click: 0.5, height: 90, ..Default::default() }; + let plan = plan(&p, &o); + assert_eq!(plan.steps.first(), Some(&Step::Still { index: 0, step: 0, frames: 5 }), "{:?}", plan.steps); + assert_eq!(plan.steps.iter().filter(|s| matches!(s, Step::Anim { .. })).count(), 10); + assert_eq!(plan.steps.last(), Some(&Step::Hold { index: 0, frames: 10 })); + // The tone plays from the slide's start, for the slide. + assert_eq!(plan.cues.len(), 1); + assert_eq!((plan.cues[0].start, plan.cues[0].until), (0.0, 2.5)); + // Before the click the text is hidden; halfway through the fade it shows faintly; at + // the end fully. + let size = frame_size(&p, &o); + let ink = |step: Step| { + let mut dark = 0usize; + frames(&p, &[step], size, &mut |px| { + dark = px.as_chunks::<4>().0.iter().filter(|q| q[0] < 200).count(); + Ok(()) + }) + .unwrap(); + dark + }; + let (before, mid, after) = ( + ink(Step::Still { index: 0, step: 0, frames: 1 }), + ink(Step::Anim { index: 0, step: 0, t: 0.5 }), + ink(Step::Hold { index: 0, frames: 1 }), + ); + assert!(before < after && after > 0, "{before} {mid} {after}"); + } + + #[test] + fn sound_is_mixed_and_encoded_as_aac() { + let p = animated_deck(); + let o = VideoOpts { fps: 10, seconds_per_slide: 1.0, seconds_per_click: 0.5, height: 72, ..Default::default() }; + let v = encode(&p, "mp4", &o).unwrap(); + assert!((v.audio_seconds - 2.5).abs() < 0.01, "{}", v.audio_seconds); + let f = deckcraft_isobmff::open(v.bytes.as_slice()).unwrap(); + assert_eq!(f.tracks.len(), 2); + let a = &f.tracks[1]; + let Some(deckcraft_isobmff::CodecConfig::Aac(cfg)) = a.entries.first().map(|e| &e.codec) else { panic!("not AAC") }; + let mut dec = deckcraft_aac::Decoder::new(&cfg.asc).unwrap(); + let mut energy = vec![]; + for i in 0..a.samples.len() { + let pcm = dec.decode(&f.read_sample(v.bytes.as_slice(), 1, i).unwrap()).unwrap(); + energy.push(pcm[0].iter().map(|x| x * x).sum::() / pcm[0].len().max(1) as f32); + } + // The one-second tone, then silence. + let loud = energy.iter().filter(|e| **e > 0.01).count(); + let quiet = energy.iter().filter(|e| **e < 1e-5).count(); + assert!(loud >= 40 && quiet >= 50, "loud {loud} quiet {quiet} of {}", energy.len()); + // PCM in a MOV. + let v = encode(&p, "mov", &VideoOpts { audio: AudioFormat::Pcm, ..o.clone() }).unwrap(); + let f = deckcraft_isobmff::open(v.bytes.as_slice()).unwrap(); + assert!(matches!(f.tracks[1].entries.first().map(|e| &e.codec), Some(deckcraft_isobmff::CodecConfig::Pcm(_)))); + assert!(encode(&p, "mp4", &VideoOpts { audio: AudioFormat::Pcm, ..o }).is_err()); + } + + /// A 64×64 H.264 MP4 at 10 fps: one second red, then one second blue. + fn red_then_blue() -> Vec { + use deckcraft_isobmff::{AvcConfig, Brand, CodecConfig, FourCc, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + let mut enc = deckcraft_h264::Encoder::new(deckcraft_h264::EncoderConfig { width: 64, height: 64, fps: 10, qp: 20 }).unwrap(); + let avc = AvcConfig::new(vec![enc.sps()], vec![enc.pps()], 4); + let entry = SampleEntry::video(FourCc::new(b"avc1"), CodecConfig::Avc(avc), 64, 64); + let mut mux = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap(); + let t = mux.add_track(TrackConfig::new(entry, 10)).unwrap(); + for k in 0..20 { + let c = if k < 10 { [230u8, 20, 20, 255] } else { [20, 20, 230, 255] }; + let px: Vec = std::iter::repeat_n(c, 64 * 64).flatten().collect(); + let pic = enc.encode_rgba(&px, k == 0 || k == 10); + mux.write_sample(t, WriteSample { data: &pic.length_prefixed(), duration: 1, composition_offset: 0, is_sync: pic.key }).unwrap(); + } + mux.finish().unwrap().into_inner() + } + + #[test] + fn video_clips_play_their_pictures() { + use serde_json::json; + let mut s = crate::Session::with_new(); + s.execute("slide.new", &json!({"layout": "blank"})).unwrap(); + let data = crate::cmd::base64_encode(&red_then_blue()); + s.execute("insert.video", &json!({"data": data, "name": "clip.mp4", "rect": [0, 0, 960, 540]})).unwrap(); + s.execute("media.options", &json!({"autoplay": true})).unwrap(); + let mut p = (*s.doc().unwrap().doc).clone(); + p.slides.retain(|sl| sl.shapes.iter().any(|sh| matches!(sh.kind, ShapeKind::Media(_)))); + for sl in &mut p.slides { + Arc::make_mut(sl).transition = None; + } + let o = VideoOpts { fps: 10, seconds_per_slide: 3.0, height: 64, audio: AudioFormat::None, ..Default::default() }; + let v = encode(&p, "mp4", &o).unwrap(); + assert_eq!(v.frames, 30); + let mut dec = deckcraft_media::VideoDecoder::open(Arc::new(v.bytes)).unwrap(); + let mut centre = vec![]; + while let Some(f) = dec.next_frame().unwrap() { + let i = ((f.height / 2 * f.width + f.width / 2) * 4) as usize; + centre.push([f.rgba[i], f.rgba[i + 1], f.rgba[i + 2]]); + } + let red = |c: &[u8; 3]| c[0] > 150 && c[2] < 100; + let blue = |c: &[u8; 3]| c[2] > 150 && c[0] < 100; + // Red for the first second, blue for the second, then the last picture held. + assert!(centre[..9].iter().all(red), "{centre:?}"); + assert!(centre[11..].iter().all(blue), "{centre:?}"); + } + + #[test] + fn progress_counts_frames_and_can_cancel() { + let p = animated_deck(); + let o = VideoOpts { fps: 10, seconds_per_slide: 1.0, seconds_per_click: 0.5, height: 64, audio: AudioFormat::None, ..Default::default() }; + let mut seen = vec![]; + let v = encode_with(&p, "mp4", &o, &mut |d, n| { + seen.push((d, n)); + true + }) + .unwrap(); + assert_eq!(seen.len() as u64, v.frames); + assert_eq!(seen.last(), Some(&(v.frames, v.frames))); + let r = encode_with(&p, "mp4", &o, &mut |d, _| d < 3); + assert_eq!(r.err().as_deref(), Some(CANCELLED)); + } +} diff --git a/crates/h264/src/encoder.rs b/crates/h264/src/encoder.rs new file mode 100644 index 0000000000..14e205a772 --- /dev/null +++ b/crates/h264/src/encoder.rs @@ -0,0 +1,920 @@ +//! A small H.264 encoder for synthetic pictures (slides): Constrained Baseline, CAVLC, 8-bit +//! 4:2:0, one slice per picture, deblocking off. Every coded macroblock is Intra 16x16 (vertical, +//! horizontal or DC prediction, chosen by SAD); in P pictures a macroblock whose source pixels are +//! unchanged since the previous picture is skipped, so a held slide costs a few bytes a frame. +//! Written from ITU-T Rec. H.264 (clauses 7.3, 8.3.3, 8.3.4, 8.5, 9.2) as the inverse of +//! [`crate::Decoder`], which the tests decode it with. + +use crate::cavlc_tables::{COEFF_TOKEN, RUN_BEFORE, TOTAL_ZEROS_2X2, TOTAL_ZEROS_4X4}; + +/// Encoder settings. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EncoderConfig { + /// Picture size in pixels; both even, at most 8192×4320. + pub width: u32, + pub height: u32, + /// Frame rate written to the VUI timing (1–240). + pub fps: u32, + /// Quantisation parameter, 10–51 (lower is better quality and larger). + pub qp: u8, +} + +/// One coded picture: its NAL units without start codes (SPS and PPS lead a key picture). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncodedPicture { + pub nals: Vec>, + pub key: bool, +} + +impl EncodedPicture { + /// Annex B byte stream (start codes). + pub fn annex_b(&self) -> Vec { + let mut out = vec![]; + for n in &self.nals { + out.extend_from_slice(&[0, 0, 0, 1]); + out.extend_from_slice(n); + } + out + } + /// The picture's NAL units with 4-byte big-endian lengths (MP4 samples), parameter sets + /// left out (they go in the `avcC` box). + pub fn length_prefixed(&self) -> Vec { + let mut out = vec![]; + for n in self.nals.iter().filter(|n| !matches!(n.first().map(|b| b & 0x1F), Some(7 | 8))) { + out.extend_from_slice(&(n.len() as u32).to_be_bytes()); + out.extend_from_slice(n); + } + out + } +} + +/// Bit writer for RBSPs. +#[derive(Default)] +struct Bits { + buf: Vec, + cur: u64, + n: u32, +} + +impl Bits { + fn put(&mut self, v: u32, bits: u32) { + if bits == 0 { + return; + } + let bits = bits.min(32); + let mask = if bits == 32 { u32::MAX } else { (1u32 << bits) - 1 }; + self.cur = (self.cur << bits) | u64::from(v & mask); + self.n += bits; + while self.n >= 8 { + self.n -= 8; + self.buf.push((self.cur >> self.n) as u8); + } + self.cur &= (1u64 << self.n) - 1; + } + fn flag(&mut self, b: bool) { + self.put(u32::from(b), 1); + } + /// A code from the tables ("0101"). + fn code(&mut self, s: &str) { + for c in s.bytes() { + self.put(u32::from(c == b'1'), 1); + } + } + fn ue(&mut self, v: u32) { + let x = u64::from(v) + 1; + let len = 64 - x.leading_zeros(); + self.put(0, len - 1); + // len ≤ 33: write the high bit and the rest separately. + self.put(1, 1); + if len > 1 { + self.put((x & ((1u64 << (len - 1)) - 1)) as u32, len - 1); + } + } + fn se(&mut self, v: i32) { + let k = if v > 0 { (v as u32) * 2 - 1 } else { v.unsigned_abs() * 2 }; + self.ue(k); + } + /// `rbsp_trailing_bits`, then the bytes. + fn finish(mut self) -> Vec { + self.put(1, 1); + if self.n > 0 { + let pad = 8 - self.n; + self.put(0, pad); + } + self.buf + } +} + +/// A NAL unit from its header byte and RBSP, with emulation prevention. +fn nal(header: u8, rbsp: &[u8]) -> Vec { + let mut out = Vec::with_capacity(rbsp.len() + rbsp.len() / 64 + 2); + out.push(header); + let mut zeros = 0; + for &b in rbsp { + if zeros >= 2 && b <= 3 { + out.push(3); + zeros = 0; + } + out.push(b); + zeros = if b == 0 { zeros + 1 } else { 0 }; + } + out +} + +/// 4x4 zig-zag scan (frame): scan index → raster index. +const ZIGZAG: [usize; 16] = [0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15]; +/// Quantisation multipliers by `qp % 6` for positions (even, even), (odd, odd) and the rest. +const MF: [[i32; 3]; 6] = [[13107, 5243, 8066], [11916, 4660, 7490], [10082, 4194, 6554], [9362, 3647, 5825], [8192, 3355, 5243], [7282, 2893, 4559]]; +/// Dequantisation scales (`normAdjust4x4`), same positions. +const V: [[i32; 3]; 6] = [[10, 16, 13], [11, 18, 14], [13, 20, 16], [14, 23, 18], [16, 25, 20], [18, 29, 23]]; +/// Chroma QP for luma QP 30..=51 (Table 8-15); below 30 they are equal. +const QPC: [u8; 22] = [29, 30, 31, 32, 32, 33, 34, 34, 35, 35, 36, 36, 37, 37, 37, 38, 38, 38, 39, 39, 39, 39]; + +fn chroma_qp(qp: u8) -> u8 { + if qp < 30 { qp } else { QPC.get(usize::from(qp - 30)).copied().unwrap_or(39) } +} + +fn pos_class(raster: usize) -> usize { + let (i, j) = (raster / 4, raster % 4); + match (i % 2, j % 2) { + (0, 0) => 0, + (1, 1) => 1, + _ => 2, + } +} + +/// Forward core transform of a 4x4 residual (raster), `Cf · X · Cfᵀ`. +fn fdct(x: &[i32; 16]) -> [i32; 16] { + let mut t = [0i32; 16]; + for i in 0..4 { + let r = &x[i * 4..i * 4 + 4]; + let (s03, d03, s12, d12) = (r[0] + r[3], r[0] - r[3], r[1] + r[2], r[1] - r[2]); + t[i * 4] = s03 + s12; + t[i * 4 + 1] = 2 * d03 + d12; + t[i * 4 + 2] = s03 - s12; + t[i * 4 + 3] = d03 - 2 * d12; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let (a, b, c, d) = (t[j], t[4 + j], t[8 + j], t[12 + j]); + let (s03, d03, s12, d12) = (a + d, a - d, b + c, b - c); + out[j] = s03 + s12; + out[4 + j] = 2 * d03 + d12; + out[8 + j] = s03 - s12; + out[12 + j] = d03 - 2 * d12; + } + out +} + +/// Inverse transform (8.5.12.2) of dequantised coefficients, giving the residual. +fn idct(d: &[i32; 16]) -> [i32; 16] { + let mut f = [0i32; 16]; + for i in 0..4 { + let r = &d[i * 4..i * 4 + 4]; + let e0 = r[0] + r[2]; + let e1 = r[0] - r[2]; + let e2 = (r[1] >> 1) - r[3]; + let e3 = r[1] + (r[3] >> 1); + f[i * 4] = e0 + e3; + f[i * 4 + 1] = e1 + e2; + f[i * 4 + 2] = e1 - e2; + f[i * 4 + 3] = e0 - e3; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let (a, b, c, dd) = (f[j], f[4 + j], f[8 + j], f[12 + j]); + let g0 = a + c; + let g1 = a - c; + let g2 = (b >> 1) - dd; + let g3 = b + (dd >> 1); + out[j] = (g0 + g3 + 32) >> 6; + out[4 + j] = (g1 + g2 + 32) >> 6; + out[8 + j] = (g1 - g2 + 32) >> 6; + out[12 + j] = (g0 - g3 + 32) >> 6; + } + out +} + +/// Largest level magnitude written (keeps every level within the 12-bit escape). +const MAX_LEVEL: i32 = 2063; + +fn quant(w: i32, mf: i32, qbits: u32, f: i32) -> i32 { + let z = ((i64::from(w.abs()) * i64::from(mf) + i64::from(f)) >> qbits) as i32; + z.min(MAX_LEVEL) * w.signum() +} + +/// Dequantise an AC (or chroma AC) coefficient (8.5.12.1, flat scaling). +fn dequant(c: i32, qp: u8, class: usize) -> i32 { + let ls = 16 * V[usize::from(qp % 6)][class]; + let q = u32::from(qp / 6); + if qp >= 24 { (c * ls) << (q - 4) } else { (c * ls + (1 << (3 - q))) >> (4 - q) } +} + +/// Write one CAVLC `residual_block` of `coeffs` (scan order) with context `nc` (−1: chroma DC). +/// Returns TotalCoeff. +fn residual_block(b: &mut Bits, coeffs: &[i32], nc: i32) -> u8 { + let max = coeffs.len(); + let nz: Vec = (0..max).filter(|&i| coeffs[i] != 0).collect(); + let tc = nz.len(); + let mut t1 = 0; + for &i in nz.iter().rev() { + if coeffs[i].abs() == 1 && t1 < 3 { + t1 += 1; + } else { + break; + } + } + let col = match nc { + -1 => 4, + n if n < 2 => 0, + n if n < 4 => 1, + n if n < 8 => 2, + _ => 3, + }; + let token = COEFF_TOKEN.iter().find(|(o, t, _)| usize::from(*o) == t1 && usize::from(*t) == tc).map(|e| e.2[col]).unwrap_or(""); + b.code(token); + if tc == 0 { + return 0; + } + // Highest frequency first. + let levels: Vec = nz.iter().rev().map(|&i| coeffs[i]).collect(); + for l in levels.iter().take(t1) { + b.flag(*l < 0); + } + let mut sl: u32 = if tc > 10 && t1 < 3 { 1 } else { 0 }; + for (k, &l) in levels.iter().enumerate().skip(t1) { + let mut code = if l > 0 { 2 * l - 2 } else { -2 * l - 1 }; + if k == t1 && t1 < 3 { + code -= 2; + } + let code = code.max(0) as u32; + if sl == 0 { + if code < 14 { + b.put(0, code); + b.put(1, 1); + } else if code < 30 { + b.put(0, 14); + b.put(1, 1); + b.put(code - 14, 4); + } else { + b.put(0, 15); + b.put(1, 1); + b.put((code - 30).min(4095), 12); + } + } else if code < (15 << sl) { + b.put(0, code >> sl); + b.put(1, 1); + b.put(code & ((1 << sl) - 1), sl); + } else { + b.put(0, 15); + b.put(1, 1); + b.put((code - (15 << sl)).min(4095), 12); + } + if sl == 0 { + sl = 1; + } + if l.unsigned_abs() > (3 << (sl - 1)) && sl < 6 { + sl += 1; + } + } + let last = nz.last().copied().unwrap_or(0); + let total_zeros = last + 1 - tc; + if tc < max { + let code = if max == 4 { + TOTAL_ZEROS_2X2.get(tc - 1).and_then(|r| r.get(total_zeros)) + } else { + TOTAL_ZEROS_4X4.get(tc - 1).and_then(|r| r.get(total_zeros)) + }; + b.code(code.copied().unwrap_or("")); + } + let mut zeros_left = total_zeros; + for w in 0..tc.saturating_sub(1) { + if zeros_left == 0 { + break; + } + // nz descending: run of zeros below the w-th highest. + let (hi, lo) = (nz[tc - 1 - w], nz[tc - 2 - w]); + let run = hi - lo - 1; + let row = zeros_left.min(7) - 1; + b.code(RUN_BEFORE.get(row).and_then(|r| r.get(run)).copied().unwrap_or("")); + zeros_left -= run; + } + tc as u8 +} + +/// A picture's planes, padded to whole macroblocks. +#[derive(Clone)] +struct Planes { + y: Vec, + u: Vec, + v: Vec, + w: usize, + h: usize, +} + +impl Planes { + fn new(w: usize, h: usize) -> Self { + Planes { y: vec![16; w * h], u: vec![128; w * h / 4], v: vec![128; w * h / 4], w, h } + } +} + +/// The encoder; feed it pictures in display order. +pub struct Encoder { + cfg: EncoderConfig, + mbw: usize, + mbh: usize, + recon: Planes, + prev: Option, + frame_num: u32, + idr_id: u32, + /// TotalCoeff per 4x4 block for this picture: luma (mbw*4 × mbh*4), Cb, Cr (mbw*2 × mbh*2). + nc_y: Vec, + nc_u: Vec, + nc_v: Vec, +} + +/// Why an encoder can't be made. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConfigError(pub &'static str); + +impl std::fmt::Display for ConfigError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0) + } +} + +impl Encoder { + pub fn new(cfg: EncoderConfig) -> Result { + if cfg.width < 16 || cfg.height < 16 || cfg.width > 8192 || cfg.height > 4320 || !cfg.width.is_multiple_of(2) || !cfg.height.is_multiple_of(2) + { + return Err(ConfigError("the picture must be 16×16 to 8192×4320 pixels, with even sides")); + } + let mbw = cfg.width.div_ceil(16) as usize; + let mbh = cfg.height.div_ceil(16) as usize; + let cfg = EncoderConfig { qp: cfg.qp.clamp(10, 51), fps: cfg.fps.clamp(1, 240), ..cfg }; + Ok(Encoder { + cfg, + mbw, + mbh, + recon: Planes::new(mbw * 16, mbh * 16), + prev: None, + frame_num: 0, + idr_id: 0, + nc_y: vec![0; mbw * mbh * 16], + nc_u: vec![0; mbw * mbh * 4], + nc_v: vec![0; mbw * mbh * 4], + }) + } + + fn level_idc(&self) -> u8 { + let mbs = (self.mbw * self.mbh) as u64; + let rate = mbs * u64::from(self.cfg.fps); + match (mbs, rate) { + (m, r) if m <= 1620 && r <= 40_500 => 30, + (m, r) if m <= 3600 && r <= 108_000 => 31, + (m, r) if m <= 8192 && r <= 245_760 => 40, + (m, r) if m <= 8704 && r <= 522_240 => 42, + (m, r) if m <= 22_080 && r <= 589_824 => 50, + _ => 51, + } + } + + /// The sequence parameter set NAL unit. + pub fn sps(&self) -> Vec { + let mut b = Bits::default(); + b.put(66, 8); // profile_idc: Baseline + b.put(0xC0, 8); // constraint_set0 and set1: Constrained Baseline + b.put(u32::from(self.level_idc()), 8); + b.ue(0); // seq_parameter_set_id + b.ue(12); // log2_max_frame_num_minus4: 16-bit frame_num + b.ue(2); // pic_order_cnt_type 2: output order is decode order + b.ue(1); // max_num_ref_frames + b.flag(false); // gaps_in_frame_num_value_allowed_flag + b.ue(self.mbw as u32 - 1); + b.ue(self.mbh as u32 - 1); + b.flag(true); // frame_mbs_only_flag + b.flag(true); // direct_8x8_inference_flag + let (cr, cb) = ((self.mbw as u32 * 16 - self.cfg.width) / 2, (self.mbh as u32 * 16 - self.cfg.height) / 2); + b.flag(cr > 0 || cb > 0); + if cr > 0 || cb > 0 { + b.ue(0); + b.ue(cr); + b.ue(0); + b.ue(cb); + } + b.flag(true); // vui_parameters_present_flag + b.flag(false); // aspect_ratio_info_present_flag + b.flag(false); // overscan_info_present_flag + b.flag(true); // video_signal_type_present_flag + b.put(5, 3); // video_format: unspecified + b.flag(false); // video_full_range_flag + b.flag(true); // colour_description_present_flag + b.put(1, 8); // BT.709 primaries, transfer and matrix + b.put(1, 8); + b.put(1, 8); + b.flag(false); // chroma_loc_info_present_flag + b.flag(true); // timing_info_present_flag + b.put(1, 32); // num_units_in_tick + b.put(self.cfg.fps * 2, 32); // time_scale + b.flag(true); // fixed_frame_rate_flag + b.flag(false); // nal_hrd_parameters_present_flag + b.flag(false); // vcl_hrd_parameters_present_flag + b.flag(false); // pic_struct_present_flag + b.flag(true); // bitstream_restriction_flag + b.flag(true); // motion_vectors_over_pic_boundaries_flag + b.ue(0); // max_bytes_per_pic_denom + b.ue(0); // max_bits_per_mb_denom + b.ue(0); // log2_max_mv_length_horizontal (every vector is zero) + b.ue(0); + b.ue(0); // max_num_reorder_frames + b.ue(1); // max_dec_frame_buffering + nal(0x67, &b.finish()) + } + + /// The picture parameter set NAL unit. + pub fn pps(&self) -> Vec { + let mut b = Bits::default(); + b.ue(0); // pic_parameter_set_id + b.ue(0); // seq_parameter_set_id + b.flag(false); // entropy_coding_mode_flag: CAVLC + b.flag(false); // bottom_field_pic_order_in_frame_present_flag + b.ue(0); // num_slice_groups_minus1 + b.ue(0); // num_ref_idx_l0_default_active_minus1 + b.ue(0); + b.flag(false); // weighted_pred_flag + b.put(0, 2); // weighted_bipred_idc + b.se(0); // pic_init_qp_minus26 + b.se(0); // pic_init_qs_minus26 + b.se(0); // chroma_qp_index_offset + b.flag(true); // deblocking_filter_control_present_flag + b.flag(false); // constrained_intra_pred_flag + b.flag(false); // redundant_pic_cnt_present_flag + nal(0x68, &b.finish()) + } + + /// Encode one RGBA picture (`width`×`height`, row-major). A key picture (IDR, with SPS and + /// PPS) is made when `key` is set and for the first picture. + pub fn encode_rgba(&mut self, rgba: &[u8], key: bool) -> EncodedPicture { + let src = self.to_yuv(rgba); + let key = key || self.prev.is_none(); + if key { + self.frame_num = 0; + self.idr_id = (self.idr_id + 1) % 65_536; + } else { + self.frame_num = (self.frame_num + 1) % 65_536; + } + self.nc_y.iter_mut().for_each(|n| *n = 0); + self.nc_u.iter_mut().for_each(|n| *n = 0); + self.nc_v.iter_mut().for_each(|n| *n = 0); + let qp = self.cfg.qp; + let mut b = Bits::default(); + b.ue(0); // first_mb_in_slice + b.ue(if key { 7 } else { 5 }); // slice_type: all I / all P + b.ue(0); // pic_parameter_set_id + b.put(self.frame_num, 16); + if key { + b.ue(self.idr_id); + } + if !key { + b.flag(false); // num_ref_idx_active_override_flag + b.flag(false); // ref_pic_list_modification_flag_l0 + } + // dec_ref_pic_marking + if key { + b.flag(false); // no_output_of_prior_pics_flag + b.flag(false); // long_term_reference_flag + } else { + b.flag(false); // adaptive_ref_pic_marking_mode_flag + } + b.se(i32::from(qp) - 26); // slice_qp_delta + b.ue(1); // disable_deblocking_filter_idc: off + let mut skip_run = 0u32; + for my in 0..self.mbh { + for mx in 0..self.mbw { + let unchanged = !key && self.prev.as_ref().is_some_and(|p| same_mb(p, &src, mx, my)); + if unchanged { + skip_run += 1; + continue; + } + if !key { + b.ue(skip_run); + skip_run = 0; + } + self.intra_mb(&mut b, &src, mx, my, !key); + } + } + if !key && skip_run > 0 { + b.ue(skip_run); + } + let slice = nal(if key { 0x65 } else { 0x41 }, &b.finish()); + self.prev = Some(src); + let nals = if key { vec![self.sps(), self.pps(), slice] } else { vec![slice] }; + EncodedPicture { nals, key } + } + + /// RGBA to padded BT.709 limited-range 4:2:0 (edges replicated into the padding). + fn to_yuv(&self, rgba: &[u8]) -> Planes { + let (w, h) = (self.cfg.width as usize, self.cfg.height as usize); + let mut p = Planes::new(self.mbw * 16, self.mbh * 16); + let px = |x: usize, y: usize| -> (i32, i32, i32) { + let i = (y.min(h - 1) * w + x.min(w - 1)) * 4; + match rgba.get(i..i + 3) { + Some(s) => (i32::from(s[0]), i32::from(s[1]), i32::from(s[2])), + None => (0, 0, 0), + } + }; + for y in 0..p.h { + for x in 0..p.w { + let (r, g, bl) = px(x, y); + p.y[y * p.w + x] = (((47 * r + 157 * g + 16 * bl + 128) >> 8) + 16).clamp(0, 255) as u8; + } + } + let cw = p.w / 2; + for y in 0..p.h / 2 { + for x in 0..cw { + let (mut su, mut sv) = (0, 0); + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let (r, g, bl) = px(x * 2 + dx, y * 2 + dy); + su += (-26 * r - 87 * g + 112 * bl + 128) >> 8; + sv += (112 * r - 102 * g - 10 * bl + 128) >> 8; + } + p.u[y * cw + x] = ((su + 2).div_euclid(4) + 128).clamp(0, 255) as u8; + p.v[y * cw + x] = ((sv + 2).div_euclid(4) + 128).clamp(0, 255) as u8; + } + } + p + } + + fn nc_luma(&self, bx: usize, by: usize) -> i32 { + let w = self.mbw * 4; + let a = (bx > 0).then(|| i32::from(self.nc_y[by * w + bx - 1])); + let t = (by > 0).then(|| i32::from(self.nc_y[(by - 1) * w + bx])); + combine(a, t) + } + + fn nc_chroma(&self, plane: &[u8], bx: usize, by: usize) -> i32 { + let w = self.mbw * 2; + let a = (bx > 0).then(|| i32::from(plane[by * w + bx - 1])); + let t = (by > 0).then(|| i32::from(plane[(by - 1) * w + bx])); + combine(a, t) + } + + fn intra_mb(&mut self, b: &mut Bits, src: &Planes, mx: usize, my: usize, in_p: bool) { + let qp = self.cfg.qp; + let stride = self.recon.w; + let (x0, y0) = (mx * 16, my * 16); + let (has_top, has_left) = (my > 0, mx > 0); + let top: Vec = if has_top { (0..16).map(|i| i32::from(self.recon.y[(y0 - 1) * stride + x0 + i])).collect() } else { vec![] }; + let left: Vec = if has_left { (0..16).map(|i| i32::from(self.recon.y[(y0 + i) * stride + x0 - 1])).collect() } else { vec![] }; + let dc = match (has_top, has_left) { + (true, true) => (top.iter().sum::() + left.iter().sum::() + 16) >> 5, + (true, false) => (top.iter().sum::() + 8) >> 4, + (false, true) => (left.iter().sum::() + 8) >> 4, + _ => 128, + }; + let s = |x: usize, y: usize| i32::from(src.y[(y0 + y) * stride + x0 + x]); + let pred_of = |mode: u32, x: usize, y: usize| match mode { + 0 => top[x], + 1 => left[y], + _ => dc, + }; + let mut modes = vec![2u32]; + if has_top { + modes.push(0); + } + if has_left { + modes.push(1); + } + let mode = + modes.iter().copied().min_by_key(|&m| (0..256).map(|i| (s(i % 16, i / 16) - pred_of(m, i % 16, i / 16)).abs()).sum::()).unwrap_or(2); + // Luma: transform each 4x4, DC through the Hadamard. + let qbits = 15 + u32::from(qp / 6); + let f = (1 << qbits) / 3; + let m6 = usize::from(qp % 6); + let mut ac = [[0i32; 16]; 16]; // [block raster (by*4+bx)][raster coeff], quantised + let mut wdc = [0i32; 16]; + for blk in 0..16 { + let (bx, by) = (blk % 4, blk / 4); + let mut r = [0i32; 16]; + for i in 0..16 { + let (x, y) = (bx * 4 + i % 4, by * 4 + i / 4); + r[i] = s(x, y) - pred_of(mode, x, y); + } + let w = fdct(&r); + wdc[blk] = w[0]; + for k in 1..16 { + ac[blk][k] = quant(w[k], MF[m6][pos_class(k)], qbits, f); + } + } + let ydc = hadamard4(&wdc); + let mut zdc = [0i32; 16]; + for k in 0..16 { + zdc[k] = quant(ydc[k] / 2, MF[m6][0], qbits + 1, 2 * f); + } + let cbp_luma = ac.iter().any(|blk| blk.iter().skip(1).any(|c| *c != 0)); + // Chroma. + let qpc = chroma_qp(qp); + let cq = qpc_planes(self, src, mx, my, qpc); + let cbp_chroma = if cq.iter().any(|c| c.ac.iter().any(|blk| blk.iter().skip(1).any(|v| *v != 0))) { + 2 + } else if cq.iter().any(|c| c.dc.iter().any(|v| *v != 0)) { + 1 + } else { + 0 + }; + // Syntax. + let mb_type = 1 + mode + 4 * cbp_chroma + if cbp_luma { 12 } else { 0 }; + b.ue(mb_type + if in_p { 5 } else { 0 }); + b.ue(0); // intra_chroma_pred_mode: DC + b.se(0); // mb_qp_delta + let (bx0, by0) = (mx * 4, my * 4); + let dc_scan: Vec = ZIGZAG.iter().map(|&r| zdc[r]).collect(); + let nc0 = self.nc_luma(bx0, by0); + residual_block(b, &dc_scan, nc0); + let lw = self.mbw * 4; + for idx in 0..16 { + let (bx, by) = blk_xy(idx); + let blk = by * 4 + bx; + let total = if cbp_luma { + let scan: Vec = ZIGZAG.iter().skip(1).map(|&r| ac[blk][r]).collect(); + let nc = self.nc_luma(bx0 + bx, by0 + by); + residual_block(b, &scan, nc) + } else { + 0 + }; + self.nc_y[(by0 + by) * lw + bx0 + bx] = total; + } + if cbp_chroma > 0 { + for c in &cq { + residual_block(b, &c.dc, -1); + } + } + let cw = self.mbw * 2; + for (ci, c) in cq.iter().enumerate() { + for blk in 0..4 { + let (bx, by) = (blk % 2, blk / 2); + let total = if cbp_chroma == 2 { + let scan: Vec = ZIGZAG.iter().skip(1).map(|&r| c.ac[blk][r]).collect(); + let plane = if ci == 0 { &self.nc_u } else { &self.nc_v }; + let nc = self.nc_chroma(plane, mx * 2 + bx, my * 2 + by); + residual_block(b, &scan, nc) + } else { + 0 + }; + let plane = if ci == 0 { &mut self.nc_u } else { &mut self.nc_v }; + plane[(my * 2 + by) * cw + mx * 2 + bx] = total; + } + } + // Reconstruction, as the decoder does it. + let fdc = hadamard4(&zdc); + let ls0 = 16 * V[m6][0]; + let q6 = u32::from(qp / 6); + for blk in 0..16 { + let (bx, by) = (blk % 4, blk / 4); + let mut d = [0i32; 16]; + d[0] = if qp >= 36 { (fdc[blk] * ls0) << (q6 - 6) } else { (fdc[blk] * ls0 + (1 << (5 - q6))) >> (6 - q6) }; + for k in 1..16 { + d[k] = dequant(ac[blk][k], qp, pos_class(k)); + } + let r = idct(&d); + for i in 0..16 { + let (x, y) = (bx * 4 + i % 4, by * 4 + i / 4); + self.recon.y[(y0 + y) * stride + x0 + x] = (pred_of(mode, x, y) + r[i]).clamp(0, 255) as u8; + } + } + for (ci, c) in cq.iter().enumerate() { + let cs = self.recon.w / 2; + let plane = if ci == 0 { &mut self.recon.u } else { &mut self.recon.v }; + // Chroma DC levels are in raster order; inverse 2x2 transform and scale. + let f2 = [ + c.dc[0] + c.dc[1] + c.dc[2] + c.dc[3], + c.dc[0] - c.dc[1] + c.dc[2] - c.dc[3], + c.dc[0] + c.dc[1] - c.dc[2] - c.dc[3], + c.dc[0] - c.dc[1] - c.dc[2] + c.dc[3], + ]; + let lsc = 16 * V[usize::from(qpc % 6)][0]; + for blk in 0..4 { + let (bx, by) = (blk % 2, blk / 2); + let mut d = [0i32; 16]; + d[0] = ((f2[blk] * lsc) << u32::from(qpc / 6)) >> 5; + for k in 1..16 { + d[k] = dequant(c.ac[blk][k], qpc, pos_class(k)); + } + let r = idct(&d); + for i in 0..16 { + let (x, y) = (bx * 4 + i % 4, by * 4 + i / 4); + plane[(my * 8 + y) * cs + mx * 8 + x] = (c.pred[blk] + r[i]).clamp(0, 255) as u8; + } + } + } + } +} + +/// One chroma component of a macroblock: DC prediction per 4x4 block, quantised DC (raster +/// 2x2) and AC levels. +struct ChromaMb { + pred: [i32; 4], + dc: [i32; 4], + ac: [[i32; 16]; 4], +} + +fn qpc_planes(e: &Encoder, src: &Planes, mx: usize, my: usize, qpc: u8) -> [ChromaMb; 2] { + let cs = e.recon.w / 2; + let (x0, y0) = (mx * 8, my * 8); + let qbits = 15 + u32::from(qpc / 6); + let f = (1 << qbits) / 3; + let m6 = usize::from(qpc % 6); + let one = |rec: &[u8], s: &[u8]| -> ChromaMb { + let top: Option> = (my > 0).then(|| (0..8).map(|i| i32::from(rec[(y0 - 1) * cs + x0 + i])).collect()); + let left: Option> = (mx > 0).then(|| (0..8).map(|i| i32::from(rec[(y0 + i) * cs + x0 - 1])).collect()); + let mut pred = [128; 4]; + for (blk, p) in pred.iter_mut().enumerate() { + let (xo, yo) = ((blk % 2) * 4, (blk / 2) * 4); + let st = top.as_ref().map(|t| t[xo..xo + 4].iter().sum::()); + let sl = left.as_ref().map(|l| l[yo..yo + 4].iter().sum::()); + *p = match ((xo, yo), st, sl) { + ((0, 0), Some(a), Some(c)) | ((4, 4), Some(a), Some(c)) => (a + c + 4) >> 3, + ((0, 0), _, Some(c)) | ((4, 4), _, Some(c)) => (c + 2) >> 2, + ((0, 0), Some(a), _) | ((4, 4), Some(a), _) => (a + 2) >> 2, + ((4, 0), Some(a), _) => (a + 2) >> 2, + ((4, 0), None, Some(c)) => (c + 2) >> 2, + ((0, 4), _, Some(c)) => (c + 2) >> 2, + ((0, 4), Some(a), None) => (a + 2) >> 2, + _ => 128, + }; + } + let mut wdc = [0i32; 4]; + let mut ac = [[0i32; 16]; 4]; + for blk in 0..4 { + let (bx, by) = (blk % 2, blk / 2); + let mut r = [0i32; 16]; + for i in 0..16 { + let (x, y) = (bx * 4 + i % 4, by * 4 + i / 4); + r[i] = i32::from(s[(y0 + y) * cs + x0 + x]) - pred[blk]; + } + let w = fdct(&r); + wdc[blk] = w[0]; + for k in 1..16 { + ac[blk][k] = quant(w[k], MF[m6][pos_class(k)], qbits, f); + } + } + let y2 = [ + wdc[0] + wdc[1] + wdc[2] + wdc[3], + wdc[0] - wdc[1] + wdc[2] - wdc[3], + wdc[0] + wdc[1] - wdc[2] - wdc[3], + wdc[0] - wdc[1] - wdc[2] + wdc[3], + ]; + let dc = y2.map(|v| quant(v, MF[m6][0], qbits + 1, 2 * f)); + ChromaMb { pred, dc, ac } + }; + [one(&e.recon.u, &src.u), one(&e.recon.v, &src.v)] +} + +/// nC from the left and top neighbours' TotalCoeff (9.2.1). +fn combine(a: Option, t: Option) -> i32 { + match (a, t) { + (Some(a), Some(t)) => (a + t + 1) >> 1, + (Some(a), None) => a, + (None, Some(t)) => t, + (None, None) => 0, + } +} + +/// `luma4x4BlkIdx` → 4x4 block position in the macroblock. +fn blk_xy(idx: usize) -> (usize, usize) { + let (b8, b4) = (idx / 4, idx % 4); + ((b8 % 2) * 2 + b4 % 2, (b8 / 2) * 2 + b4 / 2) +} + +/// 4x4 Hadamard `H · X · H` (raster, symmetric H). +fn hadamard4(x: &[i32; 16]) -> [i32; 16] { + let h = |a: i32, b: i32, c: i32, d: i32| [a + b + c + d, a + b - c - d, a - b - c + d, a - b + c - d]; + let mut t = [0i32; 16]; + for i in 0..4 { + let r = h(x[i * 4], x[i * 4 + 1], x[i * 4 + 2], x[i * 4 + 3]); + t[i * 4..i * 4 + 4].copy_from_slice(&r); + } + let mut out = [0i32; 16]; + for j in 0..4 { + let c = h(t[j], t[4 + j], t[8 + j], t[12 + j]); + for i in 0..4 { + out[i * 4 + j] = c[i]; + } + } + out +} + +/// The macroblock's source pixels are the same in both pictures. +fn same_mb(a: &Planes, b: &Planes, mx: usize, my: usize) -> bool { + let w = a.w; + let rows = |pa: &[u8], pb: &[u8], stride: usize, x0: usize, y0: usize, n: usize| { + (0..n).all(|y| pa[(y0 + y) * stride + x0..(y0 + y) * stride + x0 + n] == pb[(y0 + y) * stride + x0..(y0 + y) * stride + x0 + n]) + }; + rows(&a.y, &b.y, w, mx * 16, my * 16, 16) && rows(&a.u, &b.u, w / 2, mx * 8, my * 8, 8) && rows(&a.v, &b.v, w / 2, mx * 8, my * 8, 8) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A test card: gradients, a hard-edged box and text-like stripes. + fn card(w: u32, h: u32, shift: u32) -> Vec { + let mut out = vec![0u8; (w * h * 4) as usize]; + for y in 0..h { + for x in 0..w { + let i = ((y * w + x) * 4) as usize; + let inbox = (x + shift) % w > w / 4 && (x + shift) % w < w / 2 && y > h / 3 && y < h * 2 / 3; + let stripe = y > h * 3 / 4 && (x / 3) % 2 == 0; + let (r, g, b) = if inbox { + (230, 80, 30) + } else if stripe { + (20, 20, 20) + } else { + ((x * 255 / w) as u8, (y * 255 / h) as u8, 200) + }; + out[i..i + 4].copy_from_slice(&[r, g, b, 255]); + } + } + out + } + + fn psnr(a: &[u8], b: &[u8]) -> f64 { + let mse: f64 = a.iter().zip(b).map(|(x, y)| (f64::from(*x) - f64::from(*y)).powi(2)).sum::() / a.len().max(1) as f64; + if mse == 0.0 { 99.0 } else { 10.0 * (255.0f64 * 255.0 / mse).log10() } + } + + #[test] + fn decodes_with_our_decoder_at_good_quality() { + let (w, h) = (200, 120); // not a multiple of 16: cropping + let mut enc = Encoder::new(EncoderConfig { width: w, height: h, fps: 30, qp: 22 }).unwrap(); + let mut dec = crate::Decoder::new(); + let frames = [card(w, h, 0), card(w, h, 0), card(w, h, 17), card(w, h, 17)]; + let mut stream = vec![]; + let mut sizes = vec![]; + let mut recons = vec![]; + for (i, f) in frames.iter().enumerate() { + let p = enc.encode_rgba(f, i == 0); + sizes.push(p.length_prefixed().len()); + stream.extend(p.annex_b()); + recons.push(enc.recon.y.clone()); + } + let mut pics = dec.decode(&stream, 0).unwrap(); + pics.extend(dec.flush()); + assert_eq!(pics.len(), 4); + let src = enc.to_yuv(&frames[3]); + for (k, pic) in pics.iter().enumerate() { + assert_eq!((pic.width, pic.height), (w, h)); + // The decoder reconstructs exactly what the encoder predicted from. + let stride = enc.recon.w; + for y in 0..h as usize { + let d = &pic.y[y * pic.y_stride..y * pic.y_stride + w as usize]; + let e = &recons[k][y * stride..y * stride + w as usize]; + assert_eq!(d, e, "picture {k} row {y}"); + } + } + let last = &pics[3]; + let dy: Vec = (0..h as usize).flat_map(|y| last.y[y * last.y_stride..y * last.y_stride + w as usize].to_vec()).collect(); + let sy: Vec = (0..h as usize).flat_map(|y| src.y[y * src.w..y * src.w + w as usize].to_vec()).collect(); + assert!(psnr(&dy, &sy) > 38.0, "psnr {}", psnr(&dy, &sy)); + // An unchanged picture is all skipped: a handful of bytes. + assert!(sizes[1] < 16, "{sizes:?}"); + assert!(sizes[3] < 16 && sizes[2] < sizes[0], "{sizes:?}"); + } + + #[test] + fn rejects_bad_sizes_and_writes_levels_for_big_pictures() { + assert!(Encoder::new(EncoderConfig { width: 15, height: 16, fps: 30, qp: 26 }).is_err()); + assert!(Encoder::new(EncoderConfig { width: 101, height: 16, fps: 30, qp: 26 }).is_err()); + let e = Encoder::new(EncoderConfig { width: 1920, height: 1080, fps: 30, qp: 26 }).unwrap(); + assert_eq!(e.level_idc(), 40); + let e = Encoder::new(EncoderConfig { width: 3840, height: 2160, fps: 30, qp: 26 }).unwrap(); + assert_eq!(e.level_idc(), 51); + } + + #[test] + fn big_levels_and_flat_pictures_round_trip() { + // Black then white at a low QP: large DC levels and escapes. + let (w, h) = (64, 32); + let mut enc = Encoder::new(EncoderConfig { width: w, height: h, fps: 30, qp: 10 }).unwrap(); + let mut dec = crate::Decoder::new(); + let mut stream = vec![]; + for (i, v) in [0u8, 255, 128].iter().enumerate() { + let mut img = vec![*v; (w * h * 4) as usize]; + // Noise for many coefficients per block. + for (k, p) in img.iter_mut().enumerate() { + if k % 4 != 3 && (k * 7919) % 13 < 6 { + *p = p.wrapping_add(((k * 31) % 97) as u8); + } + } + stream.extend(enc.encode_rgba(&img, i == 0).annex_b()); + } + let mut pics = dec.decode(&stream, 0).unwrap(); + pics.extend(dec.flush()); + assert_eq!(pics.len(), 3); + let last = &pics[2]; + let stride = enc.recon.w; + for y in 0..h as usize { + assert_eq!(&last.y[y * last.y_stride..y * last.y_stride + w as usize], &enc.recon.y[y * stride..y * stride + w as usize]); + } + } +} diff --git a/crates/h264/src/lib.rs b/crates/h264/src/lib.rs index a65a18e342..7ac80f324a 100644 --- a/crates/h264/src/lib.rs +++ b/crates/h264/src/lib.rs @@ -27,6 +27,8 @@ mod cavlc; #[rustfmt::skip] mod cavlc_tables; mod deblock; +mod encoder; +pub use encoder::{ConfigError, EncodedPicture, Encoder, EncoderConfig}; mod decoder; mod dpb; mod error; diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 93b0dc955e..e3825bddb5 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -227,6 +227,8 @@ pub struct RenderOpts<'a> { pub editing_text: Option, /// Animated GIF pictures and how far into their play they are (seconds); others show frame 1. pub gif_times: &'a [(ShapeId, f64)], + /// Video clips showing a decoded frame instead of their poster (video export). + pub video_frames: &'a [(ShapeId, VideoFrame<'a>)], /// Worker threads (0 = this thread; required when effects use filters). pub threads: u16, /// Translate the slide within the output (pixels) and output size override. @@ -240,6 +242,14 @@ pub struct RenderOpts<'a> { pub now: Option, } +/// A decoded video picture: straight RGBA, `width`×`height`. +#[derive(Clone, Copy, Debug)] +pub struct VideoFrame<'a> { + pub width: u32, + pub height: u32, + pub rgba: &'a [u8], +} + impl Default for RenderOpts<'_> { fn default() -> Self { RenderOpts { @@ -251,6 +261,7 @@ impl Default for RenderOpts<'_> { skip_text: None, editing_text: None, gif_times: &[], + video_frames: &[], threads: 0, offset: (0.0, 0.0), size: None, @@ -1084,7 +1095,11 @@ impl Renderer { ShapeKind::Media(mc) => { ctx.set_transform(m); let bounds = Rect::new(0.0, 0.0, w, h); - if let Some(p) = mc.poster { + let playing = if mc.video { f.opts.video_frames.iter().find(|v| v.0 == s.id).map(|v| v.1) } else { None }; + if let Some(vf) = playing + && paint::video_frame(ctx, vf, bounds, m) + { + } else if let Some(p) = mc.poster { let pf = deckcraft_model::style::PictureFill { media: p, ..Default::default() }; paint::picture(ctx, f.pres, &pf, bounds, m, None); } else if mc.video { diff --git a/crates/render/src/paint.rs b/crates/render/src/paint.rs index ee572fd989..8cf6b45833 100644 --- a/crates/render/src/paint.rs +++ b/crates/render/src/paint.rs @@ -279,6 +279,30 @@ pub fn tile_origin(bounds: Rect, align: &str, tile: (f64, f64), offset: (f64, f6 Point::new(bounds.x0 + ax * (bounds.width() - tile.0) + ox, bounds.y0 + ay * (bounds.height() - tile.1) + oy) } +/// A decoded video frame stretched over `bounds`; `false` when it has no usable pixels. +pub fn video_frame(ctx: &mut RenderContext, f: crate::VideoFrame, bounds: Rect, m: Affine) -> bool { + let (w, h) = (f.width, f.height); + let n = (w as usize).saturating_mul(h as usize); + if w == 0 || h == 0 || w > u32::from(u16::MAX) || h > u32::from(u16::MAX) || f.rgba.len() < n.saturating_mul(4) { + return false; + } + let data: Vec = f.rgba.as_chunks::<4>().0[..n] + .iter() + .map(|p| { + let a = u16::from(p[3]); + let k = |c: u8| ((u16::from(c) * a + 127) / 255) as u8; + vello_cpu::color::PremulRgba8 { r: k(p[0]), g: k(p[1]), b: k(p[2]), a: p[3] } + }) + .collect(); + let pm = vello_cpu::Pixmap::from_parts(data, w as u16, h as u16); + let image = vello_cpu::Image { + image: vello_cpu::ImageSource::Pixmap(std::sync::Arc::new(pm)), + sampler: peniko::ImageSampler { quality: peniko::ImageQuality::High, ..Default::default() }, + }; + draw_image(ctx, &image, f64::from(w), f64::from(h), bounds, m, 1.0); + true +} + fn draw_image(ctx: &mut RenderContext, image: &vello_cpu::Image, iw: f64, ih: f64, dest: Rect, m: Affine, alpha: f64) { let t = m * Affine::translate(dest.origin().to_vec2()) * Affine::scale_non_uniform(dest.width() / iw, dest.height() / ih); ctx.set_transform(t); diff --git a/crates/render/src/transition.rs b/crates/render/src/transition.rs index bd6e331756..b16af7c758 100644 --- a/crates/render/src/transition.rs +++ b/crates/render/src/transition.rs @@ -107,6 +107,7 @@ fn copy_opts<'a>(o: &RenderOpts<'a>) -> RenderOpts<'a> { skip_text: o.skip_text, editing_text: o.editing_text, gif_times: o.gif_times, + video_frames: o.video_frames, threads: o.threads, offset: o.offset, size: o.size, diff --git a/crates/ui-egui/src/backstage.rs b/crates/ui-egui/src/backstage.rs index f7bfe25300..ce31ab92d8 100644 --- a/crates/ui-egui/src/backstage.rs +++ b/crates/ui-egui/src/backstage.rs @@ -145,6 +145,7 @@ pub enum PrintScope { pub enum ExportTask { Pdf, Handouts, + Video, FileType, } @@ -175,6 +176,14 @@ pub struct Backstage { pub file_type: &'static str, pub handouts_per_page: u8, pub pdf_hidden: bool, + /// Export › Create a Video: frame height, frames a second, seconds a slide and before each + /// click, the sound (`aac`, `pcm`, `none`), and hidden slides. + pub video_height: u32, + pub video_fps: u32, + pub video_slide_s: f64, + pub video_click_s: f64, + pub video_audio: &'static str, + pub video_hidden: bool, } impl Backstage { @@ -197,6 +206,12 @@ impl Backstage { file_type: "pptx", handouts_per_page: 3, pdf_hidden: false, + video_height: 1080, + video_fps: 30, + video_slide_s: 5.0, + video_click_s: 1.0, + video_audio: "aac", + video_hidden: false, } } @@ -1247,6 +1262,7 @@ fn export(app: &mut SlideApp, ui: &mut Ui, bs: &mut Backstage) { for (task, icon, label) in [ (ExportTask::Pdf, Icon::File, "Create PDF Document"), (ExportTask::Handouts, Icon::Notes, "Create Handouts"), + (ExportTask::Video, Icon::Video, "Create a Video"), (ExportTask::FileType, Icon::Export, "Change File Type"), ] { if place_row(ui, icon, label, bs.export == task, 244.0).clicked() { @@ -1290,6 +1306,7 @@ fn export(app: &mut SlideApp, ui: &mut Ui, bs: &mut Backstage) { app.export_as("pdf", json!({"layout": "handouts", "perPage": bs.handouts_per_page})); } } + ExportTask::Video => video_page(app, ui, bs), ExportTask::FileType => { ui.label(RichText::new("Change File Type").font(theme::font(16.0)).color(t.text)); ui.add_space(8.0); @@ -1319,6 +1336,116 @@ fn export(app: &mut SlideApp, ui: &mut Ui, bs: &mut Backstage) { }); } +/// Video resolutions as PowerPoint names them: (frame height, label). +pub const VIDEO_SIZES: &[(u32, &str)] = &[ + (2160, "Ultra HD (4K) · 3840 × 2160"), + (1080, "Full HD (1080p) · 1920 × 1080"), + (720, "HD (720p) · 1280 × 720"), + (480, "Standard (480p) · 852 × 480"), +]; +/// The sound choices: (`audio` param, label). +pub const VIDEO_AUDIO: &[(&str, &str)] = &[("aac", "AAC sound (MP4)"), ("pcm", "Uncompressed sound (MOV)"), ("none", "No sound")]; + +/// Export › Create a Video: the options, then the export with its progress and Cancel. +fn video_page(app: &mut SlideApp, ui: &mut Ui, bs: &mut Backstage) { + let t = Tokens::get(ui.ctx()); + ui.label(RichText::new("Create a Video").font(theme::font(16.0)).color(t.text)); + bullets( + ui, + &[ + "Plays the transitions, animations, narration and media", + "MP4 plays almost anywhere; MOV can keep the sound uncompressed", + "The presentation keeps its own file", + ], + ); + let running = app.video_job.is_some(); + ui.add_enabled_ui(!running, |ui| { + let label = |ui: &mut Ui, s: &str| { + ui.label(RichText::new(s).font(theme::font(12.5)).color(t.text_dim)); + }; + label(ui, "Resolution"); + let size = VIDEO_SIZES.iter().find(|v| v.0 == bs.video_height).map(|v| v.1).unwrap_or("Custom"); + egui::ComboBox::from_id_salt("backstage_video_size").width(300.0).selected_text(size).show_ui(ui, |ui| { + for (h, l) in VIDEO_SIZES { + if ui.selectable_label(bs.video_height == *h, *l).clicked() { + bs.video_height = *h; + } + } + }); + ui.add_space(6.0); + label(ui, "Frames per second"); + egui::ComboBox::from_id_salt("backstage_video_fps").width(300.0).selected_text(format!("{} fps", bs.video_fps)).show_ui(ui, |ui| { + for f in [24u32, 25, 30, 60] { + if ui.selectable_label(bs.video_fps == f, format!("{f} fps")).clicked() { + bs.video_fps = f; + } + } + }); + ui.add_space(6.0); + ui.horizontal(|ui| { + ui.add(egui::DragValue::new(&mut bs.video_slide_s).range(0.1..=3600.0).speed(0.1).max_decimals(2).suffix(" s")); + ui.label("on each slide without a timing"); + }); + ui.horizontal(|ui| { + ui.add(egui::DragValue::new(&mut bs.video_click_s).range(0.0..=3600.0).speed(0.1).max_decimals(2).suffix(" s")); + ui.label("before each animation click"); + }); + ui.add_space(6.0); + label(ui, "Sound"); + let audio = VIDEO_AUDIO.iter().find(|a| a.0 == bs.video_audio).map(|a| a.1).unwrap_or("AAC sound (MP4)"); + egui::ComboBox::from_id_salt("backstage_video_audio").width(300.0).selected_text(audio).show_ui(ui, |ui| { + for (id, l) in VIDEO_AUDIO { + if ui.selectable_label(bs.video_audio == *id, *l).clicked() { + bs.video_audio = id; + } + } + }); + ui.checkbox(&mut bs.video_hidden, "Include hidden slides"); + }); + ui.add_space(10.0); + match app.video_job.as_ref().map(|j| (j.progress(), j.cancelling())) { + None => { + if square_button(ui, Icon::Video, "Create Video", true).clicked() { + let params = json!({ + "height": bs.video_height, + "fps": bs.video_fps, + "secondsPerSlide": bs.video_slide_s, + "secondsPerClick": bs.video_click_s, + "audio": bs.video_audio, + "includeHidden": bs.video_hidden, + }); + if let Err(e) = app.run("app.exportVideo", params) { + app.set_status(e); + } + } + } + Some(((done, total), cancelling)) => { + let frac = if total > 0 { done as f32 / total as f32 } else { 0.0 }; + let text = if cancelling { + "Cancelling…".to_string() + } else if total > 0 { + format!("Frame {done} of {total}") + } else { + "Starting…".into() + }; + // A flat bar in the accent colour, the frame count beside it. + ui.horizontal(|ui| { + let (r, resp) = ui.allocate_exact_size(vec2(300.0, 8.0), Sense::hover()); + resp.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::ProgressIndicator, true, &text)); + let p = ui.painter(); + p.rect_filled(r, CornerRadius::same(4), t.border); + let fill = egui::Rect::from_min_size(r.min, vec2((r.width() * frac.clamp(0.0, 1.0)).max(8.0), r.height())); + p.rect_filled(fill, CornerRadius::same(4), t.accent); + ui.label(RichText::new(&text).font(theme::font(12.0)).color(t.text_dim)); + }); + ui.add_space(6.0); + if ui.add_enabled(!cancelling, egui::Button::new("Cancel")).clicked() { + let _ = app.run("app.cancelVideoExport", json!({})); + } + } + } +} + fn bullets(ui: &mut Ui, items: &[&str]) { ui.add_space(4.0); for b in items { diff --git a/crates/ui-egui/src/backstage_tests.rs b/crates/ui-egui/src/backstage_tests.rs index 9b3906a2a3..4a304b768b 100644 --- a/crates/ui-egui/src/backstage_tests.rs +++ b/crates/ui-egui/src/backstage_tests.rs @@ -535,3 +535,102 @@ fn print_slides_follow_the_settings() { bs.print_layout = "slides"; assert_eq!(print_page_size(&bs, &doc), (doc.slide_size.width as f32, doc.slide_size.height as f32), "slides print slide-shaped"); } + +/// Steps the window until the background video export has finished (at most ~60 s). +fn wait_for_video(h: &mut Harness<'_, SlideApp>) { + for _ in 0..3000 { + if h.state().video_job.is_none() { + return; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + h.run_steps(1); + } + panic!("the video export didn't finish"); +} + +/// The window as PNG into `$DECKCRAFT_UI_SHOTS/.png`, to look at (when a GPU renderer +/// is available and the variable is set). +fn shot(h: &mut Harness<'_, SlideApp>, name: &str) { + if let (Ok(dir), Ok(img)) = (std::env::var("DECKCRAFT_UI_SHOTS"), h.render()) { + let _ = img.save(std::path::Path::new(&dir).join(format!("{name}.png"))); + } +} + +#[test] +fn export_creates_a_video_with_the_chosen_options() { + let dir = temp_dir("video"); + let (services, asked) = saving_into(&dir); + let mut h = harness(SlideApp::new(Session::with_new(), services)); + h.state_mut().run("app.backstage", json!({"page": "export"})).unwrap(); + h.run_steps(3); + click_on_page(&h, "Create a Video"); + h.run_steps(3); + assert_eq!(h.state().backstage.as_ref().map(|b| b.export), Some(ExportTask::Video)); + for label in ["Resolution", "Frames per second", "Sound", "Include hidden slides", "Create Video"] { + assert!(h.query_all_by_label(label).next().is_some(), "no “{label}” on the page"); + } + // Pick Standard (480p) in the resolution list, then keep the rest small for the test. + h.query_all_by_role(Role::ComboBox).find(|n| !in_nav(n.rect())).expect("the resolution list").click(); + h.run_steps(3); + h.query_all_by_label_contains("Standard (480p)").last().expect("480p in the list").click(); + h.run_steps(3); + assert_eq!(h.state().backstage.as_ref().map(|b| b.video_height), Some(480)); + shot(&mut h, "video-export-page"); + if let Some(b) = h.state_mut().backstage.as_mut() { + b.video_height = 90; + b.video_fps = 10; + b.video_slide_s = 0.3; + b.video_click_s = 0.1; + b.video_audio = "pcm"; + } + h.run_steps(2); + click_on_page(&h, "Create Video"); + h.run_steps(2); + wait_for_video(&mut h); + let title = h.state().session.active().unwrap().title(); + // PCM sound means a MOV. + assert_eq!(asked.borrow().last(), Some(&format!("{title}.mov"))); + let bytes = std::fs::read(dir.join(format!("{title}.mov"))).unwrap(); + assert_eq!(&bytes[4..8], b"ftyp"); + let status = h.state().status.as_ref().map(|s| s.0.clone()).unwrap_or_default(); + assert!(status.starts_with("Exported"), "{status}"); + assert!(h.state().session.active().unwrap().path.is_none(), "exports are copies"); +} + +#[test] +fn a_video_export_shows_progress_and_can_be_cancelled() { + let dir = temp_dir("video-cancel"); + let (services, _) = saving_into(&dir); + let mut h = harness(SlideApp::new(Session::with_new(), services)); + h.state_mut().run("app.backstage", json!({"page": "export"})).unwrap(); + h.run_steps(3); + click_on_page(&h, "Create a Video"); + h.run_steps(3); + if let Some(b) = h.state_mut().backstage.as_mut() { + b.video_height = 360; + b.video_slide_s = 600.0; + } + click_on_page(&h, "Create Video"); + h.run_steps(2); + assert!(h.state().video_job.is_some()); + assert!(h.state_mut().run("app.exportVideo", json!({})).is_err(), "one export at a time"); + // The progress bar counts frames. + let mut seen = false; + for _ in 0..500 { + h.run_steps(1); + if h.query_all_by_label_contains("Frame ").next().is_some() { + seen = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert!(seen, "no progress shown"); + shot(&mut h, "video-export-progress"); + click_on_page(&h, "Cancel"); + h.run_steps(2); + wait_for_video(&mut h); + assert_eq!(h.state().status.as_ref().map(|s| s.0.as_str()), Some("Video export cancelled")); + let title = h.state().session.active().unwrap().title(); + assert!(!dir.join(format!("{title}.mp4")).exists(), "nothing written"); + assert_eq!(h.state_mut().run("app.cancelVideoExport", json!({})).unwrap(), json!({"cancelled": false})); +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 8867407062..ca5341d53f 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -35,6 +35,7 @@ pub mod theme; pub mod thumbs; pub mod toast; pub mod uifonts; +pub mod video_job; pub mod widgets; use std::collections::HashMap; @@ -406,6 +407,13 @@ pub const UI_COMMANDS: &[(&str, &str, Option<&str>, &str)] = &[ ("app.exportDialog", "Export…", None, "{}"), ("app.browseThemeDialog", "Browse for Themes…", None, "{}"), ("app.compareDialog", "Compare…", None, "{}"), + ( + "app.exportVideo", + "Create a Video", + None, + "{path?, height?, fps?, secondsPerSlide?, secondsPerClick?, audio?: aac|pcm|none, includeHidden?} → {started}", + ), + ("app.cancelVideoExport", "Cancel the Video Export", None, "{} → {cancelled}"), ("app.dialog", "Open Dialog", None, "{id}"), ("app.pasteSpecial", "Paste Special…", Some("Cmd+Alt+V"), "{}"), ("app.qat", "Quick Access Toolbar", None, "{add: command id, params?, label?, icon?} | {remove: id, params?} | {reset: true} → [buttons]"), @@ -453,6 +461,8 @@ pub struct SlideApp { pub status: Option<(String, f64)>, /// A presentation being read in the background, with its progress window ([`opening`]). pub opening: Option, + /// File › Export › Create a Video while it runs (desktop). + pub video_job: Option, /// A brief notice over the bottom of the window (missing fonts on opening a file). pub toast: Option, /// Documents (by uid) whose missing fonts have been mentioned. @@ -545,6 +555,7 @@ impl SlideApp { media: media::MediaHost::new(audio_out), status: None, opening: None, + video_job: None, toast: None, fonts_noted: Default::default(), perf: Perf { frame_ms: 0.0, render_ms: 0.0, fps: 0.0 }, @@ -781,6 +792,63 @@ impl SlideApp { self.download(&suggested, bytes) } + /// File › Export › Create a Video: asks where (unless `path` is given; `.mov` when the sound + /// is PCM) and exports in the background on the desktop, or at once as a download on the + /// web. `{started}` is false when the save dialog was dismissed. + fn export_video(&mut self, p: &Value) -> Result { + if self.video_job.is_some() { + return Err("a video is already being exported".into()); + } + let opts = deckcraft_engine::cmd::file::video_opts(p).map_err(|e| e.to_string())?; + let Some(doc) = self.session.active().map(|d| d.doc.clone()) else { return Err("no presentation is open".into()) }; + let ext = if opts.audio == deckcraft_engine::video::AudioFormat::Pcm { "mov" } else { "mp4" }; + let suggested = self.suggested_name(ext); + let path = match p.get("path").and_then(Value::as_str) { + Some(path) => Some(path.to_string()), + None => match self.services.pick_save.as_mut() { + Some(pick) => match pick(&suggested) { + Some(path) => Some(path), + None => return Ok(json!({"started": false})), + }, + None => None, + }, + }; + match path { + #[cfg(not(target_arch = "wasm32"))] + Some(path) => { + let format = if path.to_ascii_lowercase().ends_with(".mov") { "mov" } else { "mp4" }; + self.video_job = Some(video_job::start(doc, path.clone(), format.into(), opts)?); + self.set_status(format!("Exporting the video to {path}…")); + Ok(json!({"started": true, "path": path})) + } + #[cfg(target_arch = "wasm32")] + Some(_) => Err("this window can't write files; the video is a download instead".into()), + None => { + // The web: no threads, so encode now and hand the file over. + let bytes = deckcraft_engine::video::encode(&doc, ext, &opts) + .map(|v| v.bytes) + .map_err(|e| deckcraft_engine::EngineError::BadParams { cmd: "app.exportVideo".into(), msg: e }); + Ok(json!({"started": self.download(&suggested, bytes)})) + } + } + } + + /// The background video export: its outcome in the status bar once it's done. + fn tick_video_job(&mut self, ctx: &egui::Context) { + let Some(job) = &self.video_job else { return }; + match job.poll() { + Some(r) => { + self.video_job = None; + match r { + Ok(msg) => self.set_status(msg), + Err(e) if e == deckcraft_engine::video::CANCELLED => self.set_status("Video export cancelled"), + Err(e) => self.set_status(format!("Couldn't export the video: {e}")), + } + } + None => ctx.request_repaint_after(std::time::Duration::from_millis(100)), + } + } + /// Hands `bytes` to the browser as a download (web); false (with the reason in the status /// bar) when there are no bytes or no way to download. fn download(&mut self, name: &str, bytes: deckcraft_engine::Result>) -> bool { @@ -1044,6 +1112,11 @@ impl SlideApp { self.set_status("Compare needs the desktop app for now".to_string()); } } + "app.exportVideo" => return self.export_video(p), + "app.cancelVideoExport" => { + let running = self.video_job.as_ref().inspect(|j| j.cancel()).is_some(); + return Ok(json!({"cancelled": running})); + } "app.dialog" => { let d = p.get("id").and_then(Value::as_str).ok_or("missing `id`")?; self.dialog = Some(dialogs::Dialog::new(d)); @@ -1782,6 +1855,7 @@ impl SlideApp { self.tick_media(&ctx); eyedropper::tick(self, &ctx); self.opening_ui(&ctx); + self.tick_video_job(&ctx); if self.show.is_some() { self.anim_preview = None; show::ui(self, ui); diff --git a/crates/ui-egui/src/video_job.rs b/crates/ui-egui/src/video_job.rs new file mode 100644 index 0000000000..c9e10fbe41 --- /dev/null +++ b/crates/ui-egui/src/video_job.rs @@ -0,0 +1,74 @@ +//! File › Export › Create a Video in the background: the export runs on a thread of its own +//! (on the desktop) while the window shows its progress and can cancel it. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::mpsc::Receiver; + +use deckcraft_engine::video::VideoOpts; +use deckcraft_model::Presentation; + +/// A video export under way. +pub struct VideoJob { + pub path: String, + done: Arc, + total: Arc, + cancel: Arc, + rx: Receiver>, +} + +impl VideoJob { + /// Frames written and frames in all (0 until the plan is known). + pub fn progress(&self) -> (u64, u64) { + (self.done.load(Ordering::Relaxed), self.total.load(Ordering::Relaxed)) + } + /// Ask the export to stop; it does after the frame it is on. + pub fn cancel(&self) { + self.cancel.store(true, Ordering::Relaxed); + } + pub fn cancelling(&self) -> bool { + self.cancel.load(Ordering::Relaxed) + } + /// The outcome once the export has finished: a status line, or why it failed. + pub fn poll(&self) -> Option> { + match self.rx.try_recv() { + Ok(r) => Some(r), + Err(std::sync::mpsc::TryRecvError::Empty) => None, + Err(std::sync::mpsc::TryRecvError::Disconnected) => Some(Err("the video export stopped unexpectedly".into())), + } + } +} + +/// Encode `doc` as `format` (`mp4`/`mov`) with `opts` and write it to `path`, calling `progress` +/// after each frame (it returns `false` to stop). The status line for the window. +#[cfg(not(target_arch = "wasm32"))] +pub fn export(doc: &Presentation, path: &str, format: &str, opts: &VideoOpts, progress: &mut dyn FnMut(u64, u64) -> bool) -> Result { + let v = deckcraft_engine::video::encode_with(doc, format, opts, progress)?; + std::fs::write(path, &v.bytes).map_err(|e| format!("couldn't write {path}: {e}"))?; + let secs = v.frames as f64 / f64::from(opts.fps.clamp(1, 60)); + let mut msg = format!("Exported {path} ({secs:.1} s)"); + if let Some(w) = v.warnings.first() { + msg.push_str(&format!(" — {w}")); + } + Ok(msg) +} + +/// Start exporting on a thread of its own. +#[cfg(not(target_arch = "wasm32"))] +pub fn start(doc: Arc, path: String, format: String, opts: VideoOpts) -> Result { + let (done, total, cancel) = (Arc::new(AtomicU64::new(0)), Arc::new(AtomicU64::new(0)), Arc::new(AtomicBool::new(false))); + let (tx, rx) = std::sync::mpsc::channel(); + let (d, t, c, p) = (done.clone(), total.clone(), cancel.clone(), path.clone()); + std::thread::Builder::new() + .name("video-export".into()) + .spawn(move || { + let r = export(&doc, &p, &format, &opts, &mut |n, all| { + d.store(n, Ordering::Relaxed); + t.store(all, Ordering::Relaxed); + !c.load(Ordering::Relaxed) + }); + let _ = tx.send(r); + }) + .map_err(|e| format!("couldn't start the video export: {e}"))?; + Ok(VideoJob { path, done, total, cancel, rx }) +} diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index a5f72fe78f..d3633243d2 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -29,7 +29,7 @@ PowerPoint opens and saves; Viewer = opens only). | PNG / JPEG slide images | export | n/a | yes (`file.export`, `deckcraft-cli render`) | Matches the editor's renderer. PNG `pHYs` and JPEG JFIF density are `scale` × 72 dpi (default scale 2 is 144 dpi), so Preview shows the slide's real size | render and engine tests | | TIFF / BMP / GIF slide images | export | n/a | no | Easy (the `image` crate writes them) | none | | Animated GIF of the deck | export | n/a | yes (`file.export` format `gif`, Export…, `deckcraft-cli`, MCP `export`) | One frame per slide from the slide renderer; size small/medium/large/extra large or 16–3840 px wide, 0.1–600 s per slide, loop forever or play once, slide range, hidden slides skipped; a 256-colour palette per frame (exact, or NeuQuant above 256 colours). Transitions and build animations are not animated | `crates/engine/src/gif_export.rs` tests (decoded frames, size, delay, loop, hidden slides, hostile params), MCP, CLI and export dialog tests | -| MP4 / MOV video of the deck | export | n/a | no | Needs timeline render + H.264 encode | none | +| MP4 / MOV video of the deck | export | n/a | yes | `file.export` mp4/mov: transitions then each slide for its timing, 480p–4K, own H.264 encoder (intra + skip); in-slide animations on the show timeline; sound (media, narration, transition sounds) as AAC or PCM in .mov; video clips play | none | | Print | yes | n/a | no | PR #66 (browser) open | none | ## Pictures and media inside decks diff --git a/docs/gaps.md b/docs/gaps.md index 4e56495123..c289f931e8 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -138,13 +138,14 @@ Detail in [file-format-parity.md](file-format-parity.md). - **Impact:** handouts and speaker notes are printed in many workplaces. - **Estimate:** 5–8 h (print through the PDF path and each OS's print dialog). **Beta blocker.** -### No video export; the animated GIF has no transitions or builds -- **Evidence:** checklist row "Export video (MP4) / animated GIF" is partial: `file.export` writes - an animated GIF with one frame per slide (#171), but there is no MP4/MOV export, and the GIF - doesn't animate transitions or build animations. -- **Impact:** sharing a talk as a video is a common request. -- **Estimate:** 10–16 h (render the timeline frame by frame, which the GIF can then reuse for - transitions and builds; an H.264 encoder is the hard part, FilmCraft has one to copy). +### The animated GIF has no transitions or builds +- **Evidence:** `file.export` writes MP4/MOV with transitions, in-slide animations on the + slide-show timeline, video clips playing their pictures (decoded by `deckcraft-media`), and a + sound track (AAC or PCM in .mov); File › Export › Create a Video sets the options and runs it + in the background with progress and Cancel. The animated GIF (#171) still has one frame per + slide. +- **Impact:** small; GIF is a niche export. +- **Estimate:** 2–4 h (reuse the video's timeline frames for the GIF). ### Embedded MicroType Express font subsets aren't drawn - **Evidence:** `p:embeddedFontLst` is read and written. A raw TrueType/OpenType face, or one inside an Embedded OpenType wrapper, is loaded and used to draw. A MicroType Express subset (`application/x-fontdata` with neither form) is kept and written back, and the text still uses a stand-in. diff --git a/docs/parity-checklist.md b/docs/parity-checklist.md index 94928d8c07..3b00d66a41 100644 --- a/docs/parity-checklist.md +++ b/docs/parity-checklist.md @@ -65,7 +65,7 @@ Open P0 items: Contextual tabs (Shape Format, Picture Format, Table Design/Layou | Files | Export to PDF (slides, notes pages, handouts; text layer, links, bookmarks; vector artwork pending) | P0 | P | | Files | Export slides as PNG/JPEG | P0 | D | | Files | Export outline / RTF | P2 | P | -| Files | Export video (MP4) / animated GIF (GIF: one frame per slide, no transitions; no video) | P2 | P | +| Files | Export video (MP4/MOV with animations, media and sound; File › Export › Create a Video) / animated GIF (one frame per slide, no transitions) | P2 | P | | Files | Print (slides, handouts, notes, outline) | P1 | M | | Files | Password protection | P2 | M | | Files | Reuse slides from another file | P1 | P | diff --git a/docs/ui-parity.md b/docs/ui-parity.md index cc97dae477..e3e6228cc3 100644 --- a/docs/ui-parity.md +++ b/docs/ui-parity.md @@ -24,7 +24,7 @@ ends and are omitted from thumbnails, slide shows and exports. | Ribbon tabs: Home, Insert, Draw, Design, Transitions, Animations, Slide Show, Record, Review, View | 10 | 10 | Record tab is a placeholder | | Colour menus: theme and standard colours, hex field, Eyedropper | yes (plus a More Colors dialog) | yes: theme and standard colours, hex field and an Eyedropper that takes the colour drawn under the pointer on the slide (#367) | More Colors dialog; Eyedropper only takes colours from the slide, not the rest of the screen | | Contextual tabs: Shape Format, Picture Format, Table Design, Table Layout, Chart Design, Chart Format, Video/Audio Format and Playback | yes | most; checklist P0 row partial | Chart Format, media tabs thinner | -| File tab (Backstage): Home, New, Open, Info, Save, Save As, Print, Share, Export, Close, Account, Options | yes | yes (#76), see [File tab](#file-tab-backstage); no Share (cloud) | Video/GIF/CD exports, templates beyond the built-in themes | +| File tab (Backstage): Home, New, Open, Info, Save, Save As, Print, Share, Export, Close, Account, Options | yes | yes (#76), see [File tab](#file-tab-backstage); no Share (cloud) | GIF/CD exports, templates beyond the built-in themes | | Start screen: new, themes, templates, recent, open | yes, the same page as File › Home | yes: File › Home with Home, New, Open, Account and Options (#76) | | | Status bar: slide counter, language, notes, comments, views, zoom slider | yes | yes | | | Command search (Tell Me) | yes | command palette | | @@ -55,7 +55,7 @@ menu colours taken from the theme tokens (menu `chrome`, page `card`). | Save As | Places, folder, file name, type, Save, More options | Type (PowerPoint, DeckCraft, PDF, Outline, PNG, JPEG) and Save, which opens the system Save dialog (web: Download). .pptx/.deckcraft take the new file; the others are copies (`file.saveAs`, `file.export`) | | Print | Print and Copies, Printer, Print All / Selection / Current / Custom Range, Slides box, layout (Full Page Slides, Notes Pages, Outline, Handouts 1–9), sides, collation, colour, Edit Header & Footer; page preview with pager and zoom | Print, Print All Slides / Current Slide / Custom Range, Slides box (`1-3, 5`), Full Page Slides / Notes Pages / Handouts 1–9, Print Hidden Slides, Frame Slides, Edit Header & Footer; preview with pager. DeckCraft prints through a PDF (`file.export`) opened in the system viewer, which has the printers (web: downloaded). No printer list, copies, sides or colour | | Share | Cloud sharing | Not offered | -| Export | Create PDF/XPS Document, Create a Video, Create an Animated GIF, Package Presentation for CD, Create Handouts, Change File Type | Create PDF Document, Create Handouts (a PDF), Change File Type. No video, GIF or CD package | +| Export | Create PDF/XPS Document, Create a Video, Create an Animated GIF, Package Presentation for CD, Create Handouts, Change File Type | Create PDF Document, Create Handouts (a PDF), Create a Video (resolution, fps, seconds per slide and click, sound; progress and Cancel), Change File Type. No GIF or CD package | | Close | Closes; asks Save / Don't Save / Cancel about unsaved changes | Same (`file.close` after the prompt); Ctrl+W asks too | | Account | User information, sign-in, Office Background and Office Theme, product information, About | Appearance (light/dark, `view.dark`), product information, About DeckCraft. No accounts to sign in to | | Options | Opens the PowerPoint Options dialog | Opens Preferences | diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index b9bb409b25..8d739832da 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -41,6 +41,7 @@ pub const TABLE: &[(&str, Class)] = &[ ("ogg", Class::Standalone), ("opus", Class::Standalone), ("h264", Class::Layer(1)), + ("aac", Class::Layer(1)), ("hevc", Class::Layer(1)), ("vp9", Class::Layer(1)), ("av1", Class::Layer(1)),