From 2ccc66e73b624c7ec604559eb1c085b83dd9c3cc Mon Sep 17 00:00:00 2001 From: daijro Date: Fri, 9 Oct 2026 11:25:34 -0500 Subject: [PATCH 1/3] Web: the layer and footage caches stay small enough for eight stacked clips Cause: the browser build kept the desktop's cache budgets (1 GB layer cache, 1 GB footage frame cache) inside the page's memory, which wasm32 caps at 4 GiB. FilmCraft's decoders keep up to 384 MB of decoded frames per open movie beside them, so a comp of eight stacked 1080p H.264 clips grew the page by about 55 MB/s while playing and ran it out of memory after about 50 s ("unreachable": a failed allocation). Fix: in the browser Prefs::cache_budgets keeps the Layer Cache within 512 MB and the Footage Frame Cache within 256 MB (WEB_LAYER_CACHE, WEB_MEDIA_CACHE); smaller settings and the desktop are unchanged, and the RAM preview keeps its budget. Docs (web, preferences) say so. Tests: tests_m145::web_cache_budgets_leave_room_for_the_decoders: with 4 GB settings the web budgets are the caps, the desktop's are the settings, and smaller settings stay as they are. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/engine/src/prefs.rs | 19 +++++++++++++++++++ crates/engine/src/tests_m145.rs | 17 +++++++++++++++++ docs/preferences.md | 4 ++-- docs/web.md | 1 + 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/crates/engine/src/prefs.rs b/crates/engine/src/prefs.rs index 4ee6cc97..b17cb8d3 100644 --- a/crates/engine/src/prefs.rs +++ b/crates/engine/src/prefs.rs @@ -586,8 +586,18 @@ impl Prefs { /// budgets, scaled down so together they leave **RAM Reserved for Other Applications** free, /// and halved while the system is low on memory when **Reduce Cache Size When System Is Low /// on Memory** is on. `mem` = the system's memory (`None` = unknown: configured budgets). + /// In the browser the layer and footage caches also stay within [`WEB_LAYER_CACHE`] and + /// [`WEB_MEDIA_CACHE`]. pub fn cache_budgets(&self, mem: Option) -> CacheBudgets { + self.cache_budgets_in(mem, cfg!(target_arch = "wasm32")) + } + + /// [`Prefs::cache_budgets`], in the browser's memory (`web`) or not. + pub(crate) fn cache_budgets_in(&self, mem: Option, web: bool) -> CacheBudgets { let (mut l, mut m, mut p) = (self.layer_cache_bytes() as u64, self.media_cache_bytes() as u64, self.preview_cache_bytes() as u64); + if web { + (l, m) = (l.min(WEB_LAYER_CACHE), m.min(WEB_MEDIA_CACHE)); + } let floor = 64u64 << 20; let mut capped = false; let mut reduced = false; @@ -732,6 +742,15 @@ fn push_mru(list: &mut Vec, v: &str, max: usize) { list.truncate(max.max(1)); } +/// The most the layer cache holds in the browser. A wasm32 page addresses at most 4 GiB, which +/// these caches share with the app and with the decoders' own decoded frames (FilmCraft keeps +/// up to 384 MB per open movie): with the desktop's 1 GB defaults, eight stacked 1080p clips ran +/// the page out of memory a few seconds into playback. +pub const WEB_LAYER_CACHE: u64 = 512 << 20; + +/// The most the footage frame cache holds in the browser (see [`WEB_LAYER_CACHE`]). +pub const WEB_MEDIA_CACHE: u64 = 256 << 20; + /// Effective cache budgets in bytes (see [`Prefs::cache_budgets`]). #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] pub struct CacheBudgets { diff --git a/crates/engine/src/tests_m145.rs b/crates/engine/src/tests_m145.rs index a73527bb..b55442eb 100644 --- a/crates/engine/src/tests_m145.rs +++ b/crates/engine/src/tests_m145.rs @@ -202,6 +202,23 @@ fn cache_budgets_leave_reserved_ram_and_shrink_when_low() { assert!(!p.cache_budgets(Some(low)).reduced); } +/// Eight stacked 1080p clips ran the browser build out of its 4 GiB of memory: the footage and +/// layer caches kept their desktop sizes there, beside the decoders' own frames. +#[test] +fn web_cache_budgets_leave_room_for_the_decoders() { + use crate::prefs::{WEB_LAYER_CACHE, WEB_MEDIA_CACHE}; + let mut p = crate::prefs::Prefs::default(); + p.memory.layer_cache_mb = 4096; + p.memory.media_cache_mb = 4096; + let web = p.cache_budgets_in(None, true); + assert_eq!((web.layer as u64, web.media as u64), (WEB_LAYER_CACHE, WEB_MEDIA_CACHE)); + assert_eq!(web.preview, p.cache_budgets_in(None, false).preview, "the RAM preview keeps its budget"); + // Smaller settings stay as they are; the desktop is unchanged. + p.memory.media_cache_mb = 128; + assert_eq!(p.cache_budgets_in(None, true).media, 128 << 20); + assert_eq!(p.cache_budgets_in(None, false).layer, 4096 << 20); +} + #[test] fn motion_path_span_follows_the_composition_setting() { let mut p = crate::prefs::Prefs::default(); diff --git a/docs/preferences.md b/docs/preferences.md index 9e3d4147..e4185433 100644 --- a/docs/preferences.md +++ b/docs/preferences.md @@ -71,8 +71,8 @@ Commands (CLI, MCP, control channel): - `audio.outputDevice`: Default Output (device list from the desktop host) - `audio.outputLeft`: Left (Audio Output Mapping) - `audio.outputRight`: Right (Audio Output Mapping) -- `memory.layerCacheMb`: Layer Cache -- `memory.mediaCacheMb`: Footage Frame Cache +- `memory.layerCacheMb`: Layer Cache (in the browser at most 512 MB, see [web.md](web.md)) +- `memory.mediaCacheMb`: Footage Frame Cache (in the browser at most 256 MB) - `memory.previewCacheMb`: Preview (RAM) Cache - `threeD.defaultRenderer`: Default 3D Renderer - `labels.N.name` / `labels.N.color`: the 16 label names and colours, used by the Label menu, diff --git a/docs/web.md b/docs/web.md index b03086eb..e73f7a89 100644 --- a/docs/web.md +++ b/docs/web.md @@ -61,6 +61,7 @@ device of their own), `?nosw` (don't register the service worker). | config directory: settings, shortcut presets, recent projects, the recovery sentinel | `store::WebConfig` in browser storage | | auto-save folder | `/EffectCraft Auto-Save/` in browser storage (`ConfigStore::files`) | | media reads (`MediaPool`, `probe`) | `files::WebImporter`: bytes from the file table, handed to `MediaPool::add_bytes` / `probe_bytes` | +| Memory & CPU cache budgets, scaled to the system's RAM | the page's memory, at most 4 GiB (wasm32), which the caches share with the decoders' own decoded frames (FilmCraft keeps up to 384 MB per open movie): the Layer Cache stays within 512 MB and the Footage Frame Cache within 256 MB (`Prefs::cache_budgets`) | | export writes (`effectcraft-export`) | the job's `sink` (`effectcraft_host::FileExporter { sink }`): files land in the table and download after the render; several files (an image sequence) download as one stored `.zip` | | rfd file dialogs | `` for File ▸ Open / Import; drop files anywhere on the page (`.ecproj` opens, everything else imports); "Save As" / "Output To" pick a download name | | Media Browser on the file system | browser storage and folders opened with the File System Access API (below) | From 74b151cedc42baed560e253da62dafe24ace7409 Mon Sep 17 00:00:00 2001 From: daijro Date: Fri, 9 Oct 2026 11:34:28 -0500 Subject: [PATCH 2/3] Footage: reduced-resolution renders get movie frames converted at their size Cause: at Half, Third and Quarter resolution (the viewer's Auto resolution in most windows) every movie frame was converted from Y'CbCr to f32 RGBA at full size and then resampled down. In the browser (wasm, one thread) that was about 8 ms converting a 1080p frame, 6 ms resampling it and 2 ms zeroing a 33 MB buffer, per layer and frame, for a frame shown at a quarter or a ninth of its pixels; the footage frame cache held the full-size frames. Fix: FootageSource::frame_at_size asks for a frame shown at a given size (default: the full frame, as before). The renderer asks for it where it resampled footage (below 3/4 scale; not for proxies, fields or Interpret Footage's pixel options, which need the full frame). MediaPool makes opaque 4:2:0 movie frames of the footage's own size there: it box-filters them to half size in Y'CbCr before converting (each 2x2 block of luma, and the [1 6 1]/8 neighbourhood of its chroma sample, which is the mean of the upsampled chroma its pixels get), resamples the rest of the way and caches the result per size. Conversion is linear until colours clip, so this is the RGB box filter resample starts with, except in blocks where some colours clip. Other frames convert at full size and resample as before. In the browser the conversion and resampling take about 4 ms instead of 16. Docs (architecture). Tests: - convert::scaled_conversion_matches_resampling_the_converted_frame: half, third, quarter and odd sizes of an unclipped 4:2:0 frame are within 1e-5 of resampling the converted frame; larger sizes, odd frame sizes and 4:4:4 convert at full size. - tests_footage_size::reduced_renders_take_footage_resampled_by_the_source: a source that resamples frames itself renders the same frames as one that doesn't, at 8 and 32 bpc and every resolution, and is asked only below 3/4 scale, and not with Invert Alpha. - footage::h264_frames_at_reduced_sizes (ffmpeg fixture): 1080p H.264 testsrc2 frames, whose saturated colours clip, at 960x540, 640x360, 480x270 and 323x182 are 58-62 dB from the resampled full frames, and are cached per size. - convert's ignored 1080p bench also times the conversion at a third. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/media/src/convert.rs | 107 ++++++++++++++++++++++++ crates/media/src/pool.rs | 71 +++++++++++----- crates/media/tests/footage.rs | 20 +++++ crates/render/src/lib.rs | 47 ++++++++--- crates/render/src/tests_footage_size.rs | 83 ++++++++++++++++++ docs/architecture.md | 6 +- 6 files changed, 298 insertions(+), 36 deletions(-) create mode 100644 crates/render/src/tests_footage_size.rs diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index 114873e7..d9ef9323 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -104,6 +104,67 @@ pub(crate) fn frame_to_image_in(f: &VideoFrame, op: AlphaOp, mut buf: Vec) - img } +/// A FilmCraft frame for a render that shows it at `w` × `h`, at most half its size each way: +/// what `resample(&frame_to_image(f, op), w, h)` makes, converting a quarter of the pixels. +/// Opaque 4:2:0 Y'CbCr is box-filtered to half size before conversion (each 2×2 block of luma, +/// and the [1 6 1]/8 neighbourhood of its chroma sample: the mean of the upsampled chroma its +/// pixels get), then resampled the rest of the way. Conversion is linear until it clips, so +/// this is the RGB box filter `resample` starts with, except in blocks where some colours clip. +/// `None` for other frames. +pub(crate) fn frame_to_image_scaled(f: &VideoFrame, w: u32, h: u32) -> Option { + let (fw, fh) = (f.width as usize, f.height as usize); + if w == 0 || h == 0 || fw % 2 != 0 || fh % 2 != 0 || (w as usize).saturating_mul(2) > fw || (h as usize).saturating_mul(2) > fh { + return None; + } + let half = match &f.data { + PixelData::Yuv8 { planes, chroma: Chroma::C420, alpha: None } => { + half_420(fw, fh, [&planes[0][..], &planes[1][..], &planes[2][..]], &Yuv::new(f.color.matrix, f.color.range, 8))? + } + PixelData::Yuv16 { planes, chroma: Chroma::C420, bits, alpha: None } => { + half_420(fw, fh, [&planes[0][..], &planes[1][..], &planes[2][..]], &Yuv::new(f.color.matrix, f.color.range, *bits))? + } + _ => return None, + }; + Some(if (half.width, half.height) == (w, h) { half } else { effectcraft_raster::resample(&half, w, h) }) +} + +/// An opaque 4:2:0 frame of even size `w` × `h` converted at half that size (see +/// [`frame_to_image_scaled`]); `None` when the planes are too short. +fn half_420(w: usize, h: usize, [luma, cb, cr]: [&[T]; 3], k: &Yuv) -> Option { + /// Row `r` of a plane `n` samples wide (empty past its end). + fn row_of(p: &[T], r: usize, n: usize) -> &[T] { + p.get(r.saturating_mul(n)..(r + 1).saturating_mul(n)).unwrap_or_default() + } + let (cw, ch) = (w / 2, h / 2); + if luma.len() < w.checked_mul(h)? || cb.len() < cw * ch || cr.len() < cw * ch { + return None; + } + let mut img = Image::new(cw as u32, ch as u32); + img.data.par_chunks_mut(cw).enumerate().for_each_init( + // Chroma rows with their edge samples repeated on both ends, for the horizontal [1 6 1]. + || (vec![0f32; cw + 2], vec![0f32; cw + 2]), + |(uc, vc), (cy, row)| { + // The chroma rows above and below weigh 1/8, as `centred` upsampling averages out. + let rows = [cy.saturating_sub(1), cy, (cy + 1).min(ch - 1)]; + for (dst, plane) in [(&mut *uc, cb), (&mut *vc, cr)] { + let [above, at, below] = rows.map(|r| row_of(plane, r, cw)); + for (d, ((a, b), c)) in dst[1..=cw].iter_mut().zip(above.iter().zip(at).zip(below)) { + *d = ((a.f() + c.f()) * 0.125 + b.f() * 0.75 - k.c_off) * k.c_scale; + } + (dst[0], dst[cw + 1]) = (dst[1], dst[cw]); + } + let (top, bottom) = (row_of(luma, 2 * cy, w), row_of(luma, 2 * cy + 1, w)); + let pixels = top.chunks_exact(2).zip(bottom.chunks_exact(2)).zip(uc.windows(3).zip(vc.windows(3))); + for (o, ((a, b), (u, v))) in row.iter_mut().zip(pixels) { + let (u, v) = ((u[0] + u[2]) * 0.125 + u[1] * 0.75, (v[0] + v[2]) * 0.125 + v[1] * 0.75); + let yy = ((a[0].f() + a[1].f() + b[0].f() + b[1].f()) * 0.25 - k.y_off) * k.y_scale; + *o = [(yy + k.cr_r * v).clamp(0.0, 1.0), (yy - k.cr_g * v - k.cb_g * u).clamp(0.0, 1.0), (yy + k.cb_b * u).clamp(0.0, 1.0), 1.0]; + } + }, + ); + Some(img) +} + /// 0..255 → 0..1. static U8: [f32; 256] = { let mut t = [0f32; 256]; @@ -338,6 +399,47 @@ mod tests { assert!((img.data[0][0] - 110.0 / 219.0).abs() < 1e-5); } + /// A smooth opaque 4:2:0 frame whose colours never clip. + fn smooth_420(w: usize, h: usize) -> VideoFrame { + let plane = |w: usize, h: usize, base: f32, amp: f32, fx: f32, fy: f32| -> Arc> { + Arc::new((0..w * h).map(|i| (base + amp * ((i % w) as f32 * fx + (i / w) as f32 * fy).sin()) as u8).collect()) + }; + VideoFrame { + width: w as u32, + height: h as u32, + data: PixelData::Yuv8 { + planes: [plane(w, h, 125.0, 45.0, 0.31, 0.17), plane(w / 2, h / 2, 128.0, 10.0, 0.4, -0.3), plane(w / 2, h / 2, 128.0, 10.0, -0.2, 0.5)], + chroma: Chroma::C420, + alpha: None, + }, + color: filmcraft_color::ColorInfo::REC709, + par: (1, 1), + pts: filmcraft_time::Tick::ZERO, + } + } + + #[test] + fn scaled_conversion_matches_resampling_the_converted_frame() { + let f = smooth_420(96, 64); + let full = frame_to_image(&f, AlphaOp::new(AlphaMode::Straight, [0.0; 3])); + for (w, h) in [(48, 32), (32, 21), (24, 16), (13, 9)] { + let scaled = frame_to_image_scaled(&f, w, h).expect("4:2:0, at most half size"); + let want = effectcraft_raster::resample(&full, w, h); + assert_eq!((scaled.width, scaled.height), (w, h)); + let worst = scaled.data.iter().zip(&want.data).flat_map(|(a, b)| (0..4).map(move |c| (a[c] - b[c]).abs())).fold(0.0f32, f32::max); + assert!(worst < 1e-5, "{w}×{h}: {worst}"); + } + // More than half the size, odd sizes, other chroma layouts: convert the full frame. + assert!(frame_to_image_scaled(&f, 49, 32).is_none()); + assert!(frame_to_image_scaled(&smooth_420(95, 64), 40, 30).is_none()); + assert!(frame_to_image_scaled(&yuv_frame(100, 128, 128, filmcraft_color::ColorInfo::REC709), 2, 1).is_some()); + let mut f444 = smooth_420(96, 64); + if let PixelData::Yuv8 { chroma, .. } = &mut f444.data { + *chroma = Chroma::C444; + } + assert!(frame_to_image_scaled(&f444, 48, 32).is_none()); + } + #[test] fn bt709_red() { // 8-bit limited BT.709 red: Y 63, Cb 102, Cr 240 @@ -402,6 +504,11 @@ mod tests { keep.push(frame_to_image(&f, op)); } eprintln!("convert into fresh memory {:?}", t.elapsed() / 20); + let t = std::time::Instant::now(); + for _ in 0..20 { + std::hint::black_box(frame_to_image_scaled(&f, 640, 360)); + } + eprintln!("convert at a third {:?}", t.elapsed() / 20); } } } diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 3df9ec45..f10389fd 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -10,7 +10,7 @@ use effectcraft_render::FootageSource; use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; use filmcraft_media::{FrameRequest, SharedSource}; -use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in}; +use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled}; use crate::{MediaError, Result}; /// Default frame-cache budget: 1 GiB of decoded frames. @@ -41,6 +41,8 @@ struct Key { /// pixels. interp: u8, matte: [u32; 3], + /// A movie frame made at this size for a reduced-resolution render ((0, 0): as decoded). + size: (u32, u32), } struct Entry { @@ -322,7 +324,21 @@ impl MediaPool { /// Decode (or fetch from cache) the frame of `footage` at source time `t`. pub fn frame_at(&self, footage: &Footage, t: Tick) -> Result> { - Inner::frame_at(&self.inner, footage, t, true) + Inner::frame_at(&self.inner, footage, t, true, None) + } + + /// [`FootageSource::frame`] / [`FootageSource::frame_at_size`]: `None` (logged) on errors. + fn frame_sized(&self, _item: ItemId, footage: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option> { + if footage.missing { + return None; + } + match Inner::frame_at(&self.inner, footage, t, true, size) { + Ok(img) => Some(img), + Err(e) => { + log::warn!("media: frame of {} at {:?}: {e}", footage.path, t); + None + } + } } /// Settings ▸ Disk ▸ Conformed Audio Folder: write each footage file's decoded audio there @@ -469,7 +485,7 @@ impl MediaPool { /// A thumbnail of `footage` (its first frame) fitting in `max_side` × `max_side`, aspect kept. pub fn thumbnail(&self, footage: &Footage, max_side: u32) -> Result { - let img = Inner::frame_at(&self.inner, footage, Tick::ZERO, false)?; + let img = Inner::frame_at(&self.inner, footage, Tick::ZERO, false, None)?; let (w, h) = (img.width.max(1), img.height.max(1)); let s = (max_side.max(1) as f64 / w.max(h) as f64).min(1.0); if s >= 1.0 { @@ -561,7 +577,7 @@ impl Inner { }; let interp = alpha | (u8::from(footage.preserve_rgb) << 2); let matte = if footage.alpha == AlphaMode::Premultiplied { footage.premul_color.map(f32::to_bits) } else { [0; 3] }; - let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte }; + let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte, size: (0, 0) }; match footage.kind { FootageKind::Sequence if !footage.sequence.is_empty() => { let i = Self::frame_index(footage.frame_rate, t, footage.sequence_frames(), footage.loop_count); @@ -591,21 +607,27 @@ impl Inner { } } - fn frame_at(self: &Arc, footage: &Footage, t: Tick, playback: bool) -> Result> { + /// The frame of `footage` at `t`; a movie's resampled to `size` when it can be made at that + /// size directly (see [`FootageSource::frame_at_size`]). + fn frame_at(self: &Arc, footage: &Footage, t: Tick, playback: bool, size: Option<(u32, u32)>) -> Result> { if !footage.has_video { return Err(MediaError::Unsupported(format!("{}: no video", footage.path))); } - let loc = Self::locate(footage, t); + let mut loc = Self::locate(footage, t); let movie = loc.media_t.is_some(); + if movie && let Some(size) = size { + loc.key.size = size; + } let mut c = lock(&self.cache); // sequential playback of a movie: read the next frame ahead if playback && movie { let prev = c.last.insert(loc.key.path.clone(), loc.key.frame); if prev == Some(loc.key.frame - 1) && self.read_ahead.load(Ordering::Relaxed) { let next_t = t + footage.frame_rate.frame_duration(); - let next = Self::locate(footage, next_t); + let mut next = Self::locate(footage, next_t); + next.key.size = loc.key.size; if next.key != loc.key && !c.lru.map.contains_key(&next.key) && !c.inflight.contains(&next.key) { - self.prefetch(footage, next_t); + self.prefetch(footage, next_t, size); } } } @@ -638,17 +660,17 @@ impl Inner { } #[cfg(not(target_arch = "wasm32"))] - fn prefetch(self: &Arc, footage: &Footage, t: Tick) { + fn prefetch(self: &Arc, footage: &Footage, t: Tick, size: Option<(u32, u32)>) { let (me, f) = (self.clone(), footage.clone()); self.prefetched.fetch_add(1, Ordering::Relaxed); // A plain thread, not a rayon job: the decode below parallelises with rayon itself. std::thread::spawn(move || { - let _ = me.frame_at(&f, t, false); + let _ = me.frame_at(&f, t, false, size); }); } #[cfg(target_arch = "wasm32")] - fn prefetch(self: &Arc, _footage: &Footage, _t: Tick) {} + fn prefetch(self: &Arc, _footage: &Footage, _t: Tick, _size: Option<(u32, u32)>) {} fn decode(&self, loc: &Loc, footage: &Footage) -> Result { let op = AlphaOp::of(footage); @@ -670,6 +692,15 @@ impl Inner { Some(mt) => { let src = self.source(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?; let vf = src.video_frame(FrameRequest::full(filmcraft_time::Tick(mt.0))).map_err(MediaError::from)?; + // A reduced-resolution render's frame, made at its size (a frame of the footage's + // own size only, as the renderer resamples only those). + let (w, h) = loc.key.size; + if w > 0 + && (vf.width, vf.height) == (footage.width, footage.height) + && let Some(img) = frame_to_image_scaled(&vf, w, h) + { + return Ok(img); + } let buf = lock(&self.cache).lru.take_spare(vf.width as usize * vf.height as usize); Ok(frame_to_image_in(&vf, op, buf)) } @@ -696,17 +727,11 @@ impl FootageSource for MediaPool { fn cache_budget(&self) -> Option { Some(self.budget()) } - fn frame(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option> { - if footage.missing { - return None; - } - match self.frame_at(footage, t) { - Ok(img) => Some(img), - Err(e) => { - log::warn!("media: frame of {} at {:?}: {e}", footage.path, t); - None - } - } + fn frame(&self, item: ItemId, footage: &Footage, t: Tick) -> Option> { + self.frame_sized(item, footage, t, None) + } + fn frame_at_size(&self, item: ItemId, footage: &Footage, t: Tick, width: u32, height: u32) -> Option> { + self.frame_sized(item, footage, t, Some((width, height))) } fn vector_frame(&self, _item: ItemId, footage: &Footage, scale: f64) -> Option> { @@ -834,7 +859,7 @@ mod tests { fn lru_respects_budget_and_recycles() { let mut c = Lru::default(); let img = || Arc::new(Image::new(16, 16)); // 4 KiB + overhead - let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3] }; + let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3], size: (0, 0) }; let budget = 3 * (4096 + 64); for i in 0..10 { c.insert(k(i), img(), budget); diff --git a/crates/media/tests/footage.rs b/crates/media/tests/footage.rs index ed113c35..8d49f087 100644 --- a/crates/media/tests/footage.rs +++ b/crates/media/tests/footage.rs @@ -236,6 +236,26 @@ fn h264_mp4_probe_and_frames() { assert!(Arc::ptr_eq(&a, &b)); } +/// A reduced-resolution render's movie frames are converted at its size: they match resampling +/// the full frames (testsrc2's saturated colours clip, where the two differ most), and are cached +/// per size. +#[test] +fn h264_frames_at_reduced_sizes() { + let Some(p) = h264_mp4() else { return }; + let f = probe(&p).expect("probe"); + let pool = MediaPool::new(); + for (n, (w, h)) in [(0i64, (960, 540)), (1, (640, 360)), (2, (480, 270)), (30, (323, 182))] { + let t = t_of(f.frame_rate, n); + let sized = pool.frame_at_size(ItemId(1), &f, t, w, h).expect("frame"); + let full = pool.frame(ItemId(1), &f, t).expect("frame"); + assert_eq!((sized.width, sized.height), (w, h)); + let db = psnr(&sized, &effectcraft_raster::resample(&full, w, h)); + eprintln!("h264 frame {n} at {w}×{h}: {db:.2} dB against the resampled full frame"); + assert!(db > 55.0, "frame {n} at {w}×{h}: {db:.2} dB"); + assert!(Arc::ptr_eq(&sized, &pool.frame_at_size(ItemId(1), &f, t, w, h).expect("frame")), "cached"); + } +} + #[test] fn h264_audio_samples() { let Some(p) = h264_mp4() else { return }; diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 591213e6..324e2a5f 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -40,6 +40,13 @@ use rayon::prelude::*; pub trait FootageSource: Send + Sync { /// The frame of `item` at source time `t`, straight from the file (any size). fn frame(&self, item: ItemId, footage: &Footage, t: Tick) -> Option>; + /// [`FootageSource::frame`] for a render that shows it at `width` × `height`, at most half its + /// size: either that frame or, when the source can make it more cheaply than the whole frame + /// (movies, see `effectcraft-media`), the frame already resampled to that size as + /// [`effectcraft_raster::resample`] would. + fn frame_at_size(&self, item: ItemId, footage: &Footage, t: Tick, _width: u32, _height: u32) -> Option> { + self.frame(item, footage, t) + } /// `frames` interleaved stereo sample frames of `item`'s audio from source time `t` at /// `rate` Hz (`None` when unavailable). fn audio(&self, _item: ItemId, _footage: &Footage, _t: Tick, _frames: usize, _rate: u32) -> Option> { @@ -1104,8 +1111,10 @@ impl<'a> Renderer<'a> { } let lt = ctx.source_time(layer); // The proxy is decoded instead, at its own size, and fills the footage's frame. - let pf = self.proxy_for(*item).unwrap_or(f); - let img = self.footage_frame(ctx, layer, *item, pf, lt)?; + let proxy = self.proxy_for(*item); + let pf = proxy.unwrap_or(f); + let size = self.reduced_size(f.width, f.height).filter(|_| proxy.is_none()); + let img = self.footage_frame(ctx, layer, *item, pf, lt, size)?; Some(self.footage_buf(&img, pf, f.width, f.height)) } LayerSource::Text => Some(self.authored(text::render(ctx, layer, self.raster_scale(ctx, layer)))), @@ -1127,12 +1136,11 @@ impl<'a> Renderer<'a> { let s = self.opts.scale; let k = if img.width > 0 { w.max(1) as f64 / img.width as f64 } else { 1.0 }; // Keep native pixels when downsampling is small; resample otherwise. - let mut buf = if s < 0.75 && k <= 1.0 + 1e-9 { - let rw = ((w as f64 * s).round() as u32).max(1); - let rh = ((h as f64 * s).round() as u32).max(1); - Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s } - } else { - Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k } + let mut buf = match self.reduced_size(w, h) { + // The source resampled it already (`FootageSource::frame_at_size`). + Some(size) if (img.width, img.height) == size => Buf { img: img.clone(), offset: [0.0; 2], scale: s }, + Some((rw, rh)) if k <= 1.0 + 1e-9 => Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s }, + _ => Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k }, }; if let Some(c) = self.pipe.media_in(f) { color::convert(&mut buf.img, &c); @@ -1140,6 +1148,13 @@ impl<'a> Renderer<'a> { buf } + /// The size of `w` × `h` layer pixels in this render when it is reduced enough that footage + /// is resampled to it (below 3/4 scale). + fn reduced_size(&self, w: u32, h: u32) -> Option<(u32, u32)> { + let s = self.opts.scale; + (s < 0.75).then(|| (((w as f64 * s).round() as u32).max(1), ((h as f64 * s).round() as u32).max(1))) + } + /// Authored colours into the working space (linear working spaces). fn authored(&self, mut buf: Buf) -> Buf { if let Some(c) = self.pipe.authored_in() { @@ -1151,7 +1166,8 @@ impl<'a> Renderer<'a> { /// A footage frame at source time `t`, frame-blended between the two nearest source frames /// when the layer's Frame Blending switch (and the comp's Enable Frame Blending) is on and /// `t` falls between frames (footage rate ≠ comp rate, time stretch, time remapping). - fn footage_frame(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option> { + /// `size`: a plain frame may come resampled to it ([`FootageSource::frame_at_size`]). + fn footage_frame(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option> { // Interpret Footage ▸ Separate Fields: each field is a frame at twice the rate. if f.fields != effectcraft_project::FieldOrder::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) { let field_rate = FrameRate::new(f.frame_rate.num * 2, f.frame_rate.den); @@ -1162,11 +1178,13 @@ impl<'a> Renderer<'a> { let parity = if (i % 2 == 0) == dominant_upper { 0 } else { 1 }; return Some(Arc::new(interpret_pixels(&field_frame(&img, parity), f))); } - let img = self.footage_frame_raw(ctx, layer, item, f, t)?; + // Interpretation is per pixel but not linear: it runs on the frame before resampling. + let size = size.filter(|_| !(f.invert_alpha || f.linear_light_applies())); + let img = self.footage_frame_raw(ctx, layer, item, f, t, size)?; if f.invert_alpha || f.linear_light_applies() { Some(Arc::new(interpret_pixels(&img, f))) } else { Some(img) } } - fn footage_frame_raw(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option> { + fn footage_frame_raw(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option> { let mode = frame_blend_mode(ctx, layer); if mode != FrameBlend::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) { let (i, w) = frame_position(t, f.frame_rate); @@ -1194,7 +1212,10 @@ impl<'a> Renderer<'a> { } } } - self.footage.frame(item, f, t) + match size { + Some((w, h)) => self.footage.frame_at_size(item, f, t, w, h), + None => self.footage.frame(item, f, t), + } } /// Source → masks, clamped/quantised to the project depth. @@ -2152,6 +2173,8 @@ mod tests_collapse; #[cfg(test)] mod tests_color; #[cfg(test)] +mod tests_footage_size; +#[cfg(test)] mod tests_frame_blend; #[cfg(test)] mod tests_mask_blur; diff --git a/crates/render/src/tests_footage_size.rs b/crates/render/src/tests_footage_size.rs new file mode 100644 index 00000000..88df726e --- /dev/null +++ b/crates/render/src/tests_footage_size.rs @@ -0,0 +1,83 @@ +//! Footage at a reduced resolution: a source that resamples frames itself +//! ([`FootageSource::frame_at_size`], movies in `effectcraft-media`) renders what resampling the +//! full frame renders. + +use std::sync::{Arc, Mutex}; + +use rayon::prelude::*; + +use effectcraft_color::Label; +use effectcraft_project::build; +use effectcraft_project::{AlphaMode, BitDepth, Comp, Footage, FootageKind, ItemId, ItemKind, LayerSource, Project}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::{FootageSource, Image, RenderOpts, Renderer}; + +/// Footage with a textured frame; `sizes` (when set) resamples it itself and records the sizes +/// it was asked for. +struct Clip { + sizes: Option>>, +} + +impl FootageSource for Clip { + fn frame(&self, _: ItemId, f: &Footage, _: Tick) -> Option> { + let mut img = Image::new(f.width, f.height); + img.rows_mut().for_each(|(y, row)| { + for (x, p) in row.iter_mut().enumerate() { + let s = (0.5 + 0.3 * (x as f32 * 0.37 + y as f32 * 0.11).sin()) * 0.9; + *p = [s, 1.0 - s, s * 0.5, 1.0]; + } + }); + Some(Arc::new(img)) + } + fn frame_at_size(&self, item: ItemId, f: &Footage, t: Tick, w: u32, h: u32) -> Option> { + let Some(sizes) = &self.sizes else { return self.frame(item, f, t) }; + sizes.lock().unwrap().push((w, h)); + Some(Arc::new(effectcraft_raster::resample(&*self.frame(item, f, t)?, w, h))) + } +} + +fn project(depth: BitDepth, invert_alpha: bool) -> (Project, ItemId) { + let mut p = Project::default(); + p.settings.bit_depth = depth; + let comp = Comp::new(96, 54, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.clone().into())); + let f = Footage { + path: "clip.mp4".into(), + kind: FootageKind::Video, + width: 96, + height: 54, + pixel_aspect: 1.0, + frame_rate: FrameRate::FPS_30, + duration: Tick::from_seconds_f64(1.0), + has_video: true, + alpha: AlphaMode::Ignore, + loop_count: 1, + invert_alpha, + ..Default::default() + }; + let fid = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(f)); + let l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, (96, 54), None); + p.comp_mut(cid).unwrap().layers.push(l); + (p, cid) +} + +#[test] +fn reduced_renders_take_footage_resampled_by_the_source() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let (p, cid) = project(depth, false); + for (scale, asked) in [(1.0, None), (0.75, None), (0.5, Some((48, 27))), (1.0 / 3.0, Some((32, 18))), (0.25, Some((24, 14)))] { + let opts = RenderOpts { scale, ..Default::default() }; + let plain = Renderer::new(&p, &Clip { sizes: None }, opts).comp_frame(cid, Tick::ZERO); + let sized = Clip { sizes: Some(Mutex::default()) }; + let frame = Renderer::new(&p, &sized, opts).comp_frame(cid, Tick::ZERO); + assert_eq!(frame, plain, "{depth:?} at {scale}"); + assert_eq!(sized.sizes.unwrap().into_inner().unwrap(), asked.into_iter().collect::>(), "{depth:?} at {scale}"); + } + } + // Interpreted pixels (Invert Alpha) are interpreted at full size, then resampled. + let (p, cid) = project(BitDepth::Bpc8, true); + let sized = Clip { sizes: Some(Mutex::default()) }; + Renderer::new(&p, &sized, RenderOpts { scale: 0.5, ..Default::default() }).comp_frame(cid, Tick::ZERO); + assert!(sized.sizes.unwrap().into_inner().unwrap().is_empty()); +} diff --git a/docs/architecture.md b/docs/architecture.md index 33e38215..6a284b8e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -230,7 +230,11 @@ Nulls / Nulls Follow Points take pins as well as paths, and `puppet.follow` give Position expression that trails the leader by a delay. The renderer flattens nested effect groups into `Params` keys (`effects::flatten_params`). Commands: `paint.*`, `puppet.*`. -Half, Third and Quarter resolution render proportionally fewer pixels end to end. +Half, Third and Quarter resolution render proportionally fewer pixels end to end. Footage +starts at the render's size where the source can make it more cheaply than the full frame +(`FootageSource::frame_at_size`): movies in opaque 4:2:0 Y'CbCr are box-filtered to half size +before conversion (a quarter of the conversions; the result differs from converting first only +in blocks where colours clip) and resampled from there, as other frames are from full size. **GPU compositor** (`crates/gpu`; Project Settings ▸ Video Rendering and Effects ▸ Mercury GPU Acceleration, the default, or Mercury Software Only; `render.backend`). The CPU renderer is the From ddfa566d7d5ac08692b64a808cfc39556a58aa26 Mon Sep 17 00:00:00 2001 From: daijro Date: Fri, 9 Oct 2026 11:47:34 -0500 Subject: [PATCH 3/3] Render: 8 and 16 bpc quantisation rounds with floor in the browser Cause: quantize_px rounded with `(v + 0.5) as u32 as f32`. Converting back from u32 is several instructions in V8's wasm code on x86 (no unsigned conversion), so quantising a 1080p layer took about 15 ms in the browser: every 8/16 bpc layer is quantised after its source and every effect, and the CPU compositor quantises the comp after every layer. Fix: round_half_up uses floor in wasm (one instruction there) and keeps the integer conversion elsewhere (x86-64's baseline has no floor instruction, where floor is slower). The values are the same: both round non-negative values half up. About 1.5x faster quantising in the browser (15 -> 10 ms per 1080p layer in a wasm micro-benchmark), unchanged on the desktop. Tests: color::tests::rounding_forms_agree: both forms agree on 0..=levels in sixteenths (exact halves included) and on 100 001 products, at 8 and 16 bpc. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/render/src/color.rs | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/crates/render/src/color.rs b/crates/render/src/color.rs index 91abef27..19785722 100644 --- a/crates/render/src/color.rs +++ b/crates/render/src/color.rs @@ -195,14 +195,22 @@ pub fn quantize(img: &mut Image, levels: f32) { #[allow(clippy::manual_clamp)] // max/min map NaN to 0; clamp would keep NaN #[inline(always)] fn quantize_px(p: &mut [f32; 4], levels: f32, inv: f32) { - let a = (p[3].max(0.0).min(1.0) * levels + 0.5) as u32 as f32 * inv; + let a = round_half_up(p[3].max(0.0).min(1.0) * levels) * inv; let hi = [a, a, a, a]; for c in 0..3 { - p[c] = (p[c].max(0.0).min(hi[c]) * levels + 0.5) as u32 as f32 * inv; + p[c] = round_half_up(p[c].max(0.0).min(hi[c]) * levels) * inv; } p[3] = a; } +/// `v` (0..=65 535) rounded half up. The same either way; the fastest on each target: x86-64's +/// baseline converts to integers in one instruction but has no `floor` (SSE4.1), while in wasm +/// `floor` is one instruction and converting back from `u32` several (about 1.5× faster there). +#[inline(always)] +fn round_half_up(v: f32) -> f32 { + if cfg!(target_arch = "wasm32") { (v + 0.5).floor() } else { (v + 0.5) as u32 as f32 } +} + /// [`quantize`] restricted to a rectangle (already clamped to the image), row-parallel. fn quantize_rect(img: &mut Image, r: PxRect, levels: f32) { if r.x0 >= r.x1 || r.y0 >= r.y1 { @@ -268,3 +276,22 @@ pub fn output_hdr(img: &mut Image, to_linear: Option, mode: HdrMode, *p = [o[0] * a, o[1] * a, o[2] * a, a]; }); } + +#[cfg(test)] +mod tests { + /// Both forms `round_half_up` picks between agree on everything quantising feeds it: + /// 0..=levels, exact halves included. + #[test] + fn rounding_forms_agree() { + for levels in [255.0f32, 32_768.0] { + for i in 0..=(levels as u32 * 16) { + let v = i as f32 / 16.0; + assert_eq!((v + 0.5).floor(), (v + 0.5) as u32 as f32, "{v}"); + } + for i in 0..=100_000 { + let v = (i as f32 / 100_000.0) * levels; + assert_eq!((v + 0.5).floor(), (v + 0.5) as u32 as f32, "{v}"); + } + } + } +}