Merge pull request #507 from daijro/perf/footage-playback

This commit is contained in:
Claude
2026-10-10 12:49:56 +00:00
11 changed files with 366 additions and 40 deletions
+19
View File
@@ -586,8 +586,18 @@ impl Prefs {
/// budgets, scaled down so together they leave **RAM Reserved for Other Applications** free,
/// and halved while the system is low on memory when **Reduce Cache Size When System Is Low
/// on Memory** is on. `mem` = the system's memory (`None` = unknown: configured budgets).
/// In the browser the layer and footage caches also stay within [`WEB_LAYER_CACHE`] and
/// [`WEB_MEDIA_CACHE`].
pub fn cache_budgets(&self, mem: Option<crate::sysinfo::SysMemory>) -> CacheBudgets {
self.cache_budgets_in(mem, cfg!(target_arch = "wasm32"))
}
/// [`Prefs::cache_budgets`], in the browser's memory (`web`) or not.
pub(crate) fn cache_budgets_in(&self, mem: Option<crate::sysinfo::SysMemory>, web: bool) -> CacheBudgets {
let (mut l, mut m, mut p) = (self.layer_cache_bytes() as u64, self.media_cache_bytes() as u64, self.preview_cache_bytes() as u64);
if web {
(l, m) = (l.min(WEB_LAYER_CACHE), m.min(WEB_MEDIA_CACHE));
}
let floor = 64u64 << 20;
let mut capped = false;
let mut reduced = false;
@@ -732,6 +742,15 @@ fn push_mru(list: &mut Vec<String>, v: &str, max: usize) {
list.truncate(max.max(1));
}
/// The most the layer cache holds in the browser. A wasm32 page addresses at most 4 GiB, which
/// these caches share with the app and with the decoders' own decoded frames (FilmCraft keeps
/// up to 384 MB per open movie): with the desktop's 1 GB defaults, eight stacked 1080p clips ran
/// the page out of memory a few seconds into playback.
pub const WEB_LAYER_CACHE: u64 = 512 << 20;
/// The most the footage frame cache holds in the browser (see [`WEB_LAYER_CACHE`]).
pub const WEB_MEDIA_CACHE: u64 = 256 << 20;
/// Effective cache budgets in bytes (see [`Prefs::cache_budgets`]).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
pub struct CacheBudgets {
+17
View File
@@ -202,6 +202,23 @@ fn cache_budgets_leave_reserved_ram_and_shrink_when_low() {
assert!(!p.cache_budgets(Some(low)).reduced);
}
/// Eight stacked 1080p clips ran the browser build out of its 4 GiB of memory: the footage and
/// layer caches kept their desktop sizes there, beside the decoders' own frames.
#[test]
fn web_cache_budgets_leave_room_for_the_decoders() {
use crate::prefs::{WEB_LAYER_CACHE, WEB_MEDIA_CACHE};
let mut p = crate::prefs::Prefs::default();
p.memory.layer_cache_mb = 4096;
p.memory.media_cache_mb = 4096;
let web = p.cache_budgets_in(None, true);
assert_eq!((web.layer as u64, web.media as u64), (WEB_LAYER_CACHE, WEB_MEDIA_CACHE));
assert_eq!(web.preview, p.cache_budgets_in(None, false).preview, "the RAM preview keeps its budget");
// Smaller settings stay as they are; the desktop is unchanged.
p.memory.media_cache_mb = 128;
assert_eq!(p.cache_budgets_in(None, true).media, 128 << 20);
assert_eq!(p.cache_budgets_in(None, false).layer, 4096 << 20);
}
#[test]
fn motion_path_span_follows_the_composition_setting() {
let mut p = crate::prefs::Prefs::default();
+107
View File
@@ -104,6 +104,67 @@ pub(crate) fn frame_to_image_in(f: &VideoFrame, op: AlphaOp, mut buf: Vec<Px>) -
img
}
/// A FilmCraft frame for a render that shows it at `w` × `h`, at most half its size each way:
/// what `resample(&frame_to_image(f, op), w, h)` makes, converting a quarter of the pixels.
/// Opaque 4:2:0 Y'CbCr is box-filtered to half size before conversion (each 2×2 block of luma,
/// and the [1 6 1]/8 neighbourhood of its chroma sample: the mean of the upsampled chroma its
/// pixels get), then resampled the rest of the way. Conversion is linear until it clips, so
/// this is the RGB box filter `resample` starts with, except in blocks where some colours clip.
/// `None` for other frames.
pub(crate) fn frame_to_image_scaled(f: &VideoFrame, w: u32, h: u32) -> Option<Image> {
let (fw, fh) = (f.width as usize, f.height as usize);
if w == 0 || h == 0 || fw % 2 != 0 || fh % 2 != 0 || (w as usize).saturating_mul(2) > fw || (h as usize).saturating_mul(2) > fh {
return None;
}
let half = match &f.data {
PixelData::Yuv8 { planes, chroma: Chroma::C420, alpha: None } => {
half_420(fw, fh, [&planes[0][..], &planes[1][..], &planes[2][..]], &Yuv::new(f.color.matrix, f.color.range, 8))?
}
PixelData::Yuv16 { planes, chroma: Chroma::C420, bits, alpha: None } => {
half_420(fw, fh, [&planes[0][..], &planes[1][..], &planes[2][..]], &Yuv::new(f.color.matrix, f.color.range, *bits))?
}
_ => return None,
};
Some(if (half.width, half.height) == (w, h) { half } else { effectcraft_raster::resample(&half, w, h) })
}
/// An opaque 4:2:0 frame of even size `w` × `h` converted at half that size (see
/// [`frame_to_image_scaled`]); `None` when the planes are too short.
fn half_420<T: Sample>(w: usize, h: usize, [luma, cb, cr]: [&[T]; 3], k: &Yuv) -> Option<Image> {
/// Row `r` of a plane `n` samples wide (empty past its end).
fn row_of<T>(p: &[T], r: usize, n: usize) -> &[T] {
p.get(r.saturating_mul(n)..(r + 1).saturating_mul(n)).unwrap_or_default()
}
let (cw, ch) = (w / 2, h / 2);
if luma.len() < w.checked_mul(h)? || cb.len() < cw * ch || cr.len() < cw * ch {
return None;
}
let mut img = Image::new(cw as u32, ch as u32);
img.data.par_chunks_mut(cw).enumerate().for_each_init(
// Chroma rows with their edge samples repeated on both ends, for the horizontal [1 6 1].
|| (vec![0f32; cw + 2], vec![0f32; cw + 2]),
|(uc, vc), (cy, row)| {
// The chroma rows above and below weigh 1/8, as `centred` upsampling averages out.
let rows = [cy.saturating_sub(1), cy, (cy + 1).min(ch - 1)];
for (dst, plane) in [(&mut *uc, cb), (&mut *vc, cr)] {
let [above, at, below] = rows.map(|r| row_of(plane, r, cw));
for (d, ((a, b), c)) in dst[1..=cw].iter_mut().zip(above.iter().zip(at).zip(below)) {
*d = ((a.f() + c.f()) * 0.125 + b.f() * 0.75 - k.c_off) * k.c_scale;
}
(dst[0], dst[cw + 1]) = (dst[1], dst[cw]);
}
let (top, bottom) = (row_of(luma, 2 * cy, w), row_of(luma, 2 * cy + 1, w));
let pixels = top.chunks_exact(2).zip(bottom.chunks_exact(2)).zip(uc.windows(3).zip(vc.windows(3)));
for (o, ((a, b), (u, v))) in row.iter_mut().zip(pixels) {
let (u, v) = ((u[0] + u[2]) * 0.125 + u[1] * 0.75, (v[0] + v[2]) * 0.125 + v[1] * 0.75);
let yy = ((a[0].f() + a[1].f() + b[0].f() + b[1].f()) * 0.25 - k.y_off) * k.y_scale;
*o = [(yy + k.cr_r * v).clamp(0.0, 1.0), (yy - k.cr_g * v - k.cb_g * u).clamp(0.0, 1.0), (yy + k.cb_b * u).clamp(0.0, 1.0), 1.0];
}
},
);
Some(img)
}
/// 0..255 → 0..1.
static U8: [f32; 256] = {
let mut t = [0f32; 256];
@@ -338,6 +399,47 @@ mod tests {
assert!((img.data[0][0] - 110.0 / 219.0).abs() < 1e-5);
}
/// A smooth opaque 4:2:0 frame whose colours never clip.
fn smooth_420(w: usize, h: usize) -> VideoFrame {
let plane = |w: usize, h: usize, base: f32, amp: f32, fx: f32, fy: f32| -> Arc<Vec<u8>> {
Arc::new((0..w * h).map(|i| (base + amp * ((i % w) as f32 * fx + (i / w) as f32 * fy).sin()) as u8).collect())
};
VideoFrame {
width: w as u32,
height: h as u32,
data: PixelData::Yuv8 {
planes: [plane(w, h, 125.0, 45.0, 0.31, 0.17), plane(w / 2, h / 2, 128.0, 10.0, 0.4, -0.3), plane(w / 2, h / 2, 128.0, 10.0, -0.2, 0.5)],
chroma: Chroma::C420,
alpha: None,
},
color: filmcraft_color::ColorInfo::REC709,
par: (1, 1),
pts: filmcraft_time::Tick::ZERO,
}
}
#[test]
fn scaled_conversion_matches_resampling_the_converted_frame() {
let f = smooth_420(96, 64);
let full = frame_to_image(&f, AlphaOp::new(AlphaMode::Straight, [0.0; 3]));
for (w, h) in [(48, 32), (32, 21), (24, 16), (13, 9)] {
let scaled = frame_to_image_scaled(&f, w, h).expect("4:2:0, at most half size");
let want = effectcraft_raster::resample(&full, w, h);
assert_eq!((scaled.width, scaled.height), (w, h));
let worst = scaled.data.iter().zip(&want.data).flat_map(|(a, b)| (0..4).map(move |c| (a[c] - b[c]).abs())).fold(0.0f32, f32::max);
assert!(worst < 1e-5, "{w}×{h}: {worst}");
}
// More than half the size, odd sizes, other chroma layouts: convert the full frame.
assert!(frame_to_image_scaled(&f, 49, 32).is_none());
assert!(frame_to_image_scaled(&smooth_420(95, 64), 40, 30).is_none());
assert!(frame_to_image_scaled(&yuv_frame(100, 128, 128, filmcraft_color::ColorInfo::REC709), 2, 1).is_some());
let mut f444 = smooth_420(96, 64);
if let PixelData::Yuv8 { chroma, .. } = &mut f444.data {
*chroma = Chroma::C444;
}
assert!(frame_to_image_scaled(&f444, 48, 32).is_none());
}
#[test]
fn bt709_red() {
// 8-bit limited BT.709 red: Y 63, Cb 102, Cr 240
@@ -402,6 +504,11 @@ mod tests {
keep.push(frame_to_image(&f, op));
}
eprintln!("convert into fresh memory {:?}", t.elapsed() / 20);
let t = std::time::Instant::now();
for _ in 0..20 {
std::hint::black_box(frame_to_image_scaled(&f, 640, 360));
}
eprintln!("convert at a third {:?}", t.elapsed() / 20);
}
}
}
+48 -23
View File
@@ -10,7 +10,7 @@ use effectcraft_render::FootageSource;
use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick};
use filmcraft_media::{FrameRequest, SharedSource};
use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in};
use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled};
use crate::{MediaError, Result};
/// Default frame-cache budget: 1 GiB of decoded frames.
@@ -41,6 +41,8 @@ struct Key {
/// pixels.
interp: u8,
matte: [u32; 3],
/// A movie frame made at this size for a reduced-resolution render ((0, 0): as decoded).
size: (u32, u32),
}
struct Entry {
@@ -322,7 +324,21 @@ impl MediaPool {
/// Decode (or fetch from cache) the frame of `footage` at source time `t`.
pub fn frame_at(&self, footage: &Footage, t: Tick) -> Result<Arc<Image>> {
Inner::frame_at(&self.inner, footage, t, true)
Inner::frame_at(&self.inner, footage, t, true, None)
}
/// [`FootageSource::frame`] / [`FootageSource::frame_at_size`]: `None` (logged) on errors.
fn frame_sized(&self, _item: ItemId, footage: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option<Arc<Image>> {
if footage.missing {
return None;
}
match Inner::frame_at(&self.inner, footage, t, true, size) {
Ok(img) => Some(img),
Err(e) => {
log::warn!("media: frame of {} at {:?}: {e}", footage.path, t);
None
}
}
}
/// Settings ▸ Disk ▸ Conformed Audio Folder: write each footage file's decoded audio there
@@ -469,7 +485,7 @@ impl MediaPool {
/// A thumbnail of `footage` (its first frame) fitting in `max_side` × `max_side`, aspect kept.
pub fn thumbnail(&self, footage: &Footage, max_side: u32) -> Result<Image> {
let img = Inner::frame_at(&self.inner, footage, Tick::ZERO, false)?;
let img = Inner::frame_at(&self.inner, footage, Tick::ZERO, false, None)?;
let (w, h) = (img.width.max(1), img.height.max(1));
let s = (max_side.max(1) as f64 / w.max(h) as f64).min(1.0);
if s >= 1.0 {
@@ -561,7 +577,7 @@ impl Inner {
};
let interp = alpha | (u8::from(footage.preserve_rgb) << 2);
let matte = if footage.alpha == AlphaMode::Premultiplied { footage.premul_color.map(f32::to_bits) } else { [0; 3] };
let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte };
let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte, size: (0, 0) };
match footage.kind {
FootageKind::Sequence if !footage.sequence.is_empty() => {
let i = Self::frame_index(footage.frame_rate, t, footage.sequence_frames(), footage.loop_count);
@@ -591,21 +607,27 @@ impl Inner {
}
}
fn frame_at(self: &Arc<Self>, footage: &Footage, t: Tick, playback: bool) -> Result<Arc<Image>> {
/// The frame of `footage` at `t`; a movie's resampled to `size` when it can be made at that
/// size directly (see [`FootageSource::frame_at_size`]).
fn frame_at(self: &Arc<Self>, footage: &Footage, t: Tick, playback: bool, size: Option<(u32, u32)>) -> Result<Arc<Image>> {
if !footage.has_video {
return Err(MediaError::Unsupported(format!("{}: no video", footage.path)));
}
let loc = Self::locate(footage, t);
let mut loc = Self::locate(footage, t);
let movie = loc.media_t.is_some();
if movie && let Some(size) = size {
loc.key.size = size;
}
let mut c = lock(&self.cache);
// sequential playback of a movie: read the next frame ahead
if playback && movie {
let prev = c.last.insert(loc.key.path.clone(), loc.key.frame);
if prev == Some(loc.key.frame - 1) && self.read_ahead.load(Ordering::Relaxed) {
let next_t = t + footage.frame_rate.frame_duration();
let next = Self::locate(footage, next_t);
let mut next = Self::locate(footage, next_t);
next.key.size = loc.key.size;
if next.key != loc.key && !c.lru.map.contains_key(&next.key) && !c.inflight.contains(&next.key) {
self.prefetch(footage, next_t);
self.prefetch(footage, next_t, size);
}
}
}
@@ -638,17 +660,17 @@ impl Inner {
}
#[cfg(not(target_arch = "wasm32"))]
fn prefetch(self: &Arc<Self>, footage: &Footage, t: Tick) {
fn prefetch(self: &Arc<Self>, footage: &Footage, t: Tick, size: Option<(u32, u32)>) {
let (me, f) = (self.clone(), footage.clone());
self.prefetched.fetch_add(1, Ordering::Relaxed);
// A plain thread, not a rayon job: the decode below parallelises with rayon itself.
std::thread::spawn(move || {
let _ = me.frame_at(&f, t, false);
let _ = me.frame_at(&f, t, false, size);
});
}
#[cfg(target_arch = "wasm32")]
fn prefetch(self: &Arc<Self>, _footage: &Footage, _t: Tick) {}
fn prefetch(self: &Arc<Self>, _footage: &Footage, _t: Tick, _size: Option<(u32, u32)>) {}
fn decode(&self, loc: &Loc, footage: &Footage) -> Result<Image> {
let op = AlphaOp::of(footage);
@@ -670,6 +692,15 @@ impl Inner {
Some(mt) => {
let src = self.source(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?;
let vf = src.video_frame(FrameRequest::full(filmcraft_time::Tick(mt.0))).map_err(MediaError::from)?;
// A reduced-resolution render's frame, made at its size (a frame of the footage's
// own size only, as the renderer resamples only those).
let (w, h) = loc.key.size;
if w > 0
&& (vf.width, vf.height) == (footage.width, footage.height)
&& let Some(img) = frame_to_image_scaled(&vf, w, h)
{
return Ok(img);
}
let buf = lock(&self.cache).lru.take_spare(vf.width as usize * vf.height as usize);
Ok(frame_to_image_in(&vf, op, buf))
}
@@ -696,17 +727,11 @@ impl FootageSource for MediaPool {
fn cache_budget(&self) -> Option<usize> {
Some(self.budget())
}
fn frame(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<Image>> {
if footage.missing {
return None;
}
match self.frame_at(footage, t) {
Ok(img) => Some(img),
Err(e) => {
log::warn!("media: frame of {} at {:?}: {e}", footage.path, t);
None
}
}
fn frame(&self, item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<Image>> {
self.frame_sized(item, footage, t, None)
}
fn frame_at_size(&self, item: ItemId, footage: &Footage, t: Tick, width: u32, height: u32) -> Option<Arc<Image>> {
self.frame_sized(item, footage, t, Some((width, height)))
}
fn vector_frame(&self, _item: ItemId, footage: &Footage, scale: f64) -> Option<Arc<Image>> {
@@ -834,7 +859,7 @@ mod tests {
fn lru_respects_budget_and_recycles() {
let mut c = Lru::default();
let img = || Arc::new(Image::new(16, 16)); // 4 KiB + overhead
let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3] };
let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3], size: (0, 0) };
let budget = 3 * (4096 + 64);
for i in 0..10 {
c.insert(k(i), img(), budget);
+20
View File
@@ -236,6 +236,26 @@ fn h264_mp4_probe_and_frames() {
assert!(Arc::ptr_eq(&a, &b));
}
/// A reduced-resolution render's movie frames are converted at its size: they match resampling
/// the full frames (testsrc2's saturated colours clip, where the two differ most), and are cached
/// per size.
#[test]
fn h264_frames_at_reduced_sizes() {
let Some(p) = h264_mp4() else { return };
let f = probe(&p).expect("probe");
let pool = MediaPool::new();
for (n, (w, h)) in [(0i64, (960, 540)), (1, (640, 360)), (2, (480, 270)), (30, (323, 182))] {
let t = t_of(f.frame_rate, n);
let sized = pool.frame_at_size(ItemId(1), &f, t, w, h).expect("frame");
let full = pool.frame(ItemId(1), &f, t).expect("frame");
assert_eq!((sized.width, sized.height), (w, h));
let db = psnr(&sized, &effectcraft_raster::resample(&full, w, h));
eprintln!("h264 frame {n} at {w}×{h}: {db:.2} dB against the resampled full frame");
assert!(db > 55.0, "frame {n} at {w}×{h}: {db:.2} dB");
assert!(Arc::ptr_eq(&sized, &pool.frame_at_size(ItemId(1), &f, t, w, h).expect("frame")), "cached");
}
}
#[test]
fn h264_audio_samples() {
let Some(p) = h264_mp4() else { return };
+29 -2
View File
@@ -195,14 +195,22 @@ pub fn quantize(img: &mut Image, levels: f32) {
#[allow(clippy::manual_clamp)] // max/min map NaN to 0; clamp would keep NaN
#[inline(always)]
fn quantize_px(p: &mut [f32; 4], levels: f32, inv: f32) {
let a = (p[3].max(0.0).min(1.0) * levels + 0.5) as u32 as f32 * inv;
let a = round_half_up(p[3].max(0.0).min(1.0) * levels) * inv;
let hi = [a, a, a, a];
for c in 0..3 {
p[c] = (p[c].max(0.0).min(hi[c]) * levels + 0.5) as u32 as f32 * inv;
p[c] = round_half_up(p[c].max(0.0).min(hi[c]) * levels) * inv;
}
p[3] = a;
}
/// `v` (0..=65 535) rounded half up. The same either way; the fastest on each target: x86-64's
/// baseline converts to integers in one instruction but has no `floor` (SSE4.1), while in wasm
/// `floor` is one instruction and converting back from `u32` several (about 1.5× faster there).
#[inline(always)]
fn round_half_up(v: f32) -> f32 {
if cfg!(target_arch = "wasm32") { (v + 0.5).floor() } else { (v + 0.5) as u32 as f32 }
}
/// [`quantize`] restricted to a rectangle (already clamped to the image), row-parallel.
fn quantize_rect(img: &mut Image, r: PxRect, levels: f32) {
if r.x0 >= r.x1 || r.y0 >= r.y1 {
@@ -268,3 +276,22 @@ pub fn output_hdr(img: &mut Image, to_linear: Option<Conversion>, mode: HdrMode,
*p = [o[0] * a, o[1] * a, o[2] * a, a];
});
}
#[cfg(test)]
mod tests {
/// Both forms `round_half_up` picks between agree on everything quantising feeds it:
/// 0..=levels, exact halves included.
#[test]
fn rounding_forms_agree() {
for levels in [255.0f32, 32_768.0] {
for i in 0..=(levels as u32 * 16) {
let v = i as f32 / 16.0;
assert_eq!((v + 0.5).floor(), (v + 0.5) as u32 as f32, "{v}");
}
for i in 0..=100_000 {
let v = (i as f32 / 100_000.0) * levels;
assert_eq!((v + 0.5).floor(), (v + 0.5) as u32 as f32, "{v}");
}
}
}
}
+35 -12
View File
@@ -40,6 +40,13 @@ use rayon::prelude::*;
pub trait FootageSource: Send + Sync {
/// The frame of `item` at source time `t`, straight from the file (any size).
fn frame(&self, item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<Image>>;
/// [`FootageSource::frame`] for a render that shows it at `width` × `height`, at most half its
/// size: either that frame or, when the source can make it more cheaply than the whole frame
/// (movies, see `effectcraft-media`), the frame already resampled to that size as
/// [`effectcraft_raster::resample`] would.
fn frame_at_size(&self, item: ItemId, footage: &Footage, t: Tick, _width: u32, _height: u32) -> Option<Arc<Image>> {
self.frame(item, footage, t)
}
/// `frames` interleaved stereo sample frames of `item`'s audio from source time `t` at
/// `rate` Hz (`None` when unavailable). The first sample is `t.to_units_floor(rate)`;
/// a sample boundary uses the earliest representable tick that selects that sample.
@@ -1105,8 +1112,10 @@ impl<'a> Renderer<'a> {
}
let lt = ctx.source_time(layer);
// The proxy is decoded instead, at its own size, and fills the footage's frame.
let pf = self.proxy_for(*item).unwrap_or(f);
let img = self.footage_frame(ctx, layer, *item, pf, lt)?;
let proxy = self.proxy_for(*item);
let pf = proxy.unwrap_or(f);
let size = self.reduced_size(f.width, f.height).filter(|_| proxy.is_none());
let img = self.footage_frame(ctx, layer, *item, pf, lt, size)?;
Some(self.footage_buf(&img, pf, f.width, f.height))
}
LayerSource::Text => Some(self.authored(text::render(ctx, layer, self.raster_scale(ctx, layer)))),
@@ -1128,12 +1137,11 @@ impl<'a> Renderer<'a> {
let s = self.opts.scale;
let k = if img.width > 0 { w.max(1) as f64 / img.width as f64 } else { 1.0 };
// Keep native pixels when downsampling is small; resample otherwise.
let mut buf = if s < 0.75 && k <= 1.0 + 1e-9 {
let rw = ((w as f64 * s).round() as u32).max(1);
let rh = ((h as f64 * s).round() as u32).max(1);
Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s }
} else {
Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k }
let mut buf = match self.reduced_size(w, h) {
// The source resampled it already (`FootageSource::frame_at_size`).
Some(size) if (img.width, img.height) == size => Buf { img: img.clone(), offset: [0.0; 2], scale: s },
Some((rw, rh)) if k <= 1.0 + 1e-9 => Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s },
_ => Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k },
};
if let Some(c) = self.pipe.media_in(f) {
color::convert(&mut buf.img, &c);
@@ -1141,6 +1149,13 @@ impl<'a> Renderer<'a> {
buf
}
/// The size of `w` × `h` layer pixels in this render when it is reduced enough that footage
/// is resampled to it (below 3/4 scale).
fn reduced_size(&self, w: u32, h: u32) -> Option<(u32, u32)> {
let s = self.opts.scale;
(s < 0.75).then(|| (((w as f64 * s).round() as u32).max(1), ((h as f64 * s).round() as u32).max(1)))
}
/// Authored colours into the working space (linear working spaces).
fn authored(&self, mut buf: Buf) -> Buf {
if let Some(c) = self.pipe.authored_in() {
@@ -1152,7 +1167,8 @@ impl<'a> Renderer<'a> {
/// A footage frame at source time `t`, frame-blended between the two nearest source frames
/// when the layer's Frame Blending switch (and the comp's Enable Frame Blending) is on and
/// `t` falls between frames (footage rate ≠ comp rate, time stretch, time remapping).
fn footage_frame(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option<Arc<Image>> {
/// `size`: a plain frame may come resampled to it ([`FootageSource::frame_at_size`]).
fn footage_frame(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option<Arc<Image>> {
// Interpret Footage ▸ Separate Fields: each field is a frame at twice the rate.
if f.fields != effectcraft_project::FieldOrder::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) {
let field_rate = FrameRate::new(f.frame_rate.num * 2, f.frame_rate.den);
@@ -1163,11 +1179,13 @@ impl<'a> Renderer<'a> {
let parity = if (i % 2 == 0) == dominant_upper { 0 } else { 1 };
return Some(Arc::new(interpret_pixels(&field_frame(&img, parity), f)));
}
let img = self.footage_frame_raw(ctx, layer, item, f, t)?;
// Interpretation is per pixel but not linear: it runs on the frame before resampling.
let size = size.filter(|_| !(f.invert_alpha || f.linear_light_applies()));
let img = self.footage_frame_raw(ctx, layer, item, f, t, size)?;
if f.invert_alpha || f.linear_light_applies() { Some(Arc::new(interpret_pixels(&img, f))) } else { Some(img) }
}
fn footage_frame_raw(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option<Arc<Image>> {
fn footage_frame_raw(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option<Arc<Image>> {
let mode = frame_blend_mode(ctx, layer);
if mode != FrameBlend::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) {
let (i, w) = frame_position(t, f.frame_rate);
@@ -1195,7 +1213,10 @@ impl<'a> Renderer<'a> {
}
}
}
self.footage.frame(item, f, t)
match size {
Some((w, h)) => self.footage.frame_at_size(item, f, t, w, h),
None => self.footage.frame(item, f, t),
}
}
/// Source → masks, clamped/quantised to the project depth.
@@ -2153,6 +2174,8 @@ mod tests_collapse;
#[cfg(test)]
mod tests_color;
#[cfg(test)]
mod tests_footage_size;
#[cfg(test)]
mod tests_frame_blend;
#[cfg(test)]
mod tests_mask_blur;
+83
View File
@@ -0,0 +1,83 @@
//! Footage at a reduced resolution: a source that resamples frames itself
//! ([`FootageSource::frame_at_size`], movies in `effectcraft-media`) renders what resampling the
//! full frame renders.
use std::sync::{Arc, Mutex};
use rayon::prelude::*;
use effectcraft_color::Label;
use effectcraft_project::build;
use effectcraft_project::{AlphaMode, BitDepth, Comp, Footage, FootageKind, ItemId, ItemKind, LayerSource, Project};
use effectcraft_time::{FrameRate, Tick};
use crate::{FootageSource, Image, RenderOpts, Renderer};
/// Footage with a textured frame; `sizes` (when set) resamples it itself and records the sizes
/// it was asked for.
struct Clip {
sizes: Option<Mutex<Vec<(u32, u32)>>>,
}
impl FootageSource for Clip {
fn frame(&self, _: ItemId, f: &Footage, _: Tick) -> Option<Arc<Image>> {
let mut img = Image::new(f.width, f.height);
img.rows_mut().for_each(|(y, row)| {
for (x, p) in row.iter_mut().enumerate() {
let s = (0.5 + 0.3 * (x as f32 * 0.37 + y as f32 * 0.11).sin()) * 0.9;
*p = [s, 1.0 - s, s * 0.5, 1.0];
}
});
Some(Arc::new(img))
}
fn frame_at_size(&self, item: ItemId, f: &Footage, t: Tick, w: u32, h: u32) -> Option<Arc<Image>> {
let Some(sizes) = &self.sizes else { return self.frame(item, f, t) };
sizes.lock().unwrap().push((w, h));
Some(Arc::new(effectcraft_raster::resample(&*self.frame(item, f, t)?, w, h)))
}
}
fn project(depth: BitDepth, invert_alpha: bool) -> (Project, ItemId) {
let mut p = Project::default();
p.settings.bit_depth = depth;
let comp = Comp::new(96, 54, FrameRate::FPS_30, Tick::from_seconds_f64(1.0));
let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.clone().into()));
let f = Footage {
path: "clip.mp4".into(),
kind: FootageKind::Video,
width: 96,
height: 54,
pixel_aspect: 1.0,
frame_rate: FrameRate::FPS_30,
duration: Tick::from_seconds_f64(1.0),
has_video: true,
alpha: AlphaMode::Ignore,
loop_count: 1,
invert_alpha,
..Default::default()
};
let fid = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(f));
let l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, (96, 54), None);
p.comp_mut(cid).unwrap().layers.push(l);
(p, cid)
}
#[test]
fn reduced_renders_take_footage_resampled_by_the_source() {
for depth in [BitDepth::Bpc8, BitDepth::Bpc32] {
let (p, cid) = project(depth, false);
for (scale, asked) in [(1.0, None), (0.75, None), (0.5, Some((48, 27))), (1.0 / 3.0, Some((32, 18))), (0.25, Some((24, 14)))] {
let opts = RenderOpts { scale, ..Default::default() };
let plain = Renderer::new(&p, &Clip { sizes: None }, opts).comp_frame(cid, Tick::ZERO);
let sized = Clip { sizes: Some(Mutex::default()) };
let frame = Renderer::new(&p, &sized, opts).comp_frame(cid, Tick::ZERO);
assert_eq!(frame, plain, "{depth:?} at {scale}");
assert_eq!(sized.sizes.unwrap().into_inner().unwrap(), asked.into_iter().collect::<Vec<_>>(), "{depth:?} at {scale}");
}
}
// Interpreted pixels (Invert Alpha) are interpreted at full size, then resampled.
let (p, cid) = project(BitDepth::Bpc8, true);
let sized = Clip { sizes: Some(Mutex::default()) };
Renderer::new(&p, &sized, RenderOpts { scale: 0.5, ..Default::default() }).comp_frame(cid, Tick::ZERO);
assert!(sized.sizes.unwrap().into_inner().unwrap().is_empty());
}
+5 -1
View File
@@ -230,7 +230,11 @@ Nulls / Nulls Follow Points take pins as well as paths, and `puppet.follow` give
Position expression that trails the leader by a delay. The renderer flattens nested effect
groups into `Params` keys (`effects::flatten_params`). Commands: `paint.*`, `puppet.*`.
Half, Third and Quarter resolution render proportionally fewer pixels end to end.
Half, Third and Quarter resolution render proportionally fewer pixels end to end. Footage
starts at the render's size where the source can make it more cheaply than the full frame
(`FootageSource::frame_at_size`): movies in opaque 4:2:0 Y'CbCr are box-filtered to half size
before conversion (a quarter of the conversions; the result differs from converting first only
in blocks where colours clip) and resampled from there, as other frames are from full size.
**GPU compositor** (`crates/gpu`; Project Settings ▸ Video Rendering and Effects ▸ Mercury GPU
Acceleration, the default, or Mercury Software Only; `render.backend`). The CPU renderer is the
+2 -2
View File
@@ -71,8 +71,8 @@ Commands (CLI, MCP, control channel):
- `audio.outputDevice`: Default Output (device list from the desktop host)
- `audio.outputLeft`: Left (Audio Output Mapping)
- `audio.outputRight`: Right (Audio Output Mapping)
- `memory.layerCacheMb`: Layer Cache
- `memory.mediaCacheMb`: Footage Frame Cache
- `memory.layerCacheMb`: Layer Cache (in the browser at most 512 MB, see [web.md](web.md))
- `memory.mediaCacheMb`: Footage Frame Cache (in the browser at most 256 MB)
- `memory.previewCacheMb`: Preview (RAM) Cache
- `threeD.defaultRenderer`: Default 3D Renderer
- `labels.N.name` / `labels.N.color`: the 16 label names and colours, used by the Label menu,
+1
View File
@@ -61,6 +61,7 @@ device of their own), `?nosw` (don't register the service worker).
| config directory: settings, shortcut presets, recent projects, the recovery sentinel | `store::WebConfig` in browser storage |
| auto-save folder | `/EffectCraft Auto-Save/` in browser storage (`ConfigStore::files`) |
| media reads (`MediaPool`, `probe`) | `files::WebImporter`: bytes from the file table, handed to `MediaPool::add_bytes` / `probe_bytes` |
| Memory & CPU cache budgets, scaled to the system's RAM | the page's memory, at most 4 GiB (wasm32), which the caches share with the decoders' own decoded frames (FilmCraft keeps up to 384 MB per open movie): the Layer Cache stays within 512 MB and the Footage Frame Cache within 256 MB (`Prefs::cache_budgets`) |
| export writes (`effectcraft-export`) | the job's `sink` (`effectcraft_host::FileExporter { sink }`): files land in the table and download after the render; several files (an image sequence) download as one stored `.zip` |
| rfd file dialogs | `<input type=file>` for File ▸ Open / Import; drop files anywhere on the page (`.ecproj` opens, everything else imports); "Save As" / "Output To" pick a download name |
| Media Browser on the file system | browser storage and folders opened with the File System Access API (below) |