Merge pull request #507 from daijro/perf/footage-playback
This commit is contained in:
@@ -586,8 +586,18 @@ impl Prefs {
|
||||
/// budgets, scaled down so together they leave **RAM Reserved for Other Applications** free,
|
||||
/// and halved while the system is low on memory when **Reduce Cache Size When System Is Low
|
||||
/// on Memory** is on. `mem` = the system's memory (`None` = unknown: configured budgets).
|
||||
/// In the browser the layer and footage caches also stay within [`WEB_LAYER_CACHE`] and
|
||||
/// [`WEB_MEDIA_CACHE`].
|
||||
pub fn cache_budgets(&self, mem: Option<crate::sysinfo::SysMemory>) -> CacheBudgets {
|
||||
self.cache_budgets_in(mem, cfg!(target_arch = "wasm32"))
|
||||
}
|
||||
|
||||
/// [`Prefs::cache_budgets`], in the browser's memory (`web`) or not.
|
||||
pub(crate) fn cache_budgets_in(&self, mem: Option<crate::sysinfo::SysMemory>, web: bool) -> CacheBudgets {
|
||||
let (mut l, mut m, mut p) = (self.layer_cache_bytes() as u64, self.media_cache_bytes() as u64, self.preview_cache_bytes() as u64);
|
||||
if web {
|
||||
(l, m) = (l.min(WEB_LAYER_CACHE), m.min(WEB_MEDIA_CACHE));
|
||||
}
|
||||
let floor = 64u64 << 20;
|
||||
let mut capped = false;
|
||||
let mut reduced = false;
|
||||
@@ -732,6 +742,15 @@ fn push_mru(list: &mut Vec<String>, v: &str, max: usize) {
|
||||
list.truncate(max.max(1));
|
||||
}
|
||||
|
||||
/// The most the layer cache holds in the browser. A wasm32 page addresses at most 4 GiB, which
|
||||
/// these caches share with the app and with the decoders' own decoded frames (FilmCraft keeps
|
||||
/// up to 384 MB per open movie): with the desktop's 1 GB defaults, eight stacked 1080p clips ran
|
||||
/// the page out of memory a few seconds into playback.
|
||||
pub const WEB_LAYER_CACHE: u64 = 512 << 20;
|
||||
|
||||
/// The most the footage frame cache holds in the browser (see [`WEB_LAYER_CACHE`]).
|
||||
pub const WEB_MEDIA_CACHE: u64 = 256 << 20;
|
||||
|
||||
/// Effective cache budgets in bytes (see [`Prefs::cache_budgets`]).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct CacheBudgets {
|
||||
|
||||
@@ -202,6 +202,23 @@ fn cache_budgets_leave_reserved_ram_and_shrink_when_low() {
|
||||
assert!(!p.cache_budgets(Some(low)).reduced);
|
||||
}
|
||||
|
||||
/// Eight stacked 1080p clips ran the browser build out of its 4 GiB of memory: the footage and
|
||||
/// layer caches kept their desktop sizes there, beside the decoders' own frames.
|
||||
#[test]
|
||||
fn web_cache_budgets_leave_room_for_the_decoders() {
|
||||
use crate::prefs::{WEB_LAYER_CACHE, WEB_MEDIA_CACHE};
|
||||
let mut p = crate::prefs::Prefs::default();
|
||||
p.memory.layer_cache_mb = 4096;
|
||||
p.memory.media_cache_mb = 4096;
|
||||
let web = p.cache_budgets_in(None, true);
|
||||
assert_eq!((web.layer as u64, web.media as u64), (WEB_LAYER_CACHE, WEB_MEDIA_CACHE));
|
||||
assert_eq!(web.preview, p.cache_budgets_in(None, false).preview, "the RAM preview keeps its budget");
|
||||
// Smaller settings stay as they are; the desktop is unchanged.
|
||||
p.memory.media_cache_mb = 128;
|
||||
assert_eq!(p.cache_budgets_in(None, true).media, 128 << 20);
|
||||
assert_eq!(p.cache_budgets_in(None, false).layer, 4096 << 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn motion_path_span_follows_the_composition_setting() {
|
||||
let mut p = crate::prefs::Prefs::default();
|
||||
|
||||
@@ -104,6 +104,67 @@ pub(crate) fn frame_to_image_in(f: &VideoFrame, op: AlphaOp, mut buf: Vec<Px>) -
|
||||
img
|
||||
}
|
||||
|
||||
/// A FilmCraft frame for a render that shows it at `w` × `h`, at most half its size each way:
|
||||
/// what `resample(&frame_to_image(f, op), w, h)` makes, converting a quarter of the pixels.
|
||||
/// Opaque 4:2:0 Y'CbCr is box-filtered to half size before conversion (each 2×2 block of luma,
|
||||
/// and the [1 6 1]/8 neighbourhood of its chroma sample: the mean of the upsampled chroma its
|
||||
/// pixels get), then resampled the rest of the way. Conversion is linear until it clips, so
|
||||
/// this is the RGB box filter `resample` starts with, except in blocks where some colours clip.
|
||||
/// `None` for other frames.
|
||||
pub(crate) fn frame_to_image_scaled(f: &VideoFrame, w: u32, h: u32) -> Option<Image> {
|
||||
let (fw, fh) = (f.width as usize, f.height as usize);
|
||||
if w == 0 || h == 0 || fw % 2 != 0 || fh % 2 != 0 || (w as usize).saturating_mul(2) > fw || (h as usize).saturating_mul(2) > fh {
|
||||
return None;
|
||||
}
|
||||
let half = match &f.data {
|
||||
PixelData::Yuv8 { planes, chroma: Chroma::C420, alpha: None } => {
|
||||
half_420(fw, fh, [&planes[0][..], &planes[1][..], &planes[2][..]], &Yuv::new(f.color.matrix, f.color.range, 8))?
|
||||
}
|
||||
PixelData::Yuv16 { planes, chroma: Chroma::C420, bits, alpha: None } => {
|
||||
half_420(fw, fh, [&planes[0][..], &planes[1][..], &planes[2][..]], &Yuv::new(f.color.matrix, f.color.range, *bits))?
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
Some(if (half.width, half.height) == (w, h) { half } else { effectcraft_raster::resample(&half, w, h) })
|
||||
}
|
||||
|
||||
/// An opaque 4:2:0 frame of even size `w` × `h` converted at half that size (see
|
||||
/// [`frame_to_image_scaled`]); `None` when the planes are too short.
|
||||
fn half_420<T: Sample>(w: usize, h: usize, [luma, cb, cr]: [&[T]; 3], k: &Yuv) -> Option<Image> {
|
||||
/// Row `r` of a plane `n` samples wide (empty past its end).
|
||||
fn row_of<T>(p: &[T], r: usize, n: usize) -> &[T] {
|
||||
p.get(r.saturating_mul(n)..(r + 1).saturating_mul(n)).unwrap_or_default()
|
||||
}
|
||||
let (cw, ch) = (w / 2, h / 2);
|
||||
if luma.len() < w.checked_mul(h)? || cb.len() < cw * ch || cr.len() < cw * ch {
|
||||
return None;
|
||||
}
|
||||
let mut img = Image::new(cw as u32, ch as u32);
|
||||
img.data.par_chunks_mut(cw).enumerate().for_each_init(
|
||||
// Chroma rows with their edge samples repeated on both ends, for the horizontal [1 6 1].
|
||||
|| (vec![0f32; cw + 2], vec![0f32; cw + 2]),
|
||||
|(uc, vc), (cy, row)| {
|
||||
// The chroma rows above and below weigh 1/8, as `centred` upsampling averages out.
|
||||
let rows = [cy.saturating_sub(1), cy, (cy + 1).min(ch - 1)];
|
||||
for (dst, plane) in [(&mut *uc, cb), (&mut *vc, cr)] {
|
||||
let [above, at, below] = rows.map(|r| row_of(plane, r, cw));
|
||||
for (d, ((a, b), c)) in dst[1..=cw].iter_mut().zip(above.iter().zip(at).zip(below)) {
|
||||
*d = ((a.f() + c.f()) * 0.125 + b.f() * 0.75 - k.c_off) * k.c_scale;
|
||||
}
|
||||
(dst[0], dst[cw + 1]) = (dst[1], dst[cw]);
|
||||
}
|
||||
let (top, bottom) = (row_of(luma, 2 * cy, w), row_of(luma, 2 * cy + 1, w));
|
||||
let pixels = top.chunks_exact(2).zip(bottom.chunks_exact(2)).zip(uc.windows(3).zip(vc.windows(3)));
|
||||
for (o, ((a, b), (u, v))) in row.iter_mut().zip(pixels) {
|
||||
let (u, v) = ((u[0] + u[2]) * 0.125 + u[1] * 0.75, (v[0] + v[2]) * 0.125 + v[1] * 0.75);
|
||||
let yy = ((a[0].f() + a[1].f() + b[0].f() + b[1].f()) * 0.25 - k.y_off) * k.y_scale;
|
||||
*o = [(yy + k.cr_r * v).clamp(0.0, 1.0), (yy - k.cr_g * v - k.cb_g * u).clamp(0.0, 1.0), (yy + k.cb_b * u).clamp(0.0, 1.0), 1.0];
|
||||
}
|
||||
},
|
||||
);
|
||||
Some(img)
|
||||
}
|
||||
|
||||
/// 0..255 → 0..1.
|
||||
static U8: [f32; 256] = {
|
||||
let mut t = [0f32; 256];
|
||||
@@ -338,6 +399,47 @@ mod tests {
|
||||
assert!((img.data[0][0] - 110.0 / 219.0).abs() < 1e-5);
|
||||
}
|
||||
|
||||
/// A smooth opaque 4:2:0 frame whose colours never clip.
|
||||
fn smooth_420(w: usize, h: usize) -> VideoFrame {
|
||||
let plane = |w: usize, h: usize, base: f32, amp: f32, fx: f32, fy: f32| -> Arc<Vec<u8>> {
|
||||
Arc::new((0..w * h).map(|i| (base + amp * ((i % w) as f32 * fx + (i / w) as f32 * fy).sin()) as u8).collect())
|
||||
};
|
||||
VideoFrame {
|
||||
width: w as u32,
|
||||
height: h as u32,
|
||||
data: PixelData::Yuv8 {
|
||||
planes: [plane(w, h, 125.0, 45.0, 0.31, 0.17), plane(w / 2, h / 2, 128.0, 10.0, 0.4, -0.3), plane(w / 2, h / 2, 128.0, 10.0, -0.2, 0.5)],
|
||||
chroma: Chroma::C420,
|
||||
alpha: None,
|
||||
},
|
||||
color: filmcraft_color::ColorInfo::REC709,
|
||||
par: (1, 1),
|
||||
pts: filmcraft_time::Tick::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scaled_conversion_matches_resampling_the_converted_frame() {
|
||||
let f = smooth_420(96, 64);
|
||||
let full = frame_to_image(&f, AlphaOp::new(AlphaMode::Straight, [0.0; 3]));
|
||||
for (w, h) in [(48, 32), (32, 21), (24, 16), (13, 9)] {
|
||||
let scaled = frame_to_image_scaled(&f, w, h).expect("4:2:0, at most half size");
|
||||
let want = effectcraft_raster::resample(&full, w, h);
|
||||
assert_eq!((scaled.width, scaled.height), (w, h));
|
||||
let worst = scaled.data.iter().zip(&want.data).flat_map(|(a, b)| (0..4).map(move |c| (a[c] - b[c]).abs())).fold(0.0f32, f32::max);
|
||||
assert!(worst < 1e-5, "{w}×{h}: {worst}");
|
||||
}
|
||||
// More than half the size, odd sizes, other chroma layouts: convert the full frame.
|
||||
assert!(frame_to_image_scaled(&f, 49, 32).is_none());
|
||||
assert!(frame_to_image_scaled(&smooth_420(95, 64), 40, 30).is_none());
|
||||
assert!(frame_to_image_scaled(&yuv_frame(100, 128, 128, filmcraft_color::ColorInfo::REC709), 2, 1).is_some());
|
||||
let mut f444 = smooth_420(96, 64);
|
||||
if let PixelData::Yuv8 { chroma, .. } = &mut f444.data {
|
||||
*chroma = Chroma::C444;
|
||||
}
|
||||
assert!(frame_to_image_scaled(&f444, 48, 32).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bt709_red() {
|
||||
// 8-bit limited BT.709 red: Y 63, Cb 102, Cr 240
|
||||
@@ -402,6 +504,11 @@ mod tests {
|
||||
keep.push(frame_to_image(&f, op));
|
||||
}
|
||||
eprintln!("convert into fresh memory {:?}", t.elapsed() / 20);
|
||||
let t = std::time::Instant::now();
|
||||
for _ in 0..20 {
|
||||
std::hint::black_box(frame_to_image_scaled(&f, 640, 360));
|
||||
}
|
||||
eprintln!("convert at a third {:?}", t.elapsed() / 20);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+48
-23
@@ -10,7 +10,7 @@ use effectcraft_render::FootageSource;
|
||||
use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick};
|
||||
use filmcraft_media::{FrameRequest, SharedSource};
|
||||
|
||||
use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in};
|
||||
use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled};
|
||||
use crate::{MediaError, Result};
|
||||
|
||||
/// Default frame-cache budget: 1 GiB of decoded frames.
|
||||
@@ -41,6 +41,8 @@ struct Key {
|
||||
/// pixels.
|
||||
interp: u8,
|
||||
matte: [u32; 3],
|
||||
/// A movie frame made at this size for a reduced-resolution render ((0, 0): as decoded).
|
||||
size: (u32, u32),
|
||||
}
|
||||
|
||||
struct Entry {
|
||||
@@ -322,7 +324,21 @@ impl MediaPool {
|
||||
|
||||
/// Decode (or fetch from cache) the frame of `footage` at source time `t`.
|
||||
pub fn frame_at(&self, footage: &Footage, t: Tick) -> Result<Arc<Image>> {
|
||||
Inner::frame_at(&self.inner, footage, t, true)
|
||||
Inner::frame_at(&self.inner, footage, t, true, None)
|
||||
}
|
||||
|
||||
/// [`FootageSource::frame`] / [`FootageSource::frame_at_size`]: `None` (logged) on errors.
|
||||
fn frame_sized(&self, _item: ItemId, footage: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option<Arc<Image>> {
|
||||
if footage.missing {
|
||||
return None;
|
||||
}
|
||||
match Inner::frame_at(&self.inner, footage, t, true, size) {
|
||||
Ok(img) => Some(img),
|
||||
Err(e) => {
|
||||
log::warn!("media: frame of {} at {:?}: {e}", footage.path, t);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Settings ▸ Disk ▸ Conformed Audio Folder: write each footage file's decoded audio there
|
||||
@@ -469,7 +485,7 @@ impl MediaPool {
|
||||
|
||||
/// A thumbnail of `footage` (its first frame) fitting in `max_side` × `max_side`, aspect kept.
|
||||
pub fn thumbnail(&self, footage: &Footage, max_side: u32) -> Result<Image> {
|
||||
let img = Inner::frame_at(&self.inner, footage, Tick::ZERO, false)?;
|
||||
let img = Inner::frame_at(&self.inner, footage, Tick::ZERO, false, None)?;
|
||||
let (w, h) = (img.width.max(1), img.height.max(1));
|
||||
let s = (max_side.max(1) as f64 / w.max(h) as f64).min(1.0);
|
||||
if s >= 1.0 {
|
||||
@@ -561,7 +577,7 @@ impl Inner {
|
||||
};
|
||||
let interp = alpha | (u8::from(footage.preserve_rgb) << 2);
|
||||
let matte = if footage.alpha == AlphaMode::Premultiplied { footage.premul_color.map(f32::to_bits) } else { [0; 3] };
|
||||
let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte };
|
||||
let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte, size: (0, 0) };
|
||||
match footage.kind {
|
||||
FootageKind::Sequence if !footage.sequence.is_empty() => {
|
||||
let i = Self::frame_index(footage.frame_rate, t, footage.sequence_frames(), footage.loop_count);
|
||||
@@ -591,21 +607,27 @@ impl Inner {
|
||||
}
|
||||
}
|
||||
|
||||
fn frame_at(self: &Arc<Self>, footage: &Footage, t: Tick, playback: bool) -> Result<Arc<Image>> {
|
||||
/// The frame of `footage` at `t`; a movie's resampled to `size` when it can be made at that
|
||||
/// size directly (see [`FootageSource::frame_at_size`]).
|
||||
fn frame_at(self: &Arc<Self>, footage: &Footage, t: Tick, playback: bool, size: Option<(u32, u32)>) -> Result<Arc<Image>> {
|
||||
if !footage.has_video {
|
||||
return Err(MediaError::Unsupported(format!("{}: no video", footage.path)));
|
||||
}
|
||||
let loc = Self::locate(footage, t);
|
||||
let mut loc = Self::locate(footage, t);
|
||||
let movie = loc.media_t.is_some();
|
||||
if movie && let Some(size) = size {
|
||||
loc.key.size = size;
|
||||
}
|
||||
let mut c = lock(&self.cache);
|
||||
// sequential playback of a movie: read the next frame ahead
|
||||
if playback && movie {
|
||||
let prev = c.last.insert(loc.key.path.clone(), loc.key.frame);
|
||||
if prev == Some(loc.key.frame - 1) && self.read_ahead.load(Ordering::Relaxed) {
|
||||
let next_t = t + footage.frame_rate.frame_duration();
|
||||
let next = Self::locate(footage, next_t);
|
||||
let mut next = Self::locate(footage, next_t);
|
||||
next.key.size = loc.key.size;
|
||||
if next.key != loc.key && !c.lru.map.contains_key(&next.key) && !c.inflight.contains(&next.key) {
|
||||
self.prefetch(footage, next_t);
|
||||
self.prefetch(footage, next_t, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -638,17 +660,17 @@ impl Inner {
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn prefetch(self: &Arc<Self>, footage: &Footage, t: Tick) {
|
||||
fn prefetch(self: &Arc<Self>, footage: &Footage, t: Tick, size: Option<(u32, u32)>) {
|
||||
let (me, f) = (self.clone(), footage.clone());
|
||||
self.prefetched.fetch_add(1, Ordering::Relaxed);
|
||||
// A plain thread, not a rayon job: the decode below parallelises with rayon itself.
|
||||
std::thread::spawn(move || {
|
||||
let _ = me.frame_at(&f, t, false);
|
||||
let _ = me.frame_at(&f, t, false, size);
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn prefetch(self: &Arc<Self>, _footage: &Footage, _t: Tick) {}
|
||||
fn prefetch(self: &Arc<Self>, _footage: &Footage, _t: Tick, _size: Option<(u32, u32)>) {}
|
||||
|
||||
fn decode(&self, loc: &Loc, footage: &Footage) -> Result<Image> {
|
||||
let op = AlphaOp::of(footage);
|
||||
@@ -670,6 +692,15 @@ impl Inner {
|
||||
Some(mt) => {
|
||||
let src = self.source(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?;
|
||||
let vf = src.video_frame(FrameRequest::full(filmcraft_time::Tick(mt.0))).map_err(MediaError::from)?;
|
||||
// A reduced-resolution render's frame, made at its size (a frame of the footage's
|
||||
// own size only, as the renderer resamples only those).
|
||||
let (w, h) = loc.key.size;
|
||||
if w > 0
|
||||
&& (vf.width, vf.height) == (footage.width, footage.height)
|
||||
&& let Some(img) = frame_to_image_scaled(&vf, w, h)
|
||||
{
|
||||
return Ok(img);
|
||||
}
|
||||
let buf = lock(&self.cache).lru.take_spare(vf.width as usize * vf.height as usize);
|
||||
Ok(frame_to_image_in(&vf, op, buf))
|
||||
}
|
||||
@@ -696,17 +727,11 @@ impl FootageSource for MediaPool {
|
||||
fn cache_budget(&self) -> Option<usize> {
|
||||
Some(self.budget())
|
||||
}
|
||||
fn frame(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<Image>> {
|
||||
if footage.missing {
|
||||
return None;
|
||||
}
|
||||
match self.frame_at(footage, t) {
|
||||
Ok(img) => Some(img),
|
||||
Err(e) => {
|
||||
log::warn!("media: frame of {} at {:?}: {e}", footage.path, t);
|
||||
None
|
||||
}
|
||||
}
|
||||
fn frame(&self, item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<Image>> {
|
||||
self.frame_sized(item, footage, t, None)
|
||||
}
|
||||
fn frame_at_size(&self, item: ItemId, footage: &Footage, t: Tick, width: u32, height: u32) -> Option<Arc<Image>> {
|
||||
self.frame_sized(item, footage, t, Some((width, height)))
|
||||
}
|
||||
|
||||
fn vector_frame(&self, _item: ItemId, footage: &Footage, scale: f64) -> Option<Arc<Image>> {
|
||||
@@ -834,7 +859,7 @@ mod tests {
|
||||
fn lru_respects_budget_and_recycles() {
|
||||
let mut c = Lru::default();
|
||||
let img = || Arc::new(Image::new(16, 16)); // 4 KiB + overhead
|
||||
let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3] };
|
||||
let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3], size: (0, 0) };
|
||||
let budget = 3 * (4096 + 64);
|
||||
for i in 0..10 {
|
||||
c.insert(k(i), img(), budget);
|
||||
|
||||
@@ -236,6 +236,26 @@ fn h264_mp4_probe_and_frames() {
|
||||
assert!(Arc::ptr_eq(&a, &b));
|
||||
}
|
||||
|
||||
/// A reduced-resolution render's movie frames are converted at its size: they match resampling
|
||||
/// the full frames (testsrc2's saturated colours clip, where the two differ most), and are cached
|
||||
/// per size.
|
||||
#[test]
|
||||
fn h264_frames_at_reduced_sizes() {
|
||||
let Some(p) = h264_mp4() else { return };
|
||||
let f = probe(&p).expect("probe");
|
||||
let pool = MediaPool::new();
|
||||
for (n, (w, h)) in [(0i64, (960, 540)), (1, (640, 360)), (2, (480, 270)), (30, (323, 182))] {
|
||||
let t = t_of(f.frame_rate, n);
|
||||
let sized = pool.frame_at_size(ItemId(1), &f, t, w, h).expect("frame");
|
||||
let full = pool.frame(ItemId(1), &f, t).expect("frame");
|
||||
assert_eq!((sized.width, sized.height), (w, h));
|
||||
let db = psnr(&sized, &effectcraft_raster::resample(&full, w, h));
|
||||
eprintln!("h264 frame {n} at {w}×{h}: {db:.2} dB against the resampled full frame");
|
||||
assert!(db > 55.0, "frame {n} at {w}×{h}: {db:.2} dB");
|
||||
assert!(Arc::ptr_eq(&sized, &pool.frame_at_size(ItemId(1), &f, t, w, h).expect("frame")), "cached");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn h264_audio_samples() {
|
||||
let Some(p) = h264_mp4() else { return };
|
||||
|
||||
@@ -195,14 +195,22 @@ pub fn quantize(img: &mut Image, levels: f32) {
|
||||
#[allow(clippy::manual_clamp)] // max/min map NaN to 0; clamp would keep NaN
|
||||
#[inline(always)]
|
||||
fn quantize_px(p: &mut [f32; 4], levels: f32, inv: f32) {
|
||||
let a = (p[3].max(0.0).min(1.0) * levels + 0.5) as u32 as f32 * inv;
|
||||
let a = round_half_up(p[3].max(0.0).min(1.0) * levels) * inv;
|
||||
let hi = [a, a, a, a];
|
||||
for c in 0..3 {
|
||||
p[c] = (p[c].max(0.0).min(hi[c]) * levels + 0.5) as u32 as f32 * inv;
|
||||
p[c] = round_half_up(p[c].max(0.0).min(hi[c]) * levels) * inv;
|
||||
}
|
||||
p[3] = a;
|
||||
}
|
||||
|
||||
/// `v` (0..=65 535) rounded half up. The same either way; the fastest on each target: x86-64's
|
||||
/// baseline converts to integers in one instruction but has no `floor` (SSE4.1), while in wasm
|
||||
/// `floor` is one instruction and converting back from `u32` several (about 1.5× faster there).
|
||||
#[inline(always)]
|
||||
fn round_half_up(v: f32) -> f32 {
|
||||
if cfg!(target_arch = "wasm32") { (v + 0.5).floor() } else { (v + 0.5) as u32 as f32 }
|
||||
}
|
||||
|
||||
/// [`quantize`] restricted to a rectangle (already clamped to the image), row-parallel.
|
||||
fn quantize_rect(img: &mut Image, r: PxRect, levels: f32) {
|
||||
if r.x0 >= r.x1 || r.y0 >= r.y1 {
|
||||
@@ -268,3 +276,22 @@ pub fn output_hdr(img: &mut Image, to_linear: Option<Conversion>, mode: HdrMode,
|
||||
*p = [o[0] * a, o[1] * a, o[2] * a, a];
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
/// Both forms `round_half_up` picks between agree on everything quantising feeds it:
|
||||
/// 0..=levels, exact halves included.
|
||||
#[test]
|
||||
fn rounding_forms_agree() {
|
||||
for levels in [255.0f32, 32_768.0] {
|
||||
for i in 0..=(levels as u32 * 16) {
|
||||
let v = i as f32 / 16.0;
|
||||
assert_eq!((v + 0.5).floor(), (v + 0.5) as u32 as f32, "{v}");
|
||||
}
|
||||
for i in 0..=100_000 {
|
||||
let v = (i as f32 / 100_000.0) * levels;
|
||||
assert_eq!((v + 0.5).floor(), (v + 0.5) as u32 as f32, "{v}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+35
-12
@@ -40,6 +40,13 @@ use rayon::prelude::*;
|
||||
pub trait FootageSource: Send + Sync {
|
||||
/// The frame of `item` at source time `t`, straight from the file (any size).
|
||||
fn frame(&self, item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<Image>>;
|
||||
/// [`FootageSource::frame`] for a render that shows it at `width` × `height`, at most half its
|
||||
/// size: either that frame or, when the source can make it more cheaply than the whole frame
|
||||
/// (movies, see `effectcraft-media`), the frame already resampled to that size as
|
||||
/// [`effectcraft_raster::resample`] would.
|
||||
fn frame_at_size(&self, item: ItemId, footage: &Footage, t: Tick, _width: u32, _height: u32) -> Option<Arc<Image>> {
|
||||
self.frame(item, footage, t)
|
||||
}
|
||||
/// `frames` interleaved stereo sample frames of `item`'s audio from source time `t` at
|
||||
/// `rate` Hz (`None` when unavailable). The first sample is `t.to_units_floor(rate)`;
|
||||
/// a sample boundary uses the earliest representable tick that selects that sample.
|
||||
@@ -1105,8 +1112,10 @@ impl<'a> Renderer<'a> {
|
||||
}
|
||||
let lt = ctx.source_time(layer);
|
||||
// The proxy is decoded instead, at its own size, and fills the footage's frame.
|
||||
let pf = self.proxy_for(*item).unwrap_or(f);
|
||||
let img = self.footage_frame(ctx, layer, *item, pf, lt)?;
|
||||
let proxy = self.proxy_for(*item);
|
||||
let pf = proxy.unwrap_or(f);
|
||||
let size = self.reduced_size(f.width, f.height).filter(|_| proxy.is_none());
|
||||
let img = self.footage_frame(ctx, layer, *item, pf, lt, size)?;
|
||||
Some(self.footage_buf(&img, pf, f.width, f.height))
|
||||
}
|
||||
LayerSource::Text => Some(self.authored(text::render(ctx, layer, self.raster_scale(ctx, layer)))),
|
||||
@@ -1128,12 +1137,11 @@ impl<'a> Renderer<'a> {
|
||||
let s = self.opts.scale;
|
||||
let k = if img.width > 0 { w.max(1) as f64 / img.width as f64 } else { 1.0 };
|
||||
// Keep native pixels when downsampling is small; resample otherwise.
|
||||
let mut buf = if s < 0.75 && k <= 1.0 + 1e-9 {
|
||||
let rw = ((w as f64 * s).round() as u32).max(1);
|
||||
let rh = ((h as f64 * s).round() as u32).max(1);
|
||||
Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s }
|
||||
} else {
|
||||
Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k }
|
||||
let mut buf = match self.reduced_size(w, h) {
|
||||
// The source resampled it already (`FootageSource::frame_at_size`).
|
||||
Some(size) if (img.width, img.height) == size => Buf { img: img.clone(), offset: [0.0; 2], scale: s },
|
||||
Some((rw, rh)) if k <= 1.0 + 1e-9 => Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s },
|
||||
_ => Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k },
|
||||
};
|
||||
if let Some(c) = self.pipe.media_in(f) {
|
||||
color::convert(&mut buf.img, &c);
|
||||
@@ -1141,6 +1149,13 @@ impl<'a> Renderer<'a> {
|
||||
buf
|
||||
}
|
||||
|
||||
/// The size of `w` × `h` layer pixels in this render when it is reduced enough that footage
|
||||
/// is resampled to it (below 3/4 scale).
|
||||
fn reduced_size(&self, w: u32, h: u32) -> Option<(u32, u32)> {
|
||||
let s = self.opts.scale;
|
||||
(s < 0.75).then(|| (((w as f64 * s).round() as u32).max(1), ((h as f64 * s).round() as u32).max(1)))
|
||||
}
|
||||
|
||||
/// Authored colours into the working space (linear working spaces).
|
||||
fn authored(&self, mut buf: Buf) -> Buf {
|
||||
if let Some(c) = self.pipe.authored_in() {
|
||||
@@ -1152,7 +1167,8 @@ impl<'a> Renderer<'a> {
|
||||
/// A footage frame at source time `t`, frame-blended between the two nearest source frames
|
||||
/// when the layer's Frame Blending switch (and the comp's Enable Frame Blending) is on and
|
||||
/// `t` falls between frames (footage rate ≠ comp rate, time stretch, time remapping).
|
||||
fn footage_frame(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option<Arc<Image>> {
|
||||
/// `size`: a plain frame may come resampled to it ([`FootageSource::frame_at_size`]).
|
||||
fn footage_frame(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option<Arc<Image>> {
|
||||
// Interpret Footage ▸ Separate Fields: each field is a frame at twice the rate.
|
||||
if f.fields != effectcraft_project::FieldOrder::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) {
|
||||
let field_rate = FrameRate::new(f.frame_rate.num * 2, f.frame_rate.den);
|
||||
@@ -1163,11 +1179,13 @@ impl<'a> Renderer<'a> {
|
||||
let parity = if (i % 2 == 0) == dominant_upper { 0 } else { 1 };
|
||||
return Some(Arc::new(interpret_pixels(&field_frame(&img, parity), f)));
|
||||
}
|
||||
let img = self.footage_frame_raw(ctx, layer, item, f, t)?;
|
||||
// Interpretation is per pixel but not linear: it runs on the frame before resampling.
|
||||
let size = size.filter(|_| !(f.invert_alpha || f.linear_light_applies()));
|
||||
let img = self.footage_frame_raw(ctx, layer, item, f, t, size)?;
|
||||
if f.invert_alpha || f.linear_light_applies() { Some(Arc::new(interpret_pixels(&img, f))) } else { Some(img) }
|
||||
}
|
||||
|
||||
fn footage_frame_raw(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option<Arc<Image>> {
|
||||
fn footage_frame_raw(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option<Arc<Image>> {
|
||||
let mode = frame_blend_mode(ctx, layer);
|
||||
if mode != FrameBlend::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) {
|
||||
let (i, w) = frame_position(t, f.frame_rate);
|
||||
@@ -1195,7 +1213,10 @@ impl<'a> Renderer<'a> {
|
||||
}
|
||||
}
|
||||
}
|
||||
self.footage.frame(item, f, t)
|
||||
match size {
|
||||
Some((w, h)) => self.footage.frame_at_size(item, f, t, w, h),
|
||||
None => self.footage.frame(item, f, t),
|
||||
}
|
||||
}
|
||||
|
||||
/// Source → masks, clamped/quantised to the project depth.
|
||||
@@ -2153,6 +2174,8 @@ mod tests_collapse;
|
||||
#[cfg(test)]
|
||||
mod tests_color;
|
||||
#[cfg(test)]
|
||||
mod tests_footage_size;
|
||||
#[cfg(test)]
|
||||
mod tests_frame_blend;
|
||||
#[cfg(test)]
|
||||
mod tests_mask_blur;
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
//! Footage at a reduced resolution: a source that resamples frames itself
|
||||
//! ([`FootageSource::frame_at_size`], movies in `effectcraft-media`) renders what resampling the
|
||||
//! full frame renders.
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use rayon::prelude::*;
|
||||
|
||||
use effectcraft_color::Label;
|
||||
use effectcraft_project::build;
|
||||
use effectcraft_project::{AlphaMode, BitDepth, Comp, Footage, FootageKind, ItemId, ItemKind, LayerSource, Project};
|
||||
use effectcraft_time::{FrameRate, Tick};
|
||||
|
||||
use crate::{FootageSource, Image, RenderOpts, Renderer};
|
||||
|
||||
/// Footage with a textured frame; `sizes` (when set) resamples it itself and records the sizes
|
||||
/// it was asked for.
|
||||
struct Clip {
|
||||
sizes: Option<Mutex<Vec<(u32, u32)>>>,
|
||||
}
|
||||
|
||||
impl FootageSource for Clip {
|
||||
fn frame(&self, _: ItemId, f: &Footage, _: Tick) -> Option<Arc<Image>> {
|
||||
let mut img = Image::new(f.width, f.height);
|
||||
img.rows_mut().for_each(|(y, row)| {
|
||||
for (x, p) in row.iter_mut().enumerate() {
|
||||
let s = (0.5 + 0.3 * (x as f32 * 0.37 + y as f32 * 0.11).sin()) * 0.9;
|
||||
*p = [s, 1.0 - s, s * 0.5, 1.0];
|
||||
}
|
||||
});
|
||||
Some(Arc::new(img))
|
||||
}
|
||||
fn frame_at_size(&self, item: ItemId, f: &Footage, t: Tick, w: u32, h: u32) -> Option<Arc<Image>> {
|
||||
let Some(sizes) = &self.sizes else { return self.frame(item, f, t) };
|
||||
sizes.lock().unwrap().push((w, h));
|
||||
Some(Arc::new(effectcraft_raster::resample(&*self.frame(item, f, t)?, w, h)))
|
||||
}
|
||||
}
|
||||
|
||||
fn project(depth: BitDepth, invert_alpha: bool) -> (Project, ItemId) {
|
||||
let mut p = Project::default();
|
||||
p.settings.bit_depth = depth;
|
||||
let comp = Comp::new(96, 54, FrameRate::FPS_30, Tick::from_seconds_f64(1.0));
|
||||
let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.clone().into()));
|
||||
let f = Footage {
|
||||
path: "clip.mp4".into(),
|
||||
kind: FootageKind::Video,
|
||||
width: 96,
|
||||
height: 54,
|
||||
pixel_aspect: 1.0,
|
||||
frame_rate: FrameRate::FPS_30,
|
||||
duration: Tick::from_seconds_f64(1.0),
|
||||
has_video: true,
|
||||
alpha: AlphaMode::Ignore,
|
||||
loop_count: 1,
|
||||
invert_alpha,
|
||||
..Default::default()
|
||||
};
|
||||
let fid = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(f));
|
||||
let l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, (96, 54), None);
|
||||
p.comp_mut(cid).unwrap().layers.push(l);
|
||||
(p, cid)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reduced_renders_take_footage_resampled_by_the_source() {
|
||||
for depth in [BitDepth::Bpc8, BitDepth::Bpc32] {
|
||||
let (p, cid) = project(depth, false);
|
||||
for (scale, asked) in [(1.0, None), (0.75, None), (0.5, Some((48, 27))), (1.0 / 3.0, Some((32, 18))), (0.25, Some((24, 14)))] {
|
||||
let opts = RenderOpts { scale, ..Default::default() };
|
||||
let plain = Renderer::new(&p, &Clip { sizes: None }, opts).comp_frame(cid, Tick::ZERO);
|
||||
let sized = Clip { sizes: Some(Mutex::default()) };
|
||||
let frame = Renderer::new(&p, &sized, opts).comp_frame(cid, Tick::ZERO);
|
||||
assert_eq!(frame, plain, "{depth:?} at {scale}");
|
||||
assert_eq!(sized.sizes.unwrap().into_inner().unwrap(), asked.into_iter().collect::<Vec<_>>(), "{depth:?} at {scale}");
|
||||
}
|
||||
}
|
||||
// Interpreted pixels (Invert Alpha) are interpreted at full size, then resampled.
|
||||
let (p, cid) = project(BitDepth::Bpc8, true);
|
||||
let sized = Clip { sizes: Some(Mutex::default()) };
|
||||
Renderer::new(&p, &sized, RenderOpts { scale: 0.5, ..Default::default() }).comp_frame(cid, Tick::ZERO);
|
||||
assert!(sized.sizes.unwrap().into_inner().unwrap().is_empty());
|
||||
}
|
||||
@@ -230,7 +230,11 @@ Nulls / Nulls Follow Points take pins as well as paths, and `puppet.follow` give
|
||||
Position expression that trails the leader by a delay. The renderer flattens nested effect
|
||||
groups into `Params` keys (`effects::flatten_params`). Commands: `paint.*`, `puppet.*`.
|
||||
|
||||
Half, Third and Quarter resolution render proportionally fewer pixels end to end.
|
||||
Half, Third and Quarter resolution render proportionally fewer pixels end to end. Footage
|
||||
starts at the render's size where the source can make it more cheaply than the full frame
|
||||
(`FootageSource::frame_at_size`): movies in opaque 4:2:0 Y'CbCr are box-filtered to half size
|
||||
before conversion (a quarter of the conversions; the result differs from converting first only
|
||||
in blocks where colours clip) and resampled from there, as other frames are from full size.
|
||||
|
||||
**GPU compositor** (`crates/gpu`; Project Settings ▸ Video Rendering and Effects ▸ Mercury GPU
|
||||
Acceleration, the default, or Mercury Software Only; `render.backend`). The CPU renderer is the
|
||||
|
||||
+2
-2
@@ -71,8 +71,8 @@ Commands (CLI, MCP, control channel):
|
||||
- `audio.outputDevice`: Default Output (device list from the desktop host)
|
||||
- `audio.outputLeft`: Left (Audio Output Mapping)
|
||||
- `audio.outputRight`: Right (Audio Output Mapping)
|
||||
- `memory.layerCacheMb`: Layer Cache
|
||||
- `memory.mediaCacheMb`: Footage Frame Cache
|
||||
- `memory.layerCacheMb`: Layer Cache (in the browser at most 512 MB, see [web.md](web.md))
|
||||
- `memory.mediaCacheMb`: Footage Frame Cache (in the browser at most 256 MB)
|
||||
- `memory.previewCacheMb`: Preview (RAM) Cache
|
||||
- `threeD.defaultRenderer`: Default 3D Renderer
|
||||
- `labels.N.name` / `labels.N.color`: the 16 label names and colours, used by the Label menu,
|
||||
|
||||
@@ -61,6 +61,7 @@ device of their own), `?nosw` (don't register the service worker).
|
||||
| config directory: settings, shortcut presets, recent projects, the recovery sentinel | `store::WebConfig` in browser storage |
|
||||
| auto-save folder | `/EffectCraft Auto-Save/` in browser storage (`ConfigStore::files`) |
|
||||
| media reads (`MediaPool`, `probe`) | `files::WebImporter`: bytes from the file table, handed to `MediaPool::add_bytes` / `probe_bytes` |
|
||||
| Memory & CPU cache budgets, scaled to the system's RAM | the page's memory, at most 4 GiB (wasm32), which the caches share with the decoders' own decoded frames (FilmCraft keeps up to 384 MB per open movie): the Layer Cache stays within 512 MB and the Footage Frame Cache within 256 MB (`Prefs::cache_budgets`) |
|
||||
| export writes (`effectcraft-export`) | the job's `sink` (`effectcraft_host::FileExporter { sink }`): files land in the table and download after the render; several files (an image sequence) download as one stored `.zip` |
|
||||
| rfd file dialogs | `<input type=file>` for File ▸ Open / Import; drop files anywhere on the page (`.ecproj` opens, everything else imports); "Save As" / "Output To" pick a download name |
|
||||
| Media Browser on the file system | browser storage and folders opened with the File System Access API (below) |
|
||||
|
||||
Reference in New Issue
Block a user