M15.2: merge main (disk cache, 3D Camera Tracker); camera analysis in workers; no disk cache on wasm32

The 3D Camera Tracker analysis joins the offloaded jobs. The disk cache stays off in the
browser (std::env::temp_dir panics on wasm32 once a config store exists).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas
2026-10-03 00:03:43 -04:00
co-authored by Claude Opus 5.5
116 changed files with 18084 additions and 82 deletions
+2
View File
@@ -36,6 +36,8 @@ iconography, images or other Adobe assets; open licences only; every asset attri
| `crates/lottie/tests/fixtures/shapes-gradient.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (star, rectangle, gradient fill, dashed stroke, trim paths) | MIT OR Apache-2.0 |
| `crates/lottie/tests/fixtures/precomp-matte-text.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (precomp, track matte, parented text) | MIT OR Apache-2.0 |
| `crates/lottie/tests/fixtures/legacy-masks.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (legacy keyframes, masks) | MIT OR Apache-2.0 |
| `crates/svg/tests/fixtures/basic-shapes.svg` | EffectCraft contributors | Original work: hand-written SVG test fixture (basic shapes, groups, transforms) | MIT OR Apache-2.0 |
| `crates/svg/tests/fixtures/paths-gradients.svg` | EffectCraft contributors | Original work: hand-written SVG test fixture (path data, gradients, CSS, use) | MIT OR Apache-2.0 |
| `crates/model/tests/fixtures/quad.gltf` | EffectCraft contributors | Original work: glTF 2.0 test fixture (textured quad, node hierarchy, translation animation; data URIs) | MIT OR Apache-2.0 |
| `crates/model/tests/fixtures/quad.glb` | EffectCraft contributors | Original work: binary glTF test fixture (the same quad with its texture in the BIN chunk) | MIT OR Apache-2.0 |
| `crates/model/tests/fixtures/cube.obj` | EffectCraft contributors | Original work: Wavefront OBJ test fixture (unit cube, two materials) | MIT OR Apache-2.0 |
Generated
+57
View File
@@ -1214,12 +1214,15 @@ dependencies = [
"effectcraft-model",
"effectcraft-path",
"effectcraft-project",
"effectcraft-psd",
"effectcraft-raster",
"effectcraft-render",
"effectcraft-svg",
"effectcraft-text",
"effectcraft-time",
"effectcraft-track",
"image",
"kurbo",
"log",
"rayon",
"serde",
@@ -1234,14 +1237,18 @@ version = "0.1.0"
dependencies = [
"effectcraft-color",
"effectcraft-media",
"effectcraft-opusenc",
"effectcraft-project",
"effectcraft-raster",
"effectcraft-render",
"effectcraft-time",
"effectcraft-vp9enc",
"filmcraft-aac",
"filmcraft-h264enc",
"filmcraft-isobmff",
"filmcraft-opus",
"filmcraft-prores",
"filmcraft-vp9",
"gif",
"image",
"log",
@@ -1303,7 +1310,11 @@ dependencies = [
"effectcraft-expr",
"effectcraft-media",
"effectcraft-project",
"effectcraft-psd",
"effectcraft-raster",
"effectcraft-script",
"effectcraft-svg",
"effectcraft-time",
"serde_json",
]
@@ -1342,8 +1353,10 @@ version = "0.1.0"
dependencies = [
"effectcraft-model",
"effectcraft-project",
"effectcraft-psd",
"effectcraft-raster",
"effectcraft-render",
"effectcraft-svg",
"effectcraft-time",
"exr",
"filmcraft-codecs",
@@ -1369,6 +1382,10 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "effectcraft-opusenc"
version = "0.1.0"
[[package]]
name = "effectcraft-path"
version = "0.1.0"
@@ -1394,6 +1411,13 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "effectcraft-psd"
version = "0.1.0"
dependencies = [
"miniz_oxide 0.8.9",
]
[[package]]
name = "effectcraft-raster"
version = "0.1.0"
@@ -1419,6 +1443,7 @@ dependencies = [
"effectcraft-time",
"kurbo",
"log",
"lz4_flex",
"rayon",
"serde",
"web-time",
@@ -1439,6 +1464,18 @@ dependencies = [
"serde_json",
]
[[package]]
name = "effectcraft-svg"
version = "0.1.0"
dependencies = [
"effectcraft-geom",
"effectcraft-path",
"effectcraft-raster",
"kurbo",
"rayon",
"roxmltree",
]
[[package]]
name = "effectcraft-text"
version = "0.1.0"
@@ -1467,6 +1504,7 @@ dependencies = [
"rayon",
"serde",
"serde_json",
"web-time",
]
[[package]]
@@ -1497,6 +1535,10 @@ dependencies = [
"web-time",
]
[[package]]
name = "effectcraft-vp9enc"
version = "0.1.0"
[[package]]
name = "effectcraft-web"
version = "0.1.0"
@@ -2809,6 +2851,12 @@ version = "0.4.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
[[package]]
name = "lz4_flex"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
[[package]]
name = "mach2"
version = "0.4.3"
@@ -4049,6 +4097,15 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "roxmltree"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb"
dependencies = [
"memchr",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
+13
View File
@@ -29,10 +29,14 @@ effectcraft-media = { path = "crates/media" }
effectcraft-export = { path = "crates/export" }
effectcraft-expr = { path = "crates/expr" }
effectcraft-lottie = { path = "crates/lottie" }
effectcraft-psd = { path = "crates/psd" }
effectcraft-vp9enc = { path = "crates/vp9enc" }
effectcraft-svg = { path = "crates/svg" }
effectcraft-engine = { path = "crates/engine" }
effectcraft-host = { path = "crates/host" }
effectcraft-ui-egui = { path = "crates/ui-egui" }
effectcraft-automation = { path = "crates/automation" }
effectcraft-opusenc = { path = "crates/opusenc" }
effectcraft-script = { path = "crates/script" }
# foundation
@@ -69,6 +73,9 @@ filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "89c
filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" }
filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" }
filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" }
# decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency)
filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" }
filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" }
# gpu (the same wgpu eframe / egui-wgpu use)
wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] }
@@ -132,9 +139,15 @@ opt-level = 3
[profile.dev.package.effectcraft-media]
opt-level = 3
[profile.dev.package.effectcraft-opusenc]
opt-level = 3
[profile.dev.package.effectcraft-export]
opt-level = 3
[profile.dev.package.effectcraft-vp9enc]
opt-level = 3
# FilmCraft codec hot loops (same settings as FilmCraft's own workspace).
[profile.dev.package.filmcraft-h264]
opt-level = 3
+6 -7
View File
@@ -25,13 +25,12 @@ once.
## How far from full parity
Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 October 2026):
Measured feature by feature in [docs/parity.md](docs/parity.md) (late 2 October 2026):
- **≈ 79% of After Effects' features, weighted by importance** (87% of the essentials, 67% of
professional daily-use features, 21% of the long tail); 257 of 298 effects, 16 of them on the GPU.
- **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in
parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours**
by the audit's conservative per-feature figures.
- Hardest remaining pieces: the 3D camera tracker, Advanced 3D, a scripting object model (Roto Brush & Refine Edge landed in M6.7).
- **≈ 93% of After Effects' features, weighted by importance** (96% of the essentials, 90% of
professional daily-use features, 65% of the long tail); all 298 After Effects effects.
- **≈ 50–55 agent-hours of work remain**: about **11–14 hours of wall-clock time** with five Claude
Opus 5.5 agents in parallel, ≈ 6–8 hours for the essentials and daily-use features.
- Hardest remaining pieces: the web app's depth, GPU 3D, Content-Aware Fill, a plug-in API.
Come tell us what matters most to you on [Discord](https://discord.gg/artcraft).
+1 -1
View File
@@ -9,7 +9,7 @@
//! effectcraft-cli get <comp> <layer> <path> [--time S] read a property
//! effectcraft-cli set <comp> <layer> <path> <value> [--time S] [--expression E]
//! effectcraft-cli render-frame [--comp C] [--time S|--frame N] [--max-side PX|--scale K] [--out F.png]
//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|png|jpeg|tiff|exr|gif] [--start S] [--end S]
//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff] [--start S] [--end S]
//! [--work-area] [--fps N] [--resolution full|half|third|quarter|K] [--quality best|draft] [--channels rgb|rgba]
//! [--jpeg-quality N] [--bitrate KBPS] [--prores proxy|lt|standard|hq|4444|4444xq] [--audio auto|on|off]
//! effectcraft-cli render F.ecproj --queue render the project's Render Queue
+291
View File
@@ -0,0 +1,291 @@
//! 3D Camera Tracker (Effect ▸ Perspective ▸ 3D Camera Tracker; Animation ▸ Track Camera).
//!
//! The analysis runs as a background job in the engine (`camera.analyze`): step 1 tracks
//! features through the clip ([`effectcraft_track::camtrack::TrackAnalyzer`]), step 2 solves
//! the camera ([`effectcraft_track::camtrack::solve`]). Both results are stored as JSON in the
//! instance's hidden parameters — **Tracks** (with **Tracks Key**: what the frames were made
//! from, so source / In-Out / time edits invalidate them) and **Solve** (with **Solve Key**: the
//! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing
//! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve.
//!
//! The effect passes its input through; with **Render Track Points** on it draws the solved
//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create
//! commands live in the engine and the UI.
use std::collections::HashMap;
use std::sync::{Arc, Mutex, OnceLock};
use effectcraft_keyframe::Value;
use effectcraft_project::ParamUi;
use effectcraft_raster::{Image, Px};
use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color};
use crate::warp_stab::param;
use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider};
/// The effect's id.
pub const ID: &str = "ec.perspective.cameratracker";
/// Hidden parameters (see the module docs).
pub const TRACKS: &str = "tracks";
pub const TRACKS_KEY: &str = "tracksKey";
pub const SOLVE: &str = "solve";
pub const SOLVE_KEY: &str = "solveKey";
/// Deleted track point ids (JSON array).
pub const DELETED: &str = "deleted";
/// The "3D Tracker Camera" layer the instance created (layer id, 0 = none).
pub const CAMERA: &str = "camera";
pub const METHOD_USED: &str = "advanced/methodUsed";
pub const AVERAGE_ERROR: &str = "advanced/averageError";
pub const SHOT_TYPES: [&str; 3] = ["Fixed Angle of View", "Variable Zoom", "Specify Angle of View"];
pub const SHOW_POINTS: [&str; 2] = ["2D Source", "3D Solved"];
pub const SOLVE_METHODS: [&str; 4] = ["Auto Detect", "Typical", "Mostly Flat Scene", "Tripod Pan"];
pub fn specs() -> Vec<EffectSpec> {
vec![EffectSpec {
id: ID,
name: "3D Camera Tracker",
category: "Perspective",
params: vec![
p("shotType", "Shot Type", Value::Enum(0), popup(&SHOT_TYPES)),
p("horizontalAngleOfView", "Horizontal Angle of View", num(40.0), slider(1.0, 170.0, 10.0, 120.0, 1)),
p("showTrackPoints", "Show Track Points", Value::Enum(1), popup(&SHOW_POINTS)),
p("renderTrackPoints", "Render Track Points", Value::Bool(false), ParamUi::Checkbox),
p("trackPointSize", "Track Point Size", num(100.0), slider(0.0, 1000.0, 0.0, 300.0, 0)),
p("targetSize", "Target Size", num(100.0), slider(1.0, 1000.0, 10.0, 300.0, 0)),
p("advanced/solveMethod", "Solve Method", Value::Enum(0), popup(&SOLVE_METHODS)),
p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden),
p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden),
p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox),
p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox),
p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox),
p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden),
p(TRACKS_KEY, "Tracks Key", Value::Str(String::new()), ParamUi::Hidden),
p(SOLVE, "Solve", Value::Str(String::new()), ParamUi::Hidden),
p(SOLVE_KEY, "Solve Key", Value::Str(String::new()), ParamUi::Hidden),
p(DELETED, "Deleted Points", Value::Str(String::new()), ParamUi::Hidden),
p(CAMERA, "Camera Layer", num(0.0), ParamUi::Hidden),
],
render,
gpu: false,
float: true,
}]
}
fn f(params: &Params, id: &str) -> f64 {
param(params, id).map(Value::as_f64).unwrap_or(0.0)
}
fn e(params: &Params, id: &str) -> u32 {
param(params, id).map(Value::as_enum).unwrap_or(0)
}
fn b(params: &Params, id: &str) -> bool {
param(params, id).map(Value::as_bool).unwrap_or(false)
}
fn s<'a>(params: &'a Params, id: &str) -> &'a str {
match param(params, id) {
Some(Value::Str(s)) => s,
_ => "",
}
}
/// Deleted point ids of an instance (stored as a JSON array of numbers).
pub fn deleted(params: &Params) -> Vec<u32> {
s(params, DELETED).trim().trim_start_matches('[').trim_end_matches(']').split(',').filter_map(|v| v.trim().parse().ok()).collect()
}
/// The solve settings of an instance.
pub fn settings(params: &Params) -> SolveSettings {
let mut del = deleted(params);
del.sort_unstable();
del.dedup();
SolveSettings {
shot: ShotType::ALL.get(e(params, "shotType") as usize).copied().unwrap_or_default(),
hfov: f(params, "horizontalAngleOfView"),
method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(),
deleted: del,
}
}
/// What a solve depends on besides the tracks.
pub fn solve_key(st: &SolveSettings) -> String {
let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() };
format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted)
}
fn fnv(bytes: &[u8]) -> u64 {
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
for b in bytes {
h ^= *b as u64;
h = h.wrapping_mul(0x0100_0000_01b3);
}
h
}
type Cache<T> = Mutex<HashMap<u64, Arc<T>>>;
fn cached<T>(c: &'static OnceLock<Cache<T>>, json: &str, parse: impl Fn(&str) -> Option<T>) -> Option<Arc<T>> {
let key = fnv(json.as_bytes());
let c = c.get_or_init(Default::default);
if let Some(a) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&key) {
return Some(a.clone());
}
let a = Arc::new(parse(json)?);
let mut m = c.lock().unwrap_or_else(|e| e.into_inner());
if m.len() > 8 {
m.clear();
}
m.insert(key, a.clone());
Some(a)
}
/// The stored feature tracks of an instance (`None` when not analysed).
pub fn tracks(params: &Params) -> Option<Arc<CameraTracks>> {
static C: OnceLock<Cache<CameraTracks>> = OnceLock::new();
let j = s(params, TRACKS);
if j.is_empty() {
return None;
}
cached(&C, j, CameraTracks::from_json).filter(|t| !t.is_empty())
}
/// The stored camera solve of an instance (`None` when not solved).
pub fn solve(params: &Params) -> Option<Arc<CameraSolve>> {
static C: OnceLock<Cache<CameraSolve>> = OnceLock::new();
let j = s(params, SOLVE);
if j.is_empty() {
return None;
}
cached(&C, j, CameraSolve::from_json).filter(|t| !t.is_empty())
}
/// Screen size (layer pixels) of a target drawn for a point at depth `z`, relative to the
/// median depth of the frame's points.
pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 {
(6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0)
}
/// The solved points visible on frame `k`: (id, layer pixel position, camera depth).
pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> {
let Some(cam) = solve.frames.get(k) else { return vec![] };
solve
.visible(k)
.filter_map(|p| {
let c = cam.to_cam(p.pos);
cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2]))
})
.collect()
}
/// Median of the depths.
pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 {
let mut z: Vec<f64> = v.iter().map(|x| x.2).collect();
if z.is_empty() {
return 1.0;
}
z.sort_by(f64::total_cmp);
z[z.len() / 2]
}
fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf {
if !b(ctx.params, "renderTrackPoints") {
return buf;
}
let Some(sol) = solve(ctx.params) else { return buf };
if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 {
return buf;
}
let Some(k) = sol.frame_at(ctx.time) else { return buf };
let pts = projected(&sol, k);
let med = median_depth(&pts);
let size = f(ctx.params, "trackPointSize");
let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 };
for (id, q, z) in pts {
let c = point_color(id);
let r = target_radius(size, z, med) * sc;
let at = [q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]];
ring(&mut buf.img, at, r, (r * 0.25).max(1.0), [c[0], c[1], c[2], 1.0]);
disc(&mut buf.img, at, (r * 0.2).max(0.8), [c[0], c[1], c[2], 1.0]);
}
buf
}
fn lerp_px(a: Px, b: Px, t: f32) -> Px {
[a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t, a[3] + (b[3] - a[3]) * t]
}
fn cover(img: &mut Image, c: [f64; 2], r_out: f64, cov: impl Fn(f64) -> f64, col: Px) {
let (x0, x1) = ((c[0] - r_out - 1.0).floor().max(0.0) as i64, (c[0] + r_out + 1.0).ceil() as i64);
let (y0, y1) = ((c[1] - r_out - 1.0).floor().max(0.0) as i64, (c[1] + r_out + 1.0).ceil() as i64);
for y in y0..y1.min(img.height as i64) {
for x in x0..x1.min(img.width as i64) {
let d = (x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]);
let a = cov(d).clamp(0.0, 1.0) as f32;
if a > 0.0 {
let i = img.idx(x as u32, y as u32);
img.data[i] = lerp_px(img.data[i], col, a);
}
}
}
}
fn disc(img: &mut Image, c: [f64; 2], r: f64, col: Px) {
cover(img, c, r, |d| r + 0.5 - d, col);
}
fn ring(img: &mut Image, c: [f64; 2], r: f64, w: f64, col: Px) {
cover(img, c, r + w, |d| (w * 0.5 + 0.5 - (d - r).abs()).min(1.0), col);
}
/// Points of a target disc (canonical coordinates) for drawing: `n` points on the circle of
/// radius `size / 2` in the plane through `center` with normal `normal`.
pub fn target_circle(center: linalg::V3, normal: linalg::V3, radius: f64, n: usize) -> Vec<linalg::V3> {
let a = if normal[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] };
let u = linalg::normalize(linalg::cross(normal, a));
let v = linalg::cross(normal, u);
(0..n)
.map(|i| {
let t = i as f64 / n as f64 * std::f64::consts::TAU;
linalg::add(center, linalg::add(linalg::scale(u, radius * t.cos()), linalg::scale(v, radius * t.sin())))
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::EffectEnv;
use effectcraft_track::camtrack::{SolvedFrame, SolvedPoint};
fn params_with(solve: &CameraSolve, render: bool) -> Params {
let mut p = Params::default();
p.values.insert(SOLVE.into(), Value::Str(solve.to_json()));
p.values.insert("renderTrackPoints".into(), Value::Bool(render));
p.values.insert("trackPointSize".into(), Value::Scalar(100.0));
p
}
#[test]
fn render_track_points_draws_targets_and_passes_through_otherwise() {
let sol = CameraSolve {
version: 1,
size: [64.0, 48.0],
start: 0.0,
frame_duration: 1.0 / 30.0,
frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }],
points: vec![SolvedPoint { id: 3, pos: [0.0, 0.0, 1.0], error: 0.1, first: 0, last: 0 }],
..Default::default()
};
let img = Image::new(64, 48);
for on in [false, true] {
let params = params_with(&sol, on);
let ctx = EffectCtx { params: &params, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() };
let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 });
let px = out.img.data[out.img.idx(32, 24)];
assert_eq!(px[3] > 0.5, on);
}
let st = settings(&Params::default());
assert_eq!(st.shot, ShotType::FixedAngle);
assert!(solve_key(&st).contains("FixedAngle"));
}
}
+4
View File
@@ -9,6 +9,7 @@
pub mod audio_fx;
mod blur2;
mod blur3;
pub mod camera_tracker;
mod channel;
mod channel2;
mod channel3d;
@@ -382,6 +383,7 @@ pub fn registry() -> &'static [EffectSpec] {
v.extend(paint::specs());
v.extend(puppet::specs());
v.extend(warp_stab::specs());
v.extend(camera_tracker::specs());
v.extend(roto::specs());
v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name)));
for s in v.iter_mut() {
@@ -530,6 +532,8 @@ pub const TIME_DEPENDENT: &[&str] = &[
"ec.obsolete.mochashape",
// Each frame gets its own stabilizing warp.
warp_stab::ID,
// Render Track Points draws each frame's solved points.
camera_tracker::ID,
// Each frame has its own segmentation.
roto::ID,
];
+3
View File
@@ -23,6 +23,9 @@ effectcraft-render = { workspace = true }
effectcraft-track = { workspace = true }
effectcraft-model = { workspace = true }
effectcraft-lottie = { workspace = true }
effectcraft-psd = { workspace = true }
kurbo = { workspace = true }
effectcraft-svg = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
+655
View File
@@ -0,0 +1,655 @@
//! 3D Camera Tracker runtime: the analysis job (`camera.analyze`, `track.camera`), keeping
//! analyses valid, and turning a solve into composition layers (`camera.createFromSolve`).
//!
//! The job renders the layer's input to the effect (`Renderer::layer_input`) for every frame
//! between the layer's In and Out points on a background thread and tracks features through
//! them (step 1 of 2, "Analyzing in background"), then solves the camera (step 2, "Solving
//! camera"). When only the solve settings changed (Shot Type, Angle of View, Solve Method,
//! deleted points) the stored tracks are re-solved without re-tracking. Results are written into
//! the effect's hidden parameters as one undo step; edits that change the layer's frames clear
//! them and queue the effect again ([`invalidate`], run after every edit), like After Effects.
//!
//! Composition space: the solve's canonical frame maps to comp world by
//! [`CameraSolve::world`] with the zoom of the first solved frame (focal length × the layer's
//! scale in the comp), so the "3D Tracker Camera" reproduces the footage's view and created
//! layers sit on the selected points.
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use effectcraft_effects::camera_tracker::{self as ct, AVERAGE_ERROR, METHOD_USED, SOLVE, SOLVE_KEY, TRACKS, TRACKS_KEY};
use effectcraft_keyframe::{Keyframe, Value};
use effectcraft_project::{AutoOrient, GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid};
use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer};
use effectcraft_time::Tick;
use effectcraft_track::camtrack::linalg::{self, Similarity, V3};
use effectcraft_track::camtrack::{AnalyzeOpts, CameraSolve, CameraTracks, SolveSettings, Target, TrackAnalyzer};
use serde::Serialize;
use crate::offload::JobKind;
use crate::tracking::lock;
use crate::warp::{analysis_times, signature};
use crate::{Event, Session};
/// Live analysis progress (serde for agents / the control channel).
#[derive(Clone, Debug, Default, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct CameraProgress {
/// 1 = tracking features, 2 = solving the camera.
pub step: u8,
pub done: u64,
pub total: u64,
pub elapsed: f64,
pub fps: f64,
pub finished: bool,
pub cancelled: bool,
pub error: Option<String>,
/// Re-solving stored tracks (no tracking step).
pub solve_only: bool,
}
impl CameraProgress {
/// The viewer banner text, as After Effects words it.
pub fn banner(&self) -> String {
match self.step {
2 => "Solving camera".into(),
_ => {
let pc = (100 * self.done).checked_div(self.total).unwrap_or(0);
format!("Analyzing in background (step 1 of 2): {pc}%")
}
}
}
}
struct CamResult {
/// New tracks (None for a re-solve).
tracks: Option<CameraTracks>,
solve: Result<CameraSolve, String>,
}
#[derive(Default)]
pub struct CameraShared {
pub cancel: AtomicBool,
pub state: Mutex<CameraProgress>,
result: Mutex<Option<CamResult>>,
}
/// A running (or finished, not yet polled) camera analysis.
pub struct CameraJob {
pub shared: Arc<CameraShared>,
thread: Option<std::thread::JoinHandle<()>>,
pub comp: ItemId,
pub layer: LayerId,
pub effect: Uid,
/// The layer signature the tracks are made from, and the solve settings key.
key: String,
solve_key: String,
}
impl CameraJob {
pub fn is_finished(&self) -> bool {
lock(&self.shared.state).finished
}
pub fn progress(&self) -> CameraProgress {
lock(&self.shared.state).clone()
}
}
struct CamWork {
project: Arc<Project>,
footage: Arc<dyn FootageSource>,
expr: Option<Arc<dyn ExprHost>>,
cache: Arc<LayerCache>,
comp: ItemId,
layer: LayerId,
effect_index: usize,
size: [f64; 2],
detailed: bool,
times: Vec<Tick>,
/// Stored tracks to re-solve (skips step 1).
tracks: Option<Arc<CameraTracks>>,
settings: SolveSettings,
}
pub(crate) fn is_tracker(g: &PropGroup) -> bool {
matches!(&g.kind, GroupKind::Effect { effect } if effect == ct::ID)
}
fn str_of(g: &PropGroup, m: &str) -> String {
match g.prop(m).map(|p| &p.value) {
Some(Value::Str(s)) => s.clone(),
_ => String::new(),
}
}
/// What a tracker's tracks are made from: the layer's frames and Detailed Analysis.
fn tracks_key(project: &Project, cid: ItemId, l: &Layer, g: &PropGroup) -> String {
let detailed = effectcraft_effects::warp_stab::param(&static_params(g), "advanced/detailedAnalysis").is_some_and(Value::as_bool);
format!("{}{}", signature(project, cid, l), if detailed { "+detailed" } else { "" })
}
/// Parameters of an instance from static values (the tracker's settings are not animated).
pub fn static_params(g: &PropGroup) -> effectcraft_effects::Params {
effectcraft_effects::flatten_params(g, &mut |p| p.value.clone())
}
fn clear(g: &mut PropGroup, all: bool) {
let mut ids = vec![SOLVE, SOLVE_KEY, METHOD_USED];
if all {
ids.extend([TRACKS, TRACKS_KEY]);
}
for m in ids {
if let Some(p) = g.prop_mut(m) {
p.value = Value::Str(String::new());
p.keys.clear();
}
}
if let Some(p) = g.prop_mut(AVERAGE_ERROR) {
p.value = Value::Scalar(0.0);
}
}
/// Clear analyses that no longer match, after an edit: tracks made from other frames (and their
/// solve), and solves made with other settings. Returns the trackers that need (re-)analysis:
/// cleared ones and newly applied, unanalysed ones.
pub(crate) fn invalidate(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> {
let mut todo = vec![];
let mut clears = vec![];
for (cid, it) in &after.items {
let ItemKind::Comp(c) = &it.kind else { continue };
for l in &c.layers {
let Some(fx) = l.effects() else { continue };
if !fx.groups().any(is_tracker) {
continue;
}
let old_layer = before.comp(*cid).and_then(|bc| bc.layer(l.id));
for g in fx.groups().filter(|g| is_tracker(g)) {
let tracked = !str_of(g, TRACKS).is_empty();
if tracked && str_of(g, TRACKS_KEY) != tracks_key(after, *cid, l, g) {
clears.push((*cid, l.id, g.uid, true));
todo.push((*cid, l.id, g.uid));
} else if tracked && str_of(g, SOLVE_KEY) != ct::solve_key(&ct::settings(&static_params(g))) {
clears.push((*cid, l.id, g.uid, false));
todo.push((*cid, l.id, g.uid));
} else if !tracked && old_layer.and_then(|ol| ol.props.find_group(g.uid)).is_none() {
todo.push((*cid, l.id, g.uid));
}
}
}
}
for (cid, lid, uid, all) in clears {
if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) {
clear(g, all);
}
}
todo
}
fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) -> Option<(Arc<effectcraft_raster::Image>, [f64; 2])> {
let buf = r.layer_input(ctx, layer, effect_index)?;
if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 {
let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32;
let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32;
let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale];
return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off));
}
Some((Arc::new(buf.img.clone()), buf.offset))
}
fn run_work(w: CamWork, shared: &CameraShared) {
let t0 = web_time::Instant::now();
let fail = |msg: &str| {
let mut s = lock(&shared.state);
s.error = Some(msg.to_string());
s.finished = true;
};
let cancelled = || {
let mut s = lock(&shared.state);
s.cancelled = true;
s.finished = true;
};
let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") };
let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") };
let lt0 = layer.layer_time(w.times[0]).seconds();
let fd = if w.times.len() > 1 { layer.layer_time(w.times[1]).seconds() - lt0 } else { comp.frame_duration().seconds() };
let (tracks, fresh) = match w.tracks.clone() {
Some(t) => (t, false),
None => {
let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() });
r.expr = w.expr.as_deref();
r.cache = Some(&w.cache);
let r = &r;
let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO);
let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 };
let frame_at = |t: Tick| input_frame(r, &ctx0.at(t), layer, w.effect_index);
{
let mut s = lock(&shared.state);
s.step = 1;
s.total = w.times.len() as u64;
}
let mut an = TrackAnalyzer::new(w.size, AnalyzeOpts { detailed: w.detailed });
let mut next = frame_at(w.times[0]);
for k in 0..w.times.len() {
if shared.cancel.load(Ordering::Relaxed) {
return cancelled();
}
let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) };
let ((), nf) = rayon::join(
|| an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }),
|| if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None },
);
next = nf;
let mut s = lock(&shared.state);
s.done = k as u64 + 1;
s.elapsed = t0.elapsed().as_secs_f64();
s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 };
crate::offload::report(s.done, s.total);
}
(Arc::new(an.finish(lt0, fd)), true)
}
};
lock(&shared.state).step = 2;
let solve = effectcraft_track::camtrack::solve(&tracks, &w.settings, Some(&shared.cancel)).map_err(|e| e.0);
if shared.cancel.load(Ordering::Relaxed) {
return cancelled();
}
*lock(&shared.result) = Some(CamResult { tracks: fresh.then(|| (*tracks).clone()), solve });
let mut s = lock(&shared.state);
s.elapsed = t0.elapsed().as_secs_f64();
s.finished = true;
}
impl Session {
pub fn is_camera_analyzing(&self) -> bool {
self.camera_job.as_ref().is_some_and(|j| !j.is_finished()) || self.offloaded(JobKind::Camera).is_some()
}
/// Progress of the running (or just finished) analysis.
pub fn camera_progress(&self) -> Option<CameraProgress> {
self.camera_job.as_ref().map(|j| j.progress()).or_else(|| {
self.offloaded(JobKind::Camera).map(|j| {
let p = &j.progress;
CameraProgress { step: 1, done: p.done, total: p.total, elapsed: p.elapsed, fps: crate::offload::rate(p.done, p.elapsed), ..Default::default() }
})
})
}
/// The tracker being analysed: (comp, layer, effect uid).
pub fn camera_target(&self) -> Option<(ItemId, LayerId, Uid)> {
self.camera_job.as_ref().map(|j| (j.comp, j.layer, j.effect)).or_else(|| self.offloaded(JobKind::Camera).map(|j| (j.comp, j.layer, j.uid)))
}
/// Cancel the running analysis (nothing is written).
pub fn stop_camera(&mut self) -> bool {
if self.cancel_offloaded(JobKind::Camera) {
return true;
}
match &self.camera_job {
Some(j) if !j.is_finished() => {
j.shared.cancel.store(true, Ordering::Relaxed);
true
}
_ => false,
}
}
/// Start analysing 3D Camera Tracker `effect` on `layer` (re-solving the stored tracks when
/// they are still valid). `wait`: block until done (wasm32 always does). Returns the number
/// of frames.
pub fn start_camera(&mut self, comp: ItemId, layer: LayerId, effect: Uid, wait: bool) -> Result<usize, String> {
if self.is_camera_analyzing() {
return Err("a 3D Camera Tracker analysis is already running".into());
}
self.poll_camera_job();
self.camera_pending.retain(|x| *x != (comp, layer, effect));
let c = self.project.comp(comp).ok_or("no composition")?;
let l = c.layer(layer).ok_or("no layer")?;
let fx = l.effects().ok_or("the layer has no effects")?;
let index = fx.groups().position(|g| g.uid == effect).ok_or("no such effect")?;
let g = fx.groups().nth(index).filter(|g| is_tracker(g)).ok_or("not a 3D Camera Tracker")?;
if l.switches.adjustment {
return Err("3D Camera Tracker needs footage, not an adjustment layer".into());
}
let times = analysis_times(c, l);
if times.len() < 2 {
return Err("the layer must be at least two frames long".into());
}
let (w, h) = effectcraft_render::source_size(&self.project, l);
let size = if w == 0 { [c.width as f64, c.height as f64] } else { [w as f64, h as f64] };
let params = static_params(g);
let key = tracks_key(&self.project, comp, l, g);
let settings = ct::settings(&params);
let solve_key = ct::solve_key(&settings);
let tracks = ct::tracks(&params).filter(|_| str_of(g, TRACKS_KEY) == key);
let n = times.len();
let work = CamWork {
project: self.project.clone(),
footage: self.footage.clone(),
expr: self.expr.clone(),
cache: self.layer_cache.clone(),
comp,
layer,
effect_index: index,
size,
detailed: params.values.iter().any(|(k, v)| k.ends_with("detailedAnalysis") && v.as_bool()),
times,
tracks: tracks.clone(),
settings,
};
if !wait && self.offloads() {
drop(work);
self.offload_analysis(crate::offload::WorkerJob::Camera { comp, layer, effect })?;
return Ok(n);
}
let wait = wait || cfg!(target_arch = "wasm32");
let shared = Arc::new(CameraShared::default());
{
let mut st = lock(&shared.state);
st.solve_only = tracks.is_some();
st.step = if tracks.is_some() { 2 } else { 1 };
st.total = n as u64;
}
let thread = if wait {
run_work(work, &shared);
None
} else {
let sh = shared.clone();
Some(std::thread::Builder::new().name("camera-track".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?)
};
self.camera_job = Some(CameraJob { shared, thread, comp, layer, effect, key, solve_key });
self.events.push(Event::ProjectChanged { revision: self.revision });
self.poll_camera_job();
Ok(n)
}
/// Write a finished analysis into its effect (one undo step) and drop the job.
fn poll_camera_job(&mut self) -> bool {
let Some(job) = &self.camera_job else { return false };
if !job.is_finished() {
return false;
}
let mut job = self.camera_job.take().expect("job");
if let Some(t) = job.thread.take() {
let _ = t.join();
}
let st = job.progress();
let result = lock(&job.shared.result).take();
match (result, &st.error) {
(Some(res), None) if !st.cancelled => {
let (cid, lid, uid) = (job.comp, job.layer, job.effect);
let (key, skey) = (job.key.clone(), job.solve_key.clone());
let tracks_json = res.tracks.as_ref().map(CameraTracks::to_json);
let (solve_json, method, avg, msg, failed) = match &res.solve {
Ok(s) => (
s.to_json(),
s.method_used.label().to_string(),
s.average_error,
format!(
"3D Camera Tracker: solved {} frame(s), {} points, average error {:.2} px ({:.1} s)",
s.frames.len(),
s.points.len(),
s.average_error,
st.elapsed
),
false,
),
Err(e) => (String::new(), String::new(), 0.0, format!("3D Camera Tracker: {e}"), true),
};
let r = self.edit(if st.solve_only { "3D Camera Tracker Solve" } else { "3D Camera Tracker Analysis" }, None, |p, _| {
let g = p
.comp_mut(cid)
.and_then(|c| c.layer_mut(lid))
.and_then(|l| l.props.find_group_mut(uid))
.ok_or_else(|| crate::EngineError::Other("the 3D Camera Tracker was removed during analysis".into()))?;
let mut set = |m: &str, v: Value| {
if let Some(pr) = g.prop_mut(m) {
pr.value = v;
}
};
if let Some(t) = &tracks_json {
set(TRACKS, Value::Str(t.clone()));
set(TRACKS_KEY, Value::Str(key.clone()));
}
set(SOLVE, Value::Str(solve_json.clone()));
set(SOLVE_KEY, Value::Str(skey.clone()));
set(METHOD_USED, Value::Str(method.clone()));
set(AVERAGE_ERROR, Value::Scalar(avg));
Ok(())
});
let (msg, failed) = match r {
Ok(()) => (msg, failed),
Err(e) => (format!("3D Camera Tracker: {e}"), true),
};
self.events.push(Event::Toast { message: msg, error: failed });
}
_ => {
let msg = match &st.error {
Some(e) => format!("3D Camera Tracker analysis failed: {e}"),
None => "3D Camera Tracker analysis cancelled".into(),
};
self.events.push(Event::Toast { message: msg, error: st.error.is_some() });
self.bump();
}
}
true
}
/// Frontends call this every frame: finishes analyses and (with `auto`) starts queued
/// (re-)analyses in the background, like After Effects.
pub fn poll_camera(&mut self, auto: bool) -> bool {
let changed = self.poll_camera_job();
if auto && self.camera_job.is_none() && self.offloaded(JobKind::Camera).is_none() {
while let Some((c, l, e)) = self.camera_pending.first().copied() {
self.camera_pending.remove(0);
if self.start_camera(c, l, e, false).is_ok() {
return true;
}
}
}
changed
}
}
// ---------------------------------------------------------------- solve → composition
/// A tracker's solve placed in its composition.
pub struct Placed {
pub solve: Arc<CameraSolve>,
/// Canonical → comp world.
pub world: Similarity,
/// Comp pixels per layer pixel.
pub scale: f64,
/// Comp times of the analysed frames.
pub times: Vec<Tick>,
pub comp_size: [f64; 2],
}
impl Placed {
/// Comp-world position of a canonical point.
pub fn to_world(&self, x: V3) -> V3 {
self.world.apply(x)
}
/// Canonical position of a comp-world point.
pub fn from_world(&self, x: V3) -> V3 {
let r = &self.world;
linalg::scale(linalg::mtv(&r.r, linalg::sub(x, r.t)), 1.0 / r.s)
}
/// The 3D Tracker Camera on frame `k`: (position, orientation degrees, zoom).
pub fn camera(&self, k: usize) -> (V3, V3, f64) {
let c = &self.solve.frames[k];
let m = linalg::mm(&self.world.r, &linalg::mt(&c.r()));
(self.to_world(c.center), linalg::euler_xyz(&m), c.focal * self.scale)
}
/// Analysed frame at comp time `t`.
pub fn frame_at(&self, t: Tick) -> usize {
self.times.iter().enumerate().min_by_key(|(_, x)| (x.0 - t.0).abs()).map(|(i, _)| i).unwrap_or(0).min(self.solve.frames.len().saturating_sub(1))
}
}
/// Comp pixels per layer pixel of `layer` (its 2D transform at its In point).
fn layer_scale(project: &Project, cid: ItemId, layer: &Layer) -> f64 {
let Some(comp) = project.comp(cid) else { return 1.0 };
let ctx = EvalCtx::new(project, cid, comp, layer.in_point.max(Tick::ZERO));
let m = ctx.world_matrix(layer).0;
let s = m[0][0].hypot(m[1][0]);
if s.is_finite() && s > 1e-9 { s } else { 1.0 }
}
/// The solve of tracker `uid` on `lid`, placed in the comp.
pub fn placed(project: &Project, cid: ItemId, lid: LayerId, uid: Uid) -> Option<Placed> {
let comp = project.comp(cid)?;
let layer = comp.layer(lid)?;
let g = layer.props.find_group(uid).filter(|g| is_tracker(g))?;
let solve = ct::solve(&static_params(g))?;
let scale = layer_scale(project, cid, layer);
let k0 = solve.first_solved()?;
let comp_size = [comp.width as f64, comp.height as f64];
let world = solve.world(comp_size, solve.frames[k0].focal * scale);
let mut times = analysis_times(comp, layer);
times.truncate(solve.frames.len());
Some(Placed { solve, world, scale, times, comp_size })
}
/// What to create from a target (`camera.createFromSolve`).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum CreateKind {
Text,
Solid,
Null,
ShadowCatcher,
/// Only the camera.
Camera,
}
impl CreateKind {
pub fn from_name(s: &str) -> Option<CreateKind> {
match s.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() {
"text" => Some(CreateKind::Text),
"solid" => Some(CreateKind::Solid),
"null" => Some(CreateKind::Null),
"shadowcatcher" | "shadow" => Some(CreateKind::ShadowCatcher),
"camera" => Some(CreateKind::Camera),
_ => None,
}
}
}
/// Rotation (layer → world) of a layer lying on a plane with unit `normal` (towards the camera),
/// its x axis along `right` projected on the plane: the layer's front (−z) faces the camera.
pub fn plane_orientation(normal: V3, right: V3) -> V3 {
let z = linalg::scale(normal, -1.0);
let mut x = linalg::sub(right, linalg::scale(z, linalg::dot(right, z)));
if linalg::norm(x) < 1e-9 {
x = if z[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] };
x = linalg::sub(x, linalg::scale(z, linalg::dot(x, z)));
}
let x = linalg::normalize(x);
let y = linalg::cross(z, x);
let m = linalg::from_cols(x, y, z);
norm_angles(linalg::euler_xyz(&m))
}
fn norm_angles(a: V3) -> V3 {
a.map(|v| {
let r = v.rem_euclid(360.0);
if (r - 360.0).abs() < 1e-9 { 0.0 } else { r }
})
}
/// Key `path` on `l` at every `(time, value)`.
pub(crate) fn key_all(l: &mut Layer, path: &str, keys: &[(Tick, Value)]) {
if let Some(pr) = l.props.prop_mut(path) {
pr.keys = keys.iter().map(|(t, v)| Keyframe::new(*t, v.clone())).collect();
if let Some((_, v)) = keys.first() {
pr.value = v.clone();
}
}
}
/// Write the solved camera path into camera layer `l` (one-node, keyed on every analysed frame;
/// Zoom keyed for Variable Zoom).
pub(crate) fn write_camera(l: &mut Layer, pl: &Placed) {
l.auto_orient = AutoOrient::Off;
let n = pl.times.len().min(pl.solve.frames.len());
let mut pos = Vec::with_capacity(n);
let mut ori = Vec::with_capacity(n);
let mut zoom = Vec::with_capacity(n);
let mut prev: Option<V3> = None;
for k in 0..n {
let (p, o, z) = pl.camera(k);
// Unwrap the angles so per-frame keys interpolate the short way.
let o = match prev {
None => norm_angles(o),
Some(q) => [0, 1, 2].map(|i| q[i] + (o[i] - q[i] + 180.0).rem_euclid(360.0) - 180.0),
};
prev = Some(o);
let t = l.layer_time(pl.times[k]);
pos.push((t, Value::Vec3(p)));
ori.push((t, Value::Vec3(o)));
zoom.push((t, Value::Scalar(z)));
}
key_all(l, "transform/position", &pos);
key_all(l, "transform/orientation", &ori);
let variable = pl.solve.frames.iter().any(|f| (f.focal - pl.solve.frames[0].focal).abs() > 1e-6 * f.focal);
if variable {
key_all(l, "cameraOptions/zoom", &zoom);
} else if let Some((_, z)) = zoom.first()
&& let Some(pr) = l.props.prop_mut("cameraOptions/zoom")
{
pr.keys.clear();
pr.value = z.clone();
}
if let Some((_, z)) = zoom.first()
&& let Some(pr) = l.props.prop_mut("cameraOptions/focusDistance")
{
pr.value = z.clone();
}
// A one-node camera has no Point of Interest.
if let Some(tr) = l.props.sub_mut("transform") {
tr.children.retain(|c| !matches!(c, effectcraft_project::Node::Prop(p) if p.match_id == "poi"));
}
for rot in ["transform/rotationX", "transform/rotationY", "transform/rotation"] {
if let Some(pr) = l.props.prop_mut(rot) {
pr.keys.clear();
pr.value = Value::Scalar(0.0);
}
}
}
/// The target for `ids` on frame `k`: when fewer than three points are given, the nearest
/// solved neighbours (in 3D) complete the plane.
pub fn target_for(solve: &CameraSolve, ids: &[u32], k: usize) -> Option<Target> {
let mut use_ids: Vec<u32> = ids.iter().copied().filter(|i| solve.point(*i).is_some()).collect();
if use_ids.is_empty() {
return None;
}
if use_ids.len() < 3 {
let c = solve.point(use_ids[0])?.pos;
let mut others: Vec<(f64, u32)> = solve.visible(k).filter(|p| !use_ids.contains(&p.id)).map(|p| (linalg::norm(linalg::sub(p.pos, c)), p.id)).collect();
others.sort_by(|a, b| a.0.total_cmp(&b.0));
for (_, id) in others.into_iter().take(6usize.saturating_sub(use_ids.len())) {
use_ids.push(id);
}
let mut t = solve.target(&use_ids, k)?;
// The target sits on the chosen point(s).
let pts: Vec<V3> = ids.iter().filter_map(|i| solve.point(*i)).map(|p| p.pos).collect();
t.center = linalg::scale(pts.iter().fold([0.0; 3], |a, b| linalg::add(a, *b)), 1.0 / pts.len() as f64);
return Some(t);
}
solve.target(&use_ids, k)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn plane_orientation_faces_the_normal() {
let n = linalg::normalize([0.2, -0.9, -0.3]);
let o = plane_orientation(n, [1.0, 0.0, 0.0]);
let m = linalg::from_euler_xyz(o);
// Layer −z (its front) is the normal.
let front = linalg::mv(&m, [0.0, 0.0, -1.0]);
assert!(linalg::norm(linalg::sub(front, n)) < 1e-9);
}
}
+574
View File
@@ -0,0 +1,574 @@
//! 3D Camera Tracker commands: Animation ▸ Track Camera (`track.camera`), the effect's Analyze /
//! Cancel / Create Camera buttons, the viewer's track point selection and right-click menu
//! (Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … and Camera,
//! Create Multiple …, Delete Selected Points) and the status / points queries.
use effectcraft_color::Label;
use effectcraft_effects::camera_tracker::{self as ct, CAMERA, DELETED, SOLVE};
use effectcraft_keyframe::{Justify, TextDoc, Value as KV};
use effectcraft_project::build;
use effectcraft_project::{ItemId, ItemKind, Layer, LayerId, LayerSource, LightKind, Project, Solid, Uid};
use effectcraft_track::camtrack::linalg::{self, V3};
use effectcraft_track::camtrack::{ShotType, SolveMethod, Target, point_color};
use serde_json::{Value, json};
use super::{CommandSpec, b_p, bad, f_p, has_comp, layer_p, str_p};
use crate::camera_track::{CreateKind, Placed, is_tracker, placed, plane_orientation, static_params, target_for, write_camera};
use crate::{EngineError, Result, Session, cmd, query};
pub fn specs() -> Vec<CommandSpec> {
vec![
cmd!(
"track.camera",
"Track Camera",
["Animation"],
None,
"{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}",
can_track,
track_camera
),
cmd!("camera.analyze", "Analyze", [], None, "{layer?, effect?: uid|name|index, wait?: block until done}", has_comp, analyze),
cmd!("camera.cancel", "Cancel", [], None, "{}", is_analyzing, |s, _| Ok(json!({"stopped": s.stop_camera()}))),
query!("camera.solveStatus", "3D Camera Tracker Status", "{layer?, effect?}", status),
query!("camera.points", "3D Camera Tracker Points", "{layer?, effect?, time?: comp seconds} → visible solved points", points),
cmd!("camera.selectPoints", "Select Track Points", [], None, "{points: [id], add?, toggle?}", has_comp, select_points),
cmd!(
"camera.createFromSolve",
"Create from Camera Solve",
[],
None,
"{kind: text|solid|null|shadowCatcher|camera, points?: [id] (default: selected), target?: {center, normal, size?} (comp world), multiple?, layer?, effect?}",
has_comp,
create_from_solve
),
cmd!("camera.create", "Create Camera", [], None, "{layer?, effect?}", has_comp, |s, p| create_from_solve(s, &with_kind(p, "camera"))),
cmd!("camera.setGroundPlane", "Set Ground Plane and Origin", [], None, "{points?: [id] (default: selected), layer?, effect?}", has_comp, set_ground),
cmd!("camera.deletePoints", "Delete Selected Points", [], None, "{points?: [id] (default: selected), layer?, effect?, wait?}", has_comp, delete_points),
]
}
fn with_kind(p: &Value, kind: &str) -> Value {
let mut q = p.clone();
if !q.is_object() {
q = json!({});
}
q["kind"] = json!(kind);
q
}
fn can_track(s: &Session) -> std::result::Result<(), String> {
has_comp(s)?;
let c = s.active_comp().ok_or("no composition is open")?;
match s.state.selected_layers.first().and_then(|l| c.layer(*l)) {
Some(l) if l.has_video() && !l.switches.adjustment && !l.is_camera() && !l.is_light() => Ok(()),
Some(_) => Err("Track Camera needs a footage or composition layer".into()),
None => Err("select a layer to track".into()),
}
}
fn is_analyzing(s: &Session) -> std::result::Result<(), String> {
if s.is_camera_analyzing() { Ok(()) } else { Err("no 3D Camera Tracker analysis is running".into()) }
}
/// (comp, layer, effect uid) of a 3D Camera Tracker: `layer` / `effect` (uid, name or 1-based
/// index), else a selected one, else the first in the comp.
fn ct_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> {
let cid = super::comp_id(s, p)?;
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
let candidates: Vec<LayerId> = match p.get("layer") {
Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?],
None => {
let mut v: Vec<LayerId> = s.state.selected_props.iter().map(|(l, _)| *l).collect();
v.extend(s.state.selected_layers.iter().copied());
v.extend(comp.layers.iter().map(|l| l.id));
v
}
};
for lid in candidates {
let Some(fx) = comp.layer(lid).and_then(Layer::effects) else { continue };
let g = match p.get("effect") {
Some(Value::Number(n)) => {
let n = n.as_u64().unwrap_or(0);
fx.groups().find(|g| g.uid == n).or_else(|| fx.groups().nth((n as usize).saturating_sub(1)))
}
Some(Value::String(name)) => fx.groups().find(|g| &g.name == name),
_ => s
.state
.selected_props
.iter()
.find_map(|(l, u)| (*l == lid).then(|| fx.groups().find(|g| g.uid == *u && is_tracker(g))).flatten())
.or_else(|| fx.groups().find(|g| is_tracker(g))),
};
if let Some(g) = g.filter(|g| is_tracker(g)) {
return Ok((cid, lid, g.uid));
}
if p.get("layer").is_some() {
break;
}
}
Err(bad(cmd, "no 3D Camera Tracker found (apply Animation ▸ Track Camera first)"))
}
fn shot_p(p: &Value) -> Result<Option<ShotType>> {
match p.get("shotType") {
None => Ok(None),
Some(Value::Number(n)) => ShotType::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "shotType: 0..2")),
Some(Value::String(v)) => ShotType::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "shotType: fixed|variable|specify")),
Some(_) => Err(bad("track.camera", "shotType: fixed|variable|specify")),
}
}
fn method_p(p: &Value) -> Result<Option<SolveMethod>> {
match p.get("solveMethod") {
None => Ok(None),
Some(Value::Number(n)) => {
SolveMethod::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "solveMethod: 0..3"))
}
Some(Value::String(v)) => SolveMethod::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "solveMethod: auto|typical|flat|tripod")),
Some(_) => Err(bad("track.camera", "solveMethod: auto|typical|flat|tripod")),
}
}
/// Animation ▸ Track Camera / the Tracker panel button: apply the 3D Camera Tracker to the layer
/// (reusing one it already has), set the given options and start the analysis.
fn track_camera(s: &mut Session, p: &Value) -> Result<Value> {
let (cid, lid) = layer_p(s, p, "track.camera")?;
let shot = shot_p(p)?;
let method = method_p(p)?;
let aov = f_p(p, "aov").or_else(|| f_p(p, "angleOfView"));
if aov.is_some_and(|a| !(1.0..=170.0).contains(&a)) {
return Err(bad("track.camera", "aov: 1..170 degrees"));
}
let detailed = b_p(p, "detailed");
let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid));
let uid = match existing {
Some(u) => u,
None => {
let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [lid.0], "comp": cid.0}))?;
r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))?
}
};
if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() {
s.edit("3D Camera Tracker Settings", None, |proj, _| {
let g = proj
.comp_mut(cid)
.and_then(|c| c.layer_mut(lid))
.and_then(|l| l.props.find_group_mut(uid))
.ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?;
let mut set = |m: &str, v: KV| {
if let Some(pr) = g.prop_mut(m) {
pr.keys.clear();
pr.value = v;
}
};
if let Some(sh) = shot {
set("shotType", KV::Enum(ShotType::ALL.iter().position(|x| *x == sh).unwrap_or(0) as u32));
}
if let Some(a) = aov {
set("horizontalAngleOfView", KV::Scalar(a));
if shot.is_none() {
set("shotType", KV::Enum(2));
}
}
if let Some(m) = method {
set("advanced/solveMethod", KV::Enum(SolveMethod::ALL.iter().position(|x| *x == m).unwrap_or(0) as u32));
}
if let Some(d) = detailed {
set("advanced/detailedAnalysis", KV::Bool(d));
}
Ok(())
})?;
}
// Select the effect so the viewer shows its track points.
s.state.selected_props = vec![(lid, uid)];
let wait = b_p(p, "wait").unwrap_or(false);
let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?;
let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()});
if wait {
out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?;
}
Ok(out)
}
fn analyze(s: &mut Session, p: &Value) -> Result<Value> {
let (cid, lid, uid) = ct_p(s, p, "camera.analyze")?;
let wait = b_p(p, "wait").unwrap_or(false);
let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?;
let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()});
if wait {
out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?;
}
Ok(out)
}
fn status(s: &mut Session, p: &Value) -> Result<Value> {
s.poll_camera(false);
let mut out = json!({"running": s.is_camera_analyzing(), "progress": s.camera_progress(), "banner": s.camera_progress().map(|p| p.banner())});
let Ok((cid, lid, uid)) = ct_p(s, p, "camera.solveStatus") else { return Ok(out) };
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
let g = layer.props.find_group(uid).ok_or(EngineError::NoComp)?;
let params = static_params(g);
out["layer"] = json!(lid.0);
out["effect"] = json!(uid);
out["pending"] = json!(s.camera_pending.contains(&(cid, lid, uid)));
let tracks = ct::tracks(&params);
out["analyzed"] = json!(tracks.is_some());
if let Some(t) = &tracks {
out["tracks"] = json!(t.tracks.len());
out["frames"] = json!(t.frames);
}
let settings = ct::settings(&params);
out["shotType"] = json!(settings.shot.label());
out["solveMethod"] = json!(settings.method.label());
out["deleted"] = json!(settings.deleted.len());
let solve = ct::solve(&params);
out["solved"] = json!(solve.is_some());
if let Some(sv) = &solve {
let f = sv.focal();
out["methodUsed"] = json!(sv.method_used.label());
out["averageError"] = json!(sv.average_error);
out["points"] = json!(sv.points.len());
out["solvedFrames"] = json!(sv.frames.iter().filter(|c| c.solved).count());
out["focalLength"] = json!(f);
out["horizontalAngleOfView"] = json!(sv.hfov(f));
out["groundPlane"] = json!(sv.ground.is_some());
if let Some(pl) = placed(&s.project, cid, lid, uid) {
let k = pl.frame_at(s.time_of(cid));
let (pos, ori, zoom) = pl.camera(k);
out["frame"] = json!(k);
out["camera"] = json!({"position": pos, "orientation": ori, "zoom": zoom});
out["visible"] = json!(sv.visible(k).count());
}
}
let cam = params.get(CAMERA).map(|v| v.as_f64() as u64).unwrap_or(0);
out["cameraLayer"] = if cam > 0 && comp.layer(LayerId(cam)).is_some_and(Layer::is_camera) { json!(cam) } else { Value::Null };
out["selected"] = json!(s.state.camera_points);
Ok(out)
}
/// The visible solved points at the current (or given) time, with their comp positions.
fn points(s: &mut Session, p: &Value) -> Result<Value> {
let (cid, lid, uid) = ct_p(s, p, "camera.points")?;
let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad("camera.points", "the camera is not solved yet"))?;
let t = f_p(p, "time").map(effectcraft_time::Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid));
let k = pl.frame_at(t);
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
let ctx = effectcraft_render::EvalCtx::new(&s.project, cid, comp, t);
let m = ctx.world_matrix(layer);
let list: Vec<Value> = ct::projected(&pl.solve, k)
.into_iter()
.map(|(id, q, z)| {
let c = m.apply(effectcraft_geom::vec3(q[0], q[1], 0.0));
let sp = pl.solve.point(id).copied().unwrap_or_default();
let col = point_color(id);
json!({"id": id, "comp": [c.x, c.y], "layer": q, "depth": z, "world": pl.to_world(sp.pos), "error": sp.error, "color": col, "selected": s.state.camera_points.contains(&id)})
})
.collect();
Ok(json!({"frame": k, "points": list}))
}
fn ids_p(s: &Session, p: &Value) -> Vec<u32> {
match p.get("points") {
Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(),
Some(Value::Number(n)) => n.as_u64().map(|v| vec![v as u32]).unwrap_or_default(),
_ => s.state.camera_points.clone(),
}
}
fn select_points(s: &mut Session, p: &Value) -> Result<Value> {
let ids: Vec<u32> = match p.get("points") {
Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(),
_ => vec![],
};
if b_p(p, "toggle").unwrap_or(false) {
for id in ids {
match s.state.camera_points.iter().position(|x| *x == id) {
Some(i) => {
s.state.camera_points.remove(i);
}
None => s.state.camera_points.push(id),
}
}
} else if b_p(p, "add").unwrap_or(false) {
for id in ids {
if !s.state.camera_points.contains(&id) {
s.state.camera_points.push(id);
}
}
} else {
s.state.camera_points = ids;
}
s.bump();
Ok(json!({"selected": s.state.camera_points}))
}
/// The target of a create / ground command in canonical solve coordinates.
fn target_p(s: &Session, p: &Value, pl: &Placed, k: usize, cmd: &str) -> Result<Target> {
if let Some(t) = p.get("target") {
let v3 = |k: &str| -> Option<V3> {
let a = t.get(k)?.as_array()?;
Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2).and_then(Value::as_f64).unwrap_or(0.0)])
};
let (c, n) = (v3("center").ok_or_else(|| bad(cmd, "target.center: [x, y, z]"))?, v3("normal").unwrap_or([0.0, -1.0, 0.0]));
let size = t.get("size").and_then(Value::as_f64).unwrap_or(100.0);
let nc = linalg::normalize(linalg::mtv(&pl.world.r, n));
return Ok(Target { center: pl.from_world(c), normal: nc, size: size / pl.world.s });
}
let ids = ids_p(s, p);
if ids.is_empty() {
return Err(bad(cmd, "select track points (or give `points` / `target`)"));
}
target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))
}
fn next_name(c: &effectcraft_project::Comp, base: &str) -> String {
let n = (1..).find(|i| !c.layers.iter().any(|l| l.name == format!("{base} {i}"))).unwrap_or(1);
format!("{base} {n}")
}
fn place(l: &mut Layer, pos: V3, ori: V3, src: &Layer) {
l.switches.three_d = true;
l.in_point = src.in_point;
l.out_point = src.out_point;
if let Some(pr) = l.props.prop_mut("transform/position") {
pr.value = KV::Vec3(pos);
}
if let Some(pr) = l.props.prop_mut("transform/orientation") {
pr.value = KV::Vec3(ori);
}
}
/// Insert `l` above the tracked layer.
fn insert_above(proj: &mut Project, cid: ItemId, above: LayerId, l: Layer) -> Result<LayerId> {
let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?;
let at = comp.layers.iter().position(|x| x.id == above).unwrap_or(0);
let id = l.id;
comp.layers.insert(at, l);
Ok(id)
}
fn add_solid(proj: &mut Project, comp: &effectcraft_project::Comp, name: &str, size: u32, color: [f32; 3]) -> Layer {
let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder));
let sid = proj.add_item(name, Label::Red, Some(folder), ItemKind::Solid(Solid { color, width: size, height: size, pixel_aspect: 1.0 }));
build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (size, size), None)
}
/// Create Text / Solid / Null / Shadow Catcher (and Light) on the target, plus the 3D Tracker
/// Camera when the tracker has none yet — one undo step.
fn create_from_solve(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "camera.createFromSolve";
let (cid, lid, uid) = ct_p(s, p, cmd)?;
let kind = CreateKind::from_name(str_p(p, "kind").unwrap_or("null")).ok_or_else(|| bad(cmd, "kind: text|solid|null|shadowCatcher|camera"))?;
let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet (analysis still running or failed)"))?;
let k = pl.frame_at(s.time_of(cid));
let multiple = b_p(p, "multiple").unwrap_or(false);
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone();
let src = comp.layer(lid).ok_or(EngineError::NoComp)?.clone();
let g = src.props.find_group(uid).ok_or(EngineError::NoComp)?;
let params = static_params(g);
let size_pct = effectcraft_effects::warp_stab::param(&params, "targetSize").map(|v| v.as_f64()).unwrap_or(100.0) / 100.0;
let existing_cam = params.get(CAMERA).map(|v| v.as_f64() as u64).filter(|id| *id > 0 && comp.layer(LayerId(*id)).is_some_and(Layer::is_camera));
// Targets: one, or one per selected point (Create Multiple …) sharing the selection's plane.
let targets: Vec<Target> = if kind == CreateKind::Camera {
vec![]
} else {
let t = target_p(s, p, &pl, k, cmd)?;
if multiple {
let ids = ids_p(s, p);
ids.iter().filter_map(|i| pl.solve.point(*i)).map(|sp| Target { center: sp.pos, ..t }).collect()
} else {
vec![t]
}
};
// Camera right vector on this frame (world), to align created layers with the view.
let cam = &pl.solve.frames[k];
let right = linalg::mv(&pl.world.r, linalg::mtv(&cam.r(), [1.0, 0.0, 0.0]));
let label = match (kind, multiple) {
(CreateKind::Text, false) => "Create Text and Camera",
(CreateKind::Solid, false) => "Create Solid and Camera",
(CreateKind::Null, false) => "Create Null and Camera",
(CreateKind::ShadowCatcher, _) => "Create Shadow Catcher, Camera and Light",
(CreateKind::Text, true) => "Create Multiple Text Layers and Camera",
(CreateKind::Solid, true) => "Create Multiple Solids and Camera",
(CreateKind::Null, true) => "Create Multiple Nulls and Camera",
(CreateKind::Camera, _) => "Create Camera",
};
let (camera, created) = s.edit(label, None, |proj, st| {
let mut created = vec![];
// The camera: reuse the tracker's, else create "3D Tracker Camera" at the top.
let camera = match existing_cam {
Some(id) if kind != CreateKind::Camera => LayerId(id),
Some(id) => {
let l = proj.comp_mut(cid).and_then(|c| c.layer_mut(LayerId(id))).ok_or(EngineError::NoComp)?;
write_camera(l, &pl);
LayerId(id)
}
None => {
let mut l = build::layer(proj, &comp, "3D Tracker Camera", LayerSource::Camera, (comp.width, comp.height), None);
l.in_point = src.in_point;
l.out_point = src.out_point;
write_camera(&mut l, &pl);
let id = l.id;
proj.comp_mut(cid).ok_or(EngineError::NoComp)?.layers.insert(0, l);
if let Some(pr) = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).and_then(|g| g.prop_mut(CAMERA)) {
pr.value = KV::Scalar(id.0 as f64);
}
id
}
};
for t in &targets {
let center = pl.to_world(t.center);
let normal = linalg::normalize(linalg::mv(&pl.world.r, t.normal));
let ori = plane_orientation(normal, right);
let size = (t.size * pl.world.s * size_pct).max(1.0);
let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone();
match kind {
CreateKind::Text => {
let mut l = build::layer(proj, &cur, "Text", LayerSource::Text, (comp.width, comp.height), None);
let doc = TextDoc { text: "Text".into(), justify: Justify::Center, size: (size * 0.35).clamp(4.0, 2000.0), ..Default::default() };
if let Some(pr) = l.props.prop_mut("text/sourceText") {
pr.value = KV::Text(Box::new(doc));
}
place(&mut l, center, ori, &src);
created.push(insert_above(proj, cid, lid, l)?);
}
CreateKind::Solid => {
let name = next_name(&cur, "Track Solid");
let c = point_color((created.len() as u32).wrapping_add(7));
let mut l = add_solid(proj, &cur, &name, size.round().max(1.0) as u32, c);
place(&mut l, center, ori, &src);
created.push(insert_above(proj, cid, lid, l)?);
}
CreateKind::Null => {
let name = next_name(&cur, "Track Null");
let mut l = build::layer(proj, &cur, &name, LayerSource::Null, (comp.width, comp.height), None);
place(&mut l, center, ori, &src);
if let Some(pr) = l.props.prop_mut("transform/scale") {
pr.value = KV::Vec3([size, size, size]);
}
created.push(insert_above(proj, cid, lid, l)?);
}
CreateKind::ShadowCatcher => {
let name = next_name(&cur, "Shadow Catcher");
let mut l = add_solid(proj, &cur, &name, (size * 4.0).round().max(1.0) as u32, [1.0, 1.0, 1.0]);
place(&mut l, center, ori, &src);
if let Some(pr) = l.props.prop_mut("materialOptions/acceptsShadows") {
pr.value = KV::Enum(2);
}
if let Some(pr) = l.props.prop_mut("materialOptions/castsShadows") {
pr.value = KV::Enum(0);
}
let sc = created.len();
created.push(insert_above(proj, cid, lid, l)?);
let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone();
let lname = next_name(&cur, "Light");
let mut light = build::layer(proj, &cur, &lname, LayerSource::Light { kind: LightKind::Point }, (comp.width, comp.height), None);
light.in_point = src.in_point;
light.out_point = src.out_point;
let lp = linalg::add(center, linalg::add(linalg::scale(normal, size * 2.0), linalg::scale(right, size)));
if let Some(pr) = light.props.prop_mut("transform/position") {
pr.value = KV::Vec3(lp);
}
if let Some(pr) = light.props.prop_mut("lightOptions/castsShadows") {
pr.value = KV::Bool(true);
}
let _ = sc;
created.push(insert_above(proj, cid, created[created.len() - 1], light)?);
}
CreateKind::Camera => {}
}
}
st.selected_layers = if created.is_empty() { vec![camera] } else { created.clone() };
st.selected_props.clear();
Ok((camera, created))
})?;
let t0 = targets.first().map(
|t| json!({"center": pl.to_world(t.center), "normal": linalg::normalize(linalg::mv(&pl.world.r, t.normal)), "size": t.size * pl.world.s * size_pct}),
);
Ok(json!({"camera": camera.0, "layers": created.iter().map(|l| l.0).collect::<Vec<_>>(), "target": t0}))
}
fn set_ground(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "camera.setGroundPlane";
let (cid, lid, uid) = ct_p(s, p, cmd)?;
let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet"))?;
let k = pl.frame_at(s.time_of(cid));
let ids = ids_p(s, p);
if ids.is_empty() {
return Err(bad(cmd, "select track points (or give `points`)"));
}
let t = target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?;
let mut sv = (*pl.solve).clone();
let mut g = sv.ground_from(&ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?;
g.origin = t.center;
sv.ground = Some(g);
let json = sv.to_json();
s.edit("Set Ground Plane and Origin", None, |proj, _| {
let pr = proj
.comp_mut(cid)
.and_then(|c| c.layer_mut(lid))
.and_then(|l| l.props.find_group_mut(uid))
.and_then(|g| g.prop_mut(SOLVE))
.ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?;
pr.value = KV::Str(json);
Ok(())
})?;
Ok(json!({"origin": g.origin, "normal": g.normal}))
}
/// Delete points (and, with Auto-delete Points Across Time, the points of other tracks on the
/// same 3D feature), then re-solve.
fn delete_points(s: &mut Session, p: &Value) -> Result<Value> {
let cmd = "camera.deletePoints";
let (cid, lid, uid) = ct_p(s, p, cmd)?;
let ids = ids_p(s, p);
if ids.is_empty() {
return Err(bad(cmd, "select track points (or give `points`)"));
}
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
let g = comp.layer(lid).and_then(|l| l.props.find_group(uid)).ok_or(EngineError::NoComp)?;
let params = static_params(g);
let mut del = ct::deleted(&params);
let auto = effectcraft_effects::warp_stab::param(&params, "advanced/autoDeletePoints").is_some_and(KV::as_bool);
let mut add = ids.clone();
if auto && let Some(sv) = ct::solve(&params) {
// Same feature tracked again at other times: solved within 1 % of its depth.
let cam0 = sv.first_solved().map(|k| sv.frames[k].center).unwrap_or([0.0; 3]);
for id in &ids {
let Some(pt) = sv.point(*id) else { continue };
let r = linalg::norm(linalg::sub(pt.pos, cam0)) * 0.01;
for q in &sv.points {
let disjoint = q.last < pt.first || q.first > pt.last;
if q.id != *id && disjoint && linalg::norm(linalg::sub(q.pos, pt.pos)) < r {
add.push(q.id);
}
}
}
}
for id in add {
if !del.contains(&id) {
del.push(id);
}
}
del.sort_unstable();
let json = serde_json::to_string(&del).unwrap_or_default();
s.edit("Delete Track Points", None, |proj, _| {
let pr = proj
.comp_mut(cid)
.and_then(|c| c.layer_mut(lid))
.and_then(|l| l.props.find_group_mut(uid))
.and_then(|g| g.prop_mut(DELETED))
.ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?;
pr.value = KV::Str(json);
Ok(())
})?;
s.state.camera_points.retain(|x| !del.contains(x));
// Re-solve now (After Effects re-solves after deleting points).
let wait = b_p(p, "wait").unwrap_or(false);
if !s.is_camera_analyzing() {
s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?;
}
Ok(json!({"deleted": del, "running": s.is_camera_analyzing()}))
}
+139
View File
@@ -0,0 +1,139 @@
//! Layer ▸ Create: Convert to Editable Text (Photoshop type layers), Create Shapes from Text,
//! Create Masks from Text and Create Shapes from Vector Layer (SVG footage).
use effectcraft_color::Label;
use effectcraft_project::{ItemKind, LayerSource, Solid};
use serde_json::{Value, json};
use super::{CommandSpec, bad, has_layers, layers_p, str_p};
use crate::{EngineError, Result, Session, cmd};
const OPS: &str = "editableText|shapesFromText|masksFromText|shapesFromVector";
fn create(s: &mut Session, p: &Value) -> Result<Value> {
let op = str_p(p, "op").ok_or_else(|| bad("layer.create", format!("missing `op` ({OPS})")))?;
let (cid, lids) = layers_p(s, p)?;
if lids.is_empty() {
return Err(bad("layer.create", "select a layer first"));
}
let time = s.time();
let mut created = vec![];
let mut skipped = vec![];
for lid in lids {
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone();
let Some(src) = comp.layer(lid).cloned() else { continue };
let footage = src.source.item().and_then(|i| match &s.project.item(i)?.kind {
ItemKind::Footage(f) => Some(f.clone()),
_ => None,
});
match op {
"shapesFromVector" => {
let Some(f) = footage.filter(|f| f.codec == "SVG" || f.path.to_ascii_lowercase().ends_with(".svg")) else {
skipped.push(format!("{}: not a vector (SVG) footage layer", src.name));
continue;
};
let bytes = s.services.read_file(&f.path).map_err(|e| EngineError::Other(format!("cannot read {}: {e}", f.path)))?;
let doc = effectcraft_svg::parse(&bytes).map_err(|e| EngineError::Other(format!("{}: {e}", f.path)))?;
let id = s.edit("Create Shapes from Vector Layer", None, |proj, st| {
let l = crate::vector::shapes_from_vector(proj, &comp, &src, &doc);
let id = l.id;
insert_above(proj, cid, lid, l, true)?;
st.selected_layers = vec![id];
Ok(id)
})?;
created.push(id.0);
}
"shapesFromText" => {
if !matches!(src.source, LayerSource::Text) {
skipped.push(format!("{}: not a text layer", src.name));
continue;
}
let snapshot = s.project.clone();
let id = s.edit("Create Shapes from Text", None, |proj, st| {
let l = crate::vector::shapes_from_text(proj, &comp, &snapshot, cid, &src, time)
.ok_or_else(|| bad("layer.create", "the text layer has no glyphs"))?;
let id = l.id;
insert_above(proj, cid, lid, l, true)?;
st.selected_layers = vec![id];
Ok(id)
})?;
created.push(id.0);
}
"masksFromText" => {
if !matches!(src.source, LayerSource::Text) {
skipped.push(format!("{}: not a text layer", src.name));
continue;
}
let outlines = crate::vector::text_outlines_in_comp(&s.project, cid, &comp, &src, time);
if outlines.is_empty() {
skipped.push(format!("{}: no glyphs", src.name));
continue;
}
let id = s.edit("Create Masks from Text", None, |proj, st| {
let name = comp.unique_layer_name(&format!("{} Outlines", src.name));
let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder));
let sid = proj.add_item(
&name,
Label::Red,
Some(folder),
ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: comp.width, height: comp.height, pixel_aspect: 1.0 }),
);
let mut l = crate::vector::masks_layer(proj, &comp, &name, outlines, sid);
l.in_point = src.in_point;
l.out_point = src.out_point;
let id = l.id;
insert_above(proj, cid, lid, l, true)?;
st.selected_layers = vec![id];
Ok(id)
})?;
created.push(id.0);
}
"editableText" => {
let Some((f, sl)) = footage.and_then(|f| f.layer.clone().map(|l| (f, l))) else {
skipped.push(format!("{}: not a Photoshop layer", src.name));
continue;
};
let bytes = s.services.read_file(&f.path).map_err(|e| EngineError::Other(format!("cannot read {}: {e}", f.path)))?;
let psd = effectcraft_psd::Psd::parse(bytes).map_err(|e| EngineError::Other(e.to_string()))?;
let id = s.edit("Convert to Editable Text", None, |proj, st| {
let l = crate::psd_import::editable_text(proj, &comp, &src, &psd, sl.index as usize)
.ok_or_else(|| bad("layer.create", format!("{}: the Photoshop layer has no text", src.name)))?;
let id = l.id;
let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?;
let at = c.layers.iter().position(|x| x.id == lid).ok_or(EngineError::NoComp)?;
c.layers[at] = l;
st.selected_layers = vec![id];
Ok(id)
})?;
created.push(id.0);
}
other => return Err(bad("layer.create", format!("`{other}` is not available yet (supported: {OPS})"))),
}
}
if created.is_empty() && !skipped.is_empty() {
return Err(bad("layer.create", skipped.join("; ")));
}
Ok(json!({"layers": created, "skipped": skipped}))
}
/// Insert `l` directly above layer `below`; optionally switch the source's video off (as After
/// Effects does after converting).
fn insert_above(
proj: &mut effectcraft_project::Project,
cid: effectcraft_project::ItemId,
below: effectcraft_project::LayerId,
l: effectcraft_project::Layer,
hide_source: bool,
) -> Result<()> {
let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?;
let at = c.layers.iter().position(|x| x.id == below).unwrap_or(0);
if hide_source && let Some(src) = c.layers.get_mut(at) {
src.switches.video = false;
}
c.layers.insert(at, l);
Ok(())
}
pub fn specs() -> Vec<CommandSpec> {
vec![cmd!("layer.create", "Create", [], None, "{op: editableText|shapesFromText|masksFromText|shapesFromVector, layers?}", has_layers, create)]
}
+31 -1
View File
@@ -525,12 +525,41 @@ fn purge_caches(s: &mut Session, p: &Value) -> Result<Value> {
s.snapshot = None;
s.state.viewer.show_snapshot = false;
}
let memory = matches!(what.as_str(), "all" | "memoryAndDisk" | "memory" | "image");
let disk = matches!(what.as_str(), "all" | "memoryAndDisk" | "disk");
if memory {
s.layer_cache.clear();
}
let mut disk_entries = 0;
if disk && let Some(dc) = &s.disk_cache {
disk_entries = dc.stats().entries;
dc.clear();
}
if matches!(what.as_str(), "all" | "image" | "memory" | "memoryAndDisk") {
effectcraft_effects::roto::purge();
}
s.events.push(crate::Event::PurgeCaches);
s.toast(format!("Purged {what} cache"));
Ok(json!({"purged": what}))
Ok(json!({"purged": what, "diskEntries": disk_entries}))
}
/// `cache.diskStats`: the disk cache's folder, limit and contents.
pub(crate) fn disk_stats(s: &mut Session, _: &Value) -> Result<Value> {
let Some(dc) = &s.disk_cache else { return Ok(json!({"enabled": false})) };
let st = dc.stats();
Ok(json!({
"enabled": true,
"folder": dc.folder().to_string_lossy(),
"entries": st.entries,
"frames": dc.keys(effectcraft_render::disk_cache::Kind::Frame).len(),
"layers": dc.keys(effectcraft_render::disk_cache::Kind::Layer).len(),
"bytes": st.bytes,
"maxBytes": st.max_bytes,
"hits": st.hits,
"misses": st.misses,
"writes": st.writes,
"evictions": st.evictions,
}))
}
/// Path of the footage behind the selected layer or project item.
@@ -599,6 +628,7 @@ pub fn specs() -> Vec<CommandSpec> {
cmd!("edit.selectLabelGroup", "Select Label Group", ["Edit", "Label"], None, "{}", has_props_or_layers_or_items, select_label_group),
cmd!("edit.purgeUndo", "Undo", ["Edit", "Purge"], None, "{}", always_ok, purge),
cmd!("edit.purge", "Purge", [], None, "{what?: all|memoryAndDisk|memory|disk|3d|image|snapshot}", always_ok, purge_caches),
crate::query!("cache.diskStats", "Disk Cache Statistics", "{}", disk_stats),
cmd!("edit.editOriginal", "Edit Original...", ["Edit"], Some("Cmd+E"), "{}", has_selected_footage, edit_original),
]
}
+95 -3
View File
@@ -84,10 +84,48 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
Some(Value::String(x)) => vec![x.clone()],
_ => return Err(bad("file.import", "missing `paths`")),
};
let importer = s.importer.clone().ok_or_else(|| EngineError::Other("media import is not available in this build".into()))?;
// Photoshop documents as compositions (Import As: Composition / – Retain Layer Sizes).
let import_as = str_p(p, "importAs").unwrap_or("footage");
let retain = match import_as {
"footage" => None,
"composition" | "comp" => Some(false),
"compositionLayerSizes" | "compositionRetainLayerSizes" | "layerSizes" => Some(true),
other => return Err(bad("file.import", format!("importAs: footage|composition|compositionLayerSizes, not `{other}`"))),
};
let mut paths = paths;
let mut out_comps = vec![];
let mut ids = vec![];
let mut errors = vec![];
if let Some(retain) = retain {
let mut rest = vec![];
for path in paths {
let bytes = match s.services.read_file(&path) {
Ok(b) if effectcraft_psd::is_psd(&b) => b,
_ => {
rest.push(path);
continue;
}
};
match import_psd_comp(s, &path, bytes, retain) {
Ok((comp, items, warnings)) => {
out_comps.push(comp);
ids.extend(items);
errors.extend(warnings);
}
Err(e) => errors.push(format!("{path}: {e}")),
}
}
paths = rest;
if paths.is_empty() {
if let Some(c) = out_comps.first() {
s.open_comp(effectcraft_project::ItemId(*c));
}
return Ok(json!({"items": ids, "comps": out_comps, "errors": errors}));
}
}
let importer = s.importer.clone().ok_or_else(|| EngineError::Other("media import is not available in this build".into()))?;
let mut probed = vec![];
let psd_layer = p.get("layer").cloned();
let seq_rate = effectcraft_time::FrameRate::from_f64(s.prefs.import.sequence_fps);
for path in &paths {
match importer.probe(path) {
@@ -99,6 +137,18 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
f.frame_rate = r;
f.duration = r.tick_of(frames.max(1));
}
// Choose Layer: one layer of a Photoshop document (document-sized).
if let Some(sel) = &psd_layer
&& f.codec == "PSD"
{
match s.services.read_file(path).ok().and_then(|b| effectcraft_psd::Psd::parse(b).ok()).and_then(|d| find_psd_layer(&d, sel)) {
Some((index, name)) => f.layer = Some(effectcraft_project::SourceLayer { index: index as u32, name, layer_size: false }),
None => {
errors.push(format!("{path}: no layer {sel}"));
continue;
}
}
}
probed.push((path.clone(), f))
}
Err(e) => errors.push(format!("{path}: {e}")),
@@ -107,6 +157,10 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
s.edit("Import", None, |proj, st| {
for (path, f) in probed {
let name = std::path::Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or(path.clone());
let name = match &f.layer {
Some(l) => format!("{}/{name}", l.name),
None => name,
};
let label = match f.kind {
FootageKind::Still | FootageKind::Sequence => Label::Lavender,
FootageKind::Audio => Label::SeaFoam,
@@ -118,7 +172,37 @@ fn import(s: &mut Session, p: &Value) -> Result<Value> {
}
Ok(())
})?;
Ok(json!({"items": ids, "errors": errors}))
if let Some(c) = out_comps.first() {
s.open_comp(effectcraft_project::ItemId(*c));
}
Ok(json!({"items": ids, "comps": out_comps, "errors": errors}))
}
/// A Photoshop layer by index or name (pixel layers only).
fn find_psd_layer(d: &effectcraft_psd::Psd, sel: &Value) -> Option<(usize, String)> {
let l = match sel {
Value::Number(n) => d.layers.get(n.as_u64()? as usize)?,
Value::String(name) => d.layers.iter().find(|l| &l.name == name)?,
_ => return None,
};
Some((l.index, l.name.clone()))
}
/// Import a Photoshop document as a composition (one undo step). Returns (comp, items, warnings).
fn import_psd_comp(s: &mut Session, path: &str, bytes: Vec<u8>, retain: bool) -> Result<(u64, Vec<u64>, Vec<String>)> {
let psd = effectcraft_psd::Psd::parse(bytes).map_err(|e| EngineError::Other(e.to_string()))?;
let name = std::path::Path::new(path).file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "Photoshop".into());
let rate = effectcraft_time::FrameRate::FPS_29_97;
let secs = if s.prefs.import.still_footage == "seconds" { s.prefs.import.still_seconds } else { 10.0 };
let duration = rate.snap_nearest(effectcraft_time::Tick::from_seconds_f64(secs));
let r = s.edit("Import", None, |proj, st| {
let r = crate::psd_import::import(proj, &psd, path, &name, retain, rate, duration);
st.project_selection = vec![r.comp];
Ok(r)
})?;
let mut items = vec![r.folder.0];
items.extend(r.items.iter().map(|i| i.0));
Ok((r.comp.0, items, r.warnings))
}
fn project_settings(s: &mut Session, p: &Value) -> Result<Value> {
@@ -224,7 +308,15 @@ pub fn specs() -> Vec<CommandSpec> {
cmd!("file.saveAs", "Save As...", ["File", "Save As"], Some("Cmd+Shift+S"), "{path}", always, save_as),
cmd!("file.incrementAndSave", "Increment and Save", ["File"], Some("Cmd+Alt+Shift+S"), "{}", has_path, increment_save),
cmd!("file.revert", "Revert", ["File"], None, "{}", has_path, revert),
cmd!("file.import", "File...", ["File", "Import"], Some("Cmd+I"), "{paths: [string]}", always, import),
cmd!(
"file.import",
"File...",
["File", "Import"],
Some("Cmd+I"),
"{paths: [string], importAs?: footage|composition|compositionLayerSizes (Photoshop files), layer?: name|index (footage of one Photoshop layer)}",
always,
import
),
cmd!(
"file.projectSettings",
"Project Settings...",
+1
View File
@@ -107,6 +107,7 @@ fn placeholder_footage(p: &Value) -> Footage {
missing: true,
sequence: vec![],
color_profile: None,
layer: None,
}
}
+4
View File
@@ -4,8 +4,10 @@
mod anim;
pub mod anim_tools;
mod animation;
mod camera_cmds;
mod comp;
mod comp_more;
mod create;
mod edit;
mod effect;
mod file;
@@ -124,6 +126,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
v.extend(help::specs());
v.extend(query::specs());
v.extend(layer_menu::specs());
v.extend(create::specs());
v.extend(animation::specs());
v.extend(view::specs());
v.extend(viewer_cmds::specs());
@@ -135,6 +138,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
v.extend(track::specs());
v.extend(mask_interp::specs());
v.extend(warp_cmds::specs());
v.extend(camera_cmds::specs());
v.extend(roto_cmds::specs());
v.extend(paint::specs());
v.extend(puppet::specs());
+4 -4
View File
@@ -131,7 +131,7 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result<bool>
fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result<bool> {
let mut any = false;
if let Some(f) = str_p(p, "format") {
let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|png|jpeg|tiff|exr|gif"))?;
let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?;
om.set_format(f);
any = true;
}
@@ -256,7 +256,7 @@ fn add(s: &mut Session, p: &Value) -> Result<Value> {
let cid = s.comp_for_queue(p).ok_or(EngineError::NoComp)?;
let mut it = RenderQueueItem::new(0, cid);
if let Some(f) = str_p(p, "format") {
let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|png|jpeg|tiff|exr|gif"))?;
let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?;
it.output = OutputModule::for_format(f);
}
apply_settings(&mut it.settings, p, "renderQueue.add")?;
@@ -434,7 +434,7 @@ fn add_output_module(s: &mut Session, p: &Value) -> Result<Value> {
let n = it.extra_outputs.len() + 2;
let mut om = it.output.clone();
if let Some(f) = str_p(p, "format") {
let f = OutputFormat::from_name(f).ok_or_else(|| bad("render.addOutputModule", "format: h264|prores|png|jpeg|tiff|exr|gif"))?;
let f = OutputFormat::from_name(f).ok_or_else(|| bad("render.addOutputModule", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?;
om = OutputModule::for_format(f);
}
if str_p(p, "output").is_none() && str_p(p, "path").is_none() {
@@ -509,7 +509,7 @@ pub fn specs() -> Vec<CommandSpec> {
"Add to Render Queue",
["Composition"],
Some("Cmd+M"),
"{comp?: id|name, format?: h264|prores|png|jpeg|tiff|exr|gif, output?: path|template, quality?: best|draft|1-100 (jpeg), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}",
"{comp?: id|name, format?: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, output?: path|template, quality?: best|draft|1-100 (jpeg, webm), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}",
can_add,
add
),
-2
View File
@@ -29,7 +29,6 @@ pub fn specs() -> Vec<CommandSpec> {
// Mask options not in the mask model yet; pen / vertex editing (timeline milestone).
// Time (timeline milestone).
stub!("layer.alignVideoToData", "Align Video to Data", ["Layer", "Time"], "{}"),
stub!("layer.create", "Create", [], "{op}"),
stub!("layer.autoTrace", "Auto-trace...", ["Layer"], "{}"),
stub!("layer.sceneEditDetection", "Scene Edit Detection...", ["Layer"], "{}"),
// Cameras / lights / materials / 3D views (3D milestone).
@@ -51,7 +50,6 @@ pub fn specs() -> Vec<CommandSpec> {
stub!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], "{}"),
stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"),
stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"),
stub!("track.camera", "Track Camera", ["Animation"], "{}"),
// Workspaces and panels that don't exist yet.
stub!("window.unavailablePanel", "Panel", [], "{panel}"),
]
+33
View File
@@ -9,6 +9,7 @@
//! (compositions are `Arc`s, so untouched comps are shared).
pub mod autosave;
pub mod camera_track;
pub mod commands;
pub mod config;
pub mod demo;
@@ -17,11 +18,13 @@ pub mod mask_track;
pub mod menus;
pub mod offload;
pub mod prefs;
pub mod psd_import;
pub mod render_queue;
pub mod roto;
mod session_settings;
pub mod shortcuts;
pub mod tracking;
pub mod vector;
pub mod viewer;
pub mod warp;
@@ -173,6 +176,9 @@ pub struct EditorState {
/// Tracker panel ▸ Method in mask mode (Track Mask).
#[serde(default)]
pub mask_track_method: mask_track::MaskMethod,
/// 3D Camera Tracker: the selected track points (ids) in the viewer.
#[serde(default)]
pub camera_points: Vec<u32>,
/// Mask Interpolation panel options.
#[serde(default)]
pub mask_interp: commands::mask_interp::MaskInterpOptions,
@@ -269,6 +275,10 @@ pub struct Session {
pub warp_job: Option<warp::WarpJob>,
/// Warp Stabilizers waiting for (re-)analysis: (comp, layer, effect uid).
pub warp_pending: Vec<(ItemId, LayerId, Uid)>,
/// The running (or finished, not yet polled) 3D Camera Tracker analysis.
pub camera_job: Option<camera_track::CameraJob>,
/// 3D Camera Trackers waiting for (re-)analysis: (comp, layer, effect uid).
pub camera_pending: Vec<(ItemId, LayerId, Uid)>,
/// The running (or finished, not yet polled) Roto Brush propagation / Freeze.
pub roto_job: Option<roto::RotoJob>,
/// Roto Brush instances edited since their last propagation: (comp, layer, effect uid).
@@ -278,6 +288,8 @@ pub struct Session {
pub journal: Vec<(String, Value)>,
/// Processed-layer pixels reused across frames and edits (content-keyed, never stale).
pub layer_cache: Arc<LayerCache>,
/// The persistent disk cache (Settings ▸ Media & Disk Cache), when enabled.
pub disk_cache: Option<Arc<effectcraft_render::disk_cache::DiskCache>>,
/// Settings (Preferences).
pub prefs: prefs::Prefs,
/// Bumped whenever settings change (frontends re-apply theme, labels…).
@@ -340,11 +352,14 @@ impl Default for Session {
mask_job: None,
warp_job: None,
warp_pending: vec![],
camera_job: None,
camera_pending: vec![],
roto_job: None,
roto_pending: vec![],
events: vec![],
journal: vec![],
layer_cache: Arc::new(LayerCache::default()),
disk_cache: None,
prefs: prefs::Prefs::default(),
prefs_revision: 0,
config: None,
@@ -411,6 +426,12 @@ impl Session {
self.warp_pending.push(w);
}
}
// So are 3D Camera Tracker tracks (and solves made with other settings).
for w in camera_track::invalidate(&before, &mut p) {
if !self.camera_pending.contains(&w) {
self.camera_pending.push(w);
}
}
// Roto Brush Input Keys follow the frames; edited instances propagate again.
for r in roto::sync(&before, &mut p) {
if !self.roto_pending.contains(&r) {
@@ -436,6 +457,9 @@ impl Session {
pub fn bump(&mut self) {
self.revision += 1;
if self.disk_cache.is_some() {
self.layer_cache.set_disk_salt(effectcraft_render::disk_cache::footage_salt(&self.project));
}
self.events.push(Event::ProjectChanged { revision: self.revision });
}
@@ -594,6 +618,9 @@ impl Session {
self.stop_warp();
self.warp_job = None;
self.warp_pending.clear();
self.stop_camera();
self.camera_job = None;
self.camera_pending.clear();
self.stop_roto();
self.roto_job = None;
self.roto_pending.clear();
@@ -620,6 +647,10 @@ mod tests_3d;
#[cfg(test)]
mod tests_anim_tools;
#[cfg(test)]
mod tests_camera_track;
#[cfg(test)]
mod tests_disk_cache;
#[cfg(test)]
mod tests_effects;
#[cfg(test)]
mod tests_fidelity;
@@ -650,6 +681,8 @@ mod tests_timeline;
#[cfg(test)]
mod tests_track;
#[cfg(test)]
mod tests_vector_import;
#[cfg(test)]
mod tests_viewer;
/// Font families available to text layers (bundled + scanned system fonts).
+14 -1
View File
@@ -16,6 +16,7 @@
//! | Track Motion / Stabilize Motion | the tracker analysis | the tracker group |
//! | Mask tracking | the mask track | the mask group |
//! | Roto Brush Freeze | the freeze | the effect group |
//! | 3D Camera Tracker analysis | [`Session::start_camera`] | the effect's property group |
//!
//! Roto Brush propagation fills the session's segmentation cache rather than the project, so it
//! keeps running on the UI thread.
@@ -65,6 +66,11 @@ pub enum WorkerJob {
layer: LayerId,
effect: Uid,
},
Camera {
comp: ItemId,
layer: LayerId,
effect: Uid,
},
RotoFreeze {
comp: ItemId,
layer: LayerId,
@@ -95,6 +101,7 @@ impl WorkerJob {
match self {
WorkerJob::Render { .. } => JobKind::Render,
WorkerJob::Warp { .. } => JobKind::Warp,
WorkerJob::Camera { .. } => JobKind::Camera,
WorkerJob::RotoFreeze { .. } => JobKind::RotoFreeze,
WorkerJob::Track { .. } => JobKind::Track,
WorkerJob::MaskTrack { .. } => JobKind::MaskTrack,
@@ -105,7 +112,9 @@ impl WorkerJob {
pub fn target(&self) -> Option<(ItemId, LayerId, Uid)> {
match *self {
WorkerJob::Render { .. } => None,
WorkerJob::Warp { comp, layer, effect } | WorkerJob::RotoFreeze { comp, layer, effect } => Some((comp, layer, effect)),
WorkerJob::Warp { comp, layer, effect } | WorkerJob::Camera { comp, layer, effect } | WorkerJob::RotoFreeze { comp, layer, effect } => {
Some((comp, layer, effect))
}
WorkerJob::Track { comp, layer, tracker, .. } => Some((comp, layer, tracker)),
WorkerJob::MaskTrack { comp, layer, mask, .. } => Some((comp, layer, mask)),
}
@@ -117,6 +126,7 @@ impl WorkerJob {
pub enum JobKind {
Render,
Warp,
Camera,
RotoFreeze,
Track,
MaskTrack,
@@ -128,6 +138,7 @@ impl JobKind {
match self {
JobKind::Render => "Render",
JobKind::Warp => "Warp Stabilizer Analysis",
JobKind::Camera => "3D Camera Tracker Analysis",
JobKind::RotoFreeze => "Freeze",
JobKind::Track => "Analyze Track",
JobKind::MaskTrack => "Track Mask",
@@ -277,6 +288,7 @@ fn run_job(s: &mut Session, job: WorkerJob, post: Post) -> Result<(), String> {
Ok(())
}
WorkerJob::Warp { comp, layer, effect } => analysis(s, kind, target, post, |s| s.start_warp(comp, layer, effect, true).map(|_| ())),
WorkerJob::Camera { comp, layer, effect } => analysis(s, kind, target, post, |s| s.start_camera(comp, layer, effect, true).map(|_| ())),
WorkerJob::RotoFreeze { comp, layer, effect } => analysis(s, kind, target, post, |s| {
s.start_roto(comp, layer, effect, crate::roto::RotoTask::Freeze, crate::roto::Direction::Both, true).map(|_| ())
}),
@@ -554,6 +566,7 @@ mod tests {
WorkerJob::Render { items: vec![vec![(item.clone(), "/out.gif".into())]] },
WorkerJob::Warp { comp, layer: LayerId(3), effect: 9 },
WorkerJob::RotoFreeze { comp, layer: LayerId(3), effect: 9 },
WorkerJob::Camera { comp, layer: LayerId(3), effect: 9 },
WorkerJob::Track {
comp,
layer: LayerId(1),
+634
View File
@@ -0,0 +1,634 @@
//! Photoshop import as a composition (File ▸ Import ▸ File… with Import As: Composition or
//! Composition – Retain Layer Sizes), and Layer ▸ Create ▸ Convert to Editable Text.
//!
//! Every pixel layer becomes a footage item that names its layer (`Footage::layer`), placed in
//! a composition the size of the document. Groups become precomps (pass-through groups collapse
//! so their blend modes act on the layers below, like Photoshop); type layers become editable
//! text layers; layer masks are baked into the layer's alpha and vector masks become masks;
//! layer effects map to Layer Styles; adjustment layers become adjustment layers with the
//! matching effect (best effort); clipped layers preserve the transparency underneath.
use std::sync::Arc;
use effectcraft_color::BlendMode;
use effectcraft_color::Label;
use effectcraft_keyframe::{Gradient, Justify, ShapePath, TextDoc, Value};
use effectcraft_project::build::{self, Ids};
use effectcraft_project::styles::{self, STYLE_BLEND_MODES};
use effectcraft_project::{AlphaMode, Comp, Footage, FootageKind, ItemId, ItemKind, Layer, LayerSource, MaskMode, Project, PropGroup, Solid, SourceLayer};
use effectcraft_psd::{Adjustment, DValue, Descriptor, Node, Psd};
use effectcraft_time::{FrameRate, Tick};
/// Photoshop blend mode key → compositor mode (`pass` = pass through, handled by collapsing).
pub fn blend_mode(key: &str) -> BlendMode {
match key {
"diss" => BlendMode::Dissolve,
"dark" => BlendMode::Darken,
"mul " => BlendMode::Multiply,
"idiv" => BlendMode::ColorBurn,
"lbrn" => BlendMode::LinearBurn,
"dkCl" => BlendMode::DarkerColor,
"lite" => BlendMode::Lighten,
"scrn" => BlendMode::Screen,
"div " => BlendMode::ColorDodge,
"lddg" => BlendMode::LinearDodge,
"lgCl" => BlendMode::LighterColor,
"over" => BlendMode::Overlay,
"sLit" => BlendMode::SoftLight,
"hLit" => BlendMode::HardLight,
"vLit" => BlendMode::VividLight,
"lLit" => BlendMode::LinearLight,
"pLit" => BlendMode::PinLight,
"hMix" => BlendMode::HardMix,
"diff" => BlendMode::Difference,
"smud" => BlendMode::Exclusion,
"fsub" => BlendMode::Subtract,
"fdiv" => BlendMode::Divide,
"hue " => BlendMode::Hue,
"sat " => BlendMode::Saturation,
"colr" => BlendMode::Color,
"lum " => BlendMode::Luminosity,
_ => BlendMode::Normal,
}
}
/// Descriptor blend mode enum (`BlnM`) → compositor mode.
fn desc_blend(v: Option<&str>) -> Option<BlendMode> {
Some(match v? {
"Nrml" => BlendMode::Normal,
"Dslv" => BlendMode::Dissolve,
"Drkn" => BlendMode::Darken,
"Mltp" => BlendMode::Multiply,
"CBrn" => BlendMode::ColorBurn,
"linearBurn" => BlendMode::LinearBurn,
"darkerColor" => BlendMode::DarkerColor,
"Lghn" => BlendMode::Lighten,
"Scrn" => BlendMode::Screen,
"CDdg" => BlendMode::ColorDodge,
"linearDodge" => BlendMode::LinearDodge,
"lighterColor" => BlendMode::LighterColor,
"Ovrl" => BlendMode::Overlay,
"SftL" => BlendMode::SoftLight,
"HrdL" => BlendMode::HardLight,
"vividLight" => BlendMode::VividLight,
"linearLight" => BlendMode::LinearLight,
"pinLight" => BlendMode::PinLight,
"hardMix" => BlendMode::HardMix,
"Dfrn" => BlendMode::Difference,
"Xclu" => BlendMode::Exclusion,
"blendSubtraction" => BlendMode::Subtract,
"blendDivide" => BlendMode::Divide,
"H " => BlendMode::Hue,
"Strt" => BlendMode::Saturation,
"Clr " => BlendMode::Color,
"Lmns" => BlendMode::Luminosity,
_ => return None,
})
}
/// What the import created.
pub struct PsdImport {
pub comp: ItemId,
pub folder: ItemId,
pub items: Vec<ItemId>,
pub warnings: Vec<String>,
}
struct Ctx<'a> {
psd: &'a Psd,
path: &'a str,
retain: bool,
folder: ItemId,
rate: FrameRate,
duration: Tick,
items: Vec<ItemId>,
warnings: Vec<String>,
}
/// Import `psd` (read from `path`) as a composition named `name`.
pub fn import(proj: &mut Project, psd: &Psd, path: &str, name: &str, retain: bool, rate: FrameRate, duration: Tick) -> PsdImport {
let folder = proj.add_item(&format!("{name} Layers"), Label::Yellow, None, ItemKind::Folder);
let mut cx = Ctx { psd, path, retain, folder, rate, duration, items: vec![], warnings: vec![] };
let comp = build_comp(proj, &mut cx, name, &psd.tree(), None);
PsdImport { comp, folder, items: cx.items, warnings: cx.warnings }
}
fn new_comp(cx: &Ctx) -> Comp {
Comp::new(cx.psd.width, cx.psd.height, cx.rate, cx.duration)
}
fn build_comp(proj: &mut Project, cx: &mut Ctx, name: &str, nodes: &[Node], parent: Option<ItemId>) -> ItemId {
let cid = proj.add_item(name, Label::Sandstone, parent, ItemKind::Comp(Arc::new(new_comp(cx))));
cx.items.push(cid);
let mut comp = new_comp(cx);
for n in nodes {
if let Some(l) = build_layer(proj, cx, &comp, n) {
comp.layers.push(l);
}
}
styles::sync_comp(&mut comp);
if let Some(item) = proj.item_mut(cid) {
item.kind = ItemKind::Comp(Arc::new(comp));
}
cid
}
fn footage_item(proj: &mut Project, cx: &mut Ctx, index: usize, name: &str) -> (ItemId, (u32, u32)) {
let l = &cx.psd.layers[index];
let retain = cx.retain;
let (w, h) = if retain && !l.rect.is_empty() { (l.rect.width(), l.rect.height()) } else { (cx.psd.width, cx.psd.height) };
let file = std::path::Path::new(cx.path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let f = Footage {
path: cx.path.to_string(),
kind: FootageKind::Still,
width: w,
height: h,
pixel_aspect: 1.0,
frame_rate: cx.rate,
native_rate: None,
duration: Tick::ZERO,
has_video: true,
has_audio: false,
alpha: AlphaMode::Straight,
premul_color: [0.0; 3],
loop_count: 1,
codec: "PSD".into(),
missing: false,
sequence: vec![],
color_profile: None,
layer: Some(SourceLayer { index: index as u32, name: name.to_string(), layer_size: retain && !l.rect.is_empty() }),
};
let id = proj.add_item(&format!("{name}/{file}"), Label::Lavender, Some(cx.folder), ItemKind::Footage(f));
cx.items.push(id);
(id, (w, h))
}
fn build_layer(proj: &mut Project, cx: &mut Ctx, comp: &Comp, n: &Node) -> Option<Layer> {
let (index, children) = match n {
Node::Layer(i) => (*i, None),
Node::Group { layer, children } => (*layer, Some(children)),
};
let pl = cx.psd.layers[index].clone();
let name = if pl.name.is_empty() { format!("Layer {}", index + 1) } else { pl.name.clone() };
let mut layer = if let Some(children) = children {
let sub = build_comp(proj, cx, &name, children, Some(cx.folder));
let mut l = build::layer(proj, comp, &name, LayerSource::Comp { item: sub }, (comp.width, comp.height), None);
// Pass-through groups composite their layers straight into the parent.
l.switches.collapse = pl.blend_key() == "pass";
l
} else if let Some(t) = &pl.text {
text_layer(proj, comp, &name, t)
} else if let Some(adj) = &pl.adjustment {
let mut l = adjustment_layer(proj, comp, &name);
add_adjustment_effect(proj, &mut l, adj, comp, &mut cx.warnings, &name);
l
} else if pl.has_pixels() || pl.fill.is_some() {
let (item, size) = footage_item(proj, cx, index, &name);
let mut l = build::layer(proj, comp, &name, LayerSource::Footage { item }, size, None);
if cx.retain && !pl.rect.is_empty() {
set_vec2(&mut l, "transform/position", [pl.rect.left as f64 + size.0 as f64 / 2.0, pl.rect.top as f64 + size.1 as f64 / 2.0]);
}
l
} else {
// Empty layers have nothing to show.
return None;
};
layer.name = comp.unique_layer_name(&name);
layer.blend_mode = blend_mode(pl.blend_key());
layer.switches.video = !pl.hidden;
layer.preserve_transparency = pl.clipping;
set_scalar(&mut layer, "transform/opacity", pl.opacity as f64 / 255.0 * 100.0);
// Vector mask → a mask in layer space.
let origin = layer_origin(&pl, cx.retain, &layer);
if let Some(vm) = pl.vector_mask.as_ref().filter(|v| !v.disabled)
&& let Some(masks) = layer.props.sub_mut("masks")
{
let mut ids = Ids(&mut proj.next_id);
for (k, sp) in vm.subpaths.iter().enumerate() {
let n = sp.knots.len();
let mut path = ShapePath { vertices: vec![], in_tangents: vec![], out_tangents: vec![], closed: sp.closed };
for kn in &sp.knots {
let v = [kn[1][0] - origin[0], kn[1][1] - origin[1]];
path.vertices.push(v);
path.in_tangents.push([kn[0][0] - kn[1][0], kn[0][1] - kn[1][1]]);
path.out_tangents.push([kn[2][0] - kn[1][0], kn[2][1] - kn[1][1]]);
}
if n == 0 {
continue;
}
let mode = if vm.inverted { MaskMode::Subtract } else { MaskMode::Add };
masks.children.push(build::mask(&mut ids, &format!("Mask {}", k + 1), path, mode, [255, 255, 0]).into());
}
}
// Layer effects → Layer Styles; Fill Opacity → Advanced Blending.
let gl = comp.global_light.clone();
if let Some(fx) = &pl.effects
&& fx.bool("masterFXSwitch") != Some(false)
&& layer.can_have_styles()
{
apply_effects(proj, &mut layer, fx, &gl);
}
if pl.fill_opacity < 255 && layer.can_have_styles() {
let mut ids = Ids(&mut proj.next_id);
if layer.props.sub(styles::GROUP).is_none() {
let g = styles::layer_styles(&mut ids, &gl);
let at = layer.props.children.iter().position(|c| c.match_id() == "transform").map(|i| i + 1).unwrap_or(layer.props.children.len());
layer.props.children.insert(at, g.into());
}
if let Some(p) = layer.props.prop_mut("layerStyles/blendingOptions/advancedBlending/fillOpacity") {
p.value = Value::Scalar(pl.fill_opacity as f64 / 255.0 * 100.0);
}
}
Some(layer)
}
/// Where layer space's origin sits in document pixels.
fn layer_origin(pl: &effectcraft_psd::Layer, retain: bool, layer: &Layer) -> [f64; 2] {
if let Some(t) = &pl.text {
return [t.transform[4], t.transform[5]];
}
if matches!(layer.source, LayerSource::Footage { .. }) && retain && !pl.rect.is_empty() {
return [pl.rect.left as f64, pl.rect.top as f64];
}
[0.0, 0.0]
}
fn set_scalar(l: &mut Layer, path: &str, v: f64) {
if let Some(p) = l.props.prop_mut(path) {
p.value = Value::Scalar(v);
}
}
fn set_vec2(l: &mut Layer, path: &str, v: [f64; 2]) {
if let Some(p) = l.props.prop_mut(path) {
p.value = match p.value {
Value::Vec3(o) => Value::Vec3([v[0], v[1], o[2]]),
_ => Value::Vec2(v),
};
}
}
/// Split a PostScript font name into family and style (`Inter-Bold` → `Inter`, `Bold`).
fn font_names(ps: &str) -> (String, String) {
match ps.split_once('-') {
Some((fam, style)) => (fam.to_string(), style.to_string()),
None => (ps.to_string(), "Regular".to_string()),
}
}
fn text_doc(t: &effectcraft_psd::TextInfo) -> TextDoc {
let mut d = TextDoc::default();
d.set_text(&t.text);
let [xx, xy, yx, yy, _, _] = t.transform;
let scale = (xx * yy - xy * yx).abs().sqrt().max(1e-6);
if let Some(s) = t.style.size {
d.size = s * scale;
}
if let Some(f) = &t.style.font {
let (fam, style) = font_names(f);
d.font = fam;
d.style = style;
}
if let Some(c) = t.style.color {
d.fill = [c[0] as f32, c[1] as f32, c[2] as f32, 1.0];
}
d.justify = match t.style.justification {
Some(1) => Justify::Right,
Some(2) => Justify::Center,
Some(3) => Justify::JustifyLastLeft,
Some(4) => Justify::JustifyLastRight,
Some(5) => Justify::JustifyLastCenter,
Some(6) => Justify::JustifyAll,
_ => Justify::Left,
};
if let Some(tr) = t.style.tracking {
d.tracking = tr;
}
if let Some(l) = t.style.leading {
d.leading = Some(l * scale);
}
d.faux_bold = t.style.faux_bold;
d.faux_italic = t.style.faux_italic;
if t.style.box_text
&& let Some([l, top, r, b]) = t.bounds
{
d.box_size = Some([(r - l) * scale, (b - top) * scale]);
d.box_pos = [l * scale, top * scale];
}
d
}
fn text_layer(proj: &mut Project, comp: &Comp, name: &str, t: &effectcraft_psd::TextInfo) -> Layer {
let mut l = build::layer(proj, comp, name, LayerSource::Text, (0, 0), None);
if let Some(p) = l.props.prop_mut("text/sourceText") {
p.value = Value::Text(Box::new(text_doc(t)));
}
set_vec2(&mut l, "transform/position", [t.transform[4], t.transform[5]]);
let [xx, xy, _, _, _, _] = t.transform;
let rot = xy.atan2(xx).to_degrees();
if rot.abs() > 1e-6 {
set_scalar(&mut l, "transform/rotation", rot);
}
l
}
fn adjustment_layer(proj: &mut Project, comp: &Comp, name: &str) -> Layer {
let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder));
let sid = proj.add_item(
name,
Label::Red,
Some(folder),
ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: comp.width, height: comp.height, pixel_aspect: 1.0 }),
);
let mut l = build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (comp.width, comp.height), None);
l.switches.adjustment = true;
l.label = Label::Purple;
l
}
fn add_effect(proj: &mut Project, l: &mut Layer, effect: &str, comp: &Comp, params: &[(&str, Value)]) {
let Some(spec) = effectcraft_effects::lookup(effect) else { return };
let mut ids = Ids(&mut proj.next_id);
let mut g = effectcraft_effects::instantiate(spec, &mut ids, spec.name, [comp.width as f64, comp.height as f64]);
for (k, v) in params {
if let Some(p) = g.get_mut(k) {
p.value = v.clone();
}
}
if let Some(fx) = l.props.sub_mut("effects") {
fx.children.push(g.into());
}
}
fn add_adjustment_effect(proj: &mut Project, l: &mut Layer, adj: &Adjustment, comp: &Comp, warnings: &mut Vec<String>, name: &str) {
match adj {
Adjustment::Invert => add_effect(proj, l, "Invert", comp, &[]),
Adjustment::BrightnessContrast { brightness, contrast, .. } => {
add_effect(proj, l, "Brightness & Contrast", comp, &[("brightness", Value::Scalar(*brightness)), ("contrast", Value::Scalar(*contrast))])
}
Adjustment::HueSaturation { hue, saturation, lightness, colorize } => {
let mut p = vec![("hue", Value::Scalar(*hue)), ("saturation", Value::Scalar(*saturation)), ("lightness", Value::Scalar(*lightness))];
if *colorize {
p = vec![
("colorize", Value::Bool(true)),
("colorizeHue", Value::Scalar(*hue)),
("colorizeSaturation", Value::Scalar(*saturation)),
("colorizeLightness", Value::Scalar(*lightness)),
];
}
add_effect(proj, l, "Hue/Saturation", comp, &p)
}
Adjustment::Levels { in_black, in_white, gamma, out_black, out_white } => add_effect(
proj,
l,
"Levels",
comp,
&[
("inBlack", Value::Scalar(in_black / 255.0)),
("inWhite", Value::Scalar(in_white / 255.0)),
("gamma", Value::Scalar(*gamma)),
("outBlack", Value::Scalar(out_black / 255.0)),
("outWhite", Value::Scalar(out_white / 255.0)),
],
),
Adjustment::Exposure { exposure, offset, gamma } => add_effect(
proj,
l,
"Exposure",
comp,
&[("exposure", Value::Scalar(*exposure)), ("offset", Value::Scalar(*offset)), ("gamma", Value::Scalar(*gamma))],
),
Adjustment::Other(k) => warnings.push(format!("{name}: adjustment `{k}` is not converted (layer kept as an empty adjustment layer)")),
}
}
// ---------------------------------------------------------------- layer effects → styles
fn style_mode(m: BlendMode) -> Value {
Value::Enum(STYLE_BLEND_MODES.iter().position(|x| x.1 == m).unwrap_or(0) as u32)
}
fn set(g: &mut PropGroup, k: &str, v: Value) {
if let Some(p) = g.get_mut(k) {
p.value = v;
}
}
fn color4(c: [f64; 3]) -> Value {
Value::Color([c[0], c[1], c[2], 1.0])
}
/// Copy the common style settings (`Md `, `Clr `, `Opct`, angle, distance, size…).
fn common(g: &mut PropGroup, d: &Descriptor) {
if let Some(m) = desc_blend(d.enum_value("Md ")) {
set(g, "blendMode", style_mode(m));
}
if let Some(c) = d.color("Clr ") {
set(g, "color", color4(c));
}
for (k, prop) in [("Opct", "opacity"), ("Nose", "noise"), ("Dstn", "distance"), ("blur", "size"), ("lagl", "angle"), ("ShdN", "jitter"), ("Inpr", "range")]
{
if let Some(v) = d.num(k) {
set(g, prop, Value::Scalar(v));
}
}
if let Some(b) = d.bool("uglg") {
set(g, "useGlobalLight", Value::Bool(b));
}
}
fn gradient_of(d: &Descriptor) -> Option<Gradient> {
let g = d.obj("Grad")?;
let list = |k: &str| match g.get(k) {
Some(DValue::List(v)) => v.clone(),
_ => vec![],
};
let colors: Vec<(f64, [f32; 4])> = list("Clrs")
.iter()
.filter_map(|c| match c {
DValue::Descriptor(s) => {
let col = s.color("Clr ")?;
Some((s.num("Lctn").unwrap_or(0.0) / 4096.0, [col[0] as f32, col[1] as f32, col[2] as f32, 1.0]))
}
_ => None,
})
.collect();
let opacities: Vec<(f64, f32)> = list("Trns")
.iter()
.filter_map(|c| match c {
DValue::Descriptor(s) => Some((s.num("Lctn").unwrap_or(0.0) / 4096.0, (s.num("Opct").unwrap_or(100.0) / 100.0) as f32)),
_ => None,
})
.collect();
(!colors.is_empty()).then(|| Gradient { colors, opacities: if opacities.is_empty() { vec![(0.0, 1.0), (1.0, 1.0)] } else { opacities } })
}
/// The effect descriptor for `key`, or the first enabled one of its `…Multi` list.
fn effect<'a>(fx: &'a Descriptor, key: &str, multi: &str) -> Option<&'a Descriptor> {
fx.obj(key).or_else(|| match fx.get(multi) {
Some(DValue::List(v)) => v.iter().find_map(|x| match x {
DValue::Descriptor(d) if d.bool("enab") != Some(false) => Some(d),
_ => None,
}),
_ => None,
})
}
fn apply_effects(proj: &mut Project, layer: &mut Layer, fx: &Descriptor, gl: &styles::GlobalLight) {
const MAP: &[(&str, &str, &str)] = &[
("DrSh", "dropShadowMulti", "dropShadow"),
("IrSh", "innerShadowMulti", "innerShadow"),
("OrGl", "outerGlowMulti", "outerGlow"),
("IrGl", "innerGlowMulti", "innerGlow"),
("ebbl", "bevelEmbossMulti", "bevelEmboss"),
("ChFX", "satinMulti", "satin"),
("SoFi", "solidFillMulti", "colorOverlay"),
("GrFl", "gradientFillMulti", "gradientOverlay"),
("FrFX", "frameFXMulti", "stroke"),
];
for (key, multi, id) in MAP {
let Some(d) = effect(fx, key, multi) else { continue };
let enabled = d.bool("enab").unwrap_or(true);
let mut ids = Ids(&mut proj.next_id);
let Some(uid) = styles::add_style(layer, &mut ids, id, gl, enabled) else { continue };
let Some(g) = layer.props.find_group_mut(uid) else { continue };
common(g, d);
match *id {
"dropShadow" => {
if let Some(v) = d.num("Ckmt") {
set(g, "spread", Value::Scalar(v));
}
if let Some(b) = d.bool("layerConceals") {
set(g, "knocksOut", Value::Bool(b));
}
}
"innerShadow" | "innerGlow" => {
if let Some(v) = d.num("Ckmt") {
set(g, "choke", Value::Scalar(v));
}
if d.enum_value("glwS") == Some("SrcC") {
set(g, "source", Value::Enum(0));
}
}
"outerGlow" => {
if let Some(v) = d.num("Ckmt") {
set(g, "spread", Value::Scalar(v));
}
}
"bevelEmboss" => {
let style = match d.enum_value("bvlS") {
Some("OtrB") => 0,
Some("Embs") => 2,
Some("PlEb") => 3,
Some("strokeEmboss") => 4,
_ => 1,
};
set(g, "style", Value::Enum(style));
let tech = match d.enum_value("bvlT") {
Some("PrBL") => 1,
Some("Slmt") => 2,
_ => 0,
};
set(g, "technique", Value::Enum(tech));
if let Some(v) = d.num("srgR") {
set(g, "depth", Value::Scalar(v));
}
set(g, "direction", Value::Enum(u32::from(d.enum_value("bvlD") == Some("Out "))));
if let Some(v) = d.num("Sftn") {
set(g, "soften", Value::Scalar(v));
}
if let Some(v) = d.num("Lald") {
set(g, "altitude", Value::Scalar(v));
}
for (k, p) in [("hglM", "highlightMode"), ("sdwM", "shadowMode")] {
if let Some(m) = desc_blend(d.enum_value(k)) {
set(g, p, style_mode(m));
}
}
for (k, p) in [("hglC", "highlightColor"), ("sdwC", "shadowColor")] {
if let Some(c) = d.color(k) {
set(g, p, color4(c));
}
}
for (k, p) in [("hglO", "highlightOpacity"), ("sdwO", "shadowOpacity")] {
if let Some(v) = d.num(k) {
set(g, p, Value::Scalar(v));
}
}
}
"satin" => {
if let Some(b) = d.bool("Invr") {
set(g, "invert", Value::Bool(b));
}
}
"gradientOverlay" => {
if let Some(gr) = gradient_of(d) {
set(g, "colors", Value::Gradient(gr));
}
if let Some(a) = d.num("Angl") {
set(g, "angle", Value::Scalar(a));
}
let st = match d.enum_value("Type") {
Some("Rdl ") => 1,
Some("Angl") => 2,
Some("Rflc") => 3,
Some("Dmnd") => 4,
_ => 0,
};
set(g, "style", Value::Enum(st));
if let Some(b) = d.bool("Rvrs") {
set(g, "reverse", Value::Bool(b));
}
if let Some(b) = d.bool("Algn") {
set(g, "alignWithLayer", Value::Bool(b));
}
if let Some(v) = d.num("Scl ") {
set(g, "scale", Value::Scalar(v));
}
}
"stroke" => {
if let Some(v) = d.num("Sz ") {
set(g, "size", Value::Scalar(v));
}
let pos = match d.enum_value("Styl") {
Some("InsF") => 1,
Some("CtrF") => 2,
_ => 0,
};
set(g, "position", Value::Enum(pos));
}
_ => {}
}
}
}
/// Layer ▸ Create ▸ Convert to Editable Text: a footage layer showing a Photoshop type layer
/// becomes a text layer (`None` when the layer's source has no text data).
pub fn editable_text(proj: &mut Project, comp: &Comp, layer: &Layer, psd: &Psd, index: usize) -> Option<Layer> {
let t = psd.layers.get(index)?.text.as_ref()?;
let mut l = text_layer(proj, comp, &layer.name, t);
l.name = layer.name.clone();
l.blend_mode = layer.blend_mode;
l.switches.video = layer.switches.video;
l.in_point = layer.in_point;
l.out_point = layer.out_point;
l.start_time = layer.start_time;
if let (Some(o), Some(p)) = (layer.props.prop("transform/opacity"), l.props.prop_mut("transform/opacity")) {
*p = effectcraft_project::Property { uid: p.uid, ..o.clone() };
}
Some(l)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn blend_keys() {
assert_eq!(blend_mode("mul "), BlendMode::Multiply);
assert_eq!(blend_mode("smud"), BlendMode::Exclusion);
assert_eq!(blend_mode("pass"), BlendMode::Normal);
assert_eq!(desc_blend(Some("linearDodge")), Some(BlendMode::LinearDodge));
assert_eq!(font_names("Inter-Bold"), ("Inter".into(), "Bold".into()));
}
}
+1 -1
View File
@@ -159,6 +159,6 @@ fn without_exporter() {
s.execute("renderQueue.add", json!({})).unwrap();
assert!(!s.is_enabled("renderQueue.render"));
let f = s.execute("renderQueue.formats", json!({})).unwrap();
assert_eq!(f["formats"].as_array().unwrap().len(), 7);
assert_eq!(f["formats"].as_array().unwrap().len(), 10);
assert_eq!(f["formats"][0]["available"], false);
}
+38
View File
@@ -27,6 +27,7 @@ impl Session {
self.prefs.normalize();
self.layer_cache.set_budget(self.prefs.layer_cache_bytes());
self.footage.set_cache_budget(self.prefs.media_cache_bytes());
self.configure_disk_cache();
// Fewer undo levels apply right away.
let levels = self.prefs.general.undo_levels.max(1) as usize;
if self.history.undo.len() > levels {
@@ -36,6 +37,43 @@ impl Session {
self.prefs_revision += 1;
}
/// Apply Settings ▸ Media & Disk Cache ▸ Disk Cache: open (or close) the persistent cache
/// and attach it to the layer cache. Without a folder setting the platform cache folder is
/// used, but only by frontends with a settings store (headless sessions stay off disk).
pub fn configure_disk_cache(&mut self) {
use effectcraft_render::disk_cache::{DiskCache, default_folder, footage_salt};
let d = &self.prefs.disk;
let folder = if d.disk_cache_folder.trim().is_empty() {
(self.config.is_some() && !cfg!(target_arch = "wasm32")).then(default_folder)
} else {
Some(std::path::PathBuf::from(d.disk_cache_folder.trim()))
};
let max = d.disk_cache_max_gb.max(1) as u64 * (1 << 30);
// (no file system in the browser: the disk cache stays off there)
let Some(folder) = folder.filter(|_| d.disk_cache_enabled && !cfg!(target_arch = "wasm32")) else {
self.disk_cache = None;
self.layer_cache.set_disk(None);
return;
};
let dc = match &self.disk_cache {
Some(c) if c.folder() == folder.as_path() => {
c.set_max_bytes(max);
c.clone()
}
_ => match DiskCache::open(&folder, max) {
Ok(c) => c,
Err(e) => {
log::warn!("disk cache at {}: {e}", folder.display());
self.disk_cache = None;
self.layer_cache.set_disk(None);
return;
}
},
};
self.layer_cache.set_disk(Some((dc.clone(), footage_salt(&self.project))));
self.disk_cache = Some(dc);
}
/// Write settings to the config store.
pub fn save_prefs(&self) {
if let Some(c) = &self.config
+307
View File
@@ -0,0 +1,307 @@
//! The 3D Camera Tracker end to end through the commands, on synthetic footage rendered in-test:
//! textured planes (a floor, a back wall and a box face) seen through a known moving camera.
use effectcraft_effects::camera_tracker as ct;
use effectcraft_geom::vec3;
use effectcraft_project::LayerId;
use effectcraft_raster::Image;
use effectcraft_render::EvalCtx;
use effectcraft_render::three_d::camera::active_camera;
use effectcraft_track::camtrack::linalg::{self, M3, V3};
use serde_json::{Value, json};
use crate::Session;
use crate::tests_track::{H, W, frame_time, setup, texture};
const F: f64 = 260.0;
/// The true camera on frame `f`: (world → camera rotation, centre).
fn true_camera(f: u32) -> (M3, V3) {
let t = f as f64 / 24.0;
let c = [-1.0 + 2.0 * t, -0.3 + 0.2 * (t * 3.0).sin(), 0.5 * t];
let fwd = linalg::normalize(linalg::sub([0.2 * t, 0.4, 8.0], c));
let right = linalg::normalize(linalg::cross([0.0, 1.0, 0.0], fwd));
([right, linalg::cross(fwd, right), fwd], c)
}
/// Planes: (point, normal, u axis, v axis, half extent or 0 = infinite).
const PLANES: [(V3, V3, V3, V3, f64); 3] = [
([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0], 0.0),
([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], 0.0),
([-1.2, 0.2, 6.0], [0.3, 0.0, -1.0], [0.957_826, 0.0, 0.287_348], [0.0, 1.0, 0.0], 1.1),
];
fn scene_frame(f: u32) -> Image {
let (r, c) = true_camera(f);
let mut img = Image::new(W, H);
for y in 0..H {
for x in 0..W {
let d = linalg::mtv(&r, [(x as f64 + 0.5 - W as f64 / 2.0) / F, (y as f64 + 0.5 - H as f64 / 2.0) / F, 1.0]);
let mut best = (f64::INFINITY, 0.2f32);
for (i, (p0, n, ua, va, ext)) in PLANES.iter().enumerate() {
let den = linalg::dot(*n, d);
if den.abs() < 1e-9 {
continue;
}
let tt = linalg::dot(*n, linalg::sub(*p0, c)) / den;
if tt <= 0.1 || tt >= best.0 {
continue;
}
let hit = linalg::add(c, linalg::scale(d, tt));
let (u, v) = (linalg::dot(linalg::sub(hit, *p0), *ua), linalg::dot(linalg::sub(hit, *p0), *va));
if *ext > 0.0 && (u.abs() > *ext || v.abs() > *ext) {
continue;
}
best = (tt, texture(u * 40.0, v * 40.0, 11 + i as u32));
}
img.set(x, y, [best.1, best.1, best.1, 1.0]);
}
}
img
}
/// The comp with the clip tracked (blocking) with `params`.
fn tracked(params: Value) -> (Session, LayerId) {
let (mut s, clip, _) = setup(scene_frame);
s.execute("layer.select", json!({"layers": [clip.0]})).unwrap();
assert!(s.is_enabled("track.camera"));
let mut p = params;
p["wait"] = json!(true);
let r = s.execute_checked("track.camera", p).unwrap();
assert_eq!(r["frames"], 25, "{r}");
assert!(s.camera_job.is_none());
(s, clip)
}
fn status(s: &mut Session) -> Value {
s.execute("camera.solveStatus", json!({})).unwrap()
}
#[test]
fn track_camera_solves_and_creates_layers_on_the_plane() {
let (mut s, clip) = tracked(json!({}));
let st = status(&mut s);
assert_eq!(st["solved"], true, "{st}");
let err = st["averageError"].as_f64().unwrap();
assert!(err < 0.5, "average error {err}");
let f = st["focalLength"].as_f64().unwrap();
assert!((f - F).abs() / F < 0.05, "focal {f}");
assert_ne!(st["methodUsed"], "Tripod Pan");
// Method Used / Average Error are reported in the effect.
let g = s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().next().unwrap().clone();
assert!(matches!(g.prop(ct::METHOD_USED).map(|p| &p.value), Some(effectcraft_keyframe::Value::Str(m)) if !m.is_empty()));
// Pick floor points on frame 12 (their 3D positions lie on y = 1.5 in the true scene: the
// ones seen below the horizon at the bottom of the frame).
s.execute("time.set", json!({"time": frame_time(12).seconds()})).unwrap();
let pts = s.execute("camera.points", json!({})).unwrap();
let list = pts["points"].as_array().unwrap();
assert!(list.len() > 30, "{}", list.len());
let floor: Vec<u64> = list.iter().filter(|p| p["comp"][1].as_f64().unwrap() > H as f64 * 0.8).map(|p| p["id"].as_u64().unwrap()).take(8).collect();
assert!(floor.len() >= 3, "{floor:?}");
s.execute("camera.selectPoints", json!({"points": floor})).unwrap();
assert_eq!(s.state.camera_points.len(), floor.len());
// Create Solid and Camera: one undo step, camera + solid.
let r = s.execute_checked("camera.createFromSolve", json!({"kind": "solid"})).unwrap();
let cam = LayerId(r["camera"].as_u64().unwrap());
let solid = LayerId(r["layers"][0].as_u64().unwrap());
let comp = s.active_comp().unwrap().clone();
assert!(comp.layer(cam).unwrap().is_camera());
assert_eq!(comp.layer(cam).unwrap().name, "3D Tracker Camera");
assert_eq!(comp.layers[0].id, cam);
let sl = comp.layer(solid).unwrap();
assert!(sl.switches.three_d);
assert!(sl.name.starts_with("Track Solid"));
// The camera is keyed on every frame.
assert_eq!(comp.layer(cam).unwrap().props.prop("transform/position").unwrap().keys.len(), 25);
// Every solved point projects through the created camera exactly where the solve puts it,
// and onto its tracked position (the footage fills the comp, so layer pixels are comp
// pixels): within 1 px for 95 % of the observations (the rest are the solve's outliers:
// features on occlusion edges), 0.5 px on average.
let cid = s.active_comp_id().unwrap();
let uid = g.uid;
let pl = crate::camera_track::placed(&s.project, cid, clip, uid).unwrap();
let tracks = ct::tracks(&crate::camera_track::static_params(&g)).unwrap();
let mut errs = vec![];
for k in [0u32, 6, 12, 18, 24] {
let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(k));
let camst = active_camera(&ctx);
for sp in pl.solve.visible(k as usize) {
let Some(obs) = tracks.track(sp.id).and_then(|t| t.at(k)) else { continue };
let w = pl.to_world(sp.pos);
let q = camst.project(W as f64, H as f64, vec3(w[0], w[1], w[2])).unwrap();
let own = pl.solve.frames[k as usize].project(pl.solve.size, sp.pos).unwrap();
assert!((q.x - own[0]).hypot(q.y - own[1]) < 0.01, "camera layer disagrees with the solve");
errs.push((q.x - obs[0]).hypot(q.y - obs[1]));
}
}
assert!(errs.len() > 100, "{}", errs.len());
errs.sort_by(f64::total_cmp);
let p95 = errs[errs.len() * 95 / 100];
let mean = errs.iter().sum::<f64>() / errs.len() as f64;
assert!(p95 < 1.0 && mean < 0.5, "p95 {p95} px, mean {mean} px");
// The solid lies on the selected points' plane.
let tr = sl.props.sub("transform").unwrap();
let pos = tr.get("position").unwrap().value.as_vec3();
let ori = tr.get("orientation").unwrap().value.as_vec3();
let normal = linalg::mv(&linalg::from_euler_xyz(ori), [0.0, 0.0, -1.0]);
let size = r["target"]["size"].as_f64().unwrap();
for id in &s.state.camera_points {
let w = pl.to_world(pl.solve.point(*id).unwrap().pos);
let d = linalg::dot(linalg::sub(w, pos), normal).abs();
assert!(d < 0.05 * size.max(1.0) + 1.0, "point {id} is {d} px off the plane");
}
// The solid faces the camera.
let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(12));
let eye = active_camera(&ctx).eye;
assert!(linalg::dot(normal, linalg::sub([eye.x, eye.y, eye.z], pos)) > 0.0);
// Undo removes both layers; redo brings them back.
let before = comp.layers.len();
assert!(s.undo());
assert_eq!(s.active_comp().unwrap().layers.len(), before - 2);
assert!(s.redo());
assert_eq!(s.active_comp().unwrap().layers.len(), before);
// A second create reuses the camera: Create Multiple Nulls and Camera.
let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null", "multiple": true})).unwrap();
assert_eq!(r["camera"].as_u64().unwrap(), cam.0);
assert_eq!(r["layers"].as_array().unwrap().len(), floor.len());
// Text and Shadow Catcher (+ light).
let r = s.execute_checked("camera.createFromSolve", json!({"kind": "text"})).unwrap();
assert_eq!(r["layers"].as_array().unwrap().len(), 1);
let r = s.execute_checked("camera.createFromSolve", json!({"kind": "shadowCatcher"})).unwrap();
let ids: Vec<LayerId> = r["layers"].as_array().unwrap().iter().map(|v| LayerId(v.as_u64().unwrap())).collect();
let comp = s.active_comp().unwrap();
assert!(comp.layer(ids[0]).unwrap().name.starts_with("Shadow Catcher"));
assert_eq!(comp.layer(ids[0]).unwrap().props.prop("materialOptions/acceptsShadows").unwrap().value.as_enum(), 2);
assert!(comp.layer(ids[1]).unwrap().is_light());
assert!(s.execute_checked("camera.createFromSolve", json!({"kind": "teapot"})).is_err());
// Set Ground Plane and Origin: the target becomes the origin; a null created there sits at
// (0, 0, 0) lying on the X-Z plane (its front facing up, −y).
s.execute_checked("camera.setGroundPlane", json!({})).unwrap();
assert_eq!(status(&mut s)["groundPlane"], true);
let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null"})).unwrap();
let null = LayerId(r["layers"][0].as_u64().unwrap());
let tr = s.active_comp().unwrap().layer(null).unwrap().props.sub("transform").unwrap().clone();
let pos = tr.get("position").unwrap().value.as_vec3();
assert!(linalg::norm(pos) < 1e-6, "{pos:?}");
let up = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]);
assert!(linalg::norm(linalg::sub(up, [0.0, -1.0, 0.0])) < 1e-6, "{up:?}");
// An explicit target (comp world) places the layer there, its front facing the normal.
let r = s
.execute_checked("camera.createFromSolve", json!({"kind": "solid", "target": {"center": [10.0, 0.0, 20.0], "normal": [0.0, 0.0, -1.0], "size": 80.0}}))
.unwrap();
let l = s.active_comp().unwrap().layer(LayerId(r["layers"][0].as_u64().unwrap())).unwrap().clone();
let tr = l.props.sub("transform").unwrap();
let pos = tr.get("position").unwrap().value.as_vec3();
assert!(linalg::norm(linalg::sub(pos, [10.0, 0.0, 20.0])) < 1e-6, "{pos:?}");
let front = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]);
assert!(linalg::norm(linalg::sub(front, [0.0, 0.0, -1.0])) < 1e-6, "{front:?}");
// Serde keeps the analysis.
let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap();
let g2 = back.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap();
let sv = ct::solve(&crate::camera_track::static_params(g2)).unwrap();
assert!(sv.ground.is_some());
assert_eq!(sv.points.len(), pl.solve.points.len());
}
#[test]
fn invalidation_resolve_cancel_and_undo() {
let (mut s, clip) = tracked(json!({"shotType": "specify", "aov": 2.0 * (W as f64 / 2.0 / F).atan().to_degrees()}));
let st = status(&mut s);
assert_eq!(st["solved"], true, "{st}");
assert_eq!(st["shotType"], "Specify Angle of View");
assert!((st["focalLength"].as_f64().unwrap() - F).abs() < 0.01 * F);
let cid = s.active_comp_id().unwrap();
let uid = st["effect"].as_u64().unwrap();
// Undo removes the analysis (one step); redo restores it.
assert!(s.undo());
assert_eq!(status(&mut s)["analyzed"], false);
assert!(s.redo());
assert_eq!(status(&mut s)["solved"], true);
// Changing the Solve Method keeps the tracks and only re-solves.
s.execute_checked("prop.set", json!({"layer": clip.0, "path": "effects/#1/advanced/solveMethod", "value": 1})).unwrap();
let st = status(&mut s);
assert_eq!(st["analyzed"], true);
assert_eq!(st["solved"], false);
assert_eq!(st["pending"], true);
s.execute_checked("camera.analyze", json!({"wait": true})).unwrap();
let st = status(&mut s);
assert_eq!(st["solved"], true, "{st}");
assert_eq!(st["methodUsed"], "Typical");
// Deleting points re-solves without them.
let pts = s.execute("camera.points", json!({})).unwrap();
let del: Vec<u64> = pts["points"].as_array().unwrap().iter().take(3).map(|p| p["id"].as_u64().unwrap()).collect();
s.execute_checked("camera.deletePoints", json!({"points": del, "wait": true})).unwrap();
s.poll_camera(false);
let st = status(&mut s);
assert_eq!(st["solved"], true, "{st}");
assert!(st["deleted"].as_u64().unwrap() >= 3);
let g = s.project.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone();
let sv = ct::solve(&crate::camera_track::static_params(&g)).unwrap();
assert!(sv.points.iter().all(|p| !del.contains(&(p.id as u64))));
// Trimming the layer changes its frames: tracks and solve are cleared and queued.
s.edit("trim", None, |p, _| {
p.comp_mut(cid).unwrap().layer_mut(clip).unwrap().in_point = frame_time(2);
Ok(())
})
.unwrap();
let st = status(&mut s);
assert_eq!(st["analyzed"], false);
assert_eq!(st["pending"], true);
// A background analysis can be cancelled: nothing is written.
s.execute_checked("camera.analyze", json!({})).unwrap();
assert!(s.is_camera_analyzing() || s.camera_job.is_some());
let banner = status(&mut s)["banner"].as_str().map(str::to_string);
if s.is_camera_analyzing() {
assert!(banner.unwrap().starts_with("Analyzing in background (step 1 of 2)"));
}
s.execute("camera.cancel", json!({})).ok();
let t0 = std::time::Instant::now();
while s.camera_job.is_some() {
assert!(t0.elapsed().as_secs() < 120);
std::thread::sleep(std::time::Duration::from_millis(5));
s.poll_camera(false);
}
// (When the job finished before the cancel landed it wrote a valid analysis.)
let st = status(&mut s);
assert_eq!(st["running"], false);
assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "shotType": "bogus"})).is_err());
assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "aov": 500})).is_err());
}
#[test]
fn effect_parameters_match_after_effects() {
let spec = effectcraft_effects::registry().iter().find(|e| e.id == ct::ID).unwrap();
assert_eq!(spec.name, "3D Camera Tracker");
assert_eq!(spec.category, "Perspective");
let names: Vec<&str> = spec.params.iter().filter(|p| !matches!(p.ui, effectcraft_project::ParamUi::Hidden)).map(|p| p.name).collect();
for n in [
"Shot Type",
"Horizontal Angle of View",
"Show Track Points",
"Render Track Points",
"Track Point Size",
"Target Size",
"Solve Method",
"Detailed Analysis",
"Auto-delete Points Across Time",
"Hide Warning Banner",
] {
assert!(names.contains(&n), "{n}");
}
// The Animation menu entry is a real command now.
let c = crate::find_command("track.camera").unwrap();
assert_eq!(c.label, "Track Camera");
}
+121
View File
@@ -0,0 +1,121 @@
//! The persistent disk cache (PRV-2) through the session: settings, layer cache persistence
//! across restarts, invalidation on edits, purging.
use serde_json::json;
use crate::Session;
use crate::render::RenderOpts;
fn folder(name: &str) -> String {
let d = std::env::temp_dir().join(format!("effectcraft-session-disk-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
d.to_string_lossy().to_string()
}
fn session(dir: &str) -> Session {
let mut s = Session::default();
s.execute_checked("prefs.set", json!({"values": {"disk.diskCacheFolder": dir, "disk.diskCacheEnabled": true}})).unwrap();
// Write every layer buffer, however quick to render.
s.disk_cache.as_ref().unwrap().set_min_layer_ms(0);
s
}
fn project(s: &mut Session) -> effectcraft_project::ItemId {
s.execute("comp.new", json!({"name": "D", "width": 160, "height": 90, "frameRate": 30, "duration": 1})).unwrap();
s.execute("layer.newSolid", json!({"color": "#3080ff", "width": 80, "height": 40})).unwrap();
s.execute("effect.apply", json!({"layer": "#1", "effect": "Gaussian Blur"})).unwrap();
s.execute("prop.set", json!({"layer": "#1", "path": "effects/#1/blurriness", "value": 12})).unwrap();
s.active_comp_id().unwrap()
}
#[test]
fn disk_cache_settings_persistence_and_invalidation() {
let dir = folder("persist");
let mut s = session(&dir);
let st = s.execute_checked("cache.diskStats", json!({})).unwrap();
assert_eq!(st["enabled"], json!(true));
assert_eq!(st["maxBytes"], json!(100u64 << 30));
let cid = project(&mut s);
let t = s.time();
let first = s.render(cid, t, RenderOpts::default());
let dc = s.disk_cache.clone().unwrap();
dc.flush();
let written = dc.stats().entries;
assert!(written > 0, "layer buffers written");
// "Restart": a new session on the same folder and project renders from disk.
let saved = s.project.clone();
drop(s);
let mut s2 = session(&dir);
s2.project = saved;
s2.bump();
s2.open_comp(cid);
let dc2 = s2.disk_cache.clone().unwrap();
assert_eq!(dc2.stats().entries, written);
let again = s2.render(cid, t, RenderOpts::default());
assert!(dc2.stats().hits > 0, "{:?}", dc2.stats());
assert_eq!(again.data, first.data, "disk-cached pixels are exact");
// An edit changes the keys: a miss, and new entries.
let hits = dc2.stats().hits;
s2.execute("prop.set", json!({"layer": "#1", "path": "effects/#1/blurriness", "value": 20})).unwrap();
let edited = s2.render(cid, t, RenderOpts::default());
assert_ne!(edited.data, first.data);
assert_eq!(dc2.stats().hits, hits);
dc2.flush();
assert!(dc2.stats().entries > written);
// Purge ▸ All Disk Cache empties it; memory purge leaves the disk alone.
let r = s2.execute_checked("edit.purge", json!({"what": "memory"})).unwrap();
assert_eq!(r["diskEntries"], json!(0));
assert!(dc2.stats().entries > 0);
let r = s2.execute_checked("edit.purge", json!({"what": "disk"})).unwrap();
assert!(r["diskEntries"].as_u64().unwrap() > 0);
assert_eq!(dc2.stats().entries, 0);
// Disabling detaches it.
s2.execute_checked("prefs.set", json!({"key": "disk.diskCacheEnabled", "value": false})).unwrap();
assert!(s2.disk_cache.is_none());
assert!(s2.layer_cache.disk().is_none());
}
#[test]
fn footage_changes_change_the_salt() {
let dir = folder("salt");
let mut s = session(&dir);
let foot = std::env::temp_dir().join(format!("effectcraft-salt-{}.png", std::process::id()));
std::fs::write(&foot, b"one").unwrap();
s.edit("add", None, |p, _| {
let f = effectcraft_project::Footage {
path: foot.to_string_lossy().to_string(),
kind: effectcraft_project::FootageKind::Still,
width: 1,
height: 1,
pixel_aspect: 1.0,
frame_rate: effectcraft_time::FrameRate::FPS_30,
native_rate: None,
duration: effectcraft_time::Tick::ZERO,
has_video: true,
has_audio: false,
alpha: effectcraft_project::AlphaMode::Ignore,
premul_color: [0.0; 3],
loop_count: 1,
codec: "PNG".into(),
missing: false,
sequence: vec![],
color_profile: None,
layer: None,
};
p.add_item("f.png", effectcraft_color::Label::Lavender, None, effectcraft_project::ItemKind::Footage(f));
Ok(())
})
.unwrap();
let a = effectcraft_render::disk_cache::footage_salt(&s.project);
std::fs::write(&foot, b"two, longer").unwrap();
let b = effectcraft_render::disk_cache::footage_salt(&s.project);
assert_ne!(a, b);
let cid = project(&mut s);
let k1 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid);
s.execute("prop.set", json!({"layer": "#1", "path": "transform/opacity", "value": 50})).unwrap();
let k2 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid);
assert_ne!(k1, k2);
// Unrelated comps do not change it.
s.execute("comp.new", json!({"name": "Other", "width": 10, "height": 10, "frameRate": 30, "duration": 1})).unwrap();
assert_eq!(effectcraft_render::disk_cache::comp_content_key(&s.project, cid), k2);
}
+1
View File
@@ -52,6 +52,7 @@ impl Importer for ModelImporter {
missing: false,
sequence: vec![],
color_profile: None,
layer: None,
})
}
}
+2 -1
View File
@@ -257,6 +257,7 @@ impl Importer for MockImporter {
missing: false,
sequence: vec![],
color_profile: None,
layer: None,
})
}
}
@@ -331,7 +332,7 @@ fn implemented_stubs_are_enabled_and_workspaces_are_frontend() {
assert!(s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == id)));
}
// What remains a stub is still disabled.
for id in ["view.displayColorManagement", "file.createProxy", "track.camera"] {
for id in ["view.displayColorManagement", "file.createProxy"] {
assert!(!s.is_enabled(id), "{id}");
}
}
+2
View File
@@ -101,6 +101,7 @@ pub(crate) fn setup(make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Ses
missing: false,
sequence: vec![],
color_profile: None,
layer: None,
};
let item = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(footage));
let comp = p.comp(cid).unwrap().clone();
@@ -410,6 +411,7 @@ fn analyze_1080p_single_point_speed() {
missing: false,
sequence: vec![],
color_profile: None,
layer: None,
};
let item = p.add_item("hd", Label::Aqua, None, ItemKind::Footage(footage));
let comp = p.comp(cid).unwrap().clone();
+237
View File
@@ -0,0 +1,237 @@
//! Photoshop import as a composition (IO-2) and Layer ▸ Create text conversions (SHP-5).
use effectcraft_color::BlendMode;
use effectcraft_keyframe::Value as KV;
use effectcraft_project::{ItemKind, LayerSource, MaskMode};
use effectcraft_psd::write::*;
use effectcraft_psd::{DValue, Descriptor, Rect};
use serde_json::json;
use crate::Session;
use crate::render::RenderOpts;
fn tmp(name: &str) -> String {
let d = std::env::temp_dir().join(format!("effectcraft-psd-tests-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
d.join(name).to_string_lossy().to_string()
}
/// A layered document exercising groups, masks, blend modes, text, styles and adjustments.
pub(crate) fn sample_psd() -> Vec<u8> {
let mut d = WDoc::new(120, 80);
let shadow = Descriptor::new("null").with("masterFXSwitch", DValue::Bool(true)).with(
"DrSh",
DValue::Descriptor(
Descriptor::new("DrSh")
.with("enab", DValue::Bool(true))
.with("Md ", DValue::Enum("BlnM".into(), "Mltp".into()))
.with("Clr ", DValue::Descriptor(Descriptor::rgb([0.0, 0.0, 0.5])))
.with("Opct", DValue::UnitFloat("#Prc".into(), 60.0))
.with("uglg", DValue::Bool(false))
.with("lagl", DValue::UnitFloat("#Ang".into(), 45.0))
.with("Dstn", DValue::UnitFloat("#Pxl".into(), 9.0))
.with("blur", DValue::UnitFloat("#Pxl".into(), 4.0)),
),
);
let stroke = Descriptor::new("null").with(
"FrFX",
DValue::Descriptor(
Descriptor::new("FrFX")
.with("enab", DValue::Bool(true))
.with("Styl", DValue::Enum("FStl".into(), "InsF".into()))
.with("Md ", DValue::Enum("BlnM".into(), "Nrml".into()))
.with("Opct", DValue::UnitFloat("#Prc".into(), 100.0))
.with("Sz ", DValue::UnitFloat("#Pxl".into(), 2.0))
.with("Clr ", DValue::Descriptor(Descriptor::rgb([1.0, 1.0, 0.0]))),
),
);
let mut masked = WLayer::solid("Masked Square", Rect::new(10, 10, 40, 40), [0.0, 0.8, 0.2, 1.0]).with_blend(b"mul ");
masked.mask = Some(WMask {
rect: Rect::new(10, 10, 40, 40),
data: (0..1600).map(|i| if i % 40 < 20 { 1.0 } else { 0.0 }).collect(),
default_color: 0,
disabled: false,
});
let mut hidden = WLayer::solid("Hidden", Rect::new(0, 0, 10, 10), [1.0, 0.0, 1.0, 1.0]);
hidden.hidden = true;
let mut clipped = WLayer::solid("Clipped", Rect::new(60, 30, 20, 20), [1.0, 1.0, 1.0, 1.0]).with_blend(b"scrn").with_opacity(128);
clipped.clipping = true;
let tri = vec![[[70.0, 10.0]; 3], [[110.0, 10.0]; 3], [[90.0, 40.0]; 3]];
d.layers = vec![
WLayer::solid("Background", Rect::new(0, 0, 120, 80), [0.2, 0.3, 0.4, 1.0]),
WLayer::group_end(),
masked,
hidden,
WLayer::group("Group A", true, *b"pass"),
WLayer::solid("Base", Rect::new(55, 25, 30, 30), [0.9, 0.1, 0.1, 1.0]).with_block(effects_block(&shadow)),
clipped,
WLayer::solid("Vector Masked", Rect::new(60, 0, 60, 50), [0.1, 0.1, 0.9, 1.0])
.with_block(vector_mask_block(&[tri], 120, 80, false))
.with_block(effects_block(&stroke)),
WLayer::empty("Hue Shift", vec![hue_sat_block(40, 10, -5)]),
WLayer::empty("Title", vec![text_block("Hello PSD", [8.0, 70.0], "Inter-Bold", 14.0, [1.0, 1.0, 1.0], 0)]),
];
write(&d)
}
#[test]
fn psd_imports_as_composition() {
let path = tmp("sample.psd");
std::fs::write(&path, sample_psd()).unwrap();
let mut s = Session::default();
let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "composition"})).unwrap();
assert_eq!(r["errors"], json!([]), "{r}");
let cid = effectcraft_project::ItemId(r["comps"][0].as_u64().unwrap());
assert_eq!(s.state.active_comp, Some(cid));
assert_eq!(s.project.item(cid).unwrap().name, "sample");
let c = s.project.comp(cid).unwrap().clone();
assert_eq!((c.width, c.height), (120, 80));
let names: Vec<&str> = c.layers.iter().map(|l| l.name.as_str()).collect();
assert_eq!(names, ["Title", "Hue Shift", "Vector Masked", "Clipped", "Base", "Group A", "Background"]);
// Text layer: editable text, positioned at the type origin.
let title = &c.layers[0];
assert!(matches!(title.source, LayerSource::Text));
let Some(KV::Text(doc)) = title.props.prop("text/sourceText").map(|p| p.value.clone()) else { panic!("text") };
assert_eq!(doc.text, "Hello PSD");
assert_eq!(doc.size, 14.0);
assert_eq!(doc.font, "Inter");
assert_eq!(doc.style, "Bold");
assert_eq!(title.props.prop("transform/position").unwrap().value.as_vec2(), [8.0, 70.0]);
// Adjustment layer with Hue/Saturation.
let hue = &c.layers[1];
assert!(hue.switches.adjustment);
let fx = hue.effects().unwrap().groups().next().unwrap();
assert_eq!(fx.get("hue").unwrap().value.as_f64(), 40.0);
assert_eq!(fx.get("saturation").unwrap().value.as_f64(), 10.0);
// Vector mask → mask; Stroke style.
let vm = &c.layers[2];
let m = vm.masks().unwrap().groups().next().unwrap();
assert!(matches!(m.kind, effectcraft_project::GroupKind::Mask { mode: MaskMode::Add, .. }));
let Some(KV::Path(p)) = m.get("path").map(|p| p.value.clone()) else { panic!("mask path") };
assert_eq!(p.vertices.len(), 3);
assert!((p.vertices[1][0] - 110.0).abs() < 1e-3 && (p.vertices[2][1] - 40.0).abs() < 1e-3);
let st = vm.props.group("layerStyles/stroke").unwrap();
assert_eq!(st.get("size").unwrap().value.as_f64(), 2.0);
assert_eq!(st.get("position").unwrap().value, KV::Enum(1));
// Clipped → preserve transparency; opacity and blend mode.
let clipped = &c.layers[3];
assert!(clipped.preserve_transparency);
assert_eq!(clipped.blend_mode, BlendMode::Screen);
assert!((clipped.props.prop("transform/opacity").unwrap().value.as_f64() - 128.0 / 255.0 * 100.0).abs() < 1e-9);
// Drop shadow.
let ds = c.layers[4].props.group("layerStyles/dropShadow").unwrap();
assert_eq!(ds.get("distance").unwrap().value.as_f64(), 9.0);
assert_eq!(ds.get("angle").unwrap().value.as_f64(), 45.0);
assert_eq!(ds.get("useGlobalLight").unwrap().value, KV::Bool(false));
assert_eq!(ds.get("opacity").unwrap().value.as_f64(), 60.0);
// Group → collapsed precomp (pass through) with its layers.
let g = &c.layers[5];
assert!(g.switches.collapse);
let LayerSource::Comp { item } = g.source else { panic!("precomp") };
let inner = s.project.comp(item).unwrap();
let inner_names: Vec<&str> = inner.layers.iter().map(|l| l.name.as_str()).collect();
assert_eq!(inner_names, ["Hidden", "Masked Square"]);
assert!(!inner.layers[0].switches.video);
assert_eq!(inner.layers[1].blend_mode, BlendMode::Multiply);
// Footage items name their Photoshop layer; document-sized footage.
let LayerSource::Footage { item } = inner.layers[1].source else { panic!("footage") };
let ItemKind::Footage(f) = &s.project.item(item).unwrap().kind else { panic!() };
let sl = f.layer.as_ref().unwrap();
assert_eq!((sl.name.as_str(), sl.layer_size), ("Masked Square", false));
assert_eq!((f.width, f.height), (120, 80));
assert_eq!(s.project.item(item).unwrap().name, "Masked Square/sample.psd");
// One undo step removes everything.
s.execute("edit.undo", json!({})).unwrap();
assert!(s.project.comp(cid).is_none());
}
#[test]
fn psd_retain_layer_sizes() {
let path = tmp("retain.psd");
std::fs::write(&path, sample_psd()).unwrap();
let mut s = Session::default();
let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "compositionLayerSizes"})).unwrap();
let cid = effectcraft_project::ItemId(r["comps"][0].as_u64().unwrap());
let c = s.project.comp(cid).unwrap();
let base = c.layer_by_name("Base").unwrap();
let LayerSource::Footage { item } = base.source else { panic!() };
let ItemKind::Footage(f) = &s.project.item(item).unwrap().kind else { panic!() };
assert_eq!((f.width, f.height), (30, 30));
assert!(f.layer.as_ref().unwrap().layer_size);
assert_eq!(base.props.prop("transform/position").unwrap().value.as_vec2(), [70.0, 40.0]);
assert_eq!(base.props.prop("transform/anchor").unwrap().value.as_vec2(), [15.0, 15.0]);
// Vector mask vertices are in the layer's own space.
let vm = c.layer_by_name("Vector Masked").unwrap();
let Some(KV::Path(p)) = vm.masks().unwrap().groups().next().unwrap().get("path").map(|p| p.value.clone()) else { panic!() };
assert!((p.vertices[0][0] - 10.0).abs() < 1e-3, "{:?}", p.vertices);
}
#[test]
fn import_rejects_unknown_kind() {
let mut s = Session::default();
assert!(s.execute_checked("file.import", json!({"paths": ["x.psd"], "importAs": "banana"})).is_err());
}
fn text_session() -> (Session, u64) {
let mut s = Session::default();
s.execute("comp.new", json!({"name": "T", "width": 320, "height": 120, "frameRate": 30, "duration": 2})).unwrap();
let r = s.execute_checked("layer.newText", json!({"text": "Ab8", "size": 64, "position": [160, 80], "fill": "#ffcc00"})).unwrap();
(s, r["layer"].as_u64().unwrap())
}
fn alpha_diff(a: &effectcraft_raster::Image, b: &effectcraft_raster::Image) -> (f32, f32) {
let mut sum = 0.0;
let mut mass = 0.0;
for (p, q) in a.data.iter().zip(&b.data) {
sum += (p[3] - q[3]).abs();
mass += p[3];
}
(sum / a.data.len() as f32, mass)
}
#[test]
fn create_shapes_from_text_matches_text_render() {
let (mut s, lid) = text_session();
let cid = s.active_comp_id().unwrap();
let opts = RenderOpts::default();
let text_img = s.render(cid, s.time(), opts);
s.execute("layer.select", json!({"layers": [lid]})).unwrap();
let r = s.execute_checked("layer.create", json!({"op": "shapesFromText"})).unwrap();
let sid = r["layers"][0].as_u64().unwrap();
let c = s.active_comp().unwrap();
assert_eq!(c.layers[0].id.0, sid);
assert!(matches!(c.layers[0].source, LayerSource::Shape));
assert_eq!(c.layers[0].name, "Ab8 Outlines");
assert!(!c.layer(effectcraft_project::LayerId(lid)).unwrap().switches.video, "source hidden");
let groups: Vec<String> = c.layers[0].props.sub("contents").unwrap().groups().map(|g| g.name.clone()).collect();
assert_eq!(groups, ["A", "b", "8"]);
let shape_img = s.render(cid, s.time(), opts);
let (d, mass) = alpha_diff(&text_img, &shape_img);
assert!(mass > 500.0, "text drew something");
assert!(d < 0.01, "mean alpha difference {d}");
// Fill colour carried over.
let i = shape_img.data.iter().position(|p| p[3] > 0.99).unwrap();
let p = shape_img.data[i];
assert!((p[0] - 1.0).abs() < 0.02 && (p[1] - 0.8).abs() < 0.02 && p[2] < 0.02, "{p:?}");
}
#[test]
fn create_masks_from_text() {
let (mut s, lid) = text_session();
let cid = s.active_comp_id().unwrap();
let text_img = s.render(cid, s.time(), RenderOpts::default());
s.execute("layer.select", json!({"layers": [lid]})).unwrap();
let r = s.execute_checked("layer.create", json!({"op": "masksFromText"})).unwrap();
let mid = effectcraft_project::LayerId(r["layers"][0].as_u64().unwrap());
let c = s.active_comp().unwrap();
let l = c.layer(mid).unwrap();
assert!(matches!(l.source, LayerSource::Solid { .. }));
// "A", "b" and "8" have 2, 2 and 3 contours.
assert_eq!(l.masks().unwrap().groups().count(), 7);
let img = s.render(cid, s.time(), RenderOpts::default());
let (d, _) = alpha_diff(&text_img, &img);
assert!(d < 0.01, "mean alpha difference {d}");
// Other ops need the right layer type.
assert!(s.execute_checked("layer.create", json!({"op": "shapesFromVector", "layers": [lid]})).is_err());
assert!(s.execute_checked("layer.create", json!({"op": "nullTracing", "layers": [lid]})).is_err());
}
+250
View File
@@ -0,0 +1,250 @@
//! Vector conversions behind Layer ▸ Create: Create Shapes from Vector Layer (SVG footage →
//! shape layer), Create Shapes from Text and Create Masks from Text (glyph outlines).
use effectcraft_keyframe::{Gradient, ShapePath, Value};
use effectcraft_project::build::{self, Ids};
use effectcraft_project::{Comp, ItemId, Layer, LayerSource, MaskMode, Project, PropGroup};
use effectcraft_svg::{Affine, Doc, Geom, GradientKind, Node, Paint};
use kurbo::{BezPath, Point};
fn shape_paths(ids: &mut Ids, path: &BezPath) -> Vec<PropGroup> {
effectcraft_path::from_kurbo(path)
.into_iter()
.enumerate()
.map(|(i, sp)| {
let mut g = build::shape_path(ids, sp);
g.name = format!("Path {}", i + 1);
g
})
.collect()
}
fn det_scale(m: Affine) -> f64 {
let c = m.as_coeffs();
(c[0] * c[3] - c[1] * c[2]).abs().sqrt()
}
/// Axis-aligned scale + translation only (basic shapes stay parametric).
fn axis_aligned(m: Affine) -> Option<(f64, f64, f64, f64)> {
let c = m.as_coeffs();
(c[1].abs() < 1e-9 && c[2].abs() < 1e-9 && c[0] > 0.0 && c[3] > 0.0).then_some((c[0], c[3], c[4], c[5]))
}
fn gradient_value(g: &effectcraft_svg::Gradient) -> Gradient {
Gradient {
colors: g.stops.iter().map(|(o, c)| (*o, [c[0] as f32, c[1] as f32, c[2] as f32, 1.0])).collect(),
opacities: g.stops.iter().map(|(o, c)| (*o, c[3] as f32)).collect(),
}
}
/// Gradient start/end points in shape-layer space (`m`: shape user space → layer space).
fn gradient_points(g: &effectcraft_svg::Gradient, m: Affine, bbox: kurbo::Rect) -> (bool, [f64; 2], [f64; 2]) {
let unit = if g.bbox_units { Affine::new([bbox.width(), 0.0, 0.0, bbox.height(), bbox.x0, bbox.y0]) } else { Affine::IDENTITY };
let full = m * unit * g.transform;
let p = |x: f64, y: f64| {
let q = full * Point::new(x, y);
[q.x, q.y]
};
match g.kind {
GradientKind::Linear { x1, y1, x2, y2 } => (false, p(x1, y1), p(x2, y2)),
GradientKind::Radial { cx, cy, r, .. } => (true, p(cx, cy), p(cx + r, cy)),
}
}
fn shape_items(ids: &mut Ids, s: &effectcraft_svg::Shape, m: Affine) -> Vec<PropGroup> {
use kurbo::Shape as _;
let mut items = vec![];
// Geometry: parametric when the transform keeps rectangles/ellipses axis-aligned.
match (&s.geom, axis_aligned(m)) {
(Geom::Rect { x, y, w, h, rx, ry }, Some((sx, sy, tx, ty))) if (rx - ry).abs() < 1e-9 && (*rx == 0.0 || (sx - sy).abs() < 1e-9) => {
let (w2, h2) = (w * sx, h * sy);
let pos = [tx + (x + w / 2.0) * sx, ty + (y + h / 2.0) * sy];
let mut g = build::shape_rect(ids, [w2, h2], pos, rx * sx);
g.name = "Rectangle Path 1".into();
items.push(g);
}
(Geom::Ellipse { cx, cy, rx, ry }, Some((sx, sy, tx, ty))) => {
items.push(build::shape_ellipse(ids, [2.0 * rx * sx, 2.0 * ry * sy], [tx + cx * sx, ty + cy * sy]));
}
(g, _) => items.extend(shape_paths(ids, &(m * g.to_path()))),
}
let bbox = s.geom.to_path().bounding_box();
if let Some(st) = &s.stroke {
let width = st.width * det_scale(m);
let mut g = match &st.paint {
Paint::Color(c) => build::shape_stroke(ids, [c[0], c[1], c[2], 1.0], width),
Paint::Gradient(gr) => {
let (radial, a, b) = gradient_points(gr, m, bbox);
build::shape_gradient_stroke(ids, radial, a, b, gradient_value(gr), width)
}
};
set(&mut g, "opacity", Value::Scalar(st.opacity * 100.0));
set(&mut g, "cap", Value::Enum(st.cap as u32));
set(&mut g, "join", Value::Enum(st.join as u32));
set(&mut g, "miter", Value::Scalar(st.miter));
if let Some((d, off)) = &st.dash
&& let Some(dg) = g.sub_mut("dashes")
{
let k = det_scale(m);
set(dg, "dash", Value::Scalar(d.first().copied().unwrap_or(0.0) * k));
set(dg, "gap", Value::Scalar(d.get(1).or(d.first()).copied().unwrap_or(0.0) * k));
set(dg, "offset", Value::Scalar(off * k));
}
items.push(g);
}
if let Some(f) = &s.fill {
let mut g = match f {
Paint::Color(c) => build::shape_fill(ids, [c[0], c[1], c[2], 1.0]),
Paint::Gradient(gr) => {
let (radial, a, b) = gradient_points(gr, m, bbox);
build::shape_gradient_fill(ids, radial, a, b, gradient_value(gr))
}
};
set(&mut g, "opacity", Value::Scalar(s.fill_opacity * 100.0));
set(&mut g, "rule", Value::Enum(u32::from(s.fill_rule == effectcraft_svg::FillRule::EvenOdd)));
items.push(g);
}
items
}
fn set(g: &mut PropGroup, k: &str, v: Value) {
if let Some(p) = g.get_mut(k) {
p.value = v;
}
}
fn group(ids: &mut Ids, name: &str, items: Vec<PropGroup>, opacity: f64) -> PropGroup {
let mut g = build::shape_group(ids, name, items);
if let Some(p) = g.prop_mut("transform/opacity") {
p.value = Value::Scalar(opacity * 100.0);
}
g
}
/// Shape layer contents for an SVG document, in document pixels (transforms are baked into the
/// geometry; group and element opacity go to the group transforms). Top of the paint order first.
pub fn svg_contents(ids: &mut Ids, doc: &Doc) -> Vec<PropGroup> {
fn walk(ids: &mut Ids, nodes: &[Node], m: Affine) -> Vec<PropGroup> {
let mut out = vec![];
for n in nodes.iter().rev() {
match n {
Node::Group(g) => {
let items = walk(ids, &g.children, m * g.transform);
if !items.is_empty() {
out.push(group(ids, &g.name, items, g.opacity));
}
}
Node::Shape(s) => {
let items = shape_items(ids, s, m * s.transform);
if !items.is_empty() {
out.push(group(ids, &s.name, items, s.opacity));
}
}
}
}
out
}
let root = &doc.root;
let items = walk(ids, &root.children, root.transform);
if (root.opacity - 1.0).abs() > 1e-9 { vec![group(ids, "svg", items, root.opacity)] } else { items }
}
/// A copy of `src`'s transform group with fresh uids.
fn copy_transform(proj: &mut Project, src: &Layer, dst: &mut Layer) {
let Some(tr) = src.props.sub("transform") else { return };
let mut tr = tr.clone();
let mut n = proj.next_id;
tr.reassign_uids(&mut n);
proj.next_id = n + 1;
if let Some(slot) = dst.props.children.iter_mut().find(|c| c.match_id() == "transform") {
*slot = tr.into();
}
}
fn like_source(dst: &mut Layer, src: &Layer) {
dst.start_time = src.start_time;
dst.in_point = src.in_point;
dst.out_point = src.out_point;
dst.stretch = src.stretch;
dst.parent = src.parent;
dst.switches.three_d = src.switches.three_d;
}
/// Layer ▸ Create ▸ Create Shapes from Vector Layer: `doc` is the SVG shown by `src`.
pub fn shapes_from_vector(proj: &mut Project, comp: &Comp, src: &Layer, doc: &Doc) -> Layer {
let mut l = build::layer(proj, comp, &format!("{} Outlines", src.name), LayerSource::Shape, (0, 0), None);
let items = svg_contents(&mut Ids(&mut proj.next_id), doc);
if let Some(c) = l.props.sub_mut("contents") {
c.children.extend(items.into_iter().map(Into::into));
}
copy_transform(proj, src, &mut l);
like_source(&mut l, src);
l.name = comp.unique_layer_name(&format!("{} Outlines", src.name));
l
}
/// Layer ▸ Create ▸ Create Shapes from Text: one group per character with its outline, fill and
/// stroke (as laid out at the comp time `ctx` was made for).
pub fn shapes_from_text(proj: &mut Project, comp: &Comp, ctx_project: &Project, cid: ItemId, src: &Layer, time: effectcraft_time::Tick) -> Option<Layer> {
let ctx = effectcraft_render::EvalCtx::new(ctx_project, cid, comp, time);
let geom = effectcraft_render::text::text_geom(&ctx, src)?;
let chars: Vec<char> = geom.doc.text.chars().filter(|c| !c.is_whitespace()).collect();
let mut l = build::layer(proj, comp, &format!("{} Outlines", src.name), LayerSource::Shape, (0, 0), None);
let mut groups = vec![];
{
let mut ids = Ids(&mut proj.next_id);
for (i, g) in geom.glyphs.iter().enumerate() {
let path = g.path();
if path.elements().is_empty() {
continue;
}
let mut items = shape_paths(&mut ids, &path);
if g.stroke_width > 0.0 && g.stroke[3] > 0.0 {
let c = g.stroke;
items.push(build::shape_stroke(&mut ids, [c[0] as f64, c[1] as f64, c[2] as f64, 1.0], g.stroke_width));
}
if g.apply_fill {
let c = g.fill;
let mut f = build::shape_fill(&mut ids, [c[0] as f64, c[1] as f64, c[2] as f64, 1.0]);
set(&mut f, "opacity", Value::Scalar(c[3] as f64 * 100.0));
items.push(f);
}
let name = chars.get(i).map(|c| c.to_string()).unwrap_or_else(|| format!("Glyph {}", i + 1));
groups.push(group(&mut ids, &name, items, 1.0));
}
}
// First character on top, as in After Effects.
if let Some(c) = l.props.sub_mut("contents") {
c.children.extend(groups.into_iter().map(Into::into));
}
copy_transform(proj, src, &mut l);
like_source(&mut l, src);
l.name = comp.unique_layer_name(&format!("{} Outlines", src.name));
Some(l)
}
/// Glyph outlines of a text layer in comp space (for Create Masks from Text).
pub fn text_outlines_in_comp(project: &Project, cid: ItemId, comp: &Comp, src: &Layer, time: effectcraft_time::Tick) -> Vec<ShapePath> {
let ctx = effectcraft_render::EvalCtx::new(project, cid, comp, time);
let Some(geom) = effectcraft_render::text::text_geom(&ctx, src) else { return vec![] };
let (m, _) = ctx.layer_to_comp(src);
let mut out = vec![];
for g in &geom.glyphs {
let p = g.path();
let p = effectcraft_path::transform(std::slice::from_ref(&p), &m).remove(0);
out.extend(effectcraft_path::from_kurbo(&p));
}
out
}
/// A comp-sized white solid with one mask per glyph contour (Difference, so counters stay open).
pub fn masks_layer(proj: &mut Project, comp: &Comp, name: &str, outlines: Vec<ShapePath>, solid: ItemId) -> Layer {
let mut l = build::layer(proj, comp, name, LayerSource::Solid { item: solid }, (comp.width, comp.height), None);
let mut ids = Ids(&mut proj.next_id);
if let Some(masks) = l.props.sub_mut("masks") {
for (i, sp) in outlines.into_iter().enumerate() {
masks.children.push(build::mask(&mut ids, &format!("Mask {}", i + 1), sp, MaskMode::Difference, [255, 255, 0]).into());
}
}
l
}
+5 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "effectcraft-export"
description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, PNG/JPEG/TIFF/EXR sequences, animated GIF"
description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, WebM (VP9 + Opus), PNG/JPEG/TIFF/EXR sequences, animated GIF, WAV/AIFF"
version.workspace = true
edition.workspace = true
license.workspace = true
@@ -18,6 +18,8 @@ filmcraft-h264enc = { workspace = true }
filmcraft-prores = { workspace = true }
filmcraft-aac = { workspace = true }
filmcraft-isobmff = { workspace = true }
effectcraft-opusenc = { workspace = true }
effectcraft-vp9enc = { workspace = true }
image = { workspace = true, features = ["png", "jpeg", "tiff", "exr"] }
gif = { workspace = true }
rayon = { workspace = true }
@@ -28,3 +30,5 @@ log = { workspace = true }
[dev-dependencies]
effectcraft-media = { workspace = true }
effectcraft-color = { workspace = true }
filmcraft-opus = { workspace = true }
filmcraft-vp9 = { workspace = true }
+6
View File
@@ -7,6 +7,8 @@
//! | PNG / JPEG / TIFF sequence | one file per frame | 8-bit | PNG, TIFF | — |
//! | OpenEXR sequence | one file per frame | 32-bit float, linear light, premultiplied | yes | — |
//! | Animated GIF | GIF89a | 256-colour palette per frame (NeuQuant) | 1-bit | — |
//! | WebM | WebM (Matroska) | VP9 profile 0 intra frames (`effectcraft-vp9enc`), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo |
//! | WAV / AIFF | RIFF WAVE / AIFF | — | — | 16-bit PCM stereo (audio only) |
//!
//! Frames are rendered in parallel batches (rayon; one batch ≈ one frame per core) and handed to
//! the encoder in order. Progress is reported after each batch through a callback that can cancel
@@ -18,6 +20,7 @@
mod encode;
mod out;
mod webm;
use web_time::Instant;
@@ -131,6 +134,9 @@ pub fn export(job: &Job, progress: &mut dyn FnMut(&Progress) -> bool) -> Result<
let mut report = match job.output.format {
OutputFormat::H264 | OutputFormat::ProRes => encode::movie(job, comp, w, h, &mut st)?,
OutputFormat::Gif => gif_export(job, comp, w, h, &mut st)?,
OutputFormat::WebM => webm::webm(job, comp, w, h, &mut st)?,
OutputFormat::Wav => webm::audio_file(job, comp, false, &mut st)?,
OutputFormat::Aiff => webm::audio_file(job, comp, true, &mut st)?,
f if f.is_sequence() => sequence(job, comp, w, h, &mut st)?,
f => return Err(ExportError::Unsupported(f.label().into())),
};
+368
View File
@@ -0,0 +1,368 @@
//! WebM output (VP9 video, optional VP9 alpha, Opus audio) and audio-only WAV / AIFF.
//!
//! The WebM container is written from the Matroska / WebM specifications (IETF RFC 8794 EBML,
//! the Matroska element registry and the WebM container guidelines): EBML header, Segment
//! (size patched at the end), Info, Tracks and one Cluster per render batch. Video frames are
//! SimpleBlocks; with alpha (Channels: RGB + Alpha) each frame is a BlockGroup whose
//! BlockAdditional (BlockAddID 1) carries a second VP9 frame coding the alpha channel as luma,
//! with `AlphaMode` 1 on the track, as WebM defines. VP9 comes from `effectcraft-vp9enc` (every
//! frame a key frame), Opus from `effectcraft-opusenc` (CELT, 48 kHz stereo, 20 ms packets,
//! `CodecDelay` = pre-skip). WAV (RIFF, 16-bit PCM) and AIFF (big-endian 16-bit, 80-bit
//! extended sample rate) follow their published file layouts.
use std::io::{Seek, SeekFrom, Write};
use effectcraft_project::Comp;
use effectcraft_project::render_queue::Channels;
use effectcraft_time::{TICKS_PER_SECOND, Tick};
use rayon::prelude::*;
use crate::{Job, Report, Result, State, batch_size, io, render_frame, rgba8, wants_audio};
const OPUS_RATE: u32 = 48_000;
const OPUS_BITRATE: u32 = 192_000;
// Element ids (with their length markers).
const EBML: u32 = 0x1A45_DFA3;
const SEGMENT: u32 = 0x1853_8067;
const INFO: u32 = 0x1549_A966;
const TRACKS: u32 = 0x1654_AE6B;
const TRACK_ENTRY: u32 = 0xAE;
const CLUSTER: u32 = 0x1F43_B675;
fn id_bytes(id: u32) -> Vec<u8> {
let b = id.to_be_bytes();
let skip = b.iter().position(|x| *x != 0).unwrap_or(3);
b[skip..].to_vec()
}
/// EBML variable-size integer for an element data size (shortest form).
fn vint(n: u64) -> Vec<u8> {
for len in 1..=8u32 {
let max = (1u64 << (7 * len)) - 2;
if n <= max {
let marked = n | (1u64 << (7 * len));
return marked.to_be_bytes()[8 - len as usize..].to_vec();
}
}
let mut v = vec![0x01];
v.extend_from_slice(&n.to_be_bytes()[1..]);
v
}
fn el(out: &mut Vec<u8>, id: u32, data: &[u8]) {
out.extend(id_bytes(id));
out.extend(vint(data.len() as u64));
out.extend_from_slice(data);
}
fn el_uint(out: &mut Vec<u8>, id: u32, v: u64) {
let b = v.to_be_bytes();
let skip = b.iter().position(|x| *x != 0).unwrap_or(7);
el(out, id, &b[skip..]);
}
fn el_float(out: &mut Vec<u8>, id: u32, v: f64) {
el(out, id, &v.to_be_bytes());
}
fn el_str(out: &mut Vec<u8>, id: u32, s: &str) {
el(out, id, s.as_bytes());
}
/// A block payload: track number, timestamp relative to the cluster, flags, frame.
fn block(track: u64, rel: i16, flags: u8, data: &[u8]) -> Vec<u8> {
let mut b = vint(track);
b.extend_from_slice(&rel.to_be_bytes());
b.push(flags);
b.extend_from_slice(data);
b
}
struct Frame {
ms: i64,
track: u64,
data: Vec<u8>,
alpha: Option<Vec<u8>>,
}
fn cluster(frames: &mut [Frame]) -> Vec<u8> {
frames.sort_by_key(|f| (f.ms, f.track));
let base = frames.first().map_or(0, |f| f.ms);
let mut body = vec![];
el_uint(&mut body, 0xE7, base.max(0) as u64);
for f in frames.iter() {
let rel = (f.ms - base).clamp(i16::MIN as i64, i16::MAX as i64) as i16;
match &f.alpha {
None => el(&mut body, 0xA3, &block(f.track, rel, 0x80, &f.data)),
Some(a) => {
let mut g = vec![];
el(&mut g, 0xA1, &block(f.track, rel, 0x00, &f.data));
let mut more = vec![];
el_uint(&mut more, 0xEE, 1);
el(&mut more, 0xA5, a);
let mut adds = vec![];
el(&mut adds, 0xA6, &more);
el(&mut g, 0x75A1, &adds);
el(&mut body, 0xA0, &g);
}
}
}
let mut out = vec![];
el(&mut out, CLUSTER, &body);
out
}
pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result<Report> {
let rate = job.settings.rate(comp);
let alpha = job.output.channels == Channels::Rgba;
let quality = job.output.quality.clamp(1, 100);
let cfg = effectcraft_vp9enc::EncoderConfig { width: w, height: h, quality, full_range: false };
let with_audio = wants_audio(job);
let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(2, OPUS_BITRATE));
let total = st.total;
let frame_ms = |k: u64| -> i64 { ((k as i128 * 1000 * rate.den as i128 + rate.num as i128 / 2) / rate.num as i128) as i64 };
let duration_ms = frame_ms(total) as f64;
let mut file = crate::out::create(job.sink, job.path)?;
// EBML header.
let mut head = vec![];
let mut eb = vec![];
el_uint(&mut eb, 0x4286, 1);
el_uint(&mut eb, 0x42F7, 1);
el_uint(&mut eb, 0x42F2, 4);
el_uint(&mut eb, 0x42F3, 8);
el_str(&mut eb, 0x4282, "webm");
el_uint(&mut eb, 0x4287, 4);
el_uint(&mut eb, 0x4285, 2);
el(&mut head, EBML, &eb);
// Segment with an 8-byte size patched at the end.
head.extend(id_bytes(SEGMENT));
let size_pos = head.len() as u64;
head.extend_from_slice(&[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]);
let seg_start = head.len() as u64;
let mut info = vec![];
el_uint(&mut info, 0x2AD7B1, 1_000_000);
el_float(&mut info, 0x4489, duration_ms);
el_str(&mut info, 0x4D80, "EffectCraft");
el_str(&mut info, 0x5741, "EffectCraft");
el(&mut head, INFO, &info);
let mut tracks = vec![];
let mut v = vec![];
el_uint(&mut v, 0xD7, 1);
el_uint(&mut v, 0x73C5, 1);
el_uint(&mut v, 0x83, 1);
el_uint(&mut v, 0x9C, 0);
el_str(&mut v, 0x86, "V_VP9");
el_uint(&mut v, 0x23E383, (1_000_000_000u128 * rate.den as u128 / rate.num.max(1) as u128) as u64);
if alpha {
el_uint(&mut v, 0x55EE, 1);
}
let mut video = vec![];
el_uint(&mut video, 0xB0, w as u64);
el_uint(&mut video, 0xBA, h as u64);
if alpha {
el_uint(&mut video, 0x53C0, 1);
}
el(&mut v, 0xE0, &video);
el(&mut tracks, TRACK_ENTRY, &v);
if let Some(o) = &opus {
let mut a = vec![];
el_uint(&mut a, 0xD7, 2);
el_uint(&mut a, 0x73C5, 2);
el_uint(&mut a, 0x83, 2);
el_uint(&mut a, 0x9C, 0);
el_str(&mut a, 0x86, "A_OPUS");
el(&mut a, 0x63A2, &o.opus_head());
el_uint(&mut a, 0x56AA, o.pre_skip() as u64 * 1_000_000_000 / OPUS_RATE as u64);
el_uint(&mut a, 0x56BB, 80_000_000);
let mut au = vec![];
el_float(&mut au, 0xB5, OPUS_RATE as f64);
el_uint(&mut au, 0x9F, 2);
el(&mut a, 0xE1, &au);
el(&mut tracks, TRACK_ENTRY, &a);
}
el(&mut head, TRACKS, &tracks);
file.write_all(&head).map_err(io)?;
let mut seg_len = head.len() as u64 - seg_start;
// Audio: mixed in blocks, encoded in 20 ms packets.
let (span_start, span_end) = job.settings.span(comp);
let mut cursor = span_start.to_units_floor(OPUS_RATE as i64);
let mut pcm_left: Vec<f32> = vec![];
let mut packets = 0u64;
let mut audio_until = |until: Tick, frames: &mut Vec<Frame>, finish: bool| {
let Some(enc) = opus.as_mut() else { return };
let end = until.to_units_floor(OPUS_RATE as i64);
if end > cursor {
let n = (end - cursor) as usize;
let start = Tick(((cursor as i128 * TICKS_PER_SECOND as i128) / OPUS_RATE as i128) as i64);
pcm_left.extend(effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, OPUS_RATE));
cursor = end;
}
let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2;
if finish && !pcm_left.is_empty() {
// Pad the last packet (pre-skip + padding cover the encoder delay).
let pad = (fs - pcm_left.len() % fs) % fs + fs;
pcm_left.extend(std::iter::repeat_n(0.0, pad));
}
while pcm_left.len() >= fs {
let pkt = enc.encode_float(&pcm_left[..fs]);
pcm_left.drain(..fs);
frames.push(Frame { ms: packets as i64 * 20, track: 2, data: pkt, alpha: None });
packets += 1;
}
};
let batch = batch_size();
let mut i = 0;
while i < total {
let end = (i + batch).min(total);
let encoded: Vec<(Vec<u8>, Option<Vec<u8>>)> = (i..end)
.into_par_iter()
.map(|k| {
let px = rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h);
let mut enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone());
let (y, u, vv) = effectcraft_vp9enc::rgba_to_yuv420(&px, w, h, false);
let color = enc.encode_yuv420(&y, &u, &vv);
let a = alpha.then(|| {
let mut ea = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone());
let (y, u, vv) = effectcraft_vp9enc::alpha_to_yuv420(&px, w, h);
ea.encode_yuv420(&y, &u, &vv)
});
(color, a)
})
.collect();
let mut frames: Vec<Frame> =
encoded.into_iter().enumerate().map(|(j, (data, alpha))| Frame { ms: frame_ms(i + j as u64), track: 1, data, alpha }).collect();
let t_end = if end >= total { span_end } else { job.settings.frame_time(comp, end) };
audio_until(t_end, &mut frames, end >= total);
let c = cluster(&mut frames);
file.write_all(&c).map_err(io)?;
seg_len += c.len() as u64;
st.advance(end - i)?;
i = end;
}
if total == 0 {
let mut frames = vec![];
audio_until(span_end, &mut frames, true);
if !frames.is_empty() {
let c = cluster(&mut frames);
file.write_all(&c).map_err(io)?;
seg_len += c.len() as u64;
}
}
// Patch the Segment size.
let mut size = [0u8; 8];
size[0] = 0x01;
size[1..].copy_from_slice(&seg_len.to_be_bytes()[1..]);
file.seek(SeekFrom::Start(size_pos)).map_err(io)?;
file.write_all(&size).map_err(io)?;
file.seek(SeekFrom::End(0)).map_err(io)?;
let bytes = file.finish()?;
Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio })
}
// ---------------------------------------------------------------- audio-only outputs
fn ieee_extended(v: f64) -> [u8; 10] {
// 80-bit IEEE 754 extended precision (AIFF COMM sample rate).
let mut out = [0u8; 10];
if v <= 0.0 {
return out;
}
let e = v.log2().floor() as i32;
let mant = (v / 2f64.powi(e) * (1u64 << 63) as f64) as u64;
out[..2].copy_from_slice(&((e + 16383) as u16).to_be_bytes());
out[2..].copy_from_slice(&mant.to_be_bytes());
out
}
/// WAV or AIFF: the comp's audio over the render span, 16-bit stereo.
pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> Result<Report> {
let sr = job.output.audio_sample_rate.clamp(8_000, 192_000);
let (span_start, span_end) = job.settings.span(comp);
let start = span_start.to_units_floor(sr as i64);
let end = span_end.to_units_floor(sr as i64);
let n = (end - start).max(0) as usize;
let mut pcm = Vec::with_capacity(n * 4);
// Mix in one-second blocks, reporting progress in frames.
let block = sr as usize;
let mut done = 0usize;
let total_frames = st.total.max(1);
let mut reported = 0u64;
while done < n {
let k = block.min(n - done);
let t = Tick((((start + done as i64) as i128 * TICKS_PER_SECOND as i128) / sr as i128) as i64);
for s in effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, t, k, sr) {
let v = (s.clamp(-1.0, 1.0) * 32767.0).round() as i16;
pcm.extend_from_slice(&if aiff { v.to_be_bytes() } else { v.to_le_bytes() });
}
done += k;
let now = (done as u64 * total_frames / n.max(1) as u64).min(total_frames);
if now > reported && st.total > 0 {
st.advance(now - reported)?;
reported = now;
}
}
if reported < st.total {
st.advance(st.total - reported)?;
}
let mut out = vec![];
let frames = (pcm.len() / 4) as u32;
if aiff {
let mut comm = vec![];
comm.extend_from_slice(&2u16.to_be_bytes());
comm.extend_from_slice(&frames.to_be_bytes());
comm.extend_from_slice(&16u16.to_be_bytes());
comm.extend_from_slice(&ieee_extended(sr as f64));
let ssnd_len = 8 + pcm.len();
out.extend_from_slice(b"FORM");
out.extend_from_slice(&((4 + 8 + comm.len() + 8 + ssnd_len + ssnd_len % 2) as u32).to_be_bytes());
out.extend_from_slice(b"AIFF");
out.extend_from_slice(b"COMM");
out.extend_from_slice(&(comm.len() as u32).to_be_bytes());
out.extend_from_slice(&comm);
out.extend_from_slice(b"SSND");
out.extend_from_slice(&(ssnd_len as u32).to_be_bytes());
out.extend_from_slice(&[0; 8]);
out.extend_from_slice(&pcm);
if ssnd_len % 2 == 1 {
out.push(0);
}
} else {
out.extend_from_slice(b"RIFF");
out.extend_from_slice(&((36 + pcm.len()) as u32).to_le_bytes());
out.extend_from_slice(b"WAVEfmt ");
out.extend_from_slice(&16u32.to_le_bytes());
out.extend_from_slice(&1u16.to_le_bytes());
out.extend_from_slice(&2u16.to_le_bytes());
out.extend_from_slice(&sr.to_le_bytes());
out.extend_from_slice(&(sr * 4).to_le_bytes());
out.extend_from_slice(&4u16.to_le_bytes());
out.extend_from_slice(&16u16.to_le_bytes());
out.extend_from_slice(b"data");
out.extend_from_slice(&(pcm.len() as u32).to_le_bytes());
out.extend_from_slice(&pcm);
}
let mut file = crate::out::create(job.sink, job.path)?;
file.write_all(&out).map_err(io)?;
let bytes = file.finish()?;
Ok(Report { path: job.path.to_string(), frames: 0, width: 0, height: 0, seconds: 0.0, bytes, audio: true })
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn vints_and_ids() {
assert_eq!(vint(0), vec![0x80]);
assert_eq!(vint(126), vec![0xFE]);
assert_eq!(vint(127), vec![0x40, 0x7F]);
assert_eq!(id_bytes(0xA3), vec![0xA3]);
assert_eq!(id_bytes(EBML), vec![0x1A, 0x45, 0xDF, 0xA3]);
assert_eq!(ieee_extended(44100.0), [0x40, 0x0E, 0xAC, 0x44, 0, 0, 0, 0, 0, 0]);
assert_eq!(ieee_extended(48000.0)[..4], [0x40, 0x0E, 0xBB, 0x80]);
}
}
+364
View File
@@ -0,0 +1,364 @@
//! Round trip of `effectcraft-opusenc` through FilmCraft's spec-derived Opus decoder, plus an
//! ffmpeg oracle (skipped when ffmpeg is not installed) reading an Ogg Opus file written here.
use std::process::Command;
use effectcraft_opusenc::OpusEncoder;
const SR: f32 = 48_000.0;
const FRAME: usize = OpusEncoder::FRAME_SIZE;
/// Encodes interleaved `pcm`, padding the tail so the encoder's look-ahead is flushed.
fn encode(pcm: &[f32], channels: u8, bitrate: u32) -> (OpusEncoder, Vec<Vec<u8>>) {
let c = channels as usize;
let mut enc = OpusEncoder::new(channels, bitrate);
let total = pcm.len() / c + enc.pre_skip() as usize;
let frames = total.div_ceil(FRAME);
let mut padded = pcm.to_vec();
padded.resize(frames * FRAME * c, 0.0);
let packets = padded.chunks(FRAME * c).map(|f| enc.encode_float(f)).collect();
(enc, packets)
}
/// Decodes with filmcraft-opus, drops the pre-skip and returns planar channels of `len` samples.
fn decode(head: &[u8], packets: &[Vec<u8>], len: usize) -> Vec<Vec<f32>> {
let mut dec = filmcraft_opus::Decoder::new(head).expect("OpusHead");
dec.set_trim_pre_skip(true);
let mut out: Vec<Vec<f32>> = vec![Vec::new(); dec.channels()];
for p in packets {
let planes = dec.decode(Some(p)).expect("decode");
for (o, p) in out.iter_mut().zip(planes) {
o.extend_from_slice(&p);
}
}
for o in &mut out {
assert!(o.len() >= len, "decoded {} < {len}", o.len());
o.truncate(len);
}
out
}
fn roundtrip(planar: &[Vec<f32>], bitrate: u32) -> (Vec<Vec<u8>>, Vec<Vec<f32>>) {
let c = planar.len();
let len = planar[0].len();
let mut pcm = vec![0f32; len * c];
for (ch, p) in planar.iter().enumerate() {
for (i, &v) in p.iter().enumerate() {
pcm[i * c + ch] = v;
}
}
let (enc, packets) = encode(&pcm, c as u8, bitrate);
let decoded = decode(&enc.opus_head(), &packets, len);
(packets, decoded)
}
/// SNR in dB over the steady part (the first and last 20 ms are excluded).
fn snr(orig: &[f32], dec: &[f32]) -> f64 {
let (a, b) = (FRAME, orig.len() - FRAME);
let mut s = 0f64;
let mut e = 0f64;
for i in a..b {
s += (orig[i] as f64).powi(2);
e += (orig[i] as f64 - dec[i] as f64).powi(2);
}
10.0 * (s / e.max(1e-30)).log10()
}
fn sine(freq: f32, amp: f32, len: usize) -> Vec<f32> {
(0..len).map(|i| amp * (2.0 * std::f32::consts::PI * freq * i as f32 / SR).sin()).collect()
}
fn chirp(f0: f32, f1: f32, amp: f32, len: usize) -> Vec<f32> {
let dur = len as f32 / SR;
let k = (f1 - f0) / dur;
(0..len)
.map(|i| {
let t = i as f32 / SR;
amp * (2.0 * std::f32::consts::PI * (f0 * t + 0.5 * k * t * t)).sin()
})
.collect()
}
fn noise(amp: f32, len: usize, seed: u32) -> Vec<f32> {
let mut s = seed;
(0..len)
.map(|_| {
s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
amp * ((s >> 8) as f32 / (1u32 << 24) as f32 * 2.0 - 1.0)
})
.collect()
}
/// Energy (dB) of `x` in each of a few octave bands up to 16 kHz, from a Hann-windowed FFT
/// averaged over 2048-sample blocks.
fn band_energies(x: &[f32]) -> Vec<f64> {
const N: usize = 2048;
let edges = [100.0, 250.0, 500.0, 1000.0, 2000.0, 4000.0, 8000.0, 16000.0];
let mut acc = vec![0f64; edges.len() - 1];
let mut start = N;
while start + N <= x.len() - N {
let mut re: Vec<f64> = (0..N).map(|i| x[start + i] as f64 * (0.5 - 0.5 * (2.0 * std::f64::consts::PI * i as f64 / N as f64).cos())).collect();
let mut im = vec![0f64; N];
fft(&mut re, &mut im);
for k in 1..N / 2 {
let f = k as f64 * SR as f64 / N as f64;
if let Some(b) = edges.windows(2).position(|w| f >= w[0] && f < w[1]) {
acc[b] += re[k] * re[k] + im[k] * im[k];
}
}
start += N;
}
acc.iter().map(|e| 10.0 * (e + 1e-20).log10()).collect()
}
fn fft(re: &mut [f64], im: &mut [f64]) {
let n = re.len();
let mut j = 0;
for i in 1..n {
let mut bit = n >> 1;
while j & bit != 0 {
j ^= bit;
bit >>= 1;
}
j |= bit;
if i < j {
re.swap(i, j);
im.swap(i, j);
}
}
let mut len = 2;
while len <= n {
let ang = -2.0 * std::f64::consts::PI / len as f64;
for s in (0..n).step_by(len) {
for k in 0..len / 2 {
let (wr, wi) = ((ang * k as f64).cos(), (ang * k as f64).sin());
let (a, b) = (s + k, s + k + len / 2);
let tr = re[b] * wr - im[b] * wi;
let ti = re[b] * wi + im[b] * wr;
re[b] = re[a] - tr;
im[b] = im[a] - ti;
re[a] += tr;
im[a] += ti;
}
}
len <<= 1;
}
}
#[test]
fn sine_440_mono() {
let x = sine(440.0, 0.5, 48_000);
for rate in [64_000, 128_000] {
let (packets, d) = roundtrip(std::slice::from_ref(&x), rate);
assert!(packets.iter().all(|p| p.len() == rate as usize / 400));
let s = snr(&x, &d[0]);
eprintln!("sine 440 Hz mono @ {} kb/s: SNR {s:.1} dB", rate / 1000);
assert!(s > 20.0, "sine SNR {s:.1} dB at {rate}");
}
}
#[test]
fn chirp_mono() {
let x = chirp(100.0, 10_000.0, 0.4, 96_000);
for rate in [64_000, 128_000] {
let (_, d) = roundtrip(std::slice::from_ref(&x), rate);
let s = snr(&x, &d[0]);
eprintln!("chirp 100 Hz-10 kHz mono @ {} kb/s: SNR {s:.1} dB", rate / 1000);
assert!(s > 15.0, "chirp SNR {s:.1} dB at {rate}");
}
}
#[test]
fn white_noise_mono() {
let x = noise(0.3, 48_000, 12345);
for rate in [64_000, 128_000] {
let (_, d) = roundtrip(std::slice::from_ref(&x), rate);
let s = snr(&x, &d[0]);
let (eo, ed) = (band_energies(&x), band_energies(&d[0]));
let err: Vec<f32> = x.iter().zip(&d[0]).map(|(a, b)| a - b).collect();
let ee = band_energies(&err);
let octave_snr: Vec<f64> = eo.iter().zip(&ee).map(|(a, b)| a - b).collect();
let worst = eo.iter().zip(&ed).map(|(a, b)| (a - b).abs()).fold(0.0, f64::max);
let min_octave = octave_snr.iter().copied().fold(f64::MAX, f64::min);
eprintln!(
"white noise mono @ {} kb/s: full-band SNR {s:.1} dB, octave SNRs 100 Hz-16 kHz {:.1?} dB, worst octave energy error {worst:.2} dB",
rate / 1000,
octave_snr
);
// Full-band noise loses everything above 20 kHz (CELT's last band ends there) and spends
// few bits per coefficient on 16-20 kHz, so the full-band waveform SNR is low; the band
// energies must match closely and the octaves below 16 kHz must be waveform-coded.
assert!(worst < 1.5, "band energy error {worst:.2} dB at {rate}: {eo:?} vs {ed:?}");
assert!(s > 0.5, "noise SNR {s:.1} dB at {rate}");
assert!(min_octave > if rate >= 128_000 { 10.0 } else { 3.0 }, "octave SNRs {octave_snr:?} at {rate}");
}
}
#[test]
fn lowpass_noise_mono() {
// White noise through a 4-pole low-pass at about 2 kHz: a dense spectrum the encoder can
// afford to code as a waveform.
let w = noise(0.8, 48_000, 999);
let a = (-2.0 * std::f32::consts::PI * 2_000.0 / SR).exp();
let mut st = [0f32; 4];
let x: Vec<f32> = w
.iter()
.map(|&v| {
let mut y = v;
for s in st.iter_mut() {
*s = (1.0 - a) * y + a * *s;
y = *s;
}
y
})
.collect();
for rate in [64_000, 128_000] {
let (_, d) = roundtrip(std::slice::from_ref(&x), rate);
let s = snr(&x, &d[0]);
eprintln!("low-pass noise mono @ {} kb/s: SNR {s:.1} dB", rate / 1000);
assert!(s > 15.0, "low-pass noise SNR {s:.1} dB at {rate}");
}
}
#[test]
fn silence_stays_silent() {
let x = vec![0f32; 48_000];
let (packets, d) = roundtrip(&[x.clone(), x], 128_000);
assert!(packets.iter().all(|p| p.len() == 3), "silence frames are 3-byte packets");
for ch in &d {
let peak = ch.iter().fold(0f32, |m, v| m.max(v.abs()));
assert!(peak < 1e-6, "peak {peak}");
}
}
#[test]
fn stereo_distinct_channels() {
let l = sine(440.0, 0.5, 48_000);
let r = chirp(200.0, 6_000.0, 0.3, 48_000);
let (packets, d) = roundtrip(&[l.clone(), r.clone()], 256_000);
assert!(packets.iter().all(|p| p[0] == 0xFC && p.len() == 640));
let (sl, sr) = (snr(&l, &d[0]), snr(&r, &d[1]));
eprintln!("stereo @ 256 kb/s: L (sine) SNR {sl:.1} dB, R (chirp) SNR {sr:.1} dB");
assert!(sl > 20.0 && sr > 15.0, "stereo SNR L {sl:.1} R {sr:.1}");
// No leakage of the chirp into the left channel (2-4 kHz octave: chirp only).
let (el, er) = (band_energies(&d[0]), band_energies(&r));
assert!(el[4] < er[4] - 30.0, "chirp energy leaked into L: {} vs {}", el[4], er[4]);
}
#[test]
fn stereo_128k_total() {
let l = sine(1000.0, 0.4, 48_000);
let r = sine(330.0, 0.4, 48_000);
let (_, d) = roundtrip(&[l.clone(), r.clone()], 128_000);
let (sl, sr) = (snr(&l, &d[0]), snr(&r, &d[1]));
eprintln!("stereo @ 128 kb/s: L SNR {sl:.1} dB, R SNR {sr:.1} dB");
assert!(sl > 15.0 && sr > 15.0, "stereo SNR L {sl:.1} R {sr:.1}");
}
#[test]
fn signal_after_silence_and_loud_transient() {
// Silence, then a burst at full scale, then silence again: energy prediction must recover.
let mut x = vec![0f32; 24_000];
x.extend(sine(2_000.0, 0.99, 24_000));
x.extend(vec![0f32; 24_000]);
let (_, d) = roundtrip(std::slice::from_ref(&x), 96_000);
let s = snr(&x[24_000 + 2 * FRAME..48_000], &d[0][24_000 + 2 * FRAME..48_000]);
eprintln!("burst after silence @ 96 kb/s: SNR {s:.1} dB");
assert!(s > 15.0, "burst SNR {s:.1}");
let tail_peak = d[0][48_000 + 2 * FRAME..].iter().fold(0f32, |m, v| m.max(v.abs()));
assert!(tail_peak < 1e-3, "tail peak {tail_peak}");
}
// ---- Ogg Opus (RFC 3533 + RFC 7845) for the ffmpeg oracle ----
fn ogg_crc(data: &[u8]) -> u32 {
let mut crc = 0u32;
for &b in data {
crc ^= (b as u32) << 24;
for _ in 0..8 {
crc = if crc & 0x8000_0000 != 0 { (crc << 1) ^ 0x04C1_1DB7 } else { crc << 1 };
}
}
crc
}
fn ogg_page(out: &mut Vec<u8>, packet: &[u8], header_type: u8, granule: u64, seq: u32) {
let mut lacing = vec![255u8; packet.len() / 255];
lacing.push((packet.len() % 255) as u8);
let start = out.len();
out.extend_from_slice(b"OggS");
out.push(0);
out.push(header_type);
out.extend_from_slice(&granule.to_le_bytes());
out.extend_from_slice(&0x4543_4F50u32.to_le_bytes()); // serial
out.extend_from_slice(&seq.to_le_bytes());
out.extend_from_slice(&0u32.to_le_bytes()); // CRC placeholder
out.push(lacing.len() as u8);
out.extend_from_slice(&lacing);
out.extend_from_slice(packet);
let crc = ogg_crc(&out[start..]);
out[start + 22..start + 26].copy_from_slice(&crc.to_le_bytes());
}
fn ogg_opus(enc: &OpusEncoder, packets: &[Vec<u8>], len: usize) -> Vec<u8> {
let mut out = Vec::new();
ogg_page(&mut out, &enc.opus_head(), 0x02, 0, 0);
let vendor = b"effectcraft-opusenc";
let mut tags = b"OpusTags".to_vec();
tags.extend_from_slice(&(vendor.len() as u32).to_le_bytes());
tags.extend_from_slice(vendor);
tags.extend_from_slice(&0u32.to_le_bytes());
ogg_page(&mut out, &tags, 0, 0, 1);
let pre_skip = enc.pre_skip() as u64;
for (i, p) in packets.iter().enumerate() {
let last = i + 1 == packets.len();
let granule = if last { pre_skip + len as u64 } else { ((i + 1) * FRAME) as u64 };
ogg_page(&mut out, p, if last { 0x04 } else { 0 }, granule, i as u32 + 2);
}
out
}
#[test]
fn ogg_crc_known_value() {
// CRC-32 (poly 0x04C11DB7, no reflection, init 0) of "123456789".
assert_eq!(ogg_crc(b"123456789"), 0x89A1_897F);
}
#[test]
fn ffmpeg_oracle_decodes_ogg_opus() {
if Command::new("ffmpeg").arg("-version").output().is_err() {
eprintln!("ffmpeg not installed; skipping the oracle test");
return;
}
for (channels, rate) in [(1u8, 96_000u32), (2, 192_000)] {
let len = 48_000;
let planar: Vec<Vec<f32>> =
if channels == 1 { vec![chirp(150.0, 9_000.0, 0.4, len)] } else { vec![sine(440.0, 0.5, len), chirp(150.0, 9_000.0, 0.3, len)] };
let c = channels as usize;
let mut pcm = vec![0f32; len * c];
for (ch, p) in planar.iter().enumerate() {
for (i, &v) in p.iter().enumerate() {
pcm[i * c + ch] = v;
}
}
let (enc, packets) = encode(&pcm, channels, rate);
let path = std::env::temp_dir().join(format!("effectcraft-opusenc-{}-{channels}.opus", std::process::id()));
std::fs::write(&path, ogg_opus(&enc, &packets, len)).unwrap();
let out = Command::new("ffmpeg")
.args(["-v", "error", "-i"])
.arg(&path)
.args(["-f", "f32le", "-ac", &channels.to_string(), "-ar", "48000", "-"])
.output()
.expect("run ffmpeg");
let _ = std::fs::remove_file(&path);
assert!(out.status.success(), "ffmpeg failed: {}", String::from_utf8_lossy(&out.stderr));
let dec: Vec<f32> = out.stdout.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect();
assert_eq!(dec.len(), len * c, "ffmpeg must honour pre-skip and the end granule");
for ch in 0..c {
let d: Vec<f32> = (0..len).map(|i| dec[i * c + ch]).collect();
let s = snr(&planar[ch], &d);
eprintln!("ffmpeg {channels} ch @ {} kb/s, channel {ch}: SNR {s:.1} dB", rate / 1000);
assert!(s > 15.0, "ffmpeg-decoded SNR {s:.1} dB (channel {ch})");
}
}
}
+92
View File
@@ -363,3 +363,95 @@ fn movies_carry_audio() {
assert!(!effectcraft_media::probe(&path).expect("probe").has_audio);
}
}
#[test]
fn webm_vp9_opus_roundtrip() {
let d = out_dir("webm");
let wav = d.join("tone.wav");
write_tone(&wav, 2.0);
let (p, cid) = project(Some(&wav));
let pool = MediaPool::new();
let om = OutputModule { quality: 90, audio: AudioOutput::Auto, ..OutputModule::for_format(OutputFormat::WebM) };
let path = d.join("comp.webm");
let r = run(&p, cid, &pool, &om, &path);
assert!(r.audio && r.bytes > 200, "{r:?}");
// Decoded back through FilmCraft (Matroska demuxer, VP9 and Opus decoders).
let f = decode_movie(&path, FRAMES, 0.08);
assert!(f.has_audio);
let s = MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.3), 24_000, 48_000);
let l = rms(s.iter().step_by(2).copied());
assert!((l - 0.3536).abs() < 0.06, "left RMS {l}");
ffmpeg_check(&path, 0.08);
if let Some(s) = ffprobe(&path) {
assert!(s.contains("vp9|64|48|yuv420p"), "{s}");
assert!(s.contains("opus"), "{s}");
}
}
#[test]
fn webm_with_alpha() {
let d = out_dir("webm-alpha");
let (p, cid) = project(None);
// Transparent outside the red solid.
let om = OutputModule { channels: Channels::Rgba, ..OutputModule::for_format(OutputFormat::WebM) };
let path = d.join("alpha.webm");
run(&p, cid, &NoFootage, &om, &path);
let f = effectcraft_media::probe(&path).expect("probe our output");
assert_eq!((f.width, f.height), (W, H));
let bytes = std::fs::read(&path).unwrap();
// AlphaMode element (0x53C0) and BlockAdditions (0x75A1).
assert!(bytes.windows(3).any(|w| w == [0x53, 0xC0, 0x81]), "AlphaMode 1");
assert!(bytes.windows(2).any(|w| w == [0x75, 0xA1]), "BlockAdditions");
// ffmpeg's libvpx decoder reads the alpha channel.
let png = path.with_extension("alpha.png");
let ok = Command::new("ffmpeg")
.args(["-hide_banner", "-loglevel", "error", "-y", "-c:v", "libvpx-vp9", "-i"])
.arg(&path)
.args(["-vf", "select=eq(n\\,8)", "-frames:v", "1", "-pix_fmt", "rgba"])
.arg(&png)
.status()
.map(|s| s.success())
.unwrap_or(false);
if ok {
let img = image::open(&png).unwrap().to_rgba8();
assert!(img.get_pixel(2, 2)[3] < 20, "corner transparent");
assert!(img.get_pixel(W / 2, H / 2)[3] > 235, "centre opaque");
} else {
eprintln!("ffmpeg with libvpx not available: skipping alpha oracle");
}
}
#[test]
fn wav_and_aiff_audio_only() {
let d = out_dir("audio-only");
let wav = d.join("tone.wav");
write_tone(&wav, 2.0);
let (p, cid) = project(Some(&wav));
let pool = MediaPool::new();
for (fmt, name, magic) in [(OutputFormat::Wav, "out.wav", &b"RIFF"[..]), (OutputFormat::Aiff, "out.aif", &b"FORM"[..])] {
let om = OutputModule::for_format(fmt);
let path = d.join(name);
let r = run(&p, cid, &pool, &om, &path);
assert!(r.audio);
let bytes = std::fs::read(&path).unwrap();
assert_eq!(&bytes[..4], magic);
// 1.2 s of 48 kHz 16-bit stereo.
assert!((bytes.len() as i64 - 57_600 * 4).abs() < 200, "{name}: {}", bytes.len());
let l = if fmt == OutputFormat::Wav {
// Decoded by FilmCraft.
let f = effectcraft_media::probe(&path).expect("probe audio");
assert!(f.has_audio && !f.has_video, "{name}");
let s = MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.3), 24_000, 48_000);
rms(s.iter().step_by(2).copied())
} else {
// AIFF: big-endian samples after the SSND chunk's offset/block size.
let at = bytes.windows(4).position(|w| w == b"SSND").expect("SSND") + 16;
let left = bytes[at..].chunks_exact(4).skip(14_400).take(24_000).map(|c| i16::from_be_bytes([c[0], c[1]]) as f32 / 32767.0);
rms(left)
};
assert!((l - 0.3536).abs() < 0.01, "{name}: left RMS {l}");
if let Some(s) = ffprobe(&path) {
assert!(s.contains("pcm_s16"), "{s}");
}
}
}
+149
View File
@@ -0,0 +1,149 @@
//! The VP9 intra encoder (`effectcraft-vp9enc`) round-trips through FilmCraft's VP9 decoder (and
//! ffmpeg as an external oracle when installed): sizes, lossless exactness, PSNR.
use effectcraft_vp9enc::{EncoderConfig, Vp9Encoder};
use filmcraft_vp9::{Decoder, Plane};
fn plane_u8(p: &Plane) -> Vec<u8> {
match p {
Plane::U8(v) => v.clone(),
Plane::U16(v) => v.iter().map(|x| *x as u8).collect(),
}
}
/// Synthetic planes: gradients, noise and sharp edges.
fn source(w: usize, h: usize, kind: u32) -> (Vec<u8>, Vec<u8>, Vec<u8>) {
let (cw, ch) = (w.div_ceil(2), h.div_ceil(2));
let mut s = 99u32 + kind;
let mut rnd = || {
s = s.wrapping_mul(1664525).wrapping_add(1013904223);
(s >> 24) as u8
};
let y: Vec<u8> = (0..w * h)
.map(|i| {
let (x, yy) = (i % w, i / w);
match kind {
0 => ((x * 255) / w.max(1)) as u8 / 2 + ((yy * 255) / h.max(1)) as u8 / 2,
1 => rnd(),
_ => {
if (x / 7 + yy / 5) % 2 == 0 {
30
} else {
220
}
}
}
})
.collect();
let u: Vec<u8> = (0..cw * ch).map(|i| (64 + (i % cw) * 128 / cw.max(1)) as u8).collect();
let v: Vec<u8> = (0..cw * ch).map(|i| (200 - (i / cw) * 100 / ch.max(1)) as u8).collect();
(y, u, v)
}
fn decode(frame: &[u8]) -> filmcraft_vp9::Picture {
let mut d = Decoder::new();
let mut pics = d.decode(frame, 0).expect("decode");
assert_eq!(pics.len(), 1);
pics.remove(0)
}
fn crop(p: &[u8], stride: usize, w: usize, h: usize) -> Vec<u8> {
(0..h).flat_map(|y| p[y * stride..y * stride + w].to_vec()).collect()
}
fn psnr(a: &[u8], b: &[u8]) -> f64 {
let mse: f64 = a.iter().zip(b).map(|(x, y)| (*x as f64 - *y as f64).powi(2)).sum::<f64>() / a.len() as f64;
if mse == 0.0 { 99.0 } else { 10.0 * (255.0f64 * 255.0 / mse).log10() }
}
#[test]
fn lossless_is_exact_for_odd_sizes() {
for (w, h) in [(1, 1), (33, 17), (64, 64), (200, 120), (7, 70)] {
for kind in 0..3 {
let (y, u, v) = source(w, h, kind);
let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false });
let f = e.encode_yuv420(&y, &u, &v);
let p = decode(&f);
assert_eq!((p.width, p.height), (w as u32, h as u32));
let (cw, ch) = (w.div_ceil(2), h.div_ceil(2));
assert_eq!(crop(&plane_u8(&p.y), p.y_stride, w, h), y, "{w}x{h} kind {kind}: luma");
assert_eq!(crop(&plane_u8(&p.u), p.uv_stride, cw, ch), u, "{w}x{h} kind {kind}: u");
assert_eq!(crop(&plane_u8(&p.v), p.uv_stride, cw, ch), v, "{w}x{h} kind {kind}: v");
}
}
}
#[test]
fn lossy_quality_and_size() {
let (w, h) = (200, 120);
for kind in [0, 2] {
let (y, u, v) = source(w, h, kind);
let mut last = 0usize;
for q in [30u8, 60, 80, 95] {
let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: q, full_range: false });
let f = e.encode_yuv420(&y, &u, &v);
let p = decode(&f);
let got = crop(&plane_u8(&p.y), p.y_stride, w, h);
let db = psnr(&got, &y);
let min = match q {
30 => 20.0,
60 => 28.0,
80 => 32.0,
_ => 38.0,
};
assert!(db > min, "kind {kind} q{q}: PSNR {db:.1} dB, {} bytes", f.len());
assert!(f.len() >= last, "higher quality, more bytes");
last = f.len();
}
}
}
#[test]
fn wide_frames_use_tile_columns() {
// 4160 px > 64 superblocks: two tile columns are required.
let (w, h) = (4160, 16);
let (y, u, v) = source(w, h, 0);
let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 90, full_range: false });
let f = e.encode_yuv420(&y, &u, &v);
let p = decode(&f);
assert!(psnr(&crop(&plane_u8(&p.y), p.y_stride, w, h), &y) > 35.0);
}
#[test]
fn ffmpeg_decodes_our_stream() {
let (w, h) = (96, 64);
let (y, u, v) = source(w, h, 0);
let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false });
let frame = e.encode_yuv420(&y, &u, &v);
// IVF container (one frame).
let mut ivf = Vec::new();
ivf.extend_from_slice(b"DKIF");
ivf.extend_from_slice(&0u16.to_le_bytes());
ivf.extend_from_slice(&32u16.to_le_bytes());
ivf.extend_from_slice(b"VP90");
ivf.extend_from_slice(&(w as u16).to_le_bytes());
ivf.extend_from_slice(&(h as u16).to_le_bytes());
ivf.extend_from_slice(&30u32.to_le_bytes());
ivf.extend_from_slice(&1u32.to_le_bytes());
ivf.extend_from_slice(&1u32.to_le_bytes());
ivf.extend_from_slice(&0u32.to_le_bytes());
ivf.extend_from_slice(&(frame.len() as u32).to_le_bytes());
ivf.extend_from_slice(&0u64.to_le_bytes());
ivf.extend_from_slice(&frame);
let dir = std::env::temp_dir().join(format!("effectcraft-vp9-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("a.ivf");
std::fs::write(&path, &ivf).unwrap();
let out = std::process::Command::new("ffmpeg")
.args(["-hide_banner", "-loglevel", "error", "-i"])
.arg(&path)
.args(["-f", "rawvideo", "-pix_fmt", "yuv420p", "-"])
.output();
let Ok(out) = out else {
eprintln!("ffmpeg not found: skipping oracle");
return;
};
assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr));
assert_eq!(&out.stdout[..w * h], &y[..], "ffmpeg luma matches (lossless)");
assert_eq!(&out.stdout[w * h..w * h + u.len()], &u[..]);
}
+1
View File
@@ -100,6 +100,7 @@ impl Scene {
missing: false,
sequence: vec![],
color_profile: None,
layer: None,
};
let fid = self.p.add_item("Pattern", Label::Lavender, None, ItemKind::Footage(f));
build::layer(&mut self.p, &self.comp, "Pattern", LayerSource::Footage { item: fid }, (w, h), None)
+4
View File
@@ -19,3 +19,7 @@ effectcraft-export = { workspace = true }
[dev-dependencies]
serde_json = { workspace = true }
effectcraft-psd = { workspace = true }
effectcraft-svg = { workspace = true }
effectcraft-raster = { workspace = true }
effectcraft-time = { workspace = true }
+186
View File
@@ -0,0 +1,186 @@
//! Photoshop and SVG import through the fully wired session (media decoding included): pixels
//! of imported compositions, SVG footage at any scale, and Create Shapes from Vector Layer.
use effectcraft_engine::render::RenderOpts;
use effectcraft_project::ItemId;
use effectcraft_psd::Rect;
use effectcraft_psd::write::*;
use effectcraft_raster::Image;
use serde_json::json;
fn tmp(name: &str) -> String {
let d = std::env::temp_dir().join(format!("effectcraft-host-vector-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
d.join(name).to_string_lossy().to_string()
}
fn mean_diff(a: &Image, b: &Image) -> f32 {
assert_eq!((a.width, a.height), (b.width, b.height));
let mut s = 0.0;
for (p, q) in a.data.iter().zip(&b.data) {
for c in 0..4 {
s += (p[c] - q[c]).abs();
}
}
s / (a.data.len() * 4) as f32
}
fn psd_doc() -> WDoc {
let mut d = WDoc::new(64, 48);
let mut masked = WLayer::solid("Masked", Rect::new(30, 8, 24, 24), [0.1, 0.9, 0.3, 1.0]);
masked.mask = Some(WMask {
rect: Rect::new(30, 8, 24, 24),
data: (0..576).map(|i| if (i / 24) < 12 { 1.0 } else { 0.25 }).collect(),
default_color: 0,
disabled: false,
});
d.layers = vec![
WLayer::solid("Background", Rect::new(0, 0, 64, 48), [0.2, 0.25, 0.3, 1.0]),
WLayer::solid("Half Red", Rect::new(4, 4, 30, 20), [1.0, 0.0, 0.0, 1.0]).with_opacity(128),
masked,
WLayer::group_end(),
WLayer::solid("In Group", Rect::new(10, 30, 20, 12), [0.9, 0.9, 0.1, 1.0]),
WLayer::group("Group", true, *b"norm"),
];
d
}
/// Straight-alpha over of the layers as Photoshop composites them (normal blending), as the
/// merged image to compare the imported composition with.
fn reference(bytes: &[u8]) -> Vec<[f32; 4]> {
let p = effectcraft_psd::Psd::parse(bytes.to_vec()).unwrap();
let mut acc = vec![[0.0f32; 4]; (p.width * p.height) as usize];
for l in p.layers.iter().filter(|l| l.has_pixels() && !l.hidden) {
let px = p.layer_pixels(l.index, true).unwrap();
let o = l.opacity as f32 / 255.0;
for (d, s) in acc.iter_mut().zip(&px.data) {
let a = s[3] * o;
let oa = a + d[3] * (1.0 - a);
for c in 0..3 {
d[c] = if oa > 0.0 { (s[c] * a + d[c] * d[3] * (1.0 - a)) / oa } else { 0.0 };
}
d[3] = oa;
}
}
acc
}
#[test]
fn psd_composition_renders_like_the_document() {
let mut doc = psd_doc();
let bytes = write(&doc);
let merged = reference(&bytes);
doc.composite = Some(merged.clone());
let bytes = write(&doc);
let path = tmp("layers.psd");
std::fs::write(&path, &bytes).unwrap();
let mut s = effectcraft_host::session();
// Footage (merged image).
let r = s.execute_checked("file.import", json!({"paths": [path]})).unwrap();
let merged_item = r["items"][0].as_u64().unwrap();
// Composition and Composition – Retain Layer Sizes.
let mut renders = vec![];
for kind in ["composition", "compositionLayerSizes"] {
let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": kind})).unwrap();
let cid = ItemId(r["comps"][0].as_u64().unwrap());
renders.push(s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default()));
}
// The merged footage in a comp of the same size.
s.execute("comp.new", json!({"name": "Merged", "width": 64, "height": 48, "frameRate": 30, "duration": 1})).unwrap();
s.execute("layer.addItem", json!({"item": merged_item})).unwrap();
let mcid = s.active_comp_id().unwrap();
let merged_img = s.render(mcid, effectcraft_time::Tick::ZERO, RenderOpts::default());
let expected = Image { width: 64, height: 48, data: merged.iter().map(|p| [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]]).collect() };
assert!(mean_diff(&merged_img, &expected) < 2.0 / 255.0, "merged footage");
for (i, img) in renders.iter().enumerate() {
let d = mean_diff(img, &expected);
assert!(d < 2.0 / 255.0, "import kind {i}: mean diff {d}");
}
// A single layer as footage (Choose Layer).
let r = s.execute_checked("file.import", json!({"paths": [path], "layer": "Masked"})).unwrap();
let it = s.project.item(ItemId(r["items"][0].as_u64().unwrap())).unwrap();
assert_eq!(it.name, "Masked/layers.psd");
}
#[test]
fn psd_16bit_cmyk_composition_imports() {
let mut doc = psd_doc();
doc.depth = 16;
doc.mode = effectcraft_psd::ColorMode::Cmyk;
doc.rle = false;
let bytes = write(&doc);
let merged = reference(&bytes);
let path = tmp("layers16.psd");
std::fs::write(&path, &bytes).unwrap();
let mut s = effectcraft_host::session();
let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "composition"})).unwrap();
let cid = ItemId(r["comps"][0].as_u64().unwrap());
let img = s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default());
let expected = Image { width: 64, height: 48, data: merged.iter().map(|p| [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]]).collect() };
let d = mean_diff(&img, &expected);
assert!(d < 2.0 / 255.0, "mean diff {d}");
}
const SVG: &[u8] = include_bytes!("../../svg/tests/fixtures/basic-shapes.svg");
const SVG2: &[u8] = include_bytes!("../../svg/tests/fixtures/paths-gradients.svg");
#[test]
fn svg_footage_and_shapes_from_vector_layer() {
for (name, bytes) in [("basic-shapes.svg", SVG), ("paths-gradients.svg", SVG2)] {
let path = tmp(name);
std::fs::write(&path, bytes).unwrap();
let mut s = effectcraft_host::session();
let r = s.execute_checked("file.import", json!({"paths": [path]})).unwrap();
assert_eq!(r["errors"], json!([]), "{r}");
let item = r["items"][0].as_u64().unwrap();
let doc = effectcraft_svg::parse(bytes).unwrap();
let (w, h) = doc.pixel_size();
s.execute("comp.new", json!({"name": "V", "width": w, "height": h, "frameRate": 30, "duration": 1})).unwrap();
let lid = s.execute_checked("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap();
let cid = s.active_comp_id().unwrap();
let t = effectcraft_time::Tick::ZERO;
let foot = s.render(cid, t, RenderOpts::default());
let direct = effectcraft_svg::rasterize(&doc, w, h, 1.0);
assert!(mean_diff(&foot, &direct) < 1.0 / 255.0, "{name}: footage = rasterised SVG");
s.execute("layer.select", json!({"layers": [lid]})).unwrap();
let r = s.execute_checked("layer.create", json!({"op": "shapesFromVector"})).unwrap();
let sid = r["layers"][0].as_u64().unwrap();
let c = s.active_comp().unwrap();
assert_eq!(c.layers[0].id.0, sid);
assert!(!c.layers[1].switches.video);
let shapes = s.render(cid, t, RenderOpts::default());
let d = mean_diff(&shapes, &direct);
assert!(d < 0.02, "{name}: shapes vs SVG mean diff {d}");
// Scaled up 3×, the shape layer stays sharp and matches the SVG rasterised at 3×.
s.execute("prop.set", json!({"layer": sid, "path": "transform/scale", "value": [300, 300]})).unwrap();
s.execute("prop.set", json!({"layer": sid, "path": "transform/position", "value": [0, 0]})).unwrap();
s.execute("prop.set", json!({"layer": sid, "path": "transform/anchor", "value": [0, 0]})).unwrap();
let big = s.render(cid, t, RenderOpts::default());
let direct3 = effectcraft_svg::rasterize(&doc, w, h, 3.0);
let d3 = mean_diff(&big, &direct3);
assert!(d3 < 0.03, "{name}: ×3 mean diff {d3}");
}
}
#[test]
fn svg_footage_continuously_rasterizes() {
let path = tmp("cr.svg");
std::fs::write(&path, SVG).unwrap();
let doc = effectcraft_svg::parse(SVG).unwrap();
let (w, h) = doc.pixel_size();
let mut s = effectcraft_host::session();
let item = s.execute_checked("file.import", json!({"paths": [path]})).unwrap()["items"][0].as_u64().unwrap();
s.execute("comp.new", json!({"name": "C", "width": w, "height": h, "frameRate": 30, "duration": 1})).unwrap();
let lid = s.execute_checked("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap();
for (k, v) in [("transform/anchor", json!([0, 0])), ("transform/position", json!([0, 0])), ("transform/scale", json!([300, 300]))] {
s.execute("prop.set", json!({"layer": lid, "path": k, "value": v})).unwrap();
}
let cid = s.active_comp_id().unwrap();
let t = effectcraft_time::Tick::ZERO;
let want = effectcraft_svg::rasterize(&doc, w, h, 3.0);
let soft = mean_diff(&s.render(cid, t, RenderOpts::default()), &want);
s.execute("layer.setSwitch", json!({"layers": [lid], "switch": "collapse", "value": true})).unwrap();
let sharp = mean_diff(&s.render(cid, t, RenderOpts::default()), &want);
assert!(sharp < 0.004, "continuously rasterised diff {sharp}");
assert!(sharp < soft, "sharper than the upscaled pixels ({sharp} vs {soft})");
}
+1
View File
@@ -171,6 +171,7 @@ pub fn import(
missing: false,
sequence: vec![],
color_profile: None,
layer: None,
};
let item = project.add_item(&nm, Label::Lavender, Some(folder), ItemKind::Footage(foot));
cx.items.push(item);
+1
View File
@@ -592,6 +592,7 @@ fn images_embed_as_base64_and_come_back() {
missing: false,
sequence: vec![],
color_profile: None,
layer: None,
}),
);
let c = comp(&p, cid);
+2
View File
@@ -14,6 +14,8 @@ effectcraft-time = { workspace = true }
effectcraft-project = { workspace = true }
effectcraft-raster = { workspace = true }
effectcraft-render = { workspace = true }
effectcraft-psd = { workspace = true }
effectcraft-svg = { workspace = true }
effectcraft-model = { workspace = true }
filmcraft-codecs = { workspace = true }
filmcraft-media = { workspace = true }
+16
View File
@@ -284,6 +284,22 @@ pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image
out
}
/// Straight RGBA `f32` pixels (layered stills: Photoshop, SVG) to an [`Image`].
pub(crate) fn straight_to_image(w: u32, h: u32, px: &[[f32; 4]], op: AlphaOp) -> Image {
let mut out = Image::new(w, h);
let wu = w as usize;
if wu == 0 || h == 0 {
return out;
}
out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| {
for (x, o) in row.iter_mut().enumerate() {
let s = px[y * wu + x];
*o = op.apply([s[0], s[1], s[2]], s[3].clamp(0.0, 1.0));
}
});
out
}
#[cfg(test)]
mod tests {
use super::*;
+62
View File
@@ -0,0 +1,62 @@
//! Layered and vector stills: Photoshop documents (`effectcraft-psd`: the merged image, or one
//! layer for footage imported as a composition) and SVG (`effectcraft-svg`, rasterised).
use effectcraft_project::{AlphaMode, Footage};
use effectcraft_raster::Image;
use crate::convert::{AlphaOp, straight_to_image};
use crate::{MediaError, Result};
fn is_svg(path: &str, bytes: &[u8]) -> bool {
path.to_ascii_lowercase().ends_with(".svg") || effectcraft_svg::looks_like_svg(bytes)
}
/// Footage for a Photoshop or SVG file (`None` for other formats).
pub(crate) fn probe(path: &str, bytes: &[u8]) -> Result<Option<Footage>> {
if effectcraft_psd::is_psd(bytes) {
let psd = effectcraft_psd::Psd::parse(bytes.to_vec()).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?;
let mut f = crate::probe::still_footage(path, psd.width, psd.height, image::ImageFormat::Png, true);
f.codec = if psd.psb { "PSB".into() } else { "PSD".into() };
f.alpha = if psd.merged_alpha || !psd.layers.is_empty() { AlphaMode::Straight } else { AlphaMode::Ignore };
return Ok(Some(f));
}
if is_svg(path, bytes) {
let doc = effectcraft_svg::parse(bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?;
let (w, h) = doc.pixel_size();
let mut f = crate::probe::still_footage(path, w, h, image::ImageFormat::Png, true);
f.codec = "SVG".into();
f.alpha = AlphaMode::Straight;
return Ok(Some(f));
}
Ok(None)
}
/// Decode a Photoshop or SVG still (`None` for other formats).
pub(crate) fn decode(path: &str, bytes: &[u8], footage: &Footage, op: AlphaOp) -> Result<Option<Image>> {
if effectcraft_psd::is_psd(bytes) {
let psd = effectcraft_psd::Psd::parse(bytes.to_vec()).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?;
let px = match &footage.layer {
Some(l) => psd.layer_pixels(l.index as usize, !l.layer_size),
None => psd.composite(),
}
.map_err(|e| MediaError::Decode(format!("{path}: {e}")))?;
return Ok(Some(straight_to_image(px.width, px.height, &px.data, op)));
}
if is_svg(path, bytes) {
let doc = effectcraft_svg::parse(bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?;
let (w, h) = doc.pixel_size();
let img = effectcraft_svg::rasterize(&doc, w, h, 1.0);
// The rasteriser produces premultiplied pixels already.
let _ = op;
return Ok(Some(img));
}
Ok(None)
}
/// An SVG rasterised at `scale` × its pixel size (Continuously Rasterize).
pub(crate) fn rasterize_svg(bytes: &[u8], scale: f64) -> Option<Image> {
let doc = effectcraft_svg::parse(bytes).ok()?;
let (w, h) = doc.pixel_size();
let (sw, sh) = (((w as f64 * scale).ceil() as u32).clamp(1, 16384), ((h as f64 * scale).ceil() as u32).clamp(1, 16384));
Some(effectcraft_svg::rasterize(&doc, sw, sh, scale))
}
+4 -2
View File
@@ -10,10 +10,12 @@
//! Video containers (MP4/MOV, Matroska/WebM) and codecs (H.264, HEVC, VP9, AV1, ProRes, DNxHD,
//! MJPEG; AAC, Opus, PCM, MP3/FLAC/Vorbis) come from FilmCraft's pure-Rust crates (git dependency,
//! pinned; `plan/adr/0001`). FilmCraft types never leave this crate. Stills use the `image` crate
//! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR).
//! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR); Photoshop documents (merged image or one layer) come
//! from `effectcraft-psd` and SVG from `effectcraft-svg` (rasterised at any scale).
mod convert;
pub mod exr_channels;
mod layered;
mod pool;
mod probe;
@@ -45,7 +47,7 @@ impl From<filmcraft_media::MediaError> for MediaError {
}
/// File extensions recognised as stills.
pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr"];
pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr", "psd", "psb", "svg"];
/// File extensions recognised as audio-only files.
pub const AUDIO_EXTENSIONS: &[&str] = filmcraft_media::AUDIO_EXTENSIONS;
/// File extensions recognised as movies.
+16
View File
@@ -386,6 +386,11 @@ impl Inner {
let i = Self::frame_index(footage.frame_rate, t, footage.sequence.len() as i64, footage.loop_count);
Loc { key: key(&footage.sequence[i as usize], 0), media_t: None }
}
// A layer of a layered still is keyed by its layer index and size mode.
FootageKind::Still if footage.layer.is_some() => {
let l = footage.layer.as_ref().map_or(0, |l| 1 + l.index as i64 * 2 + l.layer_size as i64);
Loc { key: key(&footage.path, l), media_t: None }
}
FootageKind::Still | FootageKind::Sequence | FootageKind::Model => Loc { key: key(&footage.path, 0), media_t: None },
FootageKind::Video | FootageKind::Audio => {
let rate = footage.frame_rate;
@@ -464,6 +469,9 @@ impl Inner {
match loc.media_t {
None => {
let bytes = self.read(path)?;
if let Some(img) = crate::layered::decode(path, &bytes, footage, op)? {
return Ok(img);
}
let img = image::load_from_memory(&bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?;
Ok(dynamic_to_image(&img, op))
}
@@ -503,6 +511,14 @@ impl FootageSource for MediaPool {
}
}
fn vector_frame(&self, _item: ItemId, footage: &Footage, scale: f64) -> Option<Arc<Image>> {
if footage.missing || !effectcraft_render::is_vector_footage(footage) {
return None;
}
let bytes = self.inner.read(&footage.path).ok()?;
crate::layered::rasterize_svg(&bytes, scale).map(Arc::new)
}
fn aux(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<effectcraft_raster::AuxChannels>> {
if footage.missing || !footage.has_video {
return None;
+7 -1
View File
@@ -64,11 +64,15 @@ pub fn probe_model(path: &str, bytes: &[u8], resolve: &dyn Fn(&str) -> Option<Ve
missing: false,
sequence: Vec::new(),
color_profile: None,
layer: None,
})
}
/// Probe in-memory file contents; `path` names the footage (and is stored as `Footage::path`).
pub fn probe_bytes(path: &str, bytes: Arc<[u8]>) -> Result<Footage> {
if let Some(f) = crate::layered::probe(path, &bytes)? {
return Ok(f);
}
if crate::MODEL_EXTENSIONS.contains(&ext_of(Path::new(path)).as_str()) {
return probe_model(path, &bytes, &|_| None);
}
@@ -111,6 +115,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) -
missing: false,
sequence: Vec::new(),
color_profile: v.and_then(|v| profile_of(&v.color)),
layer: None,
}
}
@@ -127,7 +132,7 @@ fn profile_of(c: &filmcraft_color::ColorInfo) -> Option<effectcraft_project::Col
}
}
fn still_footage(path: &str, w: u32, h: u32, fmt: image::ImageFormat, has_alpha: bool) -> Footage {
pub(crate) fn still_footage(path: &str, w: u32, h: u32, fmt: image::ImageFormat, has_alpha: bool) -> Footage {
let codec = fmt.extensions_str().first().map_or("image", |s| s).to_ascii_uppercase();
// OpenEXR stores premultiplied colour by convention; other formats straight alpha.
let alpha = match (has_alpha, fmt) {
@@ -153,6 +158,7 @@ fn still_footage(path: &str, w: u32, h: u32, fmt: image::ImageFormat, has_alpha:
missing: false,
sequence: Vec::new(),
color_profile: None,
layer: None,
}
}
+12
View File
@@ -0,0 +1,12 @@
[package]
name = "effectcraft-opusenc"
description = "Clean-room pure-Rust Opus encoder (RFC 6716 CELT-only, fullband 48 kHz, 20 ms; RFC 7845 OpusHead) for WebM export audio"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
[lints]
workspace = true
[dependencies]
+33
View File
@@ -0,0 +1,33 @@
# effectcraft-opusenc
Clean-room, pure-Rust Opus encoder for EffectCraft's WebM export audio (layer L0, `std` only,
no `unsafe`).
- **CELT-only** packets (TOC configuration 31: fullband 20 kHz, 20 ms, one frame per packet),
48 kHz input, mono or stereo (stereo coded as dual L/R stereo), constant packet size from the
requested bitrate; all-zero frames become 3-byte silence packets.
- Forward MDCT with the low-overlap window, band energies with inter/intra coarse prediction
(cheaper of the two per frame), fine and final energy bits, the decoder's exact bit allocation,
spreading and recursive band splitting, greedy PVQ search and codeword indexing.
- Not implemented: SILK/hybrid modes, transient (short-block) frames, TF changes, pitch
post-filter, intensity stereo, dynamic allocation boosts, VBR rate control.
- `OpusEncoder::pre_skip()` is 120 samples (the MDCT overlap); `opus_head()` returns the RFC 7845
identification header.
```rust
let mut enc = effectcraft_opusenc::OpusEncoder::new(2, 128_000);
let packet = enc.encode_float(&vec![0.0; effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2]);
```
## Specifications
- RFC 6716, *Definition of the Opus Audio Codec* (September 2012), with the RFC 8251 updates
(October 2017) as implemented by the decoder we round-trip against.
- RFC 7845, *Ogg Encapsulation for the Opus Audio Codec* (April 2016), §5.1 `OpusHead` and
§4.2 pre-skip.
Implemented from the RFC text. Constant tables (band edges, `eMeans`, the coarse-energy Laplace
model, the allocation table, ICDFs) and the decoder-side allocation arithmetic are mirrored from
FilmCraft's first-party, spec-derived `filmcraft-opus` decoder (MIT OR Apache-2.0), which is also
the round-trip oracle in `crates/export/tests/opus_roundtrip.rs` (ffmpeg is a second, external
oracle when installed). No third-party encoder or decoder source was consulted.
+497
View File
@@ -0,0 +1,497 @@
//! Band shape coding: PVQ search and codeword indexing, spreading, recursive band splitting
//! (RFC 6716 §4.3.4, encoder direction).
//!
//! The encoder codes 20 ms long-block frames only (no transients, no TF changes), and stereo
//! as dual (independent L/R) stereo, so a band is always coded by `quant_partition` with a single
//! block. Every bit-budget computation mirrors the decoder so both sides take the same
//! split/pulse decisions.
use crate::range::{BITRES, RangeEncoder, ilog};
use crate::rate::{Mode, QTHETA_OFFSET, get_pulses};
use crate::tables::*;
pub const SPREAD_NORMAL: usize = 2;
#[inline]
fn frac_mul16(a: i32, b: i32) -> i32 {
(16384 + (a as i16 as i32) * (b as i16 as i32)) >> 15
}
fn bitexact_cos(x: i32) -> i32 {
let tmp = (4096 + x * x) >> 13;
let x2 = tmp;
let x2 = (32767 - x2) + frac_mul16(x2, -7651 + frac_mul16(x2, 8277 + frac_mul16(-626, x2)));
1 + x2
}
fn bitexact_log2tan(isin: i32, icos: i32) -> i32 {
let lc = ilog(icos as u32);
let ls = ilog(isin as u32);
let icos = icos << (15 - lc);
let isin = isin << (15 - ls);
(ls - lc) * (1 << 11) + frac_mul16(isin, frac_mul16(isin, -2597) + 7932) - frac_mul16(icos, frac_mul16(icos, -2597) + 7932)
}
fn compute_qn(n: i32, b: i32, offset: i32, pulse_cap: i32, stereo: bool) -> i32 {
const EXP2_TABLE8: [i32; 8] = [16384, 17866, 19483, 21247, 23170, 25267, 27554, 30048];
let mut n2 = 2 * n - 1;
if stereo && n == 2 {
n2 -= 1;
}
let mut qb = (b + n2 * offset) / n2;
qb = qb.min(b - pulse_cap - (4 << BITRES));
qb = qb.min(8 << BITRES);
if qb < (1 << BITRES >> 1) {
1
} else {
let qn = EXP2_TABLE8[(qb & 0x7) as usize] >> (14 - (qb >> BITRES));
(qn + 1) >> 1 << 1
}
}
/// One elementary pass of the spreading rotation, inverted (the decoder applies the forward
/// pass; the encoder undoes it before the pulse search).
fn exp_rotation1_inv(x: &mut [f32], len: usize, stride: usize, c: f32, s: f32) {
if len <= stride {
return;
}
if len >= 2 * stride + 1 {
for i in 0..=len - 2 * stride - 1 {
let x1 = x[i];
let x2 = x[i + stride];
x[i + stride] = c * x2 - s * x1;
x[i] = c * x1 + s * x2;
}
}
for i in (0..len - stride).rev() {
let x1 = x[i];
let x2 = x[i + stride];
x[i + stride] = c * x2 - s * x1;
x[i] = c * x1 + s * x2;
}
}
/// Inverse of the decoder's spreading rotation (§4.3.4.3) for one block of `len` samples.
pub fn exp_rotation_inv(x: &mut [f32], len: usize, k: usize, spread: usize) {
const SPREAD_FACTOR: [usize; 3] = [15, 10, 5];
if 2 * k >= len || spread == 0 {
return;
}
let factor = SPREAD_FACTOR[spread - 1];
let gain = len as f32 / (len + factor * k) as f32;
let theta = 0.5 * gain * gain;
let c = (0.5 * std::f32::consts::PI * theta).cos();
let s = (0.5 * std::f32::consts::PI * (1.0 - theta)).cos();
let mut stride2 = 0;
if len >= 8 {
stride2 = 1;
while stride2 * stride2 + stride2 < len {
stride2 += 1;
}
}
// The decoder applies (stride2: s, c) then (1: c, s); undo in reverse order.
exp_rotation1_inv(x, len, 1, c, s);
if stride2 != 0 {
exp_rotation1_inv(x, len, stride2, s, c);
}
}
/// Greedy PVQ search: the integer vector `y` with `sum |y| = k` closest in direction to `x`.
pub fn pvq_search(x: &[f32], k: usize, y: &mut [i32]) {
let n = x.len();
y[..n].fill(0);
let abs: Vec<f32> = x.iter().map(|v| v.abs()).collect();
let sum: f32 = abs.iter().sum();
if !sum.is_finite() || sum <= 1e-15 {
y[0] = k as i32;
return;
}
let mut placed = 0usize;
let mut xy = 0f32;
let mut yy = 0f32;
if k > n / 2 {
// Pre-project onto the pyramid (rounding down) and place the rest greedily.
let r = (k as f32 - 1.0) / sum;
for j in 0..n {
let p = (abs[j] * r).floor() as i32;
y[j] = p;
placed += p as usize;
xy += abs[j] * p as f32;
yy += (p * p) as f32;
}
}
while placed < k {
let mut best = 0usize;
let mut best_num = -1f32;
let mut best_den = 1f32;
for j in 0..n {
let num = xy + abs[j];
let num = num * num;
let den = yy + 2.0 * y[j] as f32 + 1.0;
if num * best_den > best_num * den {
best_num = num;
best_den = den;
best = j;
}
}
xy += abs[best];
yy += 2.0 * y[best] as f32 + 1.0;
y[best] += 1;
placed += 1;
}
for j in 0..n {
if x[j] < 0.0 {
y[j] = -y[j];
}
}
}
/// Encodes the PVQ codeword `y` (`sum |y| = k`) as the index the RFC 6716 §4.3.4.2 decoder maps
/// back to `y`.
pub fn encode_pulses(y: &[i32], k: usize, enc: &mut RangeEncoder, v: &mut Vec<u64>) {
let n = y.len();
let w = k + 1;
v.clear();
v.resize((n + 1) * w, 0);
v[0] = 1;
for nn in 1..=n {
v[nn * w] = 1;
for kk in 1..=k {
let a = v[(nn - 1) * w + kk];
let b = v[nn * w + kk - 1];
let c = v[(nn - 1) * w + kk - 1];
v[nn * w + kk] = a.saturating_add(b).saturating_add(c);
}
}
let total = v[n * w + k];
let vv = |nn: usize, kx: usize| v[nn * w + kx];
let mut index = 0u64;
let mut kk = k;
for j in 0..n {
if kk == 0 {
break;
}
let rem = n - j;
let t = y[j].unsigned_abs() as usize;
// Codewords with |y_j| = t start at sum_{m < kk - t} V(rem-1, m) within their sign half.
let mut base: u64 = (0..kk - t).map(|m| vv(rem - 1, m)).sum();
if y[j] < 0 {
base += (vv(rem - 1, kk) + vv(rem, kk)) / 2;
}
index += base;
kk -= t;
}
debug_assert!(total <= u32::MAX as u64);
if total >= 2 {
enc.uint(index as u32, total as u32);
}
}
pub struct BandEnc<'m, 'e> {
pub m: &'m Mode,
pub enc: &'e mut RangeEncoder,
pub i: usize,
pub spread: usize,
pub remaining_bits: i32,
pvq_v: Vec<u64>,
iy: Vec<i32>,
tmp: Vec<f32>,
}
impl BandEnc<'_, '_> {
/// `compute_theta` for a mono split (single block): measures the energy split between the
/// two halves and codes the quantised angle with the triangular pdf.
fn compute_theta_split(&mut self, xa: &[f32], xb: &[f32], b: &mut i32, lm: i32) -> (i32, i32, i32) {
let n = xa.len() as i32;
let i = self.i;
let pulse_cap = self.m.log_n[i] + lm * (1 << BITRES);
let offset = (pulse_cap >> 1) - QTHETA_OFFSET;
let qn = compute_qn(n, *b, offset, pulse_cap, false);
let tell = self.enc.tell_frac();
let mut itheta = 0i32;
if qn != 1 {
let mid = xa.iter().map(|v| v * v).sum::<f32>().sqrt();
let side = xb.iter().map(|v| v * v).sum::<f32>().sqrt();
let ang = (side as f64).atan2(mid as f64) * std::f64::consts::FRAC_2_PI;
let it = ((ang * qn as f64 + 0.5).floor() as i32).clamp(0, qn) as u32;
let q = qn as u32;
let half = q >> 1;
let ft = (half + 1) * (half + 1);
let (fl, fs) = if it <= half { (it * (it + 1) >> 1, it + 1) } else { (ft - ((q + 1 - it) * (q + 2 - it) >> 1), q + 1 - it) };
self.enc.encode(fl, fl + fs, ft);
itheta = ((it * 16384) / q) as i32;
}
let qalloc = self.enc.tell_frac() - tell;
*b -= qalloc;
let delta = if itheta == 0 {
-16384
} else if itheta == 16384 {
16384
} else {
let imid = bitexact_cos(itheta);
let iside = bitexact_cos(16384 - itheta);
frac_mul16((n - 1) << 7, bitexact_log2tan(iside, imid))
};
(itheta, delta, qalloc)
}
/// `quant_partition` (single block, encoder side).
pub fn quant_partition(&mut self, x: &[f32], mut b: i32, mut lm: i32) {
let n = x.len();
let i = self.i;
let cache = self.m.cache(lm, i);
let cache_max = cache[cache[0] as usize] as i32;
if lm != -1 && b > cache_max + 12 && n > 2 {
let half = n >> 1;
let (xa, xb) = x.split_at(half);
lm -= 1;
let (itheta, delta, qalloc) = self.compute_theta_split(xa, xb, &mut b, lm);
let mut mbits = 0.max(b.min((b - delta) / 2));
let mut sbits = b - mbits;
self.remaining_bits -= qalloc;
let mut rebalance = self.remaining_bits;
if mbits >= sbits {
self.quant_partition(xa, mbits, lm);
rebalance = mbits - (rebalance - self.remaining_bits);
if rebalance > 3 << BITRES && itheta != 0 {
sbits += rebalance - (3 << BITRES);
}
self.quant_partition(xb, sbits, lm);
} else {
self.quant_partition(xb, sbits, lm);
rebalance = sbits - (rebalance - self.remaining_bits);
if rebalance > 3 << BITRES && itheta != 16384 {
mbits += rebalance - (3 << BITRES);
}
self.quant_partition(xa, mbits, lm);
}
return;
}
let m = self.m;
let mut q = m.bits2pulses(i, lm, b);
let mut curr_bits = m.pulses2bits(i, lm, q);
self.remaining_bits -= curr_bits;
while self.remaining_bits < 0 && q > 0 {
self.remaining_bits += curr_bits;
q -= 1;
curr_bits = m.pulses2bits(i, lm, q);
self.remaining_bits -= curr_bits;
}
if q != 0 {
let k = get_pulses(q) as usize;
let mut tmp = std::mem::take(&mut self.tmp);
tmp.clear();
tmp.extend_from_slice(x);
exp_rotation_inv(&mut tmp, n, k, self.spread);
let mut iy = std::mem::take(&mut self.iy);
iy.clear();
iy.resize(n, 0);
pvq_search(&tmp, k, &mut iy);
encode_pulses(&iy, k, self.enc, &mut self.pvq_v);
self.iy = iy;
self.tmp = tmp;
}
}
}
/// Codes all band shapes (`quant_all_bands`, encoder side, long blocks). `x` holds `c` channels
/// of `n` normalised MDCT coefficients each.
#[allow(clippy::too_many_arguments)]
pub fn quant_all_bands(
m: &Mode,
start: usize,
end: usize,
x: &[f32],
n_total: usize,
c: usize,
pulses: &[i32; NB_EBANDS],
spread: usize,
dual_stereo: bool,
intensity: usize,
total_bits: i32,
mut balance: i32,
enc: &mut RangeEncoder,
lm: usize,
coded_bands: usize,
) {
let mm = 1usize << lm;
debug_assert!(c == 1 || dual_stereo || intensity <= start, "stereo frames are coded as dual stereo");
let mut ctx = BandEnc { m, enc, i: 0, spread, remaining_bits: 0, pvq_v: Vec::new(), iy: Vec::new(), tmp: Vec::new() };
for i in start..end {
ctx.i = i;
let xo = mm * EBANDS[i] as usize;
let n = mm * EBANDS[i + 1] as usize - xo;
let tell = ctx.enc.tell_frac();
if i != start {
balance -= tell;
}
let remaining_bits = total_bits - tell - 1;
ctx.remaining_bits = remaining_bits;
let b = if i < coded_bands {
let curr_balance = balance / 3.min(coded_bands as i32 - i as i32);
0.max(16383.min((remaining_bits + 1).min(pulses[i] + curr_balance)))
} else {
0
};
// Bands at and above `intensity` (= coded_bands) carry no bits, so they emit no symbols.
if i < coded_bands || c == 1 {
if c == 2 && dual_stereo && i < intensity {
ctx.quant_partition(&x[xo..xo + n], b / 2, lm as i32);
ctx.quant_partition(&x[n_total + xo..n_total + xo + n], b / 2, lm as i32);
} else {
ctx.quant_partition(&x[xo..xo + n], b, lm as i32);
}
}
balance += pulses[i] + tell;
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::range::dec::RangeDecoder;
fn exp_rotation1(x: &mut [f32], len: usize, stride: usize, c: f32, s: f32) {
let ms = -s;
if len <= stride {
return;
}
for i in 0..len - stride {
let x1 = x[i];
let x2 = x[i + stride];
x[i + stride] = c * x2 + s * x1;
x[i] = c * x1 + ms * x2;
}
if len >= 2 * stride + 1 {
for i in (0..=len - 2 * stride - 1).rev() {
let x1 = x[i];
let x2 = x[i + stride];
x[i + stride] = c * x2 + s * x1;
x[i] = c * x1 + ms * x2;
}
}
}
/// The decoder's spreading rotation (single block).
fn exp_rotation(x: &mut [f32], len: usize, k: usize, spread: usize) {
const SPREAD_FACTOR: [usize; 3] = [15, 10, 5];
if 2 * k >= len || spread == 0 {
return;
}
let factor = SPREAD_FACTOR[spread - 1];
let gain = len as f32 / (len + factor * k) as f32;
let theta = 0.5 * gain * gain;
let c = (0.5 * std::f32::consts::PI * theta).cos();
let s = (0.5 * std::f32::consts::PI * (1.0 - theta)).cos();
let mut stride2 = 0;
if len >= 8 {
stride2 = 1;
while stride2 * stride2 + stride2 < len {
stride2 += 1;
}
}
if stride2 != 0 {
exp_rotation1(x, len, stride2, s, c);
}
exp_rotation1(x, len, 1, c, s);
}
#[test]
fn spreading_inverse_is_exact() {
for &len in &[8usize, 16, 24, 48, 96, 176] {
for k in [1usize, 2, 3, 5] {
let orig: Vec<f32> = (0..len).map(|i| ((i * 31 % 17) as f32 - 8.0) / 8.0).collect();
let mut x = orig.clone();
exp_rotation_inv(&mut x, len, k, SPREAD_NORMAL);
exp_rotation(&mut x, len, k, SPREAD_NORMAL);
for (a, b) in x.iter().zip(&orig) {
assert!((a - b).abs() < 1e-4, "len={len} k={k}");
}
}
}
}
/// PVQ decoder of RFC 6716 §4.3.4.2 (as in FilmCraft's decoder).
fn decode_pulses(n: usize, k: usize, dec: &mut RangeDecoder) -> Vec<i32> {
let w = k + 1;
let mut v = vec![0u64; (n + 1) * w];
v[0] = 1;
for nn in 1..=n {
v[nn * w] = 1;
for kk in 1..=k {
v[nn * w + kk] = v[(nn - 1) * w + kk].saturating_add(v[nn * w + kk - 1]).saturating_add(v[(nn - 1) * w + kk - 1]);
}
}
let total = v[n * w + k];
let mut i = if total >= 2 { dec.uint(total as u32) as u64 } else { 0 };
let mut y = vec![0i32; n];
let mut kk = k;
for j in 0..n {
let rem = n - j;
if kk == 0 {
break;
}
let vv = |nn: usize, kx: usize| v[nn * w + kx];
let mut p = (vv(rem - 1, kk) + vv(rem, kk)) / 2;
let neg = i >= p;
if neg {
i -= p;
}
let k0 = kk;
p -= vv(rem - 1, kk);
while p > i && kk > 0 {
kk -= 1;
p -= vv(rem - 1, kk);
}
let mag = (k0 - kk) as i32;
y[j] = if neg { -mag } else { mag };
i -= p.min(i);
}
y
}
fn all_codewords(n: usize, k: usize) -> Vec<Vec<i32>> {
if n == 0 {
return if k == 0 { vec![vec![]] } else { vec![] };
}
let mut out = Vec::new();
for t in -(k as i32)..=(k as i32) {
for mut rest in all_codewords(n - 1, k - t.unsigned_abs() as usize) {
rest.insert(0, t);
out.push(rest);
}
}
out
}
#[test]
fn pvq_index_roundtrip_exhaustive() {
let mut v = Vec::new();
for n in 1..6usize {
for k in 1..6usize {
for y in all_codewords(n, k) {
let mut e = RangeEncoder::new();
encode_pulses(&y, k, &mut e, &mut v);
let buf = e.done(16);
let mut d = RangeDecoder::new(&buf);
assert_eq!(decode_pulses(n, k, &mut d), y, "n={n} k={k}");
}
}
}
}
#[test]
fn pvq_search_finds_good_direction() {
let x: Vec<f32> = (0..32).map(|i| ((i as f32) * 0.7).sin()).collect();
let mut y = vec![0i32; 32];
for k in [1usize, 4, 16, 64] {
pvq_search(&x, k, &mut y);
assert_eq!(y.iter().map(|v| v.unsigned_abs() as usize).sum::<usize>(), k);
let xy: f32 = x.iter().zip(&y).map(|(a, &b)| a * b as f32).sum();
let yy: f32 = y.iter().map(|&b| (b * b) as f32).sum();
let xx: f32 = x.iter().map(|a| a * a).sum();
let corr = xy / (xx * yy).sqrt();
assert!(corr > if k >= 16 { 0.9 } else { 0.2 }, "k={k} corr={corr}");
}
}
}
+230
View File
@@ -0,0 +1,230 @@
//! Band energy quantisation: Laplace-coded coarse energy with inter/intra prediction, fine
//! energy and the final refinement bits (RFC 6716 §4.3.2, encoder direction).
//!
//! The quantised energies are updated with exactly the decoder's arithmetic so the prediction
//! state never drifts between encoder and decoder.
use crate::range::RangeEncoder;
use crate::rate::MAX_FINE_BITS;
use crate::tables::*;
const LAPLACE_MINP: u32 = 1;
const LAPLACE_NMIN: u32 = 16;
fn laplace_get_freq1(fs0: u32, decay: u32) -> u32 {
let ft = 32768 - LAPLACE_MINP * (2 * LAPLACE_NMIN) - fs0;
(ft * (16384 - decay)) >> 15
}
/// Encodes a Laplace-distributed integer (`fs` = probability of 0 in Q15, `decay` in Q14).
/// Returns the value actually coded (very large magnitudes are clamped to the representable
/// range).
pub fn laplace_encode(enc: &mut RangeEncoder, value: i32, mut fs: u32, decay: u32) -> i32 {
let mut fl = 0u32;
let mut coded = value;
if value != 0 {
let s: i32 = if value < 0 { -1 } else { 0 };
let val = value.abs();
fl = fs;
fs = laplace_get_freq1(fs, decay);
let mut i = 1;
while fs > 0 && i < val {
fs *= 2;
fl += fs + 2 * LAPLACE_MINP;
fs = (fs * decay) >> 15;
i += 1;
}
if fs == 0 {
let ndi_max = (32768 - fl + LAPLACE_MINP - 1) as i32;
let ndi_max = (ndi_max - s) >> 1;
let di = (val - i).min(ndi_max - 1);
fl = (fl as i32 + (2 * di + 1 + s) * LAPLACE_MINP as i32) as u32;
fs = LAPLACE_MINP.min(32768 - fl);
let mag = i + di;
coded = if s < 0 { -mag } else { mag };
} else {
fs += LAPLACE_MINP;
if s == 0 {
fl += fs;
}
}
}
enc.encode_bin(fl, fl + fs, 15);
coded
}
/// `quant_coarse_energy` for one prediction mode. `e` holds the target log2 energies (relative
/// to `E_MEANS`), `old_e` the quantised energies, updated in place. `budget` is the frame size in
/// bits.
#[allow(clippy::too_many_arguments)]
pub fn quant_coarse_energy(
start: usize,
end: usize,
e: &[f32; 2 * NB_EBANDS],
old_e: &mut [f32; 2 * NB_EBANDS],
intra: bool,
enc: &mut RangeEncoder,
c: usize,
lm: usize,
budget: i32,
) {
let prob = &E_PROB_MODEL[lm][intra as usize];
let (coef, beta) = if intra { (0.0, BETA_INTRA) } else { (PRED_COEF[lm], BETA_COEF[lm]) };
let mut prev = [0f32; 2];
for i in start..end {
for ch in 0..c {
let idx = i + ch * NB_EBANDS;
let oldc = old_e[idx].max(-9.0);
let x = e[idx];
let f = x - coef * oldc - prev[ch];
let mut qi = (f + 0.5).floor().clamp(-64.0, 64.0) as i32;
let tell = enc.tell();
// Keep a few bits per remaining band in reserve when the budget runs out.
let bits_left = budget - tell - 3 * c as i32 * (end - i) as i32;
if i != start && bits_left < 30 {
if bits_left < 24 {
qi = qi.min(1);
}
if bits_left < 16 {
qi = qi.max(-1);
}
}
if budget - tell >= 15 {
let pi = 2 * i.min(20);
qi = laplace_encode(enc, qi, (prob[pi] as u32) << 7, (prob[pi + 1] as u32) << 6);
} else if budget - tell >= 2 {
qi = qi.clamp(-1, 1);
enc.icdf(if qi < 0 { (-2 * qi - 1) as usize } else { (2 * qi) as usize }, &SMALL_ENERGY_ICDF, 2);
} else if budget - tell >= 1 {
qi = qi.clamp(-1, 0);
enc.bit_logp(qi == -1, 1);
} else {
qi = -1;
}
let q = qi as f32;
old_e[idx] = oldc;
let tmp = coef * old_e[idx] + prev[ch] + q;
old_e[idx] = tmp;
prev[ch] = prev[ch] + q - beta * q;
}
}
}
/// `quant_fine_energy`: raw bits refining each band's quantised energy.
pub fn quant_fine_energy(
start: usize,
end: usize,
e: &[f32; 2 * NB_EBANDS],
old_e: &mut [f32; 2 * NB_EBANDS],
fine_quant: &[i32; NB_EBANDS],
enc: &mut RangeEncoder,
c: usize,
) {
for i in start..end {
let fq = fine_quant[i];
if fq <= 0 {
continue;
}
for ch in 0..c {
let idx = i + ch * NB_EBANDS;
let err = e[idx] - old_e[idx];
let levels = 1i32 << fq;
let q2 = (((err + 0.5) * levels as f32).floor() as i32).clamp(0, levels - 1);
enc.bits(q2 as u32, fq as u32);
let offset = (q2 as f32 + 0.5) * (1 << (14 - fq)) as f32 / 16384.0 - 0.5;
old_e[idx] += offset;
}
}
}
/// `quant_energy_finalise`: spends the bits left at the end of the frame on one more bit of
/// energy resolution per band, in priority order.
#[allow(clippy::too_many_arguments)]
pub fn quant_energy_finalise(
start: usize,
end: usize,
e: &[f32; 2 * NB_EBANDS],
old_e: &mut [f32; 2 * NB_EBANDS],
fine_quant: &[i32; NB_EBANDS],
fine_priority: &[i32; NB_EBANDS],
mut bits_left: i32,
enc: &mut RangeEncoder,
c: usize,
) {
for prio in 0..2 {
let mut i = start;
while i < end && bits_left >= c as i32 {
if fine_quant[i] >= MAX_FINE_BITS || fine_priority[i] != prio {
i += 1;
continue;
}
for ch in 0..c {
let idx = i + ch * NB_EBANDS;
let q2 = (e[idx] - old_e[idx] >= 0.0) as u32;
enc.bits(q2, 1);
let offset = (q2 as f32 - 0.5) * (1 << (14 - fine_quant[i] - 1)) as f32 / 16384.0;
old_e[idx] += offset;
bits_left -= 1;
}
i += 1;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::range::dec::RangeDecoder;
/// Laplace decoder of RFC 6716 §4.3.2.1 (as in FilmCraft's decoder).
fn laplace_decode(dec: &mut RangeDecoder, mut fs: u32, decay: u32) -> i32 {
let mut val = 0i32;
let fm = dec.decode(32768);
let mut fl = 0u32;
if fm >= fs {
val += 1;
fl = fs;
fs = laplace_get_freq1(fs, decay) + LAPLACE_MINP;
while fs > LAPLACE_MINP && fm >= fl + 2 * fs {
fs *= 2;
fl += fs;
fs = (((fs - 2 * LAPLACE_MINP) * decay) >> 15) + LAPLACE_MINP;
val += 1;
}
if fs <= LAPLACE_MINP {
let di = (fm - fl) >> 1;
val += di as i32;
fl += 2 * di * LAPLACE_MINP;
}
if fm < fl + fs {
val = -val;
} else {
fl += fs;
}
}
dec.update(fl, (fl + fs).min(32768), 32768);
val
}
#[test]
fn laplace_roundtrip_all_models() {
for lm in 0..4 {
for intra in 0..2 {
for band in 0..21 {
let prob = &E_PROB_MODEL[lm][intra];
let (fs, decay) = ((prob[2 * band] as u32) << 7, (prob[2 * band + 1] as u32) << 6);
let vals: Vec<i32> = (-64..=64).collect();
let mut e = RangeEncoder::new();
let coded: Vec<i32> = vals.iter().map(|&v| laplace_encode(&mut e, v, fs, decay)).collect();
let size = (e.tell() as usize).div_ceil(8);
let buf = e.done(size);
let mut d = RangeDecoder::new(&buf);
for (&v, &c) in vals.iter().zip(&coded) {
assert!(c == v || (c.signum() == v.signum() && c.abs() < v.abs()), "clamp {v} -> {c}");
assert_eq!(laplace_decode(&mut d, fs, decay), c, "lm={lm} intra={intra} band={band} v={v}");
}
}
}
}
}
}
+311
View File
@@ -0,0 +1,311 @@
//! Clean-room pure-Rust Opus encoder (RFC 6716 CELT-only mode; RFC 7845 `OpusHead`).
//!
//! Produces fullband (20 kHz) 48 kHz Opus packets of 20 ms, mono or stereo, at a constant
//! packet size derived from the requested bitrate. Each packet is a single CELT frame (TOC
//! configuration 31, frame-count code 0). Stereo is coded as dual (independent L/R) stereo.
//!
//! Layer L0: no dependencies beyond `std`; no `unsafe`; builds for `wasm32-unknown-unknown`.
//!
//! ```
//! use effectcraft_opusenc::OpusEncoder;
//! let mut enc = OpusEncoder::new(2, 128_000);
//! let pcm = vec![0.0f32; OpusEncoder::FRAME_SIZE * 2];
//! let packet = enc.encode_float(&pcm);
//! assert_eq!(packet[0] >> 3, 31); // CELT-only, fullband, 20 ms
//! assert_eq!(enc.opus_head().len(), 19);
//! ```
// The bit-allocation code mirrors the normative integer arithmetic; keep C-like shift expressions.
#![allow(clippy::precedence, clippy::int_plus_one)]
mod bands;
mod energy;
mod mdct;
mod range;
mod rate;
mod tables;
use std::sync::OnceLock;
use bands::{SPREAD_NORMAL, quant_all_bands};
use energy::{quant_coarse_energy, quant_energy_finalise, quant_fine_energy};
use mdct::Mdct;
use range::{BITRES, RangeEncoder};
use rate::{Mode, compute_allocation};
use tables::*;
/// Samples per channel in one 20 ms frame at 48 kHz.
const N: usize = 960;
/// LM of a 20 ms frame (8 short blocks).
const LM: usize = 3;
/// Smallest CELT payload the encoder produces (bytes).
const MIN_FRAME_BYTES: usize = 16;
/// Largest CELT payload allowed in one frame (RFC 6716 §3.4, R2).
const MAX_FRAME_BYTES: usize = 1275;
fn mode() -> &'static Mode {
static MODE: OnceLock<Mode> = OnceLock::new();
MODE.get_or_init(Mode::new)
}
/// Opus encoder for one mono or stereo stream at 48 kHz.
pub struct OpusEncoder {
channels: usize,
/// CELT payload bytes per packet (the packet is one byte longer: the TOC).
frame_bytes: usize,
mdct: Mdct,
/// Last input sample of each channel (pre-emphasis filter memory, ×32768).
preemph_mem: [f32; 2],
/// Last `OVERLAP` pre-emphasised samples of each channel.
hist: Vec<f32>,
/// Quantised band energies, exactly as the decoder holds them.
old_band_e: [f32; 2 * NB_EBANDS],
block: Vec<f32>,
freq: Vec<f32>,
}
impl OpusEncoder {
/// Samples per channel per packet (20 ms at 48 kHz).
pub const FRAME_SIZE: usize = N;
/// Creates an encoder for `channels` (1 or 2) channels of 48 kHz audio at `bitrate_bps`
/// (total for all channels; packets are constant-size, clamped to 6.8–510 kb/s).
pub fn new(channels: u8, bitrate_bps: u32) -> Self {
assert!(channels == 1 || channels == 2, "Opus family-0 streams have 1 or 2 channels");
let c = channels as usize;
let packet_bytes = (bitrate_bps as usize).div_ceil(400);
let frame_bytes = packet_bytes.saturating_sub(1).clamp(MIN_FRAME_BYTES, MAX_FRAME_BYTES);
OpusEncoder {
channels: c,
frame_bytes,
mdct: Mdct::new(N),
preemph_mem: [0.0; 2],
hist: vec![0.0; c * OVERLAP],
old_band_e: [-28.0; 2 * NB_EBANDS],
block: vec![0.0; N + OVERLAP],
freq: vec![0.0; c * N],
}
}
/// Number of channels.
pub fn channels(&self) -> u8 {
self.channels as u8
}
/// Actual bitrate in bits per second (packet size × 50 packets per second).
pub fn bitrate(&self) -> u32 {
((self.frame_bytes + 1) * 8 * 50) as u32
}
/// Samples at 48 kHz the decoder must discard from the start of the stream (the MDCT
/// overlap); this is the `OpusHead` pre-skip.
pub fn pre_skip(&self) -> u16 {
OVERLAP as u16
}
/// The 19-byte `OpusHead` identification header (RFC 7845 §5.1): version 1, channel count,
/// pre-skip, input sample rate 48000, output gain 0, channel mapping family 0.
pub fn opus_head(&self) -> Vec<u8> {
let mut h = Vec::with_capacity(19);
h.extend_from_slice(b"OpusHead");
h.push(1);
h.push(self.channels as u8);
h.extend_from_slice(&self.pre_skip().to_le_bytes());
h.extend_from_slice(&48_000u32.to_le_bytes());
h.extend_from_slice(&0i16.to_le_bytes());
h.push(0);
h
}
/// Encodes one 20 ms frame of interleaved f32 samples in [-1, 1] (exactly
/// `FRAME_SIZE * channels` values) and returns one Opus packet (TOC byte + CELT frame).
pub fn encode_float(&mut self, pcm: &[f32]) -> Vec<u8> {
let c = self.channels;
assert_eq!(pcm.len(), N * c, "encode_float takes exactly FRAME_SIZE samples per channel");
// Pre-emphasis, the MDCT and the band energies.
let mut band_e = [0f32; 2 * NB_EBANDS];
let mut log_e = [0f32; 2 * NB_EBANDS];
let mut peak = 0f32;
for ch in 0..c {
self.block[..OVERLAP].copy_from_slice(&self.hist[ch * OVERLAP..(ch + 1) * OVERLAP]);
let mut mem = self.preemph_mem[ch];
for j in 0..N {
let s = pcm[j * c + ch];
let s = if s.is_finite() { s.clamp(-2.0, 2.0) * 32768.0 } else { 0.0 };
let v = s - PREEMPH * mem;
mem = s;
self.block[OVERLAP + j] = v;
}
self.preemph_mem[ch] = mem;
self.hist[ch * OVERLAP..(ch + 1) * OVERLAP].copy_from_slice(&self.block[N..]);
peak = self.block.iter().fold(peak, |p, v| p.max(v.abs()));
let freq = &mut self.freq[ch * N..(ch + 1) * N];
self.mdct.forward(&self.block, freq);
for i in 0..NB_EBANDS {
let lo = (EBANDS[i] as usize) << LM;
let hi = (EBANDS[i + 1] as usize) << LM;
let e = (1e-27f32 + freq[lo..hi].iter().map(|v| v * v).sum::<f32>()).sqrt();
band_e[ch * NB_EBANDS + i] = e;
log_e[ch * NB_EBANDS + i] = e.log2() - E_MEANS[i];
let g = 1.0 / e;
for v in freq[lo..hi].iter_mut() {
*v *= g;
}
}
}
let silence = peak < 1e-4;
let toc = (31u8 << 3) | if c == 2 { 4 } else { 0 };
let mut packet = vec![toc];
packet.extend_from_slice(&self.encode_celt(&log_e, silence));
packet
}
/// Codes one CELT frame (RFC 6716 §4.3, in bitstream order).
fn encode_celt(&mut self, log_e: &[f32; 2 * NB_EBANDS], silence: bool) -> Vec<u8> {
let m = mode();
let c = self.channels;
let (start, end) = (0usize, NB_EBANDS);
let mut enc = RangeEncoder::new();
if c == 1 {
for i in 0..NB_EBANDS {
self.old_band_e[i] = self.old_band_e[i].max(self.old_band_e[NB_EBANDS + i]);
}
}
if silence {
// The silence flag; the decoder treats every remaining bit as consumed.
enc.bit_logp(true, 15);
self.old_band_e = [-28.0; 2 * NB_EBANDS];
return enc.done(2);
}
let len = self.frame_bytes;
let total_bits_raw = len as i32 * 8;
enc.bit_logp(false, 15);
// No pitch post-filter.
if enc.tell() + 16 <= total_bits_raw {
enc.bit_logp(false, 1);
}
// Long blocks only.
if enc.tell() + 3 <= total_bits_raw {
enc.bit_logp(false, 3);
}
// Coarse energy, choosing intra or inter prediction by trial (whichever is cheaper).
if enc.tell() + 3 <= total_bits_raw {
let mut enc_intra = enc.clone();
let mut old_intra = self.old_band_e;
enc_intra.bit_logp(true, 3);
quant_coarse_energy(start, end, log_e, &mut old_intra, true, &mut enc_intra, c, LM, total_bits_raw);
enc.bit_logp(false, 3);
let mut old_inter = self.old_band_e;
quant_coarse_energy(start, end, log_e, &mut old_inter, false, &mut enc, c, LM, total_bits_raw);
if enc_intra.tell_frac() < enc.tell_frac() {
enc = enc_intra;
self.old_band_e = old_intra;
} else {
self.old_band_e = old_inter;
}
} else {
quant_coarse_energy(start, end, log_e, &mut self.old_band_e, false, &mut enc, c, LM, total_bits_raw);
}
// TF resolution: no changes.
{
let mut budget = total_bits_raw;
let mut tell = enc.tell();
let mut logp = 4;
let tf_select_rsv = LM > 0 && tell + logp + 1 <= budget;
budget -= tf_select_rsv as i32;
for _ in start..end {
if tell + logp <= budget {
enc.bit_logp(false, logp as u32);
tell = enc.tell();
}
logp = 5;
}
if tf_select_rsv && TF_SELECT_TABLE[LM][0] != TF_SELECT_TABLE[LM][2] {
enc.bit_logp(false, 1);
}
}
if enc.tell() + 4 <= total_bits_raw {
enc.icdf(SPREAD_NORMAL, &SPREAD_ICDF, 5);
}
// Dynamic allocation: no boosts.
let cap = m.init_caps(LM, c);
let offsets = [0i32; NB_EBANDS];
let total_bits = total_bits_raw << BITRES;
let mut tellf = enc.tell_frac();
for &cp in cap.iter().take(end).skip(start) {
if tellf + (6 << BITRES) < total_bits && 0 < cp {
enc.bit_logp(false, 6);
tellf = enc.tell_frac();
}
}
let alloc_trim = 5;
if tellf + (6 << BITRES) <= total_bits {
enc.icdf(alloc_trim as usize, &TRIM_ICDF, 7);
}
let bits = (total_bits_raw << BITRES) - enc.tell_frac() - 1;
let alloc = compute_allocation(m, start, end, &offsets, &cap, alloc_trim, bits, c, LM, &mut enc);
quant_fine_energy(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &mut enc, c);
quant_all_bands(
m,
start,
end,
&self.freq,
N,
c,
&alloc.pulses,
SPREAD_NORMAL,
alloc.dual_stereo,
alloc.intensity,
total_bits_raw << BITRES,
alloc.balance,
&mut enc,
LM,
alloc.coded_bands,
);
let bits_left = total_bits_raw - enc.tell();
quant_energy_finalise(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &alloc.fine_priority, bits_left, &mut enc, c);
debug_assert!(enc.tell() <= total_bits_raw, "frame overflow: {} > {}", enc.tell(), total_bits_raw);
if c == 1 {
let (a, b) = self.old_band_e.split_at_mut(NB_EBANDS);
b.copy_from_slice(a);
}
enc.done(len)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn opus_head_layout() {
let e = OpusEncoder::new(2, 96_000);
let h = e.opus_head();
assert_eq!(&h[..8], b"OpusHead");
assert_eq!(h[8], 1);
assert_eq!(h[9], 2);
assert_eq!(u16::from_le_bytes([h[10], h[11]]), e.pre_skip());
assert_eq!(u32::from_le_bytes([h[12], h[13], h[14], h[15]]), 48_000);
assert_eq!(&h[16..], &[0, 0, 0]);
}
#[test]
fn packet_sizes_follow_bitrate() {
for (ch, rate) in [(1u8, 64_000u32), (2, 128_000), (2, 256_000)] {
let mut e = OpusEncoder::new(ch, rate);
let pcm: Vec<f32> = (0..N * ch as usize).map(|i| (i as f32 * 0.05).sin() * 0.5).collect();
let p = e.encode_float(&pcm);
assert_eq!(p.len(), rate as usize / 400);
assert_eq!(p[0], 0xF8 | if ch == 2 { 4 } else { 0 });
assert_eq!(e.bitrate(), rate);
}
}
#[test]
fn silence_is_a_tiny_packet() {
let mut e = OpusEncoder::new(1, 64_000);
let p = e.encode_float(&vec![0.0; N]);
assert_eq!(p.len(), 3);
}
}
+293
View File
@@ -0,0 +1,293 @@
//! Forward MDCT with the CELT low-overlap window (RFC 6716 §4.3.7, run in the analysis
//! direction).
//!
//! A 20 ms frame is a 1920-point MDCT (960 coefficients) whose window is zero for 420 samples,
//! rises over the 120-sample overlap, is flat for 840 samples, falls over the next 120 samples
//! and is zero again for the last 420. The input block is therefore the 120 overlap samples kept
//! from the previous frame followed by the 960 new samples. The MDCT is folded into a DCT-IV,
//! which is computed with an N/2-point complex FFT.
use crate::tables::OVERLAP;
#[derive(Clone, Copy, Default, Debug)]
pub struct Cpx {
pub re: f32,
pub im: f32,
}
impl Cpx {
#[inline]
fn mul(self, o: Cpx) -> Cpx {
Cpx { re: self.re * o.re - self.im * o.im, im: self.re * o.im + self.im * o.re }
}
#[inline]
fn add(self, o: Cpx) -> Cpx {
Cpx { re: self.re + o.re, im: self.im + o.im }
}
}
/// Mixed-radix (2, 3, 4, 5) forward complex FFT, unscaled (decimation in time, recursive).
pub struct Fft {
n: usize,
factors: Vec<usize>,
twiddle: Vec<Cpx>,
}
impl Fft {
pub fn new(n: usize) -> Fft {
let mut factors = Vec::new();
let mut m = n;
for p in [4usize, 2, 3, 5] {
while m.is_multiple_of(p) {
factors.push(p);
m /= p;
}
}
assert_eq!(m, 1, "unsupported FFT size {n}");
let twiddle = (0..n)
.map(|k| {
let ph = -2.0 * std::f64::consts::PI * k as f64 / n as f64;
Cpx { re: ph.cos() as f32, im: ph.sin() as f32 }
})
.collect();
Fft { n, factors, twiddle }
}
pub fn process(&self, data: &mut [Cpx], scratch: &mut Vec<Cpx>) {
scratch.clear();
scratch.extend_from_slice(&data[..self.n]);
self.rec(&mut data[..self.n], scratch, 1, self.n, 0, 1);
}
fn rec(&self, out: &mut [Cpx], inp: &[Cpx], in_stride: usize, n: usize, fi: usize, tw_stride: usize) {
let p = self.factors[fi];
let m = n / p;
if m == 1 {
for (i, o) in out.iter_mut().enumerate().take(p) {
*o = inp[i * in_stride];
}
} else {
for q in 0..p {
self.rec(&mut out[q * m..(q + 1) * m], &inp[q * in_stride..], in_stride * p, m, fi + 1, tw_stride * p);
}
}
let big = self.n;
let mut t = [Cpx::default(); 5];
for k in 0..m {
for (q, tq) in t.iter_mut().enumerate().take(p) {
*tq = out[q * m + k].mul(self.twiddle[(q * k * tw_stride) % big]);
}
for q2 in 0..p {
let mut acc = Cpx::default();
for (q, tq) in t.iter().enumerate().take(p) {
acc = acc.add(tq.mul(self.twiddle[((q * q2) % p) * (big / p)]));
}
out[q2 * m + k] = acc;
}
}
}
}
/// Forward MDCT producing `n` coefficients from a block of `n + OVERLAP` samples.
pub struct Mdct {
/// Number of coefficients (half the MDCT length).
n: usize,
fft: Fft,
pre: Vec<Cpx>,
post: Vec<Cpx>,
/// Rising half of the low-overlap window (RFC 6716 §4.3.7).
pub window: [f32; OVERLAP],
scale: f32,
z: Vec<Cpx>,
scratch: Vec<Cpx>,
v: Vec<f32>,
}
impl Mdct {
pub fn new(n: usize) -> Mdct {
let n2 = n / 2;
let pi = std::f64::consts::PI;
let pre = (0..n2)
.map(|m| {
let ph = -pi * m as f64 / n as f64;
Cpx { re: ph.cos() as f32, im: ph.sin() as f32 }
})
.collect();
let post = (0..n2)
.map(|p| {
let ph = -pi * (p as f64 + 0.25) / n as f64;
Cpx { re: ph.cos() as f32, im: ph.sin() as f32 }
})
.collect();
let mut window = [0f32; OVERLAP];
for (i, w) in window.iter_mut().enumerate() {
let x = std::f64::consts::FRAC_PI_2 * (i as f64 + 0.5) / OVERLAP as f64;
let s = x.sin();
*w = (std::f64::consts::FRAC_PI_2 * s * s).sin() as f32;
}
Mdct { n, fft: Fft::new(n2), pre, post, window, scale: 2.0 / n as f32, z: Vec::new(), scratch: Vec::new(), v: vec![0.0; n] }
}
/// Transforms `block` (`n + OVERLAP` samples: the previous frame's last `OVERLAP` samples, then
/// `n` new ones) into `out[..n]`, scaled so that the RFC 6716 decoder's inverse MDCT and
/// overlap-add reproduce the input.
pub fn forward(&mut self, block: &[f32], out: &mut [f32]) {
let n = self.n;
let n2 = n / 2;
debug_assert_eq!(block.len(), n + OVERLAP);
// Place the windowed block into the (virtual) 2n-sample MDCT input u at offset `pad`.
let pad = (n - OVERLAP) / 2;
let w = &self.window;
let u = |t: usize| -> f32 {
if t < pad || t >= pad + n + OVERLAP {
return 0.0;
}
let j = t - pad;
let g = if j < OVERLAP {
w[j]
} else if j >= n {
w[OVERLAP - 1 - (j - n)]
} else {
1.0
};
block[j] * g
};
// Fold to a DCT-IV input: v = (-c_r - d, a - b_r) over the four quarters of u.
for m in 0..n2 {
self.v[m] = -u(n + n2 - 1 - m) - u(n + n2 + m);
self.v[n2 + m] = u(m) - u(n2 + n2 - 1 - m);
}
// DCT-IV via an n/2-point complex FFT.
self.z.clear();
for m in 0..n2 {
let wv = Cpx { re: self.v[2 * m], im: self.v[n - 1 - 2 * m] };
self.z.push(wv.mul(self.pre[m]));
}
self.fft.process(&mut self.z, &mut self.scratch);
for p in 0..n2 {
let y = self.z[p].mul(self.post[p]);
out[2 * p] = y.re * self.scale;
out[n - 1 - 2 * p] = -y.im * self.scale;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Direct MDCT definition X[k] = sum u[t] cos(pi/N (t + 1/2 + N/2)(k + 1/2)).
fn direct(block: &[f32], n: usize, window: &[f32; OVERLAP]) -> Vec<f64> {
let pad = (n - OVERLAP) / 2;
let mut u = vec![0f64; 2 * n];
for (j, &b) in block.iter().enumerate() {
let g = if j < OVERLAP {
window[j]
} else if j >= n {
window[OVERLAP - 1 - (j - n)]
} else {
1.0
};
u[pad + j] = (b * g) as f64;
}
(0..n)
.map(|k| {
(0..2 * n).map(|t| u[t] * (std::f64::consts::PI / n as f64 * (t as f64 + 0.5 + n as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum::<f64>() * 2.0
/ n as f64
})
.collect()
}
#[test]
fn matches_direct_formula() {
for &n in &[240usize, 960] {
let block: Vec<f32> = (0..n + OVERLAP).map(|i| ((i * 37 % 101) as f32 - 50.0) / 7.0).collect();
let mut m = Mdct::new(n);
let mut out = vec![0f32; n];
m.forward(&block, &mut out);
let d = direct(&block, n, &m.window);
for k in 0..n {
assert!((out[k] as f64 - d[k]).abs() < 1e-2, "n={n} k={k}: {} vs {}", out[k], d[k]);
}
}
}
// ---- Replica of the RFC 6716 decoder inverse MDCT (as in FilmCraft's decoder) ----
fn imdct_backward(n_full: usize, input: &[f32], out: &mut [f32], window: &[f32]) {
let n2 = n_full >> 1;
let n4 = n_full >> 2;
let t: Vec<f32> = (0..n2).map(|i| (2.0 * std::f64::consts::PI * (i as f64 + 0.125) / n_full as f64).cos() as f32).collect();
let fft = Fft::new(n4);
let mut z = Vec::new();
for i in 0..n4 {
let x1 = input[2 * i];
let x2 = input[n2 - 1 - 2 * i];
let yr = x2 * t[i] + x1 * t[n4 + i];
let yi = x1 * t[i] - x2 * t[n4 + i];
z.push(Cpx { re: yi, im: yr });
}
fft.process(&mut z, &mut Vec::new());
let overlap = OVERLAP;
let base = overlap >> 1;
let y = &mut out[base..base + n2];
for i in 0..n4.div_ceil(2) {
let front = z[i];
let back = z[n4 - 1 - i];
let (re, im) = (front.im, front.re);
let (t0, t1) = (t[i], t[n4 + i]);
let yr0 = re * t0 + im * t1;
let yi0 = re * t1 - im * t0;
let (re, im) = (back.im, back.re);
let (t0, t1) = (t[n4 - i - 1], t[n2 - i - 1]);
let yr1 = re * t0 + im * t1;
let yi1 = re * t1 - im * t0;
y[2 * i] = yr0;
y[n2 - 1 - 2 * i] = yi0;
y[n2 - 2 - 2 * i] = yr1;
y[2 * i + 1] = yi1;
}
for i in 0..overlap / 2 {
let x1 = out[overlap - 1 - i];
let x2 = out[i];
let w1 = window[i];
let w2 = window[overlap - 1 - i];
out[i] = x2 * w2 - x1 * w1;
out[overlap - 1 - i] = x2 * w1 + x1 * w2;
}
}
/// MDCT analysis followed by the decoder's IMDCT + TDAC overlap-add reproduces the input
/// delayed by exactly `OVERLAP` samples.
#[test]
fn perfect_reconstruction_through_decoder_imdct() {
let n = 960;
let frames = 5;
let sig: Vec<f32> = (0..n * frames).map(|i| (i as f32 * 0.013).sin() * 1000.0 + ((i * 7919) % 211) as f32 - 105.0).collect();
let mut m = Mdct::new(n);
let mut hist = vec![0f32; OVERLAP];
// Decoder memory: [frame output n][folded tail OVERLAP/2] laid out like decode_mem.
let mut mem = vec![0f32; n + OVERLAP];
let mut decoded = Vec::new();
for f in 0..frames {
let mut block = hist.clone();
block.extend_from_slice(&sig[f * n..(f + 1) * n]);
hist.copy_from_slice(&block[n..]);
let mut coefs = vec![0f32; n];
m.forward(&block, &mut coefs);
// Shift: the previous folded tail (at n..n+OVERLAP/2) moves to 0..OVERLAP/2.
mem.copy_within(n..n + OVERLAP / 2, 0);
imdct_backward(2 * n, &coefs, &mut mem, &m.window);
decoded.extend_from_slice(&mem[..n]);
}
let mut err = 0f64;
let mut sum = 0f64;
for t in OVERLAP..n * frames {
let d = decoded[t] as f64 - sig[t - OVERLAP] as f64;
err += d * d;
sum += (sig[t - OVERLAP] as f64).powi(2);
}
let snr = 10.0 * (sum / err).log10();
assert!(snr > 80.0, "reconstruction SNR {snr:.1} dB");
}
}
+472
View File
@@ -0,0 +1,472 @@
//! Range encoder (RFC 6716 §5.1), the exact inverse of the §4.1 range decoder.
//!
//! Range-coded symbols are written from the front of the buffer and raw bits from the back; the
//! bit accounting ([`RangeEncoder::tell`], [`RangeEncoder::tell_frac`]) matches the decoder's, so
//! every budget decision the decoder takes can be mirrored exactly.
const SYM_BITS: u32 = 8;
const CODE_BITS: u32 = 32;
const SYM_MAX: u32 = (1 << SYM_BITS) - 1;
const CODE_SHIFT: u32 = CODE_BITS - SYM_BITS - 1;
const CODE_TOP: u32 = 1 << (CODE_BITS - 1);
const CODE_BOT: u32 = CODE_TOP >> SYM_BITS;
/// Bit resolution of [`RangeEncoder::tell_frac`] (1/8 bit).
pub const BITRES: u32 = 3;
/// Number of bits needed to represent `x` (0 for 0).
#[inline]
pub fn ilog(x: u32) -> i32 {
(32 - x.leading_zeros()) as i32
}
#[derive(Clone, Debug)]
pub struct RangeEncoder {
out: Vec<u8>,
end: Vec<u8>,
end_window: u32,
nend_bits: i32,
nbits_total: i32,
rng: u32,
low: u32,
rem: i32,
ext: u32,
}
impl Default for RangeEncoder {
fn default() -> Self {
Self::new()
}
}
impl RangeEncoder {
pub fn new() -> Self {
RangeEncoder {
out: Vec::new(),
end: Vec::new(),
end_window: 0,
nend_bits: 0,
nbits_total: (CODE_BITS + 1) as i32,
rng: CODE_TOP,
low: 0,
rem: -1,
ext: 0,
}
}
/// Current range (equals the decoder's range after the same symbols).
#[cfg(test)]
pub fn rng(&self) -> u32 {
self.rng
}
fn carry_out(&mut self, c: u32) {
if c != SYM_MAX {
let carry = c >> SYM_BITS;
if self.rem >= 0 {
self.out.push((self.rem as u32 + carry) as u8);
}
if self.ext > 0 {
let sym = (SYM_MAX + carry) & SYM_MAX;
for _ in 0..self.ext {
self.out.push(sym as u8);
}
self.ext = 0;
}
self.rem = (c & SYM_MAX) as i32;
} else {
self.ext += 1;
}
}
#[inline]
fn normalize(&mut self) {
while self.rng <= CODE_BOT {
self.carry_out(self.low >> CODE_SHIFT);
self.low = (self.low << SYM_BITS) & (CODE_TOP - 1);
self.rng <<= SYM_BITS;
self.nbits_total += SYM_BITS as i32;
}
}
/// Encodes the symbol with cumulative range `[fl, fh)` of total `ft`.
pub fn encode(&mut self, fl: u32, fh: u32, ft: u32) {
let r = self.rng / ft;
if fl > 0 {
self.low += self.rng - r * (ft - fl);
self.rng = r * (fh - fl);
} else {
self.rng -= r * (ft - fh);
}
self.normalize();
}
/// Like [`Self::encode`] with `ft = 1 << bits`.
pub fn encode_bin(&mut self, fl: u32, fh: u32, bits: u32) {
let r = self.rng >> bits;
let ft = 1u32 << bits;
if fl > 0 {
self.low += self.rng - r * (ft - fl);
self.rng = r * (fh - fl);
} else {
self.rng -= r * (ft - fh);
}
self.normalize();
}
/// Encodes a binary symbol whose probability of being 1 is `1/2^logp`.
pub fn bit_logp(&mut self, val: bool, logp: u32) {
let r = self.rng;
let s = r >> logp;
let r2 = r - s;
if val {
self.low += r2;
self.rng = s;
} else {
self.rng = r2;
}
self.normalize();
}
/// Encodes symbol `s` with an "inverse CDF" table (`ft = 1 << ftb`).
pub fn icdf(&mut self, s: usize, icdf: &[u8], ftb: u32) {
let r = self.rng >> ftb;
if s > 0 {
self.low += self.rng - r * icdf[s - 1] as u32;
self.rng = r * (icdf[s - 1] as u32 - icdf[s] as u32);
} else {
self.rng -= r * icdf[s] as u32;
}
self.normalize();
}
/// Encodes a uniformly distributed integer `fl` in `[0, ft)` (`ft >= 2`).
pub fn uint(&mut self, fl: u32, ft: u32) {
debug_assert!(ft > 1 && fl < ft);
let ft1 = ft - 1;
let ftb = ilog(ft1);
if ftb > 8 {
let ftb = (ftb - 8) as u32;
let top = (ft1 >> ftb) + 1;
let v = fl >> ftb;
self.encode(v, v + 1, top);
self.bits(fl & ((1 << ftb) - 1), ftb);
} else {
self.encode(fl, fl + 1, ft);
}
}
/// Writes `bits` raw bits (0..=25) to the end of the buffer.
pub fn bits(&mut self, val: u32, bits: u32) {
if bits == 0 {
return;
}
let mut window = self.end_window;
let mut used = self.nend_bits;
if used as u32 + bits > CODE_BITS {
while used >= SYM_BITS as i32 {
self.end.push((window & SYM_MAX) as u8);
window >>= SYM_BITS;
used -= SYM_BITS as i32;
}
}
window |= val << used;
used += bits as i32;
self.end_window = window;
self.nend_bits = used;
self.nbits_total += bits as i32;
}
/// Number of bits written so far (rounded up), identical to the decoder's `tell`.
#[inline]
pub fn tell(&self) -> i32 {
self.nbits_total - ilog(self.rng)
}
/// Number of 1/8 bits written so far (rounded up), identical to the decoder's `tell_frac`.
pub fn tell_frac(&self) -> i32 {
let nbits = (self.nbits_total as u32) << BITRES;
let mut l = ilog(self.rng) as u32;
let mut r = self.rng >> (l - 16);
for _ in 0..BITRES {
r = (r * r) >> 15;
let b = r >> 16;
l = (l << 1) | b;
r >>= b;
}
nbits.wrapping_sub(l) as i32
}
/// Finalises the stream into a buffer of exactly `size` bytes (range data at the front, raw
/// bits at the back, zero padding in between; a partial raw-bit byte is merged into the byte
/// just before the full raw-bit bytes).
pub fn done(mut self, size: usize) -> Vec<u8> {
let mut l = CODE_BITS as i32 - ilog(self.rng);
let mut msk = (CODE_TOP - 1) >> l;
let mut end = (self.low.wrapping_add(msk)) & !msk;
if (end | msk) >= self.low.wrapping_add(self.rng) {
l += 1;
msk >>= 1;
end = (self.low.wrapping_add(msk)) & !msk;
}
while l > 0 {
self.carry_out(end >> CODE_SHIFT);
end = (end << SYM_BITS) & (CODE_TOP - 1);
l -= SYM_BITS as i32;
}
if self.rem >= 0 || self.ext > 0 {
self.carry_out(0);
}
let mut window = self.end_window;
let mut used = self.nend_bits;
while used >= SYM_BITS as i32 {
self.end.push((window & SYM_MAX) as u8);
window >>= SYM_BITS;
used -= SYM_BITS as i32;
}
let mut buf = vec![0u8; size];
let full = self.end.len().min(size);
let room = size - full;
let n = self.out.len().min(room);
buf[..n].copy_from_slice(&self.out[..n]);
for (k, &b) in self.end.iter().take(full).enumerate() {
buf[size - 1 - k] = b;
}
if used > 0 && room > 0 {
buf[room - 1] |= window as u8;
}
buf
}
}
#[cfg(test)]
pub(crate) mod dec {
//! Minimal range decoder (RFC 6716 §4.1) used to verify the encoder.
use super::*;
const CODE_EXTRA: u32 = (CODE_BITS - 2) % SYM_BITS + 1;
pub struct RangeDecoder<'a> {
buf: &'a [u8],
offs: usize,
end_offs: usize,
end_window: u32,
nend_bits: i32,
nbits_total: i32,
rng: u32,
val: u32,
rem: u32,
}
impl<'a> RangeDecoder<'a> {
pub fn new(buf: &'a [u8]) -> Self {
let mut d = RangeDecoder {
buf,
offs: 0,
end_offs: 0,
end_window: 0,
nend_bits: 0,
nbits_total: (CODE_BITS + 1 - ((CODE_BITS - CODE_EXTRA) / SYM_BITS) * SYM_BITS) as i32,
rng: 1 << CODE_EXTRA,
val: 0,
rem: 0,
};
d.rem = d.read_byte();
d.val = d.rng - 1 - (d.rem >> (SYM_BITS - CODE_EXTRA));
d.normalize();
d
}
fn read_byte(&mut self) -> u32 {
if self.offs < self.buf.len() {
self.offs += 1;
self.buf[self.offs - 1] as u32
} else {
0
}
}
fn read_byte_from_end(&mut self) -> u32 {
if self.end_offs < self.buf.len() {
self.end_offs += 1;
self.buf[self.buf.len() - self.end_offs] as u32
} else {
0
}
}
fn normalize(&mut self) {
while self.rng <= CODE_BOT {
self.nbits_total += SYM_BITS as i32;
self.rng <<= SYM_BITS;
let sym = self.rem;
self.rem = self.read_byte();
let sym = ((sym << SYM_BITS) | self.rem) >> (SYM_BITS - CODE_EXTRA);
self.val = ((self.val << SYM_BITS).wrapping_add(SYM_MAX & !sym)) & (CODE_TOP - 1);
}
}
pub fn rng(&self) -> u32 {
self.rng
}
pub fn decode(&mut self, ft: u32) -> u32 {
let ext = self.rng / ft;
let s = self.val / ext;
ft - (s + 1).min(ft)
}
pub fn update(&mut self, fl: u32, fh: u32, ft: u32) {
let ext = self.rng / ft;
let s = ext * (ft - fh);
self.val = self.val.wrapping_sub(s);
self.rng = if fl > 0 { ext * (fh - fl) } else { self.rng - s };
self.normalize();
}
pub fn bit_logp(&mut self, logp: u32) -> bool {
let r = self.rng;
let d = self.val;
let s = r >> logp;
let ret = d < s;
if !ret {
self.val = d - s;
}
self.rng = if ret { s } else { r - s };
self.normalize();
ret
}
pub fn icdf(&mut self, icdf: &[u8], ftb: u32) -> usize {
let mut s = self.rng;
let d = self.val;
let r = s >> ftb;
let mut ret = 0usize;
let mut t;
loop {
t = s;
s = r * icdf[ret] as u32;
if d >= s {
break;
}
ret += 1;
}
self.val = d - s;
self.rng = t - s;
self.normalize();
ret
}
pub fn uint(&mut self, ft: u32) -> u32 {
let ft1 = ft - 1;
let ftb = ilog(ft1);
if ftb > 8 {
let ftb = (ftb - 8) as u32;
let top = (ft1 >> ftb) + 1;
let s = self.decode(top);
self.update(s, s + 1, top);
(s << ftb) | self.bits(ftb)
} else {
let s = self.decode(ft);
self.update(s, s + 1, ft);
s
}
}
pub fn bits(&mut self, bits: u32) -> u32 {
let mut window = self.end_window;
let mut available = self.nend_bits;
if (available as u32) < bits {
loop {
window |= self.read_byte_from_end() << available;
available += SYM_BITS as i32;
if available > (CODE_BITS - SYM_BITS) as i32 {
break;
}
}
}
let ret = window & ((1u32 << bits) - 1);
self.end_window = window >> bits;
self.nend_bits = available - bits as i32;
self.nbits_total += bits as i32;
ret
}
pub fn tell(&self) -> i32 {
self.nbits_total - ilog(self.rng)
}
}
}
#[cfg(test)]
mod tests {
use super::dec::RangeDecoder;
use super::*;
struct Lcg(u32);
impl Lcg {
fn next(&mut self) -> u32 {
self.0 = self.0.wrapping_mul(1664525).wrapping_add(1013904223);
self.0 >> 8
}
}
#[test]
fn starts_at_one_bit() {
let e = RangeEncoder::new();
assert_eq!(e.tell(), 1);
assert_eq!(e.tell_frac(), 8);
}
#[test]
fn roundtrip_mixed_symbols_and_tell_matches() {
let mut rng = Lcg(7);
let icdf: [u8; 4] = [200, 120, 30, 0];
for trial in 0..300 {
let n = 1 + (rng.next() % 300) as usize;
let mut ops = Vec::new();
for _ in 0..n {
match rng.next() % 5 {
0 => {
let ft = 2 + rng.next() % 1000;
ops.push((0u8, rng.next() % ft, ft));
}
1 => {
let logp = 1 + rng.next() % 15;
ops.push((1, rng.next().is_multiple_of(5) as u32, logp));
}
2 => {
let bits = 1 + rng.next() % 24;
ops.push((2, rng.next() & ((1 << bits) - 1), bits));
}
3 => {
let ft = 2 + rng.next() % 100_000_000;
ops.push((3, rng.next() % ft, ft));
}
_ => ops.push((4, rng.next() % 4, 0)),
}
}
let mut e = RangeEncoder::new();
let mut tells = Vec::new();
for &(k, v, p) in &ops {
match k {
0 => e.encode(v, v + 1, p),
1 => e.bit_logp(v != 0, p),
2 => e.bits(v, p),
3 => e.uint(v, p),
_ => e.icdf(v as usize, &icdf, 8),
}
tells.push(e.tell());
}
// Exactly the number of bytes `tell` says were used must suffice.
let size = (e.tell() as usize).div_ceil(8);
let final_rng = e.rng();
let buf = e.done(size);
assert_eq!(buf.len(), size);
let mut d = RangeDecoder::new(&buf);
for (i, &(k, v, p)) in ops.iter().enumerate() {
let got = match k {
0 => {
let s = d.decode(p);
d.update(s, s + 1, p);
s
}
1 => d.bit_logp(p) as u32,
2 => d.bits(p),
3 => d.uint(p),
_ => d.icdf(&icdf, 8) as u32,
};
assert_eq!(got, v, "trial {trial} op {i} kind {k}");
assert_eq!(d.tell(), tells[i], "tell mismatch trial {trial} op {i}");
}
assert_eq!(d.rng(), final_rng);
}
}
}
+561
View File
@@ -0,0 +1,561 @@
//! Pulse cache and bit allocation (RFC 6716 §4.3.3, §4.3.4.1).
//!
//! The allocation must be bit-identical to the decoder's, so this mirrors FilmCraft's
//! spec-derived `filmcraft-opus` decoder (first-party, MIT OR Apache-2.0) step by step; the only
//! difference is that the encoder *writes* the band-skip, intensity and dual-stereo symbols.
use crate::range::{BITRES, RangeEncoder, ilog};
use crate::tables::*;
const MAX_PSEUDO: i32 = 40;
const LOG_MAX_PSEUDO: i32 = 6;
pub const MAX_FINE_BITS: i32 = 8;
const FINE_OFFSET: i32 = 21;
pub const QTHETA_OFFSET: i32 = 4;
pub const QTHETA_OFFSET_TWOPHASE: i32 = 16;
const ALLOC_STEPS: i32 = 6;
/// Number of pulses represented by pseudo-pulse index `i`.
pub fn get_pulses(i: i32) -> i32 {
if i < 8 { i } else { (8 + (i & 7)) << ((i >> 3) - 1) }
}
/// Conservative (rounded up) log2 of `val` with `frac` fractional bits.
pub fn log2_frac(mut val: u32, mut frac: u32) -> i32 {
let mut l = ilog(val);
if val & val.wrapping_sub(1) != 0 {
if l > 16 {
val = ((val - 1) >> (l - 16)) + 1;
} else {
val <<= 16 - l;
}
let mut l = (l - 1) << frac;
loop {
let b = (val >> 16) as i32;
l += b << frac;
val = (val + b as u32) >> b;
val = (val * val + 0x7FFF) >> 15;
if frac == 0 {
break;
}
frac -= 1;
}
l + (val > 0x8000) as i32
} else {
l -= 1;
l << frac
}
}
/// PVQ codebook size V(N, K) (saturating far above 2^32).
pub fn pvq_v(n: usize, k: usize) -> u128 {
if k == 0 {
return 1;
}
if n == 0 {
return 0;
}
let mut row = vec![0u128; k + 1];
row[0] = 1;
for _ in 0..n {
let mut prev_diag = row[0];
for kk in 1..=k {
let up = row[kk];
let v = up.saturating_add(row[kk - 1]).saturating_add(prev_diag);
prev_diag = up;
row[kk] = v.min(u128::MAX >> 8);
}
row[0] = 1;
}
row[k]
}
fn fits_in32(n: usize, k: i32) -> bool {
pvq_v(n, k as usize) <= u32::MAX as u128
}
/// Precomputed per-mode data.
pub struct Mode {
/// log2 of band width in 1/8 bits.
pub log_n: [i32; NB_EBANDS],
cache_index: Vec<i16>,
cache_bits: Vec<u8>,
caps: Vec<u8>,
}
impl Mode {
pub fn new() -> Mode {
let mut log_n = [0i32; NB_EBANDS];
for (i, l) in log_n.iter_mut().enumerate() {
*l = log2_frac((EBANDS[i + 1] - EBANDS[i]) as u32, BITRES);
}
let (cache_index, cache_bits) = compute_pulse_cache(MAX_LM as i32);
let mut m = Mode { log_n, cache_index, cache_bits, caps: Vec::new() };
m.caps = compute_caps(&m);
m
}
#[inline]
pub fn cache(&self, lm: i32, band: usize) -> &[u8] {
let idx = self.cache_index[((lm + 1) as usize) * NB_EBANDS + band];
&self.cache_bits[idx.max(0) as usize..]
}
pub fn bits2pulses(&self, band: usize, lm: i32, bits: i32) -> i32 {
let cache = self.cache(lm, band);
let mut lo = 0i32;
let mut hi = cache[0] as i32;
let bits = bits - 1;
for _ in 0..LOG_MAX_PSEUDO {
let mid = (lo + hi + 1) >> 1;
if cache[mid as usize] as i32 >= bits {
hi = mid;
} else {
lo = mid;
}
}
let lo_bits = if lo == 0 { -1 } else { cache[lo as usize] as i32 };
if bits - lo_bits <= cache[hi as usize] as i32 - bits { lo } else { hi }
}
pub fn pulses2bits(&self, band: usize, lm: i32, pulses: i32) -> i32 {
if pulses == 0 { 0 } else { self.cache(lm, band)[pulses as usize] as i32 + 1 }
}
/// Per-band allocation caps for this frame (`init_caps`).
pub fn init_caps(&self, lm: usize, c: usize) -> [i32; NB_EBANDS] {
let mut cap = [0i32; NB_EBANDS];
for (i, cp) in cap.iter_mut().enumerate() {
let n = (EBANDS[i + 1] - EBANDS[i]) << lm;
*cp = (self.caps[NB_EBANDS * (2 * lm + c - 1) + i] as i32 + 64) * c as i32 * n >> 2;
}
cap
}
}
fn compute_pulse_cache(lm_max: i32) -> (Vec<i16>, Vec<u8>) {
let nb = NB_EBANDS;
let rows = (lm_max + 2) as usize;
let mut index = vec![-1i16; nb * rows];
let mut entries: Vec<(usize, i32, usize)> = Vec::new();
let mut curr = 0usize;
for i in 0..rows {
for j in 0..nb {
let n = (((EBANDS[j + 1] - EBANDS[j]) << i) >> 1) as usize;
index[i * nb + j] = -1;
'search: for k in 0..=i {
for nn in 0..nb {
if k == i && nn >= j {
break;
}
if n == (((EBANDS[nn + 1] - EBANDS[nn]) << k) >> 1) as usize {
index[i * nb + j] = index[k * nb + nn];
break 'search;
}
}
}
if index[i * nb + j] == -1 && n != 0 {
let mut k = 0;
while k < MAX_PSEUDO && fits_in32(n, get_pulses(k + 1)) {
k += 1;
}
index[i * nb + j] = curr as i16;
entries.push((n, k, curr));
curr += (k + 1) as usize;
}
}
}
let mut bits = vec![0u8; curr];
for &(n, k, off) in &entries {
let maxk = get_pulses(k) as usize;
let tmp = required_bits(n, maxk, BITRES);
for j in 1..=k {
bits[off + j as usize] = (tmp[get_pulses(j) as usize] - 1) as u8;
}
bits[off] = k as u8;
}
(index, bits)
}
fn required_bits(n: usize, maxk: usize, frac: u32) -> Vec<i32> {
let mut out = vec![0i32; maxk + 1];
if n == 1 {
for o in out.iter_mut().skip(1) {
*o = 1 << frac;
}
} else {
for (k, o) in out.iter_mut().enumerate().skip(1) {
*o = log2_frac(pvq_v(n, k) as u32, frac);
}
}
out
}
fn compute_caps(m: &Mode) -> Vec<u8> {
let nb = NB_EBANDS;
let mut caps = vec![0u8; (MAX_LM + 1) * 2 * nb];
for i in 0..=MAX_LM as i32 {
for c in 1..=2i32 {
for j in 0..nb {
let mut n0 = EBANDS[j + 1] - EBANDS[j];
let max_bits;
if n0 << i == 1 {
max_bits = c * (1 + MAX_FINE_BITS) << BITRES;
} else {
let mut lm0 = 0;
if n0 > 2 {
n0 >>= 1;
lm0 -= 1;
} else if n0 <= 1 {
lm0 = i.min(1);
n0 <<= lm0;
}
let pcache = m.cache(lm0, j);
let mut mb = pcache[pcache[0] as usize] as i32 + 1;
let mut n = n0;
for k in 0..(i - lm0) {
mb <<= 1;
let offset = ((m.log_n[j] + ((lm0 + k) << BITRES)) >> 1) - QTHETA_OFFSET;
let num = 459 * ((2 * n - 1) * offset + mb);
let den = ((2 * n - 1) << 9) - 459;
let qb = ((num + (den >> 1)) / den).min(57);
mb += qb;
n <<= 1;
}
if c == 2 {
mb <<= 1;
let offset = ((m.log_n[j] + (i << BITRES)) >> 1) - if n == 2 { QTHETA_OFFSET_TWOPHASE } else { QTHETA_OFFSET };
let ndof = 2 * n - 1 - (n == 2) as i32;
let p = if n == 2 { 512 } else { 487 };
let num = p * (mb + ndof * offset);
let den = (ndof << 9) - p;
let qb = ((num + (den >> 1)) / den).min(if n == 2 { 64 } else { 61 });
mb += qb;
}
let ndof = c * n + if c == 2 && n > 2 { 1 } else { 0 };
let mut offset = ((m.log_n[j] + (i << BITRES)) >> 1) - FINE_OFFSET;
if n == 2 {
offset += (1 << BITRES) >> 2;
}
let num = mb + ndof * offset;
let den = (ndof - 1) << BITRES;
let qb = ((num + (den >> 1)) / den).min(MAX_FINE_BITS);
mb += c * qb << BITRES;
max_bits = mb;
}
let w = c * ((EBANDS[j + 1] - EBANDS[j]) << i);
let v = (4 * max_bits / w) - 64;
caps[((i * 2 + c - 1) as usize) * nb + j] = v.clamp(0, 255) as u8;
}
}
}
caps
}
/// Result of the allocation process.
pub struct Allocation {
pub coded_bands: usize,
pub intensity: usize,
pub dual_stereo: bool,
pub balance: i32,
pub pulses: [i32; NB_EBANDS],
pub fine_quant: [i32; NB_EBANDS],
pub fine_priority: [i32; NB_EBANDS],
}
/// `clt_compute_allocation`, encoder side. Stereo frames are always coded as dual (L/R) stereo
/// with no intensity bands, and no band is skipped voluntarily.
#[allow(clippy::too_many_arguments)]
pub fn compute_allocation(
m: &Mode,
start: usize,
end: usize,
offsets: &[i32; NB_EBANDS],
cap: &[i32; NB_EBANDS],
alloc_trim: i32,
total: i32,
c: usize,
lm: usize,
enc: &mut RangeEncoder,
) -> Allocation {
let ci = c as i32;
let lmi = lm as i32;
let mut total = total.max(0);
let mut skip_start = start;
let skip_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 };
total -= skip_rsv;
let mut intensity_rsv = 0;
let mut dual_stereo_rsv = 0;
if c == 2 {
intensity_rsv = LOG2_FRAC_TABLE[end - start];
if intensity_rsv > total {
intensity_rsv = 0;
} else {
total -= intensity_rsv;
dual_stereo_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 };
total -= dual_stereo_rsv;
}
}
let mut bits1 = [0i32; NB_EBANDS];
let mut bits2 = [0i32; NB_EBANDS];
let mut thresh = [0i32; NB_EBANDS];
let mut trim_offset = [0i32; NB_EBANDS];
for j in start..end {
let w = EBANDS[j + 1] - EBANDS[j];
thresh[j] = (ci << BITRES).max((3 * w << lmi << BITRES) >> 4);
trim_offset[j] = ci * w * (alloc_trim - 5 - lmi) * (end as i32 - j as i32 - 1) * (1 << (lmi + BITRES as i32)) >> 6;
if w << lmi == 1 {
trim_offset[j] -= ci << BITRES;
}
}
let nb_alloc = ALLOC.len() as i32;
let mut lo = 1;
let mut hi = nb_alloc - 1;
loop {
let mut done = false;
let mut psum = 0;
let mid = (lo + hi) >> 1;
for j in (start..end).rev() {
let n = EBANDS[j + 1] - EBANDS[j];
let mut bitsj = ci * n * (ALLOC[mid as usize][j] as i32) << lmi >> 2;
if bitsj > 0 {
bitsj = (bitsj + trim_offset[j]).max(0);
}
bitsj += offsets[j];
if bitsj >= thresh[j] || done {
done = true;
psum += bitsj.min(cap[j]);
} else if bitsj >= ci << BITRES {
psum += ci << BITRES;
}
}
if psum > total {
hi = mid - 1;
} else {
lo = mid + 1;
}
if lo > hi {
break;
}
}
hi = lo;
lo -= 1;
for j in start..end {
let n = EBANDS[j + 1] - EBANDS[j];
let mut bits1j = ci * n * (ALLOC[lo as usize][j] as i32) << lmi >> 2;
let mut bits2j = if hi >= nb_alloc { cap[j] } else { ci * n * (ALLOC[hi as usize][j] as i32) << lmi >> 2 };
if bits1j > 0 {
bits1j = (bits1j + trim_offset[j]).max(0);
}
if bits2j > 0 {
bits2j = (bits2j + trim_offset[j]).max(0);
}
if lo > 0 {
bits1j += offsets[j];
}
bits2j += offsets[j];
if offsets[j] > 0 {
skip_start = j;
}
bits2j = (bits2j - bits1j).max(0);
bits1[j] = bits1j;
bits2[j] = bits2j;
}
interp_bits2pulses(m, start, end, skip_start, &bits1, &bits2, &thresh, cap, total, skip_rsv, intensity_rsv, dual_stereo_rsv, c, lm, enc)
}
#[allow(clippy::too_many_arguments)]
fn interp_bits2pulses(
m: &Mode,
start: usize,
end: usize,
skip_start: usize,
bits1: &[i32; NB_EBANDS],
bits2: &[i32; NB_EBANDS],
thresh: &[i32; NB_EBANDS],
cap: &[i32; NB_EBANDS],
mut total: i32,
skip_rsv: i32,
mut intensity_rsv: i32,
mut dual_stereo_rsv: i32,
c: usize,
lm: usize,
enc: &mut RangeEncoder,
) -> Allocation {
let ci = c as i32;
let alloc_floor = ci << BITRES;
let stereo = (c > 1) as i32;
let log_m = (lm as i32) << BITRES;
let mut lo = 0;
let mut hi = 1 << ALLOC_STEPS;
for _ in 0..ALLOC_STEPS {
let mid = (lo + hi) >> 1;
let mut psum = 0;
let mut done = false;
for j in (start..end).rev() {
let tmp = bits1[j] + (mid * bits2[j] >> ALLOC_STEPS);
if tmp >= thresh[j] || done {
done = true;
psum += tmp.min(cap[j]);
} else if tmp >= alloc_floor {
psum += alloc_floor;
}
}
if psum > total {
hi = mid;
} else {
lo = mid;
}
}
let mut bits = [0i32; NB_EBANDS];
let mut psum = 0;
let mut done = false;
for j in (start..end).rev() {
let mut tmp = bits1[j] + (lo * bits2[j] >> ALLOC_STEPS);
if tmp < thresh[j] && !done {
tmp = if tmp >= alloc_floor { alloc_floor } else { 0 };
} else {
done = true;
}
tmp = tmp.min(cap[j]);
bits[j] = tmp;
psum += tmp;
}
let s = start;
let mut coded_bands = end;
loop {
let j = coded_bands - 1;
if j <= skip_start {
total += skip_rsv;
break;
}
let mut left = total - psum;
let span = EBANDS[coded_bands] - EBANDS[s];
let percoeff = left / span;
left -= span * percoeff;
let rem = (left - (EBANDS[j] - EBANDS[s])).max(0);
let band_width = EBANDS[coded_bands] - EBANDS[j];
let band_bits = bits[j] + percoeff * band_width + rem;
if band_bits >= thresh[j].max(alloc_floor + (1 << BITRES)) {
// Keep this band (and every band below it).
enc.bit_logp(true, 1);
break;
}
psum -= bits[j] + intensity_rsv;
if intensity_rsv > 0 {
intensity_rsv = LOG2_FRAC_TABLE[j - s];
}
psum += intensity_rsv;
if band_bits >= alloc_floor {
psum += alloc_floor;
bits[j] = alloc_floor;
} else {
bits[j] = 0;
}
coded_bands -= 1;
}
let intensity = if intensity_rsv > 0 {
// No intensity stereo: every coded band is coded in full.
enc.uint((coded_bands - s) as u32, (coded_bands + 1 - s) as u32);
coded_bands
} else {
0
};
if intensity <= s {
total += dual_stereo_rsv;
dual_stereo_rsv = 0;
}
let dual_stereo = if dual_stereo_rsv > 0 {
enc.bit_logp(true, 1);
true
} else {
false
};
let mut left = total - psum;
let span = EBANDS[coded_bands] - EBANDS[s];
let percoeff = left / span;
left -= span * percoeff;
for j in s..coded_bands {
bits[j] += percoeff * (EBANDS[j + 1] - EBANDS[j]);
}
for j in s..coded_bands {
let tmp = left.min(EBANDS[j + 1] - EBANDS[j]);
bits[j] += tmp;
left -= tmp;
}
let mut ebits = [0i32; NB_EBANDS];
let mut fine_priority = [0i32; NB_EBANDS];
let mut balance = 0;
for j in s..coded_bands {
let n0 = EBANDS[j + 1] - EBANDS[j];
let n = n0 << lm;
let bit = bits[j] + balance;
let mut excess;
if n > 1 {
excess = (bit - cap[j]).max(0);
bits[j] = bit - excess;
let den = ci * n + if c == 2 && n > 2 && !dual_stereo && j < intensity { 1 } else { 0 };
let nclogn = den * (m.log_n[j] + log_m);
let mut offset = (nclogn >> 1) - den * FINE_OFFSET;
if n == 2 {
offset += (den << BITRES) >> 2;
}
if bits[j] + offset < den * 2 << BITRES {
offset += nclogn >> 2;
} else if bits[j] + offset < den * 3 << BITRES {
offset += nclogn >> 3;
}
let mut eb = (bits[j] + offset + (den << (BITRES - 1))).max(0);
eb = (eb / den) >> BITRES;
if ci * eb > (bits[j] >> BITRES) {
eb = bits[j] >> stereo >> BITRES;
}
eb = eb.min(MAX_FINE_BITS);
ebits[j] = eb;
fine_priority[j] = (eb * (den << BITRES) >= bits[j] + offset) as i32;
bits[j] -= ci * eb << BITRES;
} else {
excess = (bit - (ci << BITRES)).max(0);
bits[j] = bit - excess;
ebits[j] = 0;
fine_priority[j] = 1;
}
if excess > 0 {
let extra_fine = (excess >> (stereo + BITRES as i32)).min(MAX_FINE_BITS - ebits[j]);
ebits[j] += extra_fine;
let extra_bits = extra_fine * ci << BITRES;
fine_priority[j] = (extra_bits >= excess - balance) as i32;
excess -= extra_bits;
}
balance = excess;
}
for j in coded_bands..end {
ebits[j] = bits[j] >> stereo >> BITRES;
bits[j] = 0;
fine_priority[j] = (ebits[j] < 1) as i32;
}
Allocation { coded_bands, intensity, dual_stereo, balance, pulses: bits, fine_quant: ebits, fine_priority }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn caps_match_decoder_reference() {
let m = Mode::new();
let first_rows: [u8; 42] = [
224, 224, 224, 224, 224, 224, 224, 224, 160, 160, 160, 160, 185, 185, 185, 178, 178, 168, 134, 61, 37, 224, 224, 224, 224, 224, 224, 224, 224, 240,
240, 240, 240, 207, 207, 207, 198, 198, 183, 144, 66, 40,
];
assert_eq!(&m.caps[..42], &first_rows[..]);
}
#[test]
fn pvq_counts() {
assert_eq!(pvq_v(5, 1), 10);
assert_eq!(pvq_v(1, 7), 2);
assert_eq!(pvq_v(2, 9), 36);
assert_eq!(pvq_v(3, 2), 18);
}
}
+122
View File
@@ -0,0 +1,122 @@
//! Constant tables of the 48 kHz CELT mode (RFC 6716 §4.3, Tables 55–58 and §4.3.2).
//!
//! Transcribed from RFC 6716; copied from FilmCraft's spec-derived `filmcraft-opus` decoder
//! (first-party, MIT OR Apache-2.0) so both sides share the exact same values.
/// Number of energy bands.
pub const NB_EBANDS: usize = 21;
/// Band edges in units of 2.5 ms MDCT bins (Table 55).
pub const EBANDS: [i32; NB_EBANDS + 1] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 34, 40, 48, 60, 78, 100];
/// Overlap of the low-overlap window (samples at 48 kHz).
pub const OVERLAP: usize = 120;
/// Largest LM (20 ms frames).
pub const MAX_LM: usize = 3;
/// Pre-emphasis coefficient.
pub const PREEMPH: f32 = 0.850_006_1;
/// Static allocation table (Table 57), indexed `[quality][band]`, in 1/32 bit per MDCT bin.
pub const ALLOC: [[u8; NB_EBANDS]; 11] = transpose([
[0, 90, 110, 118, 126, 134, 144, 152, 162, 172, 200],
[0, 80, 100, 110, 119, 127, 137, 145, 155, 165, 200],
[0, 75, 90, 103, 112, 120, 130, 138, 148, 158, 200],
[0, 69, 84, 93, 104, 114, 124, 132, 142, 152, 200],
[0, 63, 78, 86, 95, 103, 113, 123, 133, 143, 200],
[0, 56, 71, 80, 89, 97, 107, 117, 127, 137, 200],
[0, 49, 65, 75, 83, 91, 101, 111, 121, 131, 200],
[0, 40, 58, 70, 78, 85, 95, 105, 115, 125, 200],
[0, 34, 51, 65, 72, 78, 88, 98, 108, 118, 198],
[0, 29, 45, 59, 66, 72, 82, 92, 102, 112, 193],
[0, 20, 39, 53, 60, 66, 76, 86, 96, 106, 188],
[0, 18, 32, 47, 54, 60, 70, 80, 90, 100, 183],
[0, 10, 26, 40, 47, 54, 64, 74, 84, 94, 178],
[0, 0, 20, 31, 39, 47, 57, 67, 77, 87, 173],
[0, 0, 12, 23, 32, 41, 51, 61, 71, 81, 168],
[0, 0, 0, 15, 25, 35, 45, 55, 65, 75, 163],
[0, 0, 0, 4, 17, 29, 39, 49, 59, 69, 158],
[0, 0, 0, 0, 12, 23, 33, 43, 53, 63, 153],
[0, 0, 0, 0, 1, 16, 26, 36, 46, 56, 148],
[0, 0, 0, 0, 0, 10, 15, 20, 30, 45, 129],
[0, 0, 0, 0, 0, 1, 1, 1, 1, 20, 104],
]);
const fn transpose(t: [[u8; 11]; NB_EBANDS]) -> [[u8; NB_EBANDS]; 11] {
let mut out = [[0u8; NB_EBANDS]; 11];
let mut b = 0;
while b < NB_EBANDS {
let mut q = 0;
while q < 11 {
out[q][b] = t[b][q];
q += 1;
}
b += 1;
}
out
}
/// Mean band energy (log2 domain) removed before coarse quantisation.
pub const E_MEANS: [f32; 25] = [
6.4375, 6.25, 5.75, 5.3125, 5.0625, 4.8125, 4.5, 4.375, 4.875, 4.6875, 4.5625, 4.4375, 4.875, 4.625, 4.3125, 4.5, 4.375, 4.625, 4.75, 4.4375, 3.75, 3.75,
3.75, 3.75, 3.75,
];
/// Inter-frame prediction coefficient (alpha) per LM, Q15.
pub const PRED_COEF: [f32; 4] = [29440.0 / 32768.0, 26112.0 / 32768.0, 21248.0 / 32768.0, 16384.0 / 32768.0];
/// Inter-band prediction coefficient (beta) per LM for inter frames, Q15.
pub const BETA_COEF: [f32; 4] = [30147.0 / 32768.0, 22282.0 / 32768.0, 12124.0 / 32768.0, 6554.0 / 32768.0];
/// Beta for intra frames.
pub const BETA_INTRA: f32 = 4915.0 / 32768.0;
/// Laplace model parameters for coarse energy: `[LM][intra][2*band + {0: prob, 1: decay}]`.
pub const E_PROB_MODEL: [[[u8; 42]; 2]; 4] = [
[
[
72, 127, 65, 129, 66, 128, 65, 128, 64, 128, 62, 128, 64, 128, 64, 128, 92, 78, 92, 79, 92, 78, 90, 79, 116, 41, 115, 40, 114, 40, 132, 26, 132,
26, 145, 17, 161, 12, 176, 10, 177, 11,
],
[
24, 179, 48, 138, 54, 135, 54, 132, 53, 134, 56, 133, 55, 132, 55, 132, 61, 114, 70, 96, 74, 88, 75, 88, 87, 74, 89, 66, 91, 67, 100, 59, 108, 50,
120, 40, 122, 37, 97, 43, 78, 50,
],
],
[
[
83, 78, 84, 81, 88, 75, 86, 74, 87, 71, 90, 73, 93, 74, 93, 74, 109, 40, 114, 36, 117, 34, 117, 34, 143, 17, 145, 18, 146, 19, 162, 12, 165, 10,
178, 7, 189, 6, 190, 8, 177, 9,
],
[
23, 178, 54, 115, 63, 102, 66, 98, 69, 99, 74, 89, 71, 91, 73, 91, 78, 89, 86, 80, 92, 66, 93, 64, 102, 59, 103, 60, 104, 60, 117, 52, 123, 44,
138, 35, 133, 31, 97, 38, 77, 45,
],
],
[
[
61, 90, 93, 60, 105, 42, 107, 41, 110, 45, 116, 38, 113, 38, 112, 38, 124, 26, 132, 27, 136, 19, 140, 20, 155, 14, 159, 16, 158, 18, 170, 13, 177,
10, 187, 8, 192, 6, 175, 9, 159, 10,
],
[
21, 178, 59, 110, 71, 86, 75, 85, 84, 83, 91, 66, 88, 73, 87, 72, 92, 75, 98, 72, 105, 58, 107, 54, 115, 52, 114, 55, 112, 56, 129, 51, 132, 40,
150, 33, 140, 29, 98, 35, 77, 42,
],
],
[
[
42, 121, 96, 66, 108, 43, 111, 40, 117, 44, 123, 32, 120, 36, 119, 33, 127, 33, 134, 34, 139, 21, 147, 23, 152, 20, 158, 25, 154, 26, 166, 21, 173,
16, 184, 13, 184, 10, 150, 13, 139, 15,
],
[
22, 178, 63, 114, 74, 82, 84, 83, 92, 82, 103, 62, 96, 72, 96, 67, 101, 73, 107, 72, 113, 55, 118, 52, 125, 52, 118, 52, 117, 55, 135, 49, 137, 39,
157, 32, 145, 29, 97, 33, 77, 40,
],
],
];
/// Inverse CDF for coarse energy when only a few bits remain.
pub const SMALL_ENERGY_ICDF: [u8; 3] = [2, 1, 0];
/// Conservative log2 in 1/8 bits of the number of coded bands (intensity reservation).
pub const LOG2_FRAC_TABLE: [i32; 24] = [0, 8, 13, 16, 19, 21, 23, 24, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 34, 35, 36, 36, 37, 37];
/// TF resolution changes: `[LM][4*transient + 2*tf_select + tf_flag]`.
pub const TF_SELECT_TABLE: [[i8; 8]; 4] = [[0, -1, 0, -1, 0, -1, 0, -1], [0, -1, 0, -2, 1, 0, 1, -1], [0, -2, 0, -3, 2, 0, 1, -1], [0, -2, 0, -3, 3, 0, 1, -1]];
/// Spreading decision `{7, 2, 21, 2}/32`.
pub const SPREAD_ICDF: [u8; 4] = [25, 23, 2, 0];
/// Allocation trim (Table 58).
pub const TRIM_ICDF: [u8; 11] = [126, 124, 119, 109, 87, 41, 19, 9, 4, 2, 0];
+15
View File
@@ -629,6 +629,21 @@ pub struct Footage {
/// Colour profile (from the file's metadata, or Interpret Footage). `None` = sRGB.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub color_profile: Option<ColorSpace>,
/// One layer of a layered file (Photoshop) instead of its merged image.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub layer: Option<SourceLayer>,
}
/// A layer of a layered still (a Photoshop document) used as footage
/// (File ▸ Import ▸ Composition / Composition – Retain Layer Sizes, or Choose Layer).
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SourceLayer {
/// Layer record index in the file (bottom of the stack = 0).
pub index: u32,
pub name: String,
/// The footage is the layer's own bounds (Retain Layer Sizes) rather than the document size.
#[serde(default)]
pub layer_size: bool,
}
fn one() -> u32 {
+26 -4
View File
@@ -136,17 +136,26 @@ pub enum OutputFormat {
ExrSequence,
/// Animated GIF.
Gif,
/// WebM: VP9 video (alpha with RGB + Alpha) + Opus audio.
WebM,
/// Audio only: WAV (16-bit PCM).
Wav,
/// Audio only: AIFF (16-bit PCM).
Aiff,
}
impl OutputFormat {
pub const ALL: [OutputFormat; 7] = [
pub const ALL: [OutputFormat; 10] = [
OutputFormat::H264,
OutputFormat::ProRes,
OutputFormat::WebM,
OutputFormat::PngSequence,
OutputFormat::JpegSequence,
OutputFormat::TiffSequence,
OutputFormat::ExrSequence,
OutputFormat::Gif,
OutputFormat::Wav,
OutputFormat::Aiff,
];
pub fn label(self) -> &'static str {
@@ -158,6 +167,9 @@ impl OutputFormat {
OutputFormat::TiffSequence => "TIFF Sequence",
OutputFormat::ExrSequence => "OpenEXR Sequence",
OutputFormat::Gif => "Animated GIF",
OutputFormat::WebM => "WebM (VP9 + Opus)",
OutputFormat::Wav => "WAV",
OutputFormat::Aiff => "AIFF",
}
}
pub fn extension(self) -> &'static str {
@@ -169,19 +181,26 @@ impl OutputFormat {
OutputFormat::TiffSequence => "tif",
OutputFormat::ExrSequence => "exr",
OutputFormat::Gif => "gif",
OutputFormat::WebM => "webm",
OutputFormat::Wav => "wav",
OutputFormat::Aiff => "aif",
}
}
pub fn is_sequence(self) -> bool {
matches!(self, OutputFormat::PngSequence | OutputFormat::JpegSequence | OutputFormat::TiffSequence | OutputFormat::ExrSequence)
}
pub fn is_movie(self) -> bool {
matches!(self, OutputFormat::H264 | OutputFormat::ProRes)
matches!(self, OutputFormat::H264 | OutputFormat::ProRes | OutputFormat::WebM)
}
/// Audio-only outputs (no video is rendered).
pub fn is_audio_only(self) -> bool {
matches!(self, OutputFormat::Wav | OutputFormat::Aiff)
}
pub fn supports_alpha(self) -> bool {
!matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence)
!matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence | OutputFormat::Wav | OutputFormat::Aiff)
}
pub fn supports_audio(self) -> bool {
self.is_movie()
self.is_movie() || self.is_audio_only()
}
/// The written frame size for a rendered size: H.264 needs even dimensions (rounded down,
/// at least 2).
@@ -201,6 +220,9 @@ impl OutputFormat {
"tif" | "tiff" | "tiffsequence" => OutputFormat::TiffSequence,
"exr" | "openexr" | "openexrsequence" => OutputFormat::ExrSequence,
"gif" | "animatedgif" => OutputFormat::Gif,
"webm" | "vp9" | "webmvp9opus" => OutputFormat::WebM,
"wav" | "wave" => OutputFormat::Wav,
"aif" | "aiff" | "aifc" => OutputFormat::Aiff,
_ => return None,
})
}
+13
View File
@@ -0,0 +1,13 @@
[package]
name = "effectcraft-psd"
description = "Photoshop PSD/PSB reader (layers, groups, masks, blend modes, text, layer effects, adjustment layers; 8/16/32-bit, RGB/CMYK/Gray/Lab/Indexed, raw/RLE/ZIP) and a minimal writer, from Adobe's published file format specification"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
[lints]
workspace = true
[dependencies]
miniz_oxide = "0.8"
+17
View File
@@ -0,0 +1,17 @@
# effectcraft-psd
Reads Photoshop documents (`.psd`, and `.psb` large documents) for File ▸ Import (as footage, as a
composition, or as a composition with layer sizes retained), and writes a minimal subset (used to
generate test fixtures).
Implemented from Adobe's public **Adobe Photoshop File Formats Specification** (November 2019
edition): file header, colour mode data, image resources, layer and mask information (layer
records, channel image data, layer mask data, additional layer information: `luni`, `lsct`/`lsdk`,
`lyid`, `iOpa`, `TySh`, `lfx2`, `SoCo`, `nvrt`, `brit`, `hue2`, `levl`, `expA`, `vmsk`/`vsms`,
`Lr16`/`Lr32`), the descriptor (action) structure, and the merged image data. Raw, RLE (PackBits)
and ZIP (with and without prediction, inflated with `miniz_oxide`) compression; 1/8/16/32-bit;
Bitmap, Grayscale, Indexed, RGB, CMYK (naive conversion) and Lab colour modes.
Text engine data (`EngineData`) is read best-effort for the font, size, fill colour and
justification of the first style run. No Adobe code, sample files or assets were used; test
fixtures are generated in-test by the writer in this crate ([`write`](src/write.rs)).
+430
View File
@@ -0,0 +1,430 @@
//! The descriptor ("action descriptor") structure: typed key/value trees used by text layers,
//! layer effects, fill layers and newer adjustment layers (spec: "Descriptor structure").
use crate::read::Reader;
use crate::{Error, Result};
/// A descriptor: a class and a list of keyed items (in file order).
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Descriptor {
pub class_name: String,
pub class_id: String,
pub items: Vec<(String, DValue)>,
}
/// One descriptor value.
#[derive(Clone, Debug, PartialEq)]
pub enum DValue {
Descriptor(Descriptor),
List(Vec<DValue>),
Double(f64),
/// Unit (`#Pxl`, `#Prc`, `#Ang`, `#Pnt`…) and value.
UnitFloat(String, f64),
UnitFloats(String, Vec<f64>),
Text(String),
/// Enumerated value: type and value ids.
Enum(String, String),
Integer(i32),
LargeInteger(i64),
Bool(bool),
Class(String, String),
Raw(Vec<u8>),
Reference(Vec<RefItem>),
}
/// An item of a reference (`obj `).
#[derive(Clone, Debug, PartialEq)]
pub enum RefItem {
Property(String, String),
Class(String),
Enum(String, String, String),
Offset(String, i32),
Identifier(i32),
Index(i32),
Name(String, String),
}
impl Descriptor {
pub fn get(&self, key: &str) -> Option<&DValue> {
self.items.iter().find(|(k, _)| k == key).map(|(_, v)| v)
}
pub fn obj(&self, key: &str) -> Option<&Descriptor> {
match self.get(key)? {
DValue::Descriptor(d) => Some(d),
_ => None,
}
}
/// A number: doubles, unit floats, integers.
pub fn num(&self, key: &str) -> Option<f64> {
match self.get(key)? {
DValue::Double(v) | DValue::UnitFloat(_, v) => Some(*v),
DValue::Integer(v) => Some(*v as f64),
DValue::LargeInteger(v) => Some(*v as f64),
_ => None,
}
}
pub fn bool(&self, key: &str) -> Option<bool> {
match self.get(key)? {
DValue::Bool(b) => Some(*b),
_ => None,
}
}
pub fn text(&self, key: &str) -> Option<&str> {
match self.get(key)? {
DValue::Text(s) => Some(s),
_ => None,
}
}
/// The value id of an enumerated item.
pub fn enum_value(&self, key: &str) -> Option<&str> {
match self.get(key)? {
DValue::Enum(_, v) => Some(v),
_ => None,
}
}
pub fn raw(&self, key: &str) -> Option<&[u8]> {
match self.get(key)? {
DValue::Raw(v) => Some(v),
_ => None,
}
}
/// An RGB colour object (`RGBC` with `Rd `/`Grn `/`Bl ` in 0..255) as 0..1 values; also
/// reads grayscale (`Gry `) colours.
pub fn color(&self, key: &str) -> Option<[f64; 3]> {
let c = self.obj(key)?;
if let (Some(r), Some(g), Some(b)) = (c.num("Rd "), c.num("Grn "), c.num("Bl ")) {
return Some([r / 255.0, g / 255.0, b / 255.0]);
}
if let (Some(r), Some(g), Some(b)) = (c.num("redFloat"), c.num("greenFloat"), c.num("blueFloat")) {
return Some([r, g, b]);
}
c.num("Gry ").map(|g| {
let v = 1.0 - g / 100.0;
[v, v, v]
})
}
}
/// A key or class id: length-prefixed string, or a 4-character code when the length is 0.
pub(crate) fn read_id(r: &mut Reader) -> Result<String> {
let n = r.u32()? as usize;
let n = if n == 0 { 4 } else { n };
let b = r.bytes(n)?;
Ok(String::from_utf8_lossy(b).into_owned())
}
pub(crate) fn read_unicode(r: &mut Reader) -> Result<String> {
let n = r.u32()? as usize;
let mut v = Vec::with_capacity(n);
for _ in 0..n {
v.push(r.u16()?);
}
while v.last() == Some(&0) {
v.pop();
}
Ok(String::from_utf16_lossy(&v))
}
/// Read a descriptor (after its version field).
pub fn read_descriptor(r: &mut Reader) -> Result<Descriptor> {
let class_name = read_unicode(r)?;
let class_id = read_id(r)?;
let n = r.u32()? as usize;
if n > 100_000 {
return Err(Error::Invalid("descriptor item count".into()));
}
let mut items = Vec::with_capacity(n.min(64));
for _ in 0..n {
let key = read_id(r)?;
let ty = r.tag()?;
let v = read_value(r, &ty, 0)?;
items.push((key, v));
}
Ok(Descriptor { class_name, class_id, items })
}
fn read_value(r: &mut Reader, ty: &[u8; 4], depth: usize) -> Result<DValue> {
if depth > 64 {
return Err(Error::Invalid("descriptor nesting".into()));
}
Ok(match ty {
b"Objc" | b"GlbO" => DValue::Descriptor(read_descriptor(r)?),
b"VlLs" => {
let n = r.u32()? as usize;
if n > 1_000_000 {
return Err(Error::Invalid("descriptor list length".into()));
}
let mut v = Vec::with_capacity(n.min(256));
for _ in 0..n {
let t = r.tag()?;
v.push(read_value(r, &t, depth + 1)?);
}
DValue::List(v)
}
b"doub" => DValue::Double(r.f64()?),
b"UntF" => {
let unit = String::from_utf8_lossy(&r.tag()?).into_owned();
DValue::UnitFloat(unit, r.f64()?)
}
b"UnFl" => {
let unit = String::from_utf8_lossy(&r.tag()?).into_owned();
let n = r.u32()? as usize;
let mut v = Vec::with_capacity(n.min(1024));
for _ in 0..n {
v.push(r.f64()?);
}
DValue::UnitFloats(unit, v)
}
b"TEXT" => DValue::Text(read_unicode(r)?),
b"enum" => {
let t = read_id(r)?;
DValue::Enum(t, read_id(r)?)
}
b"long" => DValue::Integer(r.u32()? as i32),
b"comp" => DValue::LargeInteger(r.u64()? as i64),
b"bool" => DValue::Bool(r.u8()? != 0),
b"type" | b"GlbC" => {
let name = read_unicode(r)?;
DValue::Class(name, read_id(r)?)
}
b"alis" | b"tdta" | b"Pth " => {
let n = r.u32()? as usize;
DValue::Raw(r.bytes(n)?.to_vec())
}
b"obj " => {
let n = r.u32()? as usize;
let mut items = Vec::with_capacity(n.min(16));
for _ in 0..n {
let t = r.tag()?;
items.push(match &t {
b"prop" => {
let _name = read_unicode(r)?;
let class = read_id(r)?;
RefItem::Property(class, read_id(r)?)
}
b"Clss" => {
let _name = read_unicode(r)?;
RefItem::Class(read_id(r)?)
}
b"Enmr" => {
let _name = read_unicode(r)?;
let class = read_id(r)?;
let ty = read_id(r)?;
RefItem::Enum(class, ty, read_id(r)?)
}
b"rele" => {
let _name = read_unicode(r)?;
let class = read_id(r)?;
RefItem::Offset(class, r.u32()? as i32)
}
b"Idnt" => RefItem::Identifier(r.u32()? as i32),
b"indx" => RefItem::Index(r.u32()? as i32),
b"name" => {
let _name = read_unicode(r)?;
let class = read_id(r)?;
RefItem::Name(class, read_unicode(r)?)
}
other => return Err(Error::Invalid(format!("reference item `{}`", String::from_utf8_lossy(other)))),
});
}
DValue::Reference(items)
}
other => return Err(Error::Invalid(format!("descriptor type `{}`", String::from_utf8_lossy(other)))),
})
}
// ---------------------------------------------------------------- writing
fn put_u32(out: &mut Vec<u8>, v: u32) {
out.extend_from_slice(&v.to_be_bytes());
}
fn put_id(out: &mut Vec<u8>, id: &str) {
if id.len() == 4 {
put_u32(out, 0);
} else {
put_u32(out, id.len() as u32);
}
out.extend_from_slice(id.as_bytes());
}
pub(crate) fn put_unicode(out: &mut Vec<u8>, s: &str) {
let v: Vec<u16> = s.encode_utf16().chain(std::iter::once(0)).collect();
put_u32(out, v.len() as u32);
for c in v {
out.extend_from_slice(&c.to_be_bytes());
}
}
/// Serialise a descriptor (without the version field).
pub fn write_descriptor(out: &mut Vec<u8>, d: &Descriptor) {
put_unicode(out, &d.class_name);
put_id(out, &d.class_id);
put_u32(out, d.items.len() as u32);
for (k, v) in &d.items {
put_id(out, k);
write_value(out, v);
}
}
fn write_value(out: &mut Vec<u8>, v: &DValue) {
match v {
DValue::Descriptor(d) => {
out.extend_from_slice(b"Objc");
write_descriptor(out, d);
}
DValue::List(l) => {
out.extend_from_slice(b"VlLs");
put_u32(out, l.len() as u32);
for x in l {
write_value(out, x);
}
}
DValue::Double(x) => {
out.extend_from_slice(b"doub");
out.extend_from_slice(&x.to_be_bytes());
}
DValue::UnitFloat(u, x) => {
out.extend_from_slice(b"UntF");
out.extend_from_slice(&tag4(u));
out.extend_from_slice(&x.to_be_bytes());
}
DValue::UnitFloats(u, xs) => {
out.extend_from_slice(b"UnFl");
out.extend_from_slice(&tag4(u));
put_u32(out, xs.len() as u32);
for x in xs {
out.extend_from_slice(&x.to_be_bytes());
}
}
DValue::Text(s) => {
out.extend_from_slice(b"TEXT");
put_unicode(out, s);
}
DValue::Enum(t, e) => {
out.extend_from_slice(b"enum");
put_id(out, t);
put_id(out, e);
}
DValue::Integer(i) => {
out.extend_from_slice(b"long");
out.extend_from_slice(&i.to_be_bytes());
}
DValue::LargeInteger(i) => {
out.extend_from_slice(b"comp");
out.extend_from_slice(&i.to_be_bytes());
}
DValue::Bool(b) => {
out.extend_from_slice(b"bool");
out.push(*b as u8);
}
DValue::Class(n, c) => {
out.extend_from_slice(b"type");
put_unicode(out, n);
put_id(out, c);
}
DValue::Raw(b) => {
out.extend_from_slice(b"tdta");
put_u32(out, b.len() as u32);
out.extend_from_slice(b);
}
DValue::Reference(items) => {
out.extend_from_slice(b"obj ");
put_u32(out, items.len() as u32);
for it in items {
match it {
RefItem::Property(c, k) => {
out.extend_from_slice(b"prop");
put_unicode(out, "");
put_id(out, c);
put_id(out, k);
}
RefItem::Class(c) => {
out.extend_from_slice(b"Clss");
put_unicode(out, "");
put_id(out, c);
}
RefItem::Enum(c, t, e) => {
out.extend_from_slice(b"Enmr");
put_unicode(out, "");
put_id(out, c);
put_id(out, t);
put_id(out, e);
}
RefItem::Offset(c, o) => {
out.extend_from_slice(b"rele");
put_unicode(out, "");
put_id(out, c);
out.extend_from_slice(&o.to_be_bytes());
}
RefItem::Identifier(i) => {
out.extend_from_slice(b"Idnt");
out.extend_from_slice(&i.to_be_bytes());
}
RefItem::Index(i) => {
out.extend_from_slice(b"indx");
out.extend_from_slice(&i.to_be_bytes());
}
RefItem::Name(c, n) => {
out.extend_from_slice(b"name");
put_unicode(out, "");
put_id(out, c);
put_unicode(out, n);
}
}
}
}
}
}
fn tag4(s: &str) -> [u8; 4] {
let mut t = *b" ";
for (i, b) in s.bytes().take(4).enumerate() {
t[i] = b;
}
t
}
/// Builder helpers for descriptors (tests and the writer).
impl Descriptor {
pub fn new(class_id: &str) -> Descriptor {
Descriptor { class_name: String::new(), class_id: class_id.into(), items: vec![] }
}
pub fn with(mut self, key: &str, v: DValue) -> Descriptor {
self.items.push((key.into(), v));
self
}
/// An `RGBC` colour object from 0..1 values.
pub fn rgb(c: [f64; 3]) -> Descriptor {
Descriptor::new("RGBC").with("Rd ", DValue::Double(c[0] * 255.0)).with("Grn ", DValue::Double(c[1] * 255.0)).with("Bl ", DValue::Double(c[2] * 255.0))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn round_trip() {
let d = Descriptor::new("null")
.with("Txt ", DValue::Text("Héllo".into()))
.with("Clr ", DValue::Descriptor(Descriptor::rgb([1.0, 0.5, 0.0])))
.with("Opct", DValue::UnitFloat("#Prc".into(), 75.0))
.with("Md ", DValue::Enum("BlnM".into(), "Mltp".into()))
.with("enab", DValue::Bool(true))
.with("longKeyName", DValue::List(vec![DValue::Integer(3), DValue::Double(1.5)]))
.with("EngineData", DValue::Raw(b"<< >>".to_vec()))
.with("ref", DValue::Reference(vec![RefItem::Enum("Lyr ".into(), "Ordn".into(), "Trgt".into()), RefItem::Index(2)]));
let mut out = vec![];
write_descriptor(&mut out, &d);
let back = read_descriptor(&mut Reader::new(&out)).unwrap();
assert_eq!(back.class_id, "null");
assert_eq!(back.text("Txt "), Some("Héllo"));
assert_eq!(back.color("Clr "), Some([1.0, 0.5, 0.0]));
assert_eq!(back.num("Opct"), Some(75.0));
assert_eq!(back.enum_value("Md "), Some("Mltp"));
assert_eq!(back.bool("enab"), Some(true));
assert_eq!(back.raw("EngineData"), Some(&b"<< >>"[..]));
assert_eq!(back, d);
}
}
+288
View File
@@ -0,0 +1,288 @@
//! The text engine data of type layers (`EngineData` in the `TySh` descriptor): a PostScript-like
//! tree of dictionaries (`<< /Key value >>`), arrays (`[ ]`), numbers, names, booleans and
//! strings (`(þÿ` + UTF-16BE with backslash escapes `)`). Read best-effort.
#[derive(Clone, Debug, PartialEq)]
pub enum EValue {
Dict(Vec<(String, EValue)>),
Array(Vec<EValue>),
Num(f64),
Bool(bool),
Name(String),
Str(String),
}
impl EValue {
pub fn get(&self, key: &str) -> Option<&EValue> {
match self {
EValue::Dict(v) => v.iter().find(|(k, _)| k == key).map(|(_, v)| v),
_ => None,
}
}
pub fn idx(&self, i: usize) -> Option<&EValue> {
match self {
EValue::Array(v) => v.get(i),
_ => None,
}
}
/// Follow a `/`-separated path; numeric segments index arrays.
pub fn path(&self, p: &str) -> Option<&EValue> {
let mut cur = self;
for seg in p.split('/').filter(|s| !s.is_empty()) {
cur = match seg.parse::<usize>() {
Ok(i) => cur.idx(i)?,
Err(_) => cur.get(seg)?,
};
}
Some(cur)
}
pub fn num(&self) -> Option<f64> {
match self {
EValue::Num(n) => Some(*n),
_ => None,
}
}
pub fn str(&self) -> Option<&str> {
match self {
EValue::Str(s) | EValue::Name(s) => Some(s),
_ => None,
}
}
}
struct P<'a> {
b: &'a [u8],
i: usize,
}
impl P<'_> {
fn ws(&mut self) {
while self.i < self.b.len() && (self.b[self.i].is_ascii_whitespace() || self.b[self.i] == 0) {
self.i += 1;
}
}
fn peek(&self, s: &[u8]) -> bool {
self.b[self.i..].starts_with(s)
}
fn value(&mut self, depth: usize) -> Option<EValue> {
if depth > 128 {
return None;
}
self.ws();
if self.i >= self.b.len() {
return None;
}
if self.peek(b"<<") {
self.i += 2;
let mut v = vec![];
loop {
self.ws();
if self.i >= self.b.len() {
return Some(EValue::Dict(v));
}
if self.peek(b">>") {
self.i += 2;
return Some(EValue::Dict(v));
}
if self.b[self.i] != b'/' {
// Unexpected token: skip it.
self.i += 1;
continue;
}
let key = self.name();
let val = self.value(depth + 1)?;
v.push((key, val));
}
}
match self.b[self.i] {
b'[' => {
self.i += 1;
let mut v = vec![];
loop {
self.ws();
if self.i >= self.b.len() {
return Some(EValue::Array(v));
}
if self.b[self.i] == b']' {
self.i += 1;
return Some(EValue::Array(v));
}
let start = self.i;
match self.value(depth + 1) {
Some(x) => v.push(x),
None => return Some(EValue::Array(v)),
}
if self.i == start {
self.i += 1;
}
}
}
b'/' => Some(EValue::Name(self.name())),
b'(' => Some(EValue::Str(self.string())),
_ => {
let start = self.i;
while self.i < self.b.len() && !self.b[self.i].is_ascii_whitespace() && !b"[]<>/(".contains(&self.b[self.i]) {
self.i += 1;
}
let tok = std::str::from_utf8(&self.b[start..self.i]).unwrap_or("");
if self.i == start {
self.i += 1;
return Some(EValue::Name(String::new()));
}
Some(match tok {
"true" => EValue::Bool(true),
"false" => EValue::Bool(false),
t => t.parse::<f64>().map(EValue::Num).unwrap_or_else(|_| EValue::Name(t.to_string())),
})
}
}
}
fn name(&mut self) -> String {
self.i += 1; // '/'
let start = self.i;
while self.i < self.b.len() && !self.b[self.i].is_ascii_whitespace() && !b"[]<>/(".contains(&self.b[self.i]) {
self.i += 1;
}
String::from_utf8_lossy(&self.b[start..self.i]).into_owned()
}
fn string(&mut self) -> String {
self.i += 1; // '('
let mut raw = vec![];
while self.i < self.b.len() {
let c = self.b[self.i];
self.i += 1;
match c {
b'\\' if self.i < self.b.len() => {
let e = self.b[self.i];
self.i += 1;
raw.push(match e {
b'n' => b'\n',
b'r' => b'\r',
b't' => b'\t',
other => other,
});
}
b')' => break,
_ => raw.push(c),
}
}
if raw.starts_with(&[0xFE, 0xFF]) {
let u: Vec<u16> = raw[2..].chunks_exact(2).map(|c| u16::from_be_bytes([c[0], c[1]])).collect();
String::from_utf16_lossy(&u).trim_end_matches('\0').to_string()
} else {
String::from_utf8_lossy(&raw).into_owned()
}
}
}
/// Parse engine data (returns an empty dictionary on garbage).
pub fn parse(b: &[u8]) -> EValue {
let mut p = P { b, i: 0 };
p.value(0).unwrap_or(EValue::Dict(vec![]))
}
/// Encode a string the way engine data stores it (UTF-16BE with a BOM, escaped).
pub fn encode_string(s: &str) -> Vec<u8> {
let mut out = vec![b'(', 0xFE, 0xFF];
for u in s.encode_utf16() {
for b in u.to_be_bytes() {
if matches!(b, b'(' | b')' | b'\\') {
out.push(b'\\');
}
out.push(b);
}
}
out.push(b')');
out
}
/// The first style run's character attributes.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct TextStyle {
pub font: Option<String>,
pub size: Option<f64>,
/// RGB 0..1.
pub color: Option<[f64; 3]>,
/// 0 left, 1 right, 2 centre, 3+ justified variants.
pub justification: Option<u8>,
pub tracking: Option<f64>,
pub leading: Option<f64>,
pub faux_bold: bool,
pub faux_italic: bool,
/// Paragraph (box) text rather than point text.
pub box_text: bool,
}
pub fn text_style(ed: &EValue) -> TextStyle {
let engine = ed.get("EngineDict").unwrap_or(ed);
let resources = ed.get("ResourceDict").or_else(|| ed.get("DocumentResources"));
let run = engine.path("StyleRun/RunArray/0/StyleSheet/StyleSheetData");
let normal = resources.and_then(|r| r.path("StyleSheetSet/0/StyleSheetData"));
let attr = |k: &str| run.and_then(|r| r.get(k)).or_else(|| normal.and_then(|n| n.get(k)));
let font =
attr("Font").and_then(EValue::num).and_then(|i| resources?.path(&format!("FontSet/{}/Name", i as usize))).and_then(EValue::str).map(str::to_string);
let color = attr("FillColor").and_then(|c| c.get("Values")).and_then(|v| match v {
EValue::Array(a) if a.len() >= 4 => Some([a[1].num()?, a[2].num()?, a[3].num()?]),
_ => None,
});
let para = engine.path("ParagraphRun/RunArray/0/ParagraphSheet/Properties");
let para_normal = resources.and_then(|r| r.path("ParagraphSheetSet/0/Properties"));
let justification =
para.and_then(|p| p.get("Justification")).or_else(|| para_normal.and_then(|p| p.get("Justification"))).and_then(EValue::num).map(|n| n as u8);
let auto_leading = attr("AutoLeading").is_none_or(|v| *v != EValue::Bool(false));
let box_text = engine.path("Rendered/Shapes/Children/0/ShapeType").and_then(EValue::num) == Some(1.0);
TextStyle {
font,
size: attr("FontSize").and_then(EValue::num),
color,
justification,
tracking: attr("Tracking").and_then(EValue::num),
leading: if auto_leading { None } else { attr("Leading").and_then(EValue::num) },
faux_bold: attr("FauxBold") == Some(&EValue::Bool(true)),
faux_italic: attr("FauxItalic") == Some(&EValue::Bool(true)),
box_text,
}
}
/// Minimal engine data for a type layer with one style run (fixtures, and what a writer needs).
pub fn minimal(text: &str, font: &str, size: f64, color: [f64; 3], justification: u8) -> Vec<u8> {
let mut out = Vec::new();
out.extend_from_slice(b"\n\n<<\n\t/EngineDict\n\t<<\n\t\t/Editor << /Text ");
out.extend_from_slice(&encode_string(&format!("{text}\r")));
out.extend_from_slice(b" >>\n\t\t/ParagraphRun << /RunArray [ << /ParagraphSheet << /Properties << /Justification ");
out.extend_from_slice(justification.to_string().as_bytes());
out.extend_from_slice(b" >> >> >> ] >>\n\t\t/StyleRun << /RunArray [ << /StyleSheet << /StyleSheetData << /Font 0 /FontSize ");
out.extend_from_slice(format!("{size:.1}").as_bytes());
out.extend_from_slice(
format!(" /FillColor << /Type 1 /Values [ 1.0 {:.5} {:.5} {:.5} ] >> /AutoLeading true >> >> >> ] >>\n\t>>\n", color[0], color[1], color[2]).as_bytes(),
);
out.extend_from_slice(b"\t/ResourceDict << /FontSet [ << /Name ");
out.extend_from_slice(&encode_string(font));
out.extend_from_slice(b" /Type 0 >> ] >>\n>>");
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_minimal() {
let b = minimal("Hi (there)", "Arial-BoldMT", 36.0, [1.0, 0.5, 0.0], 2);
let v = parse(&b);
assert_eq!(v.path("EngineDict/Editor/Text").and_then(EValue::str), Some("Hi (there)\r"));
let st = text_style(&v);
assert_eq!(st.font.as_deref(), Some("Arial-BoldMT"));
assert_eq!(st.size, Some(36.0));
assert_eq!(st.color, Some([1.0, 0.5, 0.0]));
assert_eq!(st.justification, Some(2));
assert!(!st.box_text);
}
#[test]
fn garbage_is_harmless() {
let _ = parse(b"<< /A [ 1 2 << /B (unterminated");
let _ = parse(b"]]]>>>> /x");
let _ = parse(&[0xFF; 64]);
}
}
+965
View File
@@ -0,0 +1,965 @@
//! Photoshop document (PSD / PSB) reader, implemented from Adobe's public *Photoshop File Formats
//! Specification* (see README.md), and a minimal writer ([`write`]) for fixtures.
//!
//! [`Psd::parse`] reads the structure (header, layer records and their additional information,
//! the layer tree) without decoding pixels; [`Psd::composite`] decodes the merged image and
//! [`Psd::layer_pixels`] one layer (with its layer mask applied to alpha) on demand. Pixels come
//! out as straight (unpremultiplied) RGBA `f32` in 0..1, display-encoded (32-bit documents,
//! which are linear, are encoded to sRGB).
pub mod descriptor;
pub mod engine_data;
mod read;
pub mod write;
use std::sync::Arc;
pub use descriptor::{DValue, Descriptor};
use read::Reader;
#[derive(Debug, Clone, PartialEq)]
pub enum Error {
NotPsd,
Truncated,
Unsupported(String),
Invalid(String),
}
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Error::NotPsd => write!(f, "not a Photoshop document"),
Error::Truncated => write!(f, "truncated Photoshop document"),
Error::Unsupported(s) => write!(f, "unsupported Photoshop feature: {s}"),
Error::Invalid(s) => write!(f, "invalid Photoshop document: {s}"),
}
}
}
impl std::error::Error for Error {}
pub type Result<T> = std::result::Result<T, Error>;
/// Whether `bytes` start like a Photoshop document.
pub fn is_psd(bytes: &[u8]) -> bool {
bytes.len() >= 6 && &bytes[..4] == b"8BPS" && matches!(u16::from_be_bytes([bytes[4], bytes[5]]), 1 | 2)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ColorMode {
Bitmap,
Grayscale,
Indexed,
Rgb,
Cmyk,
Multichannel,
Duotone,
Lab,
}
impl ColorMode {
pub fn from_u16(v: u16) -> Option<ColorMode> {
Some(match v {
0 => ColorMode::Bitmap,
1 => ColorMode::Grayscale,
2 => ColorMode::Indexed,
3 => ColorMode::Rgb,
4 => ColorMode::Cmyk,
7 => ColorMode::Multichannel,
8 => ColorMode::Duotone,
9 => ColorMode::Lab,
_ => return None,
})
}
pub fn to_u16(self) -> u16 {
match self {
ColorMode::Bitmap => 0,
ColorMode::Grayscale => 1,
ColorMode::Indexed => 2,
ColorMode::Rgb => 3,
ColorMode::Cmyk => 4,
ColorMode::Multichannel => 7,
ColorMode::Duotone => 8,
ColorMode::Lab => 9,
}
}
/// Number of colour channels (before alpha).
pub fn color_channels(self) -> usize {
match self {
ColorMode::Rgb | ColorMode::Lab => 3,
ColorMode::Cmyk => 4,
_ => 1,
}
}
}
/// A rectangle in document pixels (`right`/`bottom` exclusive).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Rect {
pub top: i32,
pub left: i32,
pub bottom: i32,
pub right: i32,
}
impl Rect {
pub fn new(left: i32, top: i32, width: i32, height: i32) -> Rect {
Rect { top, left, bottom: top + height, right: left + width }
}
pub fn width(&self) -> u32 {
(self.right - self.left).max(0) as u32
}
pub fn height(&self) -> u32 {
(self.bottom - self.top).max(0) as u32
}
pub fn is_empty(&self) -> bool {
self.width() == 0 || self.height() == 0
}
}
/// One channel of a layer: id (0.. colour, -1 transparency, -2 layer mask, -3 real user mask) and
/// where its data (compression + image data) sits in the file.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ChannelInfo {
pub id: i16,
pub offset: usize,
pub len: usize,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct LayerMask {
pub rect: Rect,
/// Value outside the mask rectangle (0 or 255).
pub default_color: u8,
pub disabled: bool,
pub inverted: bool,
}
/// Layer kind from the section divider setting (`lsct`).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum Section {
#[default]
Layer,
OpenFolder,
ClosedFolder,
/// The hidden "</Layer group>" record that closes a group (below its children).
Divider,
}
/// A type layer (`TySh`).
#[derive(Clone, Debug, Default, PartialEq)]
pub struct TextInfo {
/// Text with `\n` line breaks.
pub text: String,
/// Affine transform (xx, xy, yx, yy, tx, ty): the text origin sits at (tx, ty).
pub transform: [f64; 6],
pub style: engine_data::TextStyle,
/// Text bounds (left, top, right, bottom) relative to the origin, when stored.
pub bounds: Option<[f64; 4]>,
}
/// Adjustment layers (best effort).
#[derive(Clone, Debug, PartialEq)]
pub enum Adjustment {
Invert,
BrightnessContrast {
brightness: f64,
contrast: f64,
legacy: bool,
},
HueSaturation {
hue: f64,
saturation: f64,
lightness: f64,
colorize: bool,
},
Levels {
in_black: f64,
in_white: f64,
gamma: f64,
out_black: f64,
out_white: f64,
},
Exposure {
exposure: f64,
offset: f64,
gamma: f64,
},
/// Another adjustment (its tagged-block key, e.g. `curv`, `blnc`).
Other(String),
}
/// A vector mask path: sub-paths of (in tangent, vertex, out tangent) points in document pixels.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct VectorMask {
pub inverted: bool,
pub disabled: bool,
pub subpaths: Vec<SubPath>,
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct SubPath {
pub closed: bool,
/// Per knot: [control point preceding, anchor, control point leaving] as (x, y).
pub knots: Vec<[[f64; 2]; 3]>,
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Layer {
/// Index in [`Psd::layers`] (file order: bottom of the stack first).
pub index: usize,
pub name: String,
pub id: Option<u32>,
pub rect: Rect,
/// Blend mode key (`norm`, `mul `, `pass`…).
pub blend: [u8; 4],
pub opacity: u8,
/// Fill opacity (`iOpa`), 255 when absent.
pub fill_opacity: u8,
/// Clipped to the layer below.
pub clipping: bool,
pub hidden: bool,
pub protect_transparency: bool,
pub section: Section,
pub channels: Vec<ChannelInfo>,
pub mask: Option<LayerMask>,
pub text: Option<TextInfo>,
pub adjustment: Option<Adjustment>,
/// Solid colour fill layer (`SoCo`), RGB 0..1.
pub fill: Option<[f64; 3]>,
/// Layer effects (`lfx2` descriptor).
pub effects: Option<Descriptor>,
pub vector_mask: Option<VectorMask>,
/// Keys of every additional-information block the layer carries.
pub keys: Vec<[u8; 4]>,
}
impl Layer {
pub fn is_group(&self) -> bool {
matches!(self.section, Section::OpenFolder | Section::ClosedFolder)
}
pub fn blend_key(&self) -> &str {
std::str::from_utf8(&self.blend).unwrap_or("norm")
}
pub fn has_pixels(&self) -> bool {
!self.rect.is_empty() && self.channels.iter().any(|c| c.id >= 0)
}
}
/// The layer tree, top of the stack first.
#[derive(Clone, Debug, PartialEq)]
pub enum Node {
Layer(usize),
Group { layer: usize, children: Vec<Node> },
}
/// Decoded pixels: straight RGBA, row-major.
#[derive(Clone, Debug, PartialEq)]
pub struct Pixels {
pub width: u32,
pub height: u32,
pub data: Vec<[f32; 4]>,
}
/// A parsed Photoshop document.
#[derive(Clone, Debug)]
pub struct Psd {
/// PSB (large document format).
pub psb: bool,
pub width: u32,
pub height: u32,
pub channels: u16,
pub depth: u16,
pub mode: ColorMode,
/// Bottom of the stack first (file order).
pub layers: Vec<Layer>,
/// The merged image's first extra channel is transparency.
pub merged_alpha: bool,
/// Indexed colour palette (256 R, then G, then B).
pub palette: Vec<u8>,
data: Arc<[u8]>,
merged_offset: usize,
}
impl Psd {
pub fn parse(data: impl Into<Arc<[u8]>>) -> Result<Psd> {
let data: Arc<[u8]> = data.into();
let mut r = Reader::new(&data);
if r.bytes(4).ok() != Some(b"8BPS") {
return Err(Error::NotPsd);
}
let version = r.u16()?;
let psb = match version {
1 => false,
2 => true,
_ => return Err(Error::NotPsd),
};
r.skip(6)?;
let channels = r.u16()?;
let height = r.u32()?;
let width = r.u32()?;
let depth = r.u16()?;
let mode_v = r.u16()?;
let mode = ColorMode::from_u16(mode_v).ok_or_else(|| Error::Unsupported(format!("colour mode {mode_v}")))?;
if !matches!(depth, 1 | 8 | 16 | 32) {
return Err(Error::Unsupported(format!("{depth}-bit")));
}
if width == 0 || height == 0 || width > 300_000 || height > 300_000 {
return Err(Error::Invalid(format!("size {width}x{height}")));
}
// Colour mode data.
let n = r.u32()? as usize;
let palette = r.bytes(n)?.to_vec();
// Image resources.
let n = r.u32()? as usize;
r.skip(n)?;
// Layer and mask information.
let mut psd = Psd { psb, width, height, channels, depth, mode, layers: vec![], merged_alpha: false, palette, data: data.clone(), merged_offset: 0 };
let lm_len = r.length(psb)?;
let lm_end = r.pos.checked_add(lm_len).filter(|&e| e <= data.len()).ok_or(Error::Truncated)?;
if lm_len > 0 {
let li_len = r.length(psb)?;
let li_end = r.pos.checked_add(li_len).filter(|&e| e <= lm_end).ok_or(Error::Truncated)?;
if li_len > 0 {
psd.layer_info(&data, r.pos, li_end)?;
}
r.pos = li_end;
if r.pos + 4 <= lm_end {
let g = r.u32()? as usize;
r.pos = (r.pos + g).min(lm_end);
}
// Global additional layer information (16/32-bit layer info lives here).
while let Some((key, start, end)) = next_block(&data, &mut r.pos, lm_end, psb) {
if matches!(&key, b"Lr16" | b"Lr32" | b"Layr") && psd.layers.is_empty() && end > start {
// The block holds the layer info, normally without its length field; accept
// both layouts.
if psd.layer_info(&data, start, end).is_err() || psd.layers.is_empty() {
psd.layers.clear();
let skip = if psb { 8 } else { 4 };
if start + skip < end && psd.layer_info(&data, start + skip, end).is_err() {
psd.layers.clear();
}
}
}
r.pos = end;
}
}
psd.merged_offset = lm_end;
Ok(psd)
}
fn layer_info(&mut self, data: &[u8], start: usize, end: usize) -> Result<()> {
let mut r = Reader::at(&data[..end], start);
let count = r.i16()?;
self.merged_alpha = count < 0;
let n = count.unsigned_abs() as usize;
let mut layers = Vec::with_capacity(n);
for i in 0..n {
layers.push(self.layer_record(&mut r, i)?);
}
// Channel image data follows in record order.
for l in &mut layers {
for c in &mut l.channels {
c.offset = r.pos;
r.skip(c.len)?;
}
}
self.layers = layers;
Ok(())
}
fn layer_record(&self, r: &mut Reader, index: usize) -> Result<Layer> {
let psb = self.psb;
let (top, left, bottom, right) = (r.i32()?, r.i32()?, r.i32()?, r.i32()?);
let nch = r.u16()? as usize;
if nch > 64 {
return Err(Error::Invalid("channel count".into()));
}
let mut channels = Vec::with_capacity(nch);
for _ in 0..nch {
let id = r.i16()?;
let len = r.length(psb)?;
channels.push(ChannelInfo { id, offset: 0, len });
}
if r.bytes(4)? != b"8BIM" {
return Err(Error::Invalid("layer record signature".into()));
}
let blend = r.tag()?;
let opacity = r.u8()?;
let clipping = r.u8()? != 0;
let flags = r.u8()?;
r.skip(1)?;
let extra = r.u32()? as usize;
let extra_end = r.pos.checked_add(extra).filter(|&e| e <= r.data.len()).ok_or(Error::Truncated)?;
let mut l = Layer {
index,
rect: Rect { top, left, bottom, right },
blend,
opacity,
fill_opacity: 255,
clipping,
hidden: flags & 2 != 0,
protect_transparency: flags & 1 != 0,
channels,
..Default::default()
};
// Layer mask data.
let mlen = r.u32()? as usize;
let mstart = r.pos;
if mlen >= 18 {
let rect = Rect { top: r.i32()?, left: r.i32()?, bottom: r.i32()?, right: r.i32()? };
let default_color = r.u8()?;
let mflags = r.u8()?;
l.mask = Some(LayerMask { rect, default_color, disabled: mflags & 2 != 0, inverted: mflags & 4 != 0 });
}
r.pos = mstart + mlen;
// Blending ranges.
let blen = r.u32()? as usize;
r.skip(blen)?;
l.name = r.pascal(4)?;
let mut pos = r.pos;
let data = r.data;
while let Some((key, start, end)) = next_block(data, &mut pos, extra_end, psb) {
l.keys.push(key);
let block = &data[start..end];
// A malformed block only loses that block's information.
let _ = self.layer_block(&mut l, &key, block);
pos = end;
}
r.pos = extra_end;
Ok(l)
}
fn layer_block(&self, l: &mut Layer, key: &[u8; 4], b: &[u8]) -> Result<()> {
let mut r = Reader::new(b);
match key {
b"luni" => {
let name = descriptor::read_unicode(&mut r)?;
if !name.is_empty() {
l.name = name;
}
}
b"lyid" => l.id = Some(r.u32()?),
b"iOpa" => l.fill_opacity = r.u8()?,
b"lsct" | b"lsdk" => {
l.section = match r.u32()? {
1 => Section::OpenFolder,
2 => Section::ClosedFolder,
3 => Section::Divider,
_ => Section::Layer,
};
if b.len() >= 12 && &b[4..8] == b"8BIM" {
l.blend.copy_from_slice(&b[8..12]);
}
}
b"TySh" => {
let _ver = r.u16()?;
let mut t = [0.0; 6];
for v in &mut t {
*v = r.f64()?;
}
let _text_ver = r.u16()?;
let _desc_ver = r.u32()?;
let d = descriptor::read_descriptor(&mut r)?;
let text = d.text("Txt ").unwrap_or_default().replace("\r\n", "\n").replace('\r', "\n");
let text = text.strip_suffix('\n').map(str::to_string).unwrap_or(text);
let style = d.raw("EngineData").map(|e| engine_data::text_style(&engine_data::parse(e))).unwrap_or_default();
let bounds =
d.obj("bounds").or_else(|| d.obj("boundingBox")).and_then(|o| Some([o.num("Left")?, o.num("Top ")?, o.num("Rght")?, o.num("Btom")?]));
l.text = Some(TextInfo { text, transform: t, style, bounds });
}
b"lfx2" => {
let _ver = r.u32()?;
let _dver = r.u32()?;
l.effects = Some(descriptor::read_descriptor(&mut r)?);
}
b"SoCo" => {
let _ver = r.u32()?;
let d = descriptor::read_descriptor(&mut r)?;
l.fill = d.color("Clr ");
}
b"nvrt" => l.adjustment = Some(Adjustment::Invert),
b"brit" => {
let brightness = r.i16()? as f64;
let contrast = r.i16()? as f64;
if l.adjustment.is_none() {
l.adjustment = Some(Adjustment::BrightnessContrast { brightness, contrast, legacy: true });
}
}
b"CgEd" => {
let _ver = r.u32()?;
let d = descriptor::read_descriptor(&mut r)?;
l.adjustment = Some(Adjustment::BrightnessContrast {
brightness: d.num("Brgh").unwrap_or(0.0),
contrast: d.num("Cntr").unwrap_or(0.0),
legacy: d.bool("useLegacy").unwrap_or(false),
});
}
b"hue2" | b"hue " => {
let _ver = r.u16()?;
let colorize = r.u8()? != 0;
r.skip(1)?;
let ch = (r.i16()? as f64, r.i16()? as f64, r.i16()? as f64);
let m = (r.i16()? as f64, r.i16()? as f64, r.i16()? as f64);
let (hue, saturation, lightness) = if colorize { ch } else { m };
l.adjustment = Some(Adjustment::HueSaturation { hue, saturation, lightness, colorize });
}
b"levl" => {
let _ver = r.u16()?;
let (ib, iw, ob, ow, g) = (r.i16()?, r.i16()?, r.i16()?, r.i16()?, r.i16()?);
l.adjustment =
Some(Adjustment::Levels { in_black: ib as f64, in_white: iw as f64, gamma: g as f64 / 100.0, out_black: ob as f64, out_white: ow as f64 });
}
b"expA" => {
let _ver = r.u16()?;
let exposure = r.f32()? as f64;
let offset = r.f32()? as f64;
let gamma = r.f32()? as f64;
l.adjustment = Some(Adjustment::Exposure { exposure, offset, gamma });
}
b"curv" | b"blnc" | b"post" | b"thrs" | b"selc" | b"mixr" | b"phfl" | b"vibA" | b"blwh" | b"grdm" | b"clrL" => {
if l.adjustment.is_none() {
l.adjustment = Some(Adjustment::Other(String::from_utf8_lossy(key).into_owned()));
}
}
b"vmsk" | b"vsms" => {
let _ver = r.u32()?;
let flags = r.u32()?;
l.vector_mask = Some(self.vector_paths(&mut r, flags)?);
}
_ => {}
}
Ok(())
}
fn vector_paths(&self, r: &mut Reader, flags: u32) -> Result<VectorMask> {
let (w, h) = (self.width as f64, self.height as f64);
let mut vm = VectorMask { inverted: flags & 1 != 0, disabled: flags & 4 != 0, subpaths: vec![] };
let fixed = |r: &mut Reader| -> Result<f64> { Ok(r.i32()? as f64 / (1 << 24) as f64) };
while r.remaining() >= 26 {
let sel = r.u16()?;
match sel {
0 | 3 => {
let _n = r.u16()?;
r.skip(22)?;
vm.subpaths.push(SubPath { closed: sel == 0, knots: vec![] });
}
1 | 2 | 4 | 5 => {
let mut pts = [[0.0; 2]; 3];
for p in &mut pts {
let y = fixed(r)?;
let x = fixed(r)?;
*p = [x * w, y * h];
}
if let Some(sp) = vm.subpaths.last_mut() {
sp.knots.push(pts);
}
}
_ => r.skip(24)?,
}
}
vm.subpaths.retain(|s| !s.knots.is_empty());
Ok(vm)
}
/// The layer tree, top of the stack first.
pub fn tree(&self) -> Vec<Node> {
let mut stack: Vec<Vec<Node>> = vec![vec![]];
for l in &self.layers {
match l.section {
Section::Divider => stack.push(vec![]),
Section::OpenFolder | Section::ClosedFolder => {
let children = if stack.len() > 1 { stack.pop().unwrap_or_default() } else { vec![] };
if let Some(top) = stack.last_mut() {
top.push(Node::Group { layer: l.index, children });
}
}
Section::Layer => {
if let Some(top) = stack.last_mut() {
top.push(Node::Layer(l.index));
}
}
}
}
while stack.len() > 1 {
let c = stack.pop().unwrap_or_default();
if let Some(top) = stack.last_mut() {
top.extend(c);
}
}
fn rev(v: &mut Vec<Node>) {
v.reverse();
for n in v {
if let Node::Group { children, .. } = n {
rev(children);
}
}
}
let mut out = stack.pop().unwrap_or_default();
rev(&mut out);
out
}
fn bytes_per_sample(&self) -> usize {
match self.depth {
16 => 2,
32 => 4,
_ => 1,
}
}
/// Decode one channel of `w`×`h` samples at `offset` (compression field first). Values 0..1.
fn decode_channel(&self, offset: usize, len: usize, w: usize, h: usize) -> Result<Vec<f32>> {
let end = offset.checked_add(len).filter(|&e| e <= self.data.len()).ok_or(Error::Truncated)?;
let mut r = Reader::at(&self.data[..end], offset);
let comp = r.u16()?;
let rowb = self.row_bytes(w);
let raw = match comp {
0 => r.bytes(rowb * h)?.to_vec(),
1 => {
let mut counts = Vec::with_capacity(h);
for _ in 0..h {
counts.push(if self.psb { r.u32()? as usize } else { r.u16()? as usize });
}
let mut out = Vec::with_capacity(rowb * h);
for c in counts {
unpack_bits(r.bytes(c)?, rowb, &mut out)?;
}
out
}
2 | 3 => {
let rest = r.bytes(r.remaining())?;
let mut v = miniz_oxide::inflate::decompress_to_vec_zlib(rest).map_err(|e| Error::Invalid(format!("zip: {e:?}")))?;
v.resize(rowb * h, 0);
if comp == 3 {
self.unpredict(&mut v, w, h);
}
v
}
c => return Err(Error::Unsupported(format!("compression {c}"))),
};
Ok(self.samples(&raw, w, h))
}
fn row_bytes(&self, w: usize) -> usize {
if self.depth == 1 { w.div_ceil(8) } else { w * self.bytes_per_sample() }
}
fn unpredict(&self, v: &mut [u8], w: usize, h: usize) {
match self.depth {
16 => {
for row in v.chunks_exact_mut(w * 2).take(h) {
let mut prev = 0u16;
for c in row.chunks_exact_mut(2) {
let x = u16::from_be_bytes([c[0], c[1]]).wrapping_add(prev);
c.copy_from_slice(&x.to_be_bytes());
prev = x;
}
}
}
32 => {
for row in v.chunks_exact_mut(w * 4).take(h) {
for i in 1..row.len() {
row[i] = row[i].wrapping_add(row[i - 1]);
}
// Bytes are stored plane by plane (all first bytes, then all second bytes…).
let planar = row.to_vec();
for x in 0..w {
for b in 0..4 {
row[x * 4 + b] = planar[b * w + x];
}
}
}
}
_ => {
for row in v.chunks_exact_mut(w).take(h) {
for i in 1..row.len() {
row[i] = row[i].wrapping_add(row[i - 1]);
}
}
}
}
}
fn samples(&self, raw: &[u8], w: usize, h: usize) -> Vec<f32> {
match self.depth {
1 => {
let rowb = w.div_ceil(8);
let mut out = Vec::with_capacity(w * h);
for y in 0..h {
for x in 0..w {
let b = raw.get(y * rowb + x / 8).copied().unwrap_or(0);
// 1 = black.
out.push(if b & (0x80 >> (x % 8)) != 0 { 0.0 } else { 1.0 });
}
}
out
}
16 => raw.chunks_exact(2).take(w * h).map(|c| u16::from_be_bytes([c[0], c[1]]) as f32 / 65535.0).collect(),
32 => raw.chunks_exact(4).take(w * h).map(|c| f32::from_be_bytes([c[0], c[1], c[2], c[3]])).collect(),
_ => raw.iter().take(w * h).map(|&b| b as f32 / 255.0).collect(),
}
}
/// Convert colour channels (in the document's mode) to RGB.
fn to_rgb(&self, ch: &[Option<Vec<f32>>], i: usize) -> [f32; 3] {
let g = |k: usize| ch.get(k).and_then(|c| c.as_ref()).map_or(0.0, |c| c[i]);
let rgb = match self.mode {
ColorMode::Rgb => [g(0), g(1), g(2)],
ColorMode::Cmyk => {
// Stored inverted (1 = no ink).
let k = g(3);
[g(0) * k, g(1) * k, g(2) * k]
}
ColorMode::Indexed if self.palette.len() >= 768 => {
let idx = (g(0) * 255.0).round() as usize & 255;
[self.palette[idx] as f32 / 255.0, self.palette[256 + idx] as f32 / 255.0, self.palette[512 + idx] as f32 / 255.0]
}
ColorMode::Lab => lab_to_rgb(g(0) * 100.0, g(1) * 255.0 - 128.0, g(2) * 255.0 - 128.0),
_ => [g(0); 3],
};
if self.depth == 32 && self.mode != ColorMode::Lab { rgb.map(srgb_encode) } else { rgb }
}
/// The merged (flattened) image.
pub fn composite(&self) -> Result<Pixels> {
let (w, h) = (self.width as usize, self.height as usize);
let mut r = Reader::at(&self.data, self.merged_offset);
let comp = r.u16()?;
let nch = self.channels as usize;
let cc = self.mode.color_channels();
let want = (cc + usize::from(self.merged_alpha)).min(nch);
let rowb = self.row_bytes(w);
let mut planes: Vec<Option<Vec<f32>>> = vec![None; nch];
match comp {
0 => {
for p in planes.iter_mut().take(want) {
*p = Some(self.samples(r.bytes(rowb * h)?, w, h));
}
}
1 => {
let mut counts = Vec::with_capacity(nch * h);
for _ in 0..nch * h {
counts.push(if self.psb { r.u32()? as usize } else { r.u16()? as usize });
}
for (c, p) in planes.iter_mut().enumerate().take(want) {
let mut out = Vec::with_capacity(rowb * h);
for row in 0..h {
unpack_bits(r.bytes(counts[c * h + row])?, rowb, &mut out)?;
}
*p = Some(self.samples(&out, w, h));
}
}
2 | 3 => {
let rest = r.bytes(r.remaining())?;
let v = miniz_oxide::inflate::decompress_to_vec_zlib(rest).map_err(|e| Error::Invalid(format!("zip: {e:?}")))?;
for (c, p) in planes.iter_mut().enumerate().take(want) {
let mut plane = v.get(c * rowb * h..(c + 1) * rowb * h).ok_or(Error::Truncated)?.to_vec();
if comp == 3 {
self.unpredict(&mut plane, w, h);
}
*p = Some(self.samples(&plane, w, h));
}
}
c => return Err(Error::Unsupported(format!("compression {c}"))),
}
let alpha = if self.merged_alpha && nch > cc { planes[cc].take() } else { None };
let mut data = Vec::with_capacity(w * h);
for i in 0..w * h {
let [r, g, b] = self.to_rgb(&planes, i);
let a = alpha.as_ref().map_or(1.0, |a| a[i]);
data.push([r, g, b, a]);
}
Ok(Pixels { width: w as u32, height: h as u32, data })
}
/// One layer's pixels with its layer mask (and vector-less fill colour) applied to alpha.
/// `canvas`: document-sized with the layer in place; otherwise the layer's own bounds.
pub fn layer_pixels(&self, index: usize, canvas: bool) -> Result<Pixels> {
let l = self.layers.get(index).ok_or_else(|| Error::Invalid(format!("no layer {index}")))?;
let doc = Rect { top: 0, left: 0, bottom: self.height as i32, right: self.width as i32 };
// Fill layers often have no pixel data: they cover the canvas (or the mask).
let fill_rect =
if l.fill.is_some() && !l.has_pixels() { l.mask.filter(|m| !m.rect.is_empty() && m.default_color == 0).map_or(doc, |m| m.rect) } else { l.rect };
let out_rect = if canvas {
doc
} else if fill_rect.is_empty() {
Rect { top: 0, left: 0, bottom: 1, right: 1 }
} else {
fill_rect
};
let (ow, oh) = (out_rect.width() as usize, out_rect.height() as usize);
let mut out = vec![[0.0f32; 4]; ow * oh];
let src = fill_rect;
let (sw, sh) = (src.width() as usize, src.height() as usize);
if sw > 0 && sh > 0 {
let mut planes: Vec<Option<Vec<f32>>> = vec![None; self.mode.color_channels()];
let mut alpha = None;
if l.has_pixels() {
for c in &l.channels {
match c.id {
-1 => alpha = Some(self.decode_channel(c.offset, c.len, sw, sh)?),
id if id >= 0 && (id as usize) < planes.len() => planes[id as usize] = Some(self.decode_channel(c.offset, c.len, sw, sh)?),
_ => {}
}
}
}
let fill = l.fill.map(|c| c.map(|v| v as f32));
for y in 0..sh {
let dy = src.top + y as i32 - out_rect.top;
if dy < 0 || dy >= oh as i32 {
continue;
}
for x in 0..sw {
let dx = src.left + x as i32 - out_rect.left;
if dx < 0 || dx >= ow as i32 {
continue;
}
let i = y * sw + x;
let (rgb, a) = match fill {
Some(f) if !l.has_pixels() => (f, 1.0),
_ => (self.to_rgb(&planes, i), alpha.as_ref().map_or(1.0, |a| a[i])),
};
out[dy as usize * ow + dx as usize] = [rgb[0], rgb[1], rgb[2], a];
}
}
}
// Layer mask: multiplies alpha (also outside the mask rect, by its default colour).
if let Some(m) = l.mask.filter(|m| !m.disabled) {
let mc = l.channels.iter().find(|c| c.id == -2);
let (mw, mh) = (m.rect.width() as usize, m.rect.height() as usize);
let mask = match mc {
Some(c) if mw > 0 && mh > 0 => Some(self.decode_channel(c.offset, c.len, mw, mh)?),
_ => None,
};
let def = m.default_color as f32 / 255.0;
for y in 0..oh {
let py = out_rect.top + y as i32;
for x in 0..ow {
let px = out_rect.left + x as i32;
let inside = py >= m.rect.top && py < m.rect.bottom && px >= m.rect.left && px < m.rect.right;
let mut v = match (&mask, inside) {
(Some(mk), true) => mk[(py - m.rect.top) as usize * mw + (px - m.rect.left) as usize],
_ => def,
};
if m.inverted {
v = 1.0 - v;
}
out[y * ow + x][3] *= v;
}
}
}
Ok(Pixels { width: ow as u32, height: oh as u32, data: out })
}
}
/// The next additional-information block at `*pos` (signature `8BIM`/`8B64`, key, length):
/// returns (key, data start, data end). Skips up to 3 padding bytes before a signature.
fn next_block(data: &[u8], pos: &mut usize, end: usize, psb: bool) -> Option<([u8; 4], usize, usize)> {
let mut p = *pos;
for _ in 0..4 {
if p + 12 > end {
return None;
}
if &data[p..p + 4] == b"8BIM" || &data[p..p + 4] == b"8B64" {
break;
}
p += 1;
}
if p + 12 > end || !(&data[p..p + 4] == b"8BIM" || &data[p..p + 4] == b"8B64") {
return None;
}
let mut key = [0u8; 4];
key.copy_from_slice(&data[p + 4..p + 8]);
let wide =
psb && matches!(&key, b"LMsk" | b"Lr16" | b"Lr32" | b"Layr" | b"Mt16" | b"Mt32" | b"Mtrn" | b"Alph" | b"FMsk" | b"lnk2" | b"FEid" | b"FXid" | b"PxSD");
let mut r = Reader::at(&data[..end], p + 8);
let len = r.length(wide).ok()?;
let start = r.pos;
let stop = start.checked_add(len)?;
if stop > end {
return None;
}
*pos = stop;
Some((key, start, stop))
}
/// PackBits decoding of one row (appends exactly `row` bytes, zero-padded).
fn unpack_bits(src: &[u8], row: usize, out: &mut Vec<u8>) -> Result<()> {
let start = out.len();
let mut i = 0;
while i < src.len() && out.len() - start < row {
let n = src[i] as i8;
i += 1;
if n >= 0 {
let k = n as usize + 1;
let s = src.get(i..i + k).ok_or(Error::Truncated)?;
out.extend_from_slice(s);
i += k;
} else if n != -128 {
let k = (1 - n as isize) as usize;
let v = *src.get(i).ok_or(Error::Truncated)?;
out.extend(std::iter::repeat_n(v, k));
i += 1;
}
}
out.resize(start + row, 0);
Ok(())
}
/// PackBits encoding of one row.
pub(crate) fn pack_bits(src: &[u8]) -> Vec<u8> {
let mut out = vec![];
let mut i = 0;
while i < src.len() {
// run of identical bytes
let mut run = 1;
while i + run < src.len() && src[i + run] == src[i] && run < 128 {
run += 1;
}
if run >= 2 {
out.push((1 - run as isize) as i8 as u8);
out.push(src[i]);
i += run;
continue;
}
// literal run
let start = i;
let mut n = 0;
while i < src.len() && n < 128 {
if i + 1 < src.len() && src[i + 1] == src[i] {
break;
}
i += 1;
n += 1;
}
if n == 0 {
i += 1;
n = 1;
}
out.push((n - 1) as u8);
out.extend_from_slice(&src[start..start + n]);
}
out
}
fn srgb_encode(v: f32) -> f32 {
if v <= 0.003_130_8 { v * 12.92 } else { 1.055 * v.powf(1.0 / 2.4) - 0.055 }
}
/// CIE L*a*b* (D50) to sRGB (D65 via the Bradford-adapted matrix), clamped.
fn lab_to_rgb(l: f32, a: f32, b: f32) -> [f32; 3] {
let fy = (l + 16.0) / 116.0;
let fx = fy + a / 500.0;
let fz = fy - b / 200.0;
let f = |t: f32| if t > 6.0 / 29.0 { t * t * t } else { 3.0 * (6.0f32 / 29.0).powi(2) * (t - 4.0 / 29.0) };
let (x, y, z) = (0.964_22 * f(fx), f(fy), 0.825_21 * f(fz));
// XYZ (D50) → linear sRGB.
let r = 3.133_856 * x - 1.616_867 * y - 0.490_615 * z;
let g = -0.978_768 * x + 1.916_142 * y + 0.033_454 * z;
let bb = 0.071_945 * x - 0.228_991 * y + 1.405_243 * z;
[r, g, bb].map(|v| srgb_encode(v.clamp(0.0, 1.0)))
}
#[cfg(test)]
mod tests;
+86
View File
@@ -0,0 +1,86 @@
//! Big-endian byte reader.
use crate::{Error, Result};
#[derive(Clone)]
pub struct Reader<'a> {
pub data: &'a [u8],
pub pos: usize,
}
impl<'a> Reader<'a> {
pub fn new(data: &'a [u8]) -> Self {
Reader { data, pos: 0 }
}
pub fn at(data: &'a [u8], pos: usize) -> Self {
Reader { data, pos }
}
pub fn remaining(&self) -> usize {
self.data.len().saturating_sub(self.pos)
}
pub fn bytes(&mut self, n: usize) -> Result<&'a [u8]> {
if self.remaining() < n {
return Err(Error::Truncated);
}
let s = &self.data[self.pos..self.pos + n];
self.pos += n;
Ok(s)
}
pub fn skip(&mut self, n: usize) -> Result<()> {
self.bytes(n).map(|_| ())
}
pub fn u8(&mut self) -> Result<u8> {
Ok(self.bytes(1)?[0])
}
pub fn u16(&mut self) -> Result<u16> {
let b = self.bytes(2)?;
Ok(u16::from_be_bytes([b[0], b[1]]))
}
pub fn i16(&mut self) -> Result<i16> {
Ok(self.u16()? as i16)
}
pub fn u32(&mut self) -> Result<u32> {
let b = self.bytes(4)?;
Ok(u32::from_be_bytes([b[0], b[1], b[2], b[3]]))
}
pub fn i32(&mut self) -> Result<i32> {
Ok(self.u32()? as i32)
}
pub fn u64(&mut self) -> Result<u64> {
let b = self.bytes(8)?;
let mut a = [0u8; 8];
a.copy_from_slice(b);
Ok(u64::from_be_bytes(a))
}
pub fn f32(&mut self) -> Result<f32> {
Ok(f32::from_bits(self.u32()?))
}
pub fn f64(&mut self) -> Result<f64> {
Ok(f64::from_bits(self.u64()?))
}
pub fn tag(&mut self) -> Result<[u8; 4]> {
let b = self.bytes(4)?;
Ok([b[0], b[1], b[2], b[3]])
}
/// A length field: 4 bytes in PSD, 8 bytes in PSB (`wide`).
pub fn length(&mut self, wide: bool) -> Result<usize> {
let v = if wide { self.u64()? } else { self.u32()? as u64 };
usize::try_from(v).map_err(|_| Error::Invalid("length".into()))
}
/// A Pascal string padded so that length byte + text is a multiple of `pad`.
pub fn pascal(&mut self, pad: usize) -> Result<String> {
let n = self.u8()? as usize;
let s = self.bytes(n)?;
let total = 1 + n;
let padded = total.div_ceil(pad) * pad;
self.skip(padded - total)?;
// Pascal strings are in the system (Mac Roman) encoding; ASCII covers the common case.
Ok(s.iter().map(|&b| if b < 0x80 { b as char } else { mac_roman(b) }).collect())
}
}
/// Mac OS Roman upper half (0x80..=0xFF), as published by the Unicode Consortium mapping table.
fn mac_roman(b: u8) -> char {
const T: &str = "ÄÅÇÉÑÖÜáàâäãåçéèêëíìîïñóòôöõúùûü†°¢£§•¶ß®©™´¨≠ÆØ∞±≤≥¥µ∂∑∏π∫ªºΩæø¿¡¬√ƒ≈∆«»… ÀÃÕŒœ–—“”‘’÷◊ÿŸ⁄€‹›fifl‡·‚„‰ÂÊÁËÈÍÎÏÌÓÔ\u{F8FF}ÒÚÛÙıˆ˜¯˘˙˚¸˝˛ˇ";
T.chars().nth((b - 0x80) as usize).unwrap_or('?')
}
+189
View File
@@ -0,0 +1,189 @@
use super::write::*;
use super::*;
fn gradient(w: u32, h: u32) -> Vec<[f32; 4]> {
(0..w * h).map(|i| [(i % w) as f32 / (w - 1).max(1) as f32, (i / w) as f32 / (h - 1).max(1) as f32, 0.25, 1.0]).collect()
}
fn close(a: &[[f32; 4]], b: &[[f32; 4]], tol: f32) -> bool {
a.len() == b.len() && a.iter().zip(b).all(|(x, y)| (0..4).all(|c| (x[c] - y[c]).abs() <= tol))
}
fn doc_with_layers(depth: u16, mode: ColorMode, rle: bool) -> WDoc {
let mut d = WDoc::new(40, 30);
d.depth = depth;
d.mode = mode;
d.rle = rle;
let mut bg = WLayer::pixels("Background", Rect::new(0, 0, 40, 30), gradient(40, 30));
bg.opacity = 255;
let mut red = WLayer::solid("Red box", Rect::new(5, 6, 10, 8), [1.0, 0.0, 0.0, 1.0]).with_blend(b"mul ").with_opacity(128);
red.hidden = true;
let mut masked = WLayer::solid("Masked", Rect::new(20, 10, 12, 12), [0.0, 0.5, 1.0, 1.0]).with_blend(b"scrn");
// Mask: left half opaque, right half transparent; default (outside) black.
let mrect = Rect::new(20, 10, 12, 12);
masked.mask = Some(WMask { rect: mrect, data: (0..144).map(|i| if i % 12 < 6 { 1.0 } else { 0.0 }).collect(), default_color: 0, disabled: false });
d.layers = vec![bg, red, masked];
d
}
#[test]
fn layers_round_trip_8bit_rle() {
let bytes = write(&doc_with_layers(8, ColorMode::Rgb, true));
assert!(is_psd(&bytes));
let p = Psd::parse(bytes).unwrap();
assert_eq!((p.width, p.height, p.depth, p.mode), (40, 30, 8, ColorMode::Rgb));
assert_eq!(p.layers.len(), 3);
let names: Vec<&str> = p.layers.iter().map(|l| l.name.as_str()).collect();
assert_eq!(names, ["Background", "Red box", "Masked"]);
assert_eq!(p.layers[1].blend_key(), "mul ");
assert_eq!(p.layers[1].opacity, 128);
assert!(p.layers[1].hidden && !p.layers[0].hidden);
assert_eq!(p.layers[2].blend_key(), "scrn");
assert_eq!(p.layers[1].rect, Rect::new(5, 6, 10, 8));
// Pixels (8-bit quantisation).
let bg = p.layer_pixels(0, false).unwrap();
assert!(close(&bg.data, &gradient(40, 30), 1.0 / 255.0));
// Retain layer size vs canvas.
let red = p.layer_pixels(1, false).unwrap();
assert_eq!((red.width, red.height), (10, 8));
let red_c = p.layer_pixels(1, true).unwrap();
assert_eq!((red_c.width, red_c.height), (40, 30));
assert_eq!(red_c.data[6 * 40 + 5], [1.0, 0.0, 0.0, 1.0]);
assert_eq!(red_c.data[0][3], 0.0);
// Layer mask becomes alpha.
let m = p.layer_pixels(2, false).unwrap();
assert_eq!(m.data[0][3], 1.0);
assert_eq!(m.data[11][3], 0.0);
// Merged composite.
let c = p.composite().unwrap();
assert_eq!((c.width, c.height), (40, 30));
assert!(p.merged_alpha);
}
#[test]
fn layers_round_trip_16bit_raw_and_cmyk_gray() {
for (depth, mode, rle) in
[(16, ColorMode::Rgb, false), (16, ColorMode::Rgb, true), (8, ColorMode::Cmyk, true), (16, ColorMode::Cmyk, false), (8, ColorMode::Grayscale, false)]
{
let bytes = write(&doc_with_layers(depth, mode, rle));
let p = Psd::parse(bytes).unwrap_or_else(|e| panic!("{depth} {mode:?} {rle}: {e}"));
assert_eq!(p.layers.len(), 3, "{depth} {mode:?}");
assert_eq!(p.depth, depth);
let bg = p.layer_pixels(0, true).unwrap();
let tol = if depth == 16 { 1.0 / 30000.0 } else { 1.5 / 255.0 };
let want: Vec<[f32; 4]> =
if mode == ColorMode::Grayscale { gradient(40, 30).iter().map(|p| [p[0], p[0], p[0], 1.0]).collect() } else { gradient(40, 30) };
assert!(close(&bg.data, &want, tol), "{depth} {mode:?} {rle}: {:?} vs {:?}", bg.data[41], want[41]);
let c = p.composite().unwrap();
assert_eq!(c.data.len(), 1200);
}
}
#[test]
fn groups_build_a_tree() {
let mut d = WDoc::new(16, 16);
d.layers = vec![
WLayer::solid("Bottom", Rect::new(0, 0, 16, 16), [1.0; 4]),
WLayer::group_end(),
WLayer::solid("Inner A", Rect::new(0, 0, 4, 4), [1.0, 0.0, 0.0, 1.0]),
WLayer::group_end(),
WLayer::solid("Deep", Rect::new(2, 2, 4, 4), [0.0, 1.0, 0.0, 1.0]),
WLayer::group("Nested", false, *b"norm"),
WLayer::group("Group 1", true, *b"pass"),
WLayer::solid("Top", Rect::new(8, 8, 4, 4), [0.0, 0.0, 1.0, 1.0]),
];
let p = Psd::parse(write(&d)).unwrap();
assert_eq!(p.layers[6].section, Section::OpenFolder);
assert_eq!(p.layers[6].blend_key(), "pass");
assert_eq!(p.layers[5].section, Section::ClosedFolder);
let t = p.tree();
assert_eq!(
t,
vec![
Node::Layer(7),
Node::Group { layer: 6, children: vec![Node::Group { layer: 5, children: vec![Node::Layer(4)] }, Node::Layer(2)] },
Node::Layer(0)
]
);
}
#[test]
fn text_effects_fill_and_adjustments() {
let mut d = WDoc::new(64, 32);
let fx = Descriptor::new("null").with("masterFXSwitch", DValue::Bool(true)).with(
"DrSh",
DValue::Descriptor(
Descriptor::new("DrSh")
.with("enab", DValue::Bool(true))
.with("Md ", DValue::Enum("BlnM".into(), "Mltp".into()))
.with("Clr ", DValue::Descriptor(Descriptor::rgb([0.0, 0.0, 1.0])))
.with("Opct", DValue::UnitFloat("#Prc".into(), 60.0))
.with("Dstn", DValue::UnitFloat("#Pxl".into(), 7.0)),
),
);
d.layers = vec![
WLayer::empty("Fill", vec![solid_color_block([0.0, 1.0, 0.0])]),
WLayer::empty("Title", vec![text_block("Hello\nWorld", [10.0, 20.0], "Inter-Bold", 18.0, [1.0, 0.0, 0.0], 2)]).with_block(effects_block(&fx)),
WLayer::empty("Hue", vec![hue_sat_block(30, -20, 10)]),
WLayer::empty("Lev", vec![levels_block(10, 240, 5, 250, 1.2)]),
WLayer::empty("Inv", vec![invert_block()]),
WLayer::empty("BC", vec![brightness_block(20, -10)]),
WLayer::empty("Exp", vec![exposure_block(1.5, 0.0, 1.0)]),
WLayer::solid("Vec", Rect::new(0, 0, 64, 32), [1.0; 4]).with_block(vector_mask_block(
&[vec![[[8.0, 4.0]; 3], [[40.0, 4.0]; 3], [[40.0, 20.0]; 3]]],
64,
32,
false,
)),
];
let p = Psd::parse(write(&d)).unwrap();
assert_eq!(p.layers[0].fill, Some([0.0, 1.0, 0.0]));
let fill = p.layer_pixels(0, false).unwrap();
assert_eq!((fill.width, fill.height), (64, 32));
assert_eq!(fill.data[5], [0.0, 1.0, 0.0, 1.0]);
let t = p.layers[1].text.as_ref().unwrap();
assert_eq!(t.text, "Hello\nWorld");
assert_eq!(t.transform[4..], [10.0, 20.0]);
assert_eq!(t.style.font.as_deref(), Some("Inter-Bold"));
assert_eq!(t.style.size, Some(18.0));
assert_eq!(t.style.color, Some([1.0, 0.0, 0.0]));
assert_eq!(t.style.justification, Some(2));
let e = p.layers[1].effects.as_ref().unwrap();
let ds = e.obj("DrSh").unwrap();
assert_eq!(ds.enum_value("Md "), Some("Mltp"));
assert_eq!(ds.num("Dstn"), Some(7.0));
assert_eq!(p.layers[2].adjustment, Some(Adjustment::HueSaturation { hue: 30.0, saturation: -20.0, lightness: 10.0, colorize: false }));
assert_eq!(p.layers[3].adjustment, Some(Adjustment::Levels { in_black: 10.0, in_white: 240.0, gamma: 1.2, out_black: 5.0, out_white: 250.0 }));
assert_eq!(p.layers[4].adjustment, Some(Adjustment::Invert));
assert_eq!(p.layers[5].adjustment, Some(Adjustment::BrightnessContrast { brightness: 20.0, contrast: -10.0, legacy: true }));
assert_eq!(p.layers[6].adjustment, Some(Adjustment::Exposure { exposure: 1.5, offset: 0.0, gamma: 1.0 }));
let vm = p.layers[7].vector_mask.as_ref().unwrap();
assert_eq!(vm.subpaths.len(), 1);
assert!(vm.subpaths[0].closed);
let k = &vm.subpaths[0].knots;
assert_eq!(k.len(), 3);
assert!((k[1][1][0] - 40.0).abs() < 1e-4 && (k[2][1][1] - 20.0).abs() < 1e-4);
}
#[test]
fn packbits_round_trip() {
for src in [vec![], vec![1u8], vec![5; 300], (0..=255).collect::<Vec<u8>>(), vec![1, 1, 2, 3, 3, 3, 4, 5, 5]] {
let enc = pack_bits(&src);
let mut out = vec![];
unpack_bits(&enc, src.len(), &mut out).unwrap();
assert_eq!(out, src);
}
}
#[test]
fn truncated_and_garbage_input_errors() {
let bytes = write(&doc_with_layers(8, ColorMode::Rgb, true));
for n in [0, 3, 10, 26, 40, 100, bytes.len() / 2] {
let r = Psd::parse(bytes[..n].to_vec());
if let Ok(p) = r {
let _ = p.composite();
}
}
assert!(Psd::parse(b"8BPS\0\x09garbage".to_vec()).is_err());
assert!(Psd::parse(b"PNG".to_vec()).is_err());
}
+491
View File
@@ -0,0 +1,491 @@
//! A minimal Photoshop document writer, per the same specification as the reader: RGB, CMYK or
//! Grayscale, 8 or 16 bits, raw or RLE channels, pixel layers with layer masks, groups and any
//! additional-information blocks (helpers below build text, effects, fill and adjustment
//! blocks). Used to generate test fixtures; not a full exporter.
use crate::descriptor::{self, DValue, Descriptor};
use crate::{ColorMode, Rect, Section, engine_data, pack_bits};
/// A layer mask for [`WLayer`].
#[derive(Clone, Debug)]
pub struct WMask {
pub rect: Rect,
/// `rect.width() * rect.height()` values in 0..1.
pub data: Vec<f32>,
pub default_color: u8,
pub disabled: bool,
}
/// A layer to write.
#[derive(Clone, Debug)]
pub struct WLayer {
pub name: String,
pub rect: Rect,
/// Straight RGBA, `rect.width() * rect.height()` pixels.
pub pixels: Vec<[f32; 4]>,
pub blend: [u8; 4],
pub opacity: u8,
pub hidden: bool,
pub clipping: bool,
pub mask: Option<WMask>,
pub section: Section,
/// Extra additional-information blocks (key, data).
pub blocks: Vec<([u8; 4], Vec<u8>)>,
}
impl WLayer {
/// A pixel layer.
pub fn pixels(name: &str, rect: Rect, pixels: Vec<[f32; 4]>) -> WLayer {
WLayer {
name: name.into(),
rect,
pixels,
blend: *b"norm",
opacity: 255,
hidden: false,
clipping: false,
mask: None,
section: Section::Layer,
blocks: vec![],
}
}
/// A layer filled with one colour.
pub fn solid(name: &str, rect: Rect, rgba: [f32; 4]) -> WLayer {
let n = (rect.width() * rect.height()) as usize;
WLayer::pixels(name, rect, vec![rgba; n])
}
/// The top record of a group (its children come *before* it in file order, after a
/// [`WLayer::group_end`]).
pub fn group(name: &str, open: bool, blend: [u8; 4]) -> WLayer {
let mut l = WLayer::pixels(name, Rect::default(), vec![]);
l.section = if open { Section::OpenFolder } else { Section::ClosedFolder };
l.blend = blend;
l
}
/// The hidden record that closes a group (written below its children).
pub fn group_end() -> WLayer {
let mut l = WLayer::pixels("</Layer group>", Rect::default(), vec![]);
l.section = Section::Divider;
l
}
/// A layer without pixels carrying `blocks` (adjustment and fill layers, type layers).
pub fn empty(name: &str, blocks: Vec<([u8; 4], Vec<u8>)>) -> WLayer {
let mut l = WLayer::pixels(name, Rect::default(), vec![]);
l.blocks = blocks;
l
}
pub fn with_blend(mut self, key: &[u8; 4]) -> WLayer {
self.blend = *key;
self
}
pub fn with_opacity(mut self, o: u8) -> WLayer {
self.opacity = o;
self
}
pub fn with_block(mut self, b: ([u8; 4], Vec<u8>)) -> WLayer {
self.blocks.push(b);
self
}
}
/// A document to write.
#[derive(Clone, Debug)]
pub struct WDoc {
pub width: u32,
pub height: u32,
/// 8 or 16.
pub depth: u16,
/// Rgb, Cmyk or Grayscale.
pub mode: ColorMode,
pub rle: bool,
/// Bottom of the stack first (file order).
pub layers: Vec<WLayer>,
/// Merged image (straight RGBA, document-sized); `None` composites visible pixel layers
/// with Normal blending.
pub composite: Option<Vec<[f32; 4]>>,
}
impl WDoc {
pub fn new(width: u32, height: u32) -> WDoc {
WDoc { width, height, depth: 8, mode: ColorMode::Rgb, rle: true, layers: vec![], composite: None }
}
}
fn u16b(out: &mut Vec<u8>, v: u16) {
out.extend_from_slice(&v.to_be_bytes());
}
fn u32b(out: &mut Vec<u8>, v: u32) {
out.extend_from_slice(&v.to_be_bytes());
}
/// Colour channels for a straight RGB pixel in the document mode.
fn mode_channels(mode: ColorMode, p: [f32; 4]) -> Vec<f32> {
match mode {
ColorMode::Cmyk => {
// Stored values are inverted ink amounts: k = max, c = r / max (reader: r = c * k).
let m = p[0].max(p[1]).max(p[2]);
if m <= 0.0 { vec![0.0, 0.0, 0.0, 0.0] } else { vec![p[0] / m, p[1] / m, p[2] / m, m] }
}
ColorMode::Grayscale => vec![p[0]],
_ => vec![p[0], p[1], p[2]],
}
}
fn encode_samples(v: &[f32], depth: u16) -> Vec<u8> {
let mut out = Vec::with_capacity(v.len() * if depth == 16 { 2 } else { 1 });
for &x in v {
let x = x.clamp(0.0, 1.0);
if depth == 16 {
u16b(&mut out, (x * 65535.0).round() as u16);
} else {
out.push((x * 255.0).round() as u8);
}
}
out
}
/// One channel's data (compression field + data) for a `w`×`h` plane.
fn channel_data(v: &[f32], w: usize, h: usize, depth: u16, rle: bool) -> Vec<u8> {
let raw = encode_samples(v, depth);
let mut out = vec![];
let rowb = raw.len() / h.max(1);
if !rle || w == 0 || h == 0 {
u16b(&mut out, 0);
out.extend_from_slice(&raw);
return out;
}
u16b(&mut out, 1);
let rows: Vec<Vec<u8>> = raw.chunks(rowb.max(1)).map(pack_bits).collect();
for r in &rows {
u16b(&mut out, r.len() as u16);
}
for r in rows {
out.extend_from_slice(&r);
}
out
}
fn pascal4(out: &mut Vec<u8>, s: &str) {
let b: Vec<u8> = s.bytes().map(|c| if c < 0x80 { c } else { b'?' }).take(255).collect();
out.push(b.len() as u8);
out.extend_from_slice(&b);
let total = 1 + b.len();
out.resize(out.len() + total.div_ceil(4) * 4 - total, 0);
}
fn block(out: &mut Vec<u8>, key: &[u8; 4], data: &[u8]) {
out.extend_from_slice(b"8BIM");
out.extend_from_slice(key);
let padded = data.len().div_ceil(2) * 2;
u32b(out, padded as u32);
out.extend_from_slice(data);
out.resize(out.len() + padded - data.len(), 0);
}
fn layer_info(doc: &WDoc) -> Vec<u8> {
let mut out = vec![];
out.extend_from_slice(&(-(doc.layers.len() as i16)).to_be_bytes());
let mut chan_data: Vec<Vec<(i16, Vec<u8>)>> = vec![];
for l in &doc.layers {
let (w, h) = (l.rect.width() as usize, l.rect.height() as usize);
let mut chans: Vec<(i16, Vec<u8>)> = vec![];
let n = w * h;
let alpha: Vec<f32> = l.pixels.iter().take(n).map(|p| p[3]).collect();
chans.push((-1, channel_data(&alpha, w, h, doc.depth, doc.rle)));
let cc = doc.mode.color_channels();
let planes: Vec<Vec<f32>> = l.pixels.iter().take(n).map(|p| mode_channels(doc.mode, *p)).collect();
for c in 0..cc {
let plane: Vec<f32> = planes.iter().map(|v| v[c]).collect();
chans.push((c as i16, channel_data(&plane, w, h, doc.depth, doc.rle)));
}
if let Some(m) = &l.mask {
chans.push((-2, channel_data(&m.data, m.rect.width() as usize, m.rect.height() as usize, doc.depth, doc.rle)));
}
// record
for v in [l.rect.top, l.rect.left, l.rect.bottom, l.rect.right] {
out.extend_from_slice(&v.to_be_bytes());
}
u16b(&mut out, chans.len() as u16);
for (id, d) in &chans {
out.extend_from_slice(&id.to_be_bytes());
u32b(&mut out, d.len() as u32);
}
out.extend_from_slice(b"8BIM");
out.extend_from_slice(&l.blend);
out.push(l.opacity);
out.push(l.clipping as u8);
let mut flags = 8u8;
if l.hidden {
flags |= 2;
}
out.push(flags);
out.push(0);
let mut extra = vec![];
match &l.mask {
Some(m) => {
u32b(&mut extra, 20);
for v in [m.rect.top, m.rect.left, m.rect.bottom, m.rect.right] {
extra.extend_from_slice(&v.to_be_bytes());
}
extra.push(m.default_color);
extra.push(if m.disabled { 2 } else { 0 });
extra.extend_from_slice(&[0, 0]);
}
None => u32b(&mut extra, 0),
}
u32b(&mut extra, 0); // blending ranges
pascal4(&mut extra, &l.name);
let luni = unicode_plain(&l.name);
block(&mut extra, b"luni", &luni);
match l.section {
Section::Layer => {}
s => {
let mut d = vec![];
u32b(
&mut d,
match s {
Section::OpenFolder => 1,
Section::ClosedFolder => 2,
_ => 3,
},
);
d.extend_from_slice(b"8BIM");
d.extend_from_slice(&l.blend);
block(&mut extra, b"lsct", &d);
}
}
for (k, d) in &l.blocks {
block(&mut extra, k, d);
}
u32b(&mut out, extra.len() as u32);
out.extend_from_slice(&extra);
chan_data.push(chans);
}
for chans in chan_data {
for (_, d) in chans {
out.extend_from_slice(&d);
}
}
if out.len() % 2 == 1 {
out.push(0);
}
out
}
fn unicode_plain(s: &str) -> Vec<u8> {
let v: Vec<u16> = s.encode_utf16().collect();
let mut out = vec![];
u32b(&mut out, v.len() as u32);
for c in v {
u16b(&mut out, c);
}
out
}
/// Normal "over" of the visible pixel layers (groups ignored), straight alpha.
fn flatten(doc: &WDoc) -> Vec<[f32; 4]> {
let (w, h) = (doc.width as i32, doc.height as i32);
let mut acc = vec![[0.0f32; 4]; (w * h) as usize];
for l in doc.layers.iter().filter(|l| !l.hidden && l.section == Section::Layer) {
let lw = l.rect.width() as i32;
for (i, p) in l.pixels.iter().enumerate() {
let (x, y) = (l.rect.left + i as i32 % lw.max(1), l.rect.top + i as i32 / lw.max(1));
if x < 0 || y < 0 || x >= w || y >= h {
continue;
}
let a = p[3] * l.opacity as f32 / 255.0;
let d = &mut acc[(y * w + x) as usize];
let oa = a + d[3] * (1.0 - a);
for c in 0..3 {
d[c] = if oa > 0.0 { (p[c] * a + d[c] * d[3] * (1.0 - a)) / oa } else { 0.0 };
}
d[3] = oa;
}
}
acc
}
/// Serialise a document.
pub fn write(doc: &WDoc) -> Vec<u8> {
let cc = doc.mode.color_channels();
let mut out = vec![];
out.extend_from_slice(b"8BPS");
u16b(&mut out, 1);
out.extend_from_slice(&[0; 6]);
u16b(&mut out, (cc + 1) as u16);
u32b(&mut out, doc.height);
u32b(&mut out, doc.width);
u16b(&mut out, doc.depth);
u16b(&mut out, doc.mode.to_u16());
u32b(&mut out, 0); // colour mode data
u32b(&mut out, 0); // image resources
// Layer and mask information.
let mut lm = vec![];
if !doc.layers.is_empty() {
let li = layer_info(doc);
if doc.depth == 16 {
u32b(&mut lm, 0);
u32b(&mut lm, 0); // global layer mask info
let mut padded = li;
while !padded.len().is_multiple_of(4) {
padded.push(0);
}
lm.extend_from_slice(b"8BIM");
lm.extend_from_slice(b"Lr16");
u32b(&mut lm, padded.len() as u32);
lm.extend_from_slice(&padded);
} else {
u32b(&mut lm, li.len() as u32);
lm.extend_from_slice(&li);
u32b(&mut lm, 0);
}
}
u32b(&mut out, lm.len() as u32);
out.extend_from_slice(&lm);
// Merged image.
let comp = doc.composite.clone().unwrap_or_else(|| flatten(doc));
let (w, h) = (doc.width as usize, doc.height as usize);
let planes: Vec<Vec<f32>> = comp.iter().map(|p| mode_channels(doc.mode, *p)).collect();
let mut chans: Vec<Vec<f32>> = (0..cc).map(|c| planes.iter().map(|v| v[c]).collect()).collect();
chans.push(comp.iter().map(|p| p[3]).collect());
if doc.rle {
u16b(&mut out, 1);
let rows: Vec<Vec<Vec<u8>>> =
chans.iter().map(|c| encode_samples(c, doc.depth).chunks((w * if doc.depth == 16 { 2 } else { 1 }).max(1)).map(pack_bits).collect()).collect();
for c in &rows {
for r in c {
u16b(&mut out, r.len() as u16);
}
}
for c in rows {
for r in c {
out.extend_from_slice(&r);
}
}
} else {
u16b(&mut out, 0);
for c in &chans {
out.extend_from_slice(&encode_samples(c, doc.depth));
}
}
let _ = h;
out
}
// ---------------------------------------------------------------- block helpers
/// `TySh`: a type layer with one style run; the text origin sits at `origin`.
pub fn text_block(text: &str, origin: [f64; 2], font: &str, size: f64, color: [f64; 3], justification: u8) -> ([u8; 4], Vec<u8>) {
let mut d = vec![];
u16b(&mut d, 1);
for v in [1.0, 0.0, 0.0, 1.0, origin[0], origin[1]] {
d.extend_from_slice(&f64::to_be_bytes(v));
}
u16b(&mut d, 50);
u32b(&mut d, 16);
let desc = Descriptor::new("TxLr")
.with("Txt ", DValue::Text(text.replace('\n', "\r")))
.with("textGridding", DValue::Enum("textGridding".into(), "None".into()))
.with("Ornt", DValue::Enum("Ornt".into(), "Hrzn".into()))
.with("EngineData", DValue::Raw(engine_data::minimal(text, font, size, color, justification)));
descriptor::write_descriptor(&mut d, &desc);
// Warp: version, descriptor version, an empty warp descriptor.
u16b(&mut d, 1);
u32b(&mut d, 16);
descriptor::write_descriptor(&mut d, &Descriptor::new("warp").with("warpStyle", DValue::Enum("warpStyle".into(), "warpNone".into())));
for _ in 0..4 {
u32b(&mut d, 0);
}
(*b"TySh", d)
}
/// `lfx2`: layer effects from an effects descriptor (`null` class with `DrSh`, `FrFX`…).
pub fn effects_block(desc: &Descriptor) -> ([u8; 4], Vec<u8>) {
let mut d = vec![];
u32b(&mut d, 0);
u32b(&mut d, 16);
descriptor::write_descriptor(&mut d, desc);
(*b"lfx2", d)
}
/// `SoCo`: solid colour fill.
pub fn solid_color_block(rgb: [f64; 3]) -> ([u8; 4], Vec<u8>) {
let mut d = vec![];
u32b(&mut d, 16);
descriptor::write_descriptor(&mut d, &Descriptor::new("null").with("Clr ", DValue::Descriptor(Descriptor::rgb(rgb))));
(*b"SoCo", d)
}
/// `hue2`: Hue/Saturation (master).
pub fn hue_sat_block(hue: i16, sat: i16, light: i16) -> ([u8; 4], Vec<u8>) {
let mut d = vec![];
u16b(&mut d, 2);
d.extend_from_slice(&[0, 0]);
for v in [0i16, 0, 0, hue, sat, light] {
d.extend_from_slice(&v.to_be_bytes());
}
// Six colour ranges (unused).
d.resize(d.len() + 6 * 24, 0);
(*b"hue2", d)
}
/// `levl`: Levels (composite record).
pub fn levels_block(in_black: i16, in_white: i16, out_black: i16, out_white: i16, gamma: f64) -> ([u8; 4], Vec<u8>) {
let mut d = vec![];
u16b(&mut d, 2);
for v in [in_black, in_white, out_black, out_white, (gamma * 100.0).round() as i16] {
d.extend_from_slice(&v.to_be_bytes());
}
for _ in 0..28 {
for v in [0i16, 255, 0, 255, 100] {
d.extend_from_slice(&v.to_be_bytes());
}
}
(*b"levl", d)
}
pub fn invert_block() -> ([u8; 4], Vec<u8>) {
(*b"nvrt", vec![])
}
pub fn brightness_block(brightness: i16, contrast: i16) -> ([u8; 4], Vec<u8>) {
let mut d = vec![];
for v in [brightness, contrast, 127] {
d.extend_from_slice(&v.to_be_bytes());
}
d.push(0);
(*b"brit", d)
}
pub fn exposure_block(exposure: f32, offset: f32, gamma: f32) -> ([u8; 4], Vec<u8>) {
let mut d = vec![];
u16b(&mut d, 1);
for v in [exposure, offset, gamma] {
d.extend_from_slice(&v.to_be_bytes());
}
(*b"expA", d)
}
/// `vmsk`: a vector mask from closed polygons/curves given as (in, anchor, out) knots in pixels.
pub fn vector_mask_block(subpaths: &[Vec<[[f64; 2]; 3]>], width: u32, height: u32, inverted: bool) -> ([u8; 4], Vec<u8>) {
let mut d = vec![];
u32b(&mut d, 3);
u32b(&mut d, inverted as u32);
// Path fill rule record.
u16b(&mut d, 6);
d.resize(d.len() + 24, 0);
let fx = |v: f64| ((v * (1 << 24) as f64).round() as i32).to_be_bytes();
for sp in subpaths {
u16b(&mut d, 0);
u16b(&mut d, sp.len() as u16);
d.resize(d.len() + 22, 0);
for k in sp {
u16b(&mut d, 1);
for p in k {
d.extend_from_slice(&fx(p[1] / height as f64));
d.extend_from_slice(&fx(p[0] / width as f64));
}
}
}
(*b"vmsk", d)
}
+1
View File
@@ -19,6 +19,7 @@ effectcraft-raster = { workspace = true }
effectcraft-path = { workspace = true }
effectcraft-text = { workspace = true }
effectcraft-effects = { workspace = true }
lz4_flex = { version = "0.11", default-features = false, features = ["std", "safe-encode", "safe-decode", "checked-decode"] }
effectcraft-model = { workspace = true }
kurbo = { workspace = true }
rayon = { workspace = true }
+1
View File
@@ -330,6 +330,7 @@ mod tests {
missing: false,
sequence: vec![],
color_profile: None,
layer: None,
};
let fid = p.add_item("x.wav", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(f));
let comp = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0));
+59 -3
View File
@@ -25,9 +25,13 @@ use effectcraft_project::{ItemKind, Layer, LayerSource, Node, PropGroup};
use crate::eval::EvalCtx;
/// Thread-safe, memory-budgeted cache of processed layer buffers.
/// Thread-safe, memory-budgeted cache of processed layer buffers, optionally backed by the
/// persistent [`DiskCache`](crate::disk_cache::DiskCache): a memory miss reads the disk, and
/// buffers that were slow to render are written there (keys salted per project, see
/// [`LayerCache::set_disk`]).
pub struct LayerCache {
inner: Mutex<Inner>,
disk: Mutex<Option<(Arc<crate::disk_cache::DiskCache>, u64)>>,
}
struct Entry {
@@ -43,6 +47,8 @@ struct Inner {
clock: u64,
hits: u64,
misses: u64,
/// When each recent miss happened (to time the render that follows it).
missed_at: HashMap<u64, web_time::Instant>,
}
/// Hit/miss counters and memory use (for perf readouts and tests).
@@ -63,7 +69,37 @@ impl Default for LayerCache {
impl LayerCache {
/// A cache holding at most `budget` bytes of pixels.
pub fn new(budget: usize) -> LayerCache {
LayerCache { inner: Mutex::new(Inner { map: HashMap::new(), bytes: 0, budget, clock: 0, hits: 0, misses: 0 }) }
LayerCache {
inner: Mutex::new(Inner { map: HashMap::new(), bytes: 0, budget, clock: 0, hits: 0, misses: 0, missed_at: HashMap::new() }),
disk: Mutex::new(None),
}
}
/// Attach (or detach) the disk cache. `salt` separates projects and footage versions (see
/// `footage_salt`); keys on disk are `(salt, key)`.
pub fn set_disk(&self, disk: Option<(Arc<crate::disk_cache::DiskCache>, u64)>) {
if let Ok(mut d) = self.disk.lock() {
*d = disk;
}
}
/// Change the disk salt (after footage changed) keeping the attached disk cache.
pub fn set_disk_salt(&self, salt: u64) {
if let Ok(mut d) = self.disk.lock()
&& let Some((_, s)) = d.as_mut()
{
*s = salt;
}
}
pub fn disk(&self) -> Option<Arc<crate::disk_cache::DiskCache>> {
self.disk.lock().ok()?.as_ref().map(|d| d.0.clone())
}
fn disk_key(&self, key: u64) -> Option<(Arc<crate::disk_cache::DiskCache>, u128)> {
let d = self.disk.lock().ok()?;
let (dc, salt) = d.as_ref()?;
Some((dc.clone(), ((*salt as u128) << 64) | key as u128))
}
pub fn get(&self, key: u64) -> Option<Arc<Buf>> {
@@ -79,12 +115,32 @@ impl LayerCache {
}
None => {
g.misses += 1;
None
if g.missed_at.len() > 4096 {
g.missed_at.clear();
}
g.missed_at.insert(key, web_time::Instant::now());
drop(g);
// Disk cache: a hit comes back into memory.
let (dc, dk) = self.disk_key(key)?;
let buf = Arc::new(dc.get_layer(dk)?);
self.insert_mem(key, buf.clone());
Some(buf)
}
}
}
pub fn insert(&self, key: u64, buf: Arc<Buf>) {
let started = self.inner.lock().ok().and_then(|mut g| g.missed_at.remove(&key));
if let Some((dc, dk)) = self.disk_key(key) {
let slow = started.is_some_and(|t| t.elapsed().as_secs_f64() * 1e3 >= dc.min_layer_ms() as f64);
if slow {
dc.put_layer(dk, &buf);
}
}
self.insert_mem(key, buf);
}
fn insert_mem(&self, key: u64, buf: Arc<Buf>) {
let bytes = buf.img.data.len() * std::mem::size_of::<effectcraft_raster::Px>();
let Ok(mut g) = self.inner.lock() else { return };
if bytes > g.budget / 4 {
+831
View File
@@ -0,0 +1,831 @@
//! The persistent disk cache (Settings ▸ Media & Disk Cache ▸ Disk Cache): RAM-preview frames
//! and layer cache entries kept on disk between sessions, like After Effects' disk cache.
//!
//! * Entries are files keyed by a 128-bit content hash (`<folder>/v1/<kind>/<2 hex>/<32 hex>.ecc`):
//! frames by the hash of the composition's content (the comp and everything it uses, footage
//! files' size and modification time included) plus frame, scale and view; layer buffers by
//! the layer cache's content key salted with the project's footage fingerprint. A changed
//! project therefore never reads stale pixels: its keys simply change.
//! * Payloads are LZ4-compressed (`lz4_flex`, MIT): frames as 8-bit premultiplied RGBA, layer
//! buffers as `f32` RGBA with the bytes shuffled into planes (exact, so cached and fresh
//! renders are bit-identical). Every file ends in a checksum; damaged files read as misses and
//! are deleted.
//! * Size limit with least-recently-used eviction. The in-memory index is rebuilt from the
//! folder when the cache opens (file modification times order it), so the cache survives
//! restarts; reads refresh a file's time.
//! * Safe concurrent access: writes go to a unique temporary file that is renamed into place
//! (atomic on every desktop OS), so readers in this or another process never see a partial
//! entry; a missing or damaged file is just a miss. Writes run on a background thread with a
//! bounded queue (a full queue drops the write rather than stalling rendering).
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Condvar, Mutex};
use effectcraft_effects::Buf;
use effectcraft_raster::Image;
/// File format version (part of the folder layout).
const VERSION: &str = "v1";
const MAGIC: &[u8; 4] = b"ECDC";
/// Pending writes before new ones are dropped.
#[cfg(not(target_arch = "wasm32"))]
const QUEUE_LIMIT: usize = 64;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Kind {
Frame,
Layer,
}
impl Kind {
fn dir(self) -> &'static str {
match self {
Kind::Frame => "frames",
Kind::Layer => "layers",
}
}
fn tag(self) -> u8 {
match self {
Kind::Frame => 1,
Kind::Layer => 2,
}
}
}
/// A cached viewer frame: 8-bit premultiplied RGBA.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Frame8 {
pub width: u32,
pub height: u32,
pub rgba: Vec<u8>,
}
/// Counters for tests, the Info panel and `cache.diskStats`.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct DiskStats {
pub entries: usize,
pub bytes: u64,
pub max_bytes: u64,
pub hits: u64,
pub misses: u64,
pub writes: u64,
pub evictions: u64,
pub dropped: u64,
}
struct Meta {
bytes: u64,
last_use: u64,
}
#[derive(Default)]
struct Index {
map: HashMap<(Kind, u128), Meta>,
total: u64,
clock: u64,
}
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
enum Job {
Write(Kind, u128, Vec<u8>),
}
struct Queue {
jobs: std::collections::VecDeque<Job>,
busy: bool,
}
pub struct DiskCache {
dir: PathBuf,
index: Mutex<Index>,
max_bytes: AtomicU64,
hits: AtomicU64,
misses: AtomicU64,
writes: AtomicU64,
evictions: AtomicU64,
dropped: AtomicU64,
tmp_counter: AtomicU64,
queue: Arc<(Mutex<Queue>, Condvar)>,
/// Layer buffers that took less than this to render are not written (ms).
min_layer_ms: AtomicU64,
}
/// FNV-1a 128-bit (content keys).
#[derive(Clone, Copy)]
pub struct Hash128(u128);
impl Default for Hash128 {
fn default() -> Self {
Hash128(0x6c62_272e_07bb_0142_62b8_2175_6295_c58d)
}
}
impl Hash128 {
pub fn write(&mut self, bytes: &[u8]) {
for &b in bytes {
self.0 ^= b as u128;
self.0 = self.0.wrapping_mul(0x0000_0000_0100_0000_0000_0000_0000_013b);
}
}
pub fn write_u64(&mut self, v: u64) {
self.write(&v.to_le_bytes());
}
pub fn finish(&self) -> u128 {
self.0
}
}
fn checksum(data: &[u8]) -> u64 {
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
for &b in data {
h ^= b as u64;
h = h.wrapping_mul(0x0000_0100_0000_01b3);
}
h
}
/// The platform's default disk cache folder.
pub fn default_folder() -> PathBuf {
let home = std::env::var_os("HOME").map(PathBuf::from);
if cfg!(target_os = "macos")
&& let Some(h) = &home
{
return h.join("Library/Caches/EffectCraft/Disk Cache");
}
if cfg!(windows)
&& let Some(l) = std::env::var_os("LOCALAPPDATA")
{
return PathBuf::from(l).join("EffectCraft").join("Disk Cache");
}
if let Some(x) = std::env::var_os("XDG_CACHE_HOME") {
return PathBuf::from(x).join("effectcraft/disk-cache");
}
home.map(|h| h.join(".cache/effectcraft/disk-cache")).unwrap_or_else(|| std::env::temp_dir().join("effectcraft-disk-cache"))
}
impl DiskCache {
/// Open (or create) the cache in `folder`, holding at most `max_bytes`.
pub fn open(folder: impl AsRef<Path>, max_bytes: u64) -> std::io::Result<Arc<DiskCache>> {
let dir = folder.as_ref().join(VERSION);
std::fs::create_dir_all(&dir)?;
let dc = Arc::new(DiskCache {
dir,
index: Mutex::new(Index::default()),
max_bytes: AtomicU64::new(max_bytes),
hits: AtomicU64::new(0),
misses: AtomicU64::new(0),
writes: AtomicU64::new(0),
evictions: AtomicU64::new(0),
dropped: AtomicU64::new(0),
tmp_counter: AtomicU64::new(0),
queue: Arc::new((Mutex::new(Queue { jobs: Default::default(), busy: false }), Condvar::new())),
min_layer_ms: AtomicU64::new(30),
});
dc.rescan();
dc.evict();
#[cfg(not(target_arch = "wasm32"))]
{
let weak = Arc::downgrade(&dc);
let q = dc.queue.clone();
std::thread::Builder::new().name("ec-disk-cache".into()).spawn(move || writer(weak, q))?;
}
Ok(dc)
}
pub fn folder(&self) -> &Path {
self.dir.parent().unwrap_or(&self.dir)
}
/// Rebuild the index from the folder (oldest files first).
pub fn rescan(&self) {
let mut found: Vec<(std::time::SystemTime, Kind, u128, u64)> = vec![];
for kind in [Kind::Frame, Kind::Layer] {
let Ok(shards) = std::fs::read_dir(self.dir.join(kind.dir())) else { continue };
for shard in shards.flatten() {
let Ok(files) = std::fs::read_dir(shard.path()) else { continue };
for f in files.flatten() {
let p = f.path();
let name = p.file_name().and_then(|n| n.to_str()).unwrap_or("");
if name.contains(".tmp") {
// A writer that died mid-write; old temporaries are removed.
if f.metadata().and_then(|m| m.modified()).is_ok_and(|t| t.elapsed().is_ok_and(|e| e.as_secs() > 600)) {
let _ = std::fs::remove_file(&p);
}
continue;
}
let Some(hex) = name.strip_suffix(".ecc") else { continue };
let Ok(key) = u128::from_str_radix(hex, 16) else { continue };
let Ok(md) = f.metadata() else { continue };
found.push((md.modified().unwrap_or(std::time::UNIX_EPOCH), kind, key, md.len()));
}
}
}
found.sort_by_key(|e| e.0);
let mut idx = Index::default();
for (_, kind, key, bytes) in found {
idx.clock += 1;
let clock = idx.clock;
idx.total += bytes;
idx.map.insert((kind, key), Meta { bytes, last_use: clock });
}
if let Ok(mut g) = self.index.lock() {
*g = idx;
}
}
fn path(&self, kind: Kind, key: u128) -> PathBuf {
let hex = format!("{key:032x}");
self.dir.join(kind.dir()).join(&hex[..2]).join(format!("{hex}.ecc"))
}
pub fn max_bytes(&self) -> u64 {
self.max_bytes.load(Ordering::Relaxed)
}
/// Change the size limit (evicts the least recently used entries when shrinking).
pub fn set_max_bytes(&self, b: u64) {
self.max_bytes.store(b, Ordering::Relaxed);
self.evict();
}
/// Layer buffers faster to render than this are not worth a disk write.
pub fn set_min_layer_ms(&self, ms: u64) {
self.min_layer_ms.store(ms, Ordering::Relaxed);
}
pub fn min_layer_ms(&self) -> u64 {
self.min_layer_ms.load(Ordering::Relaxed)
}
pub fn contains(&self, kind: Kind, key: u128) -> bool {
self.index.lock().is_ok_and(|g| g.map.contains_key(&(kind, key)))
}
/// Keys of one kind (unordered).
pub fn keys(&self, kind: Kind) -> Vec<u128> {
self.index.lock().map(|g| g.map.keys().filter(|k| k.0 == kind).map(|k| k.1).collect()).unwrap_or_default()
}
pub fn stats(&self) -> DiskStats {
let (entries, bytes) = self.index.lock().map(|g| (g.map.len(), g.total)).unwrap_or_default();
DiskStats {
entries,
bytes,
max_bytes: self.max_bytes(),
hits: self.hits.load(Ordering::Relaxed),
misses: self.misses.load(Ordering::Relaxed),
writes: self.writes.load(Ordering::Relaxed),
evictions: self.evictions.load(Ordering::Relaxed),
dropped: self.dropped.load(Ordering::Relaxed),
}
}
/// Delete every entry (Edit ▸ Purge ▸ All Disk Cache / Empty Disk Cache).
pub fn clear(&self) {
self.flush();
if let Ok(mut g) = self.index.lock() {
*g = Index::default();
}
for kind in [Kind::Frame, Kind::Layer] {
let _ = std::fs::remove_dir_all(self.dir.join(kind.dir()));
}
}
/// Wait until queued writes are on disk.
pub fn flush(&self) {
let (m, cv) = &*self.queue;
let Ok(mut q) = m.lock() else { return };
#[cfg(target_arch = "wasm32")]
{
q.jobs.clear();
}
while !q.jobs.is_empty() || q.busy {
q = match cv.wait(q) {
Ok(q) => q,
Err(_) => return,
};
}
}
fn read(&self, kind: Kind, key: u128) -> Option<Vec<u8>> {
if !self.contains(kind, key) {
self.misses.fetch_add(1, Ordering::Relaxed);
return None;
}
let path = self.path(kind, key);
let data = std::fs::read(&path).ok();
let ok = data.as_deref().and_then(|d| verify(d, kind));
match ok {
Some(payload) => {
let payload = payload.to_vec();
self.hits.fetch_add(1, Ordering::Relaxed);
if let Ok(mut g) = self.index.lock() {
g.clock += 1;
let c = g.clock;
if let Some(m) = g.map.get_mut(&(kind, key)) {
m.last_use = c;
}
}
// Persist the recency for the next session (best effort).
if let Ok(f) = std::fs::File::options().append(true).open(&path) {
let _ = f.set_modified(std::time::SystemTime::now());
}
Some(payload)
}
None => {
self.misses.fetch_add(1, Ordering::Relaxed);
// Gone (another process evicted it) or damaged: forget it.
if data.is_some() {
let _ = std::fs::remove_file(&path);
}
self.forget(kind, key);
None
}
}
}
fn forget(&self, kind: Kind, key: u128) {
if let Ok(mut g) = self.index.lock()
&& let Some(m) = g.map.remove(&(kind, key))
{
g.total -= m.bytes;
}
}
fn enqueue(&self, kind: Kind, key: u128, payload: Vec<u8>) {
#[cfg(target_arch = "wasm32")]
{
self.write_now(kind, key, &payload);
}
#[cfg(not(target_arch = "wasm32"))]
{
let (m, cv) = &*self.queue;
let Ok(mut q) = m.lock() else { return };
if q.jobs.len() >= QUEUE_LIMIT {
self.dropped.fetch_add(1, Ordering::Relaxed);
return;
}
q.jobs.push_back(Job::Write(kind, key, payload));
cv.notify_all();
}
}
/// Write an entry now (atomic rename), then evict down to the limit.
fn write_now(&self, kind: Kind, key: u128, payload: &[u8]) {
if self.contains(kind, key) {
return;
}
let path = self.path(kind, key);
let Some(parent) = path.parent() else { return };
if std::fs::create_dir_all(parent).is_err() {
return;
}
let mut data = Vec::with_capacity(payload.len() + 16);
data.extend_from_slice(MAGIC);
data.push(kind.tag());
data.extend_from_slice(&[0, 0, 0]);
data.extend_from_slice(payload);
let sum = checksum(&data);
data.extend_from_slice(&sum.to_le_bytes());
let n = self.tmp_counter.fetch_add(1, Ordering::Relaxed);
let tmp = parent.join(format!("{key:032x}.tmp.{}.{n}", std::process::id()));
if std::fs::write(&tmp, &data).is_err() {
let _ = std::fs::remove_file(&tmp);
return;
}
if std::fs::rename(&tmp, &path).is_err() {
let _ = std::fs::remove_file(&tmp);
return;
}
self.writes.fetch_add(1, Ordering::Relaxed);
if let Ok(mut g) = self.index.lock() {
g.clock += 1;
let c = g.clock;
let bytes = data.len() as u64;
if let Some(old) = g.map.insert((kind, key), Meta { bytes, last_use: c }) {
g.total -= old.bytes;
}
g.total += bytes;
}
self.evict();
}
/// Remove least recently used entries until the cache fits its limit.
fn evict(&self) {
let max = self.max_bytes();
let victims: Vec<(Kind, u128)> = {
let Ok(mut g) = self.index.lock() else { return };
if g.total <= max {
return;
}
let mut by_age: Vec<(u64, (Kind, u128), u64)> = g.map.iter().map(|(k, m)| (m.last_use, *k, m.bytes)).collect();
by_age.sort_unstable_by_key(|e| e.0);
let mut out = vec![];
for (_, k, bytes) in by_age {
if g.total <= max {
break;
}
g.map.remove(&k);
g.total -= bytes;
out.push(k);
}
out
};
for (kind, key) in victims {
let _ = std::fs::remove_file(self.path(kind, key));
self.evictions.fetch_add(1, Ordering::Relaxed);
}
}
// ------------------------------------------------------------ typed entries
pub fn get_frame(&self, key: u128) -> Option<Frame8> {
let p = self.read(Kind::Frame, key)?;
decode_frame(&p)
}
/// Queue a frame for writing (8-bit premultiplied RGBA, `width * height * 4` bytes).
pub fn put_frame(&self, key: u128, width: u32, height: u32, rgba: &[u8]) {
if self.contains(Kind::Frame, key) || rgba.len() != width as usize * height as usize * 4 {
return;
}
self.enqueue(Kind::Frame, key, encode_frame(width, height, rgba));
}
pub fn get_layer(&self, key: u128) -> Option<Buf> {
let p = self.read(Kind::Layer, key)?;
decode_layer(&p)
}
/// Queue a layer buffer for writing.
pub fn put_layer(&self, key: u128, buf: &Buf) {
if self.contains(Kind::Layer, key) {
return;
}
self.enqueue(Kind::Layer, key, encode_layer(buf));
}
/// Write a layer buffer synchronously (tests).
pub fn put_layer_now(&self, key: u128, buf: &Buf) {
self.write_now(Kind::Layer, key, &encode_layer(buf));
}
/// Write a frame synchronously (tests).
pub fn put_frame_now(&self, key: u128, width: u32, height: u32, rgba: &[u8]) {
self.write_now(Kind::Frame, key, &encode_frame(width, height, rgba));
}
}
#[cfg(not(target_arch = "wasm32"))]
fn writer(dc: std::sync::Weak<DiskCache>, q: Arc<(Mutex<Queue>, Condvar)>) {
let (m, cv) = &*q;
loop {
let job = {
let Ok(mut g) = m.lock() else { return };
loop {
if let Some(j) = g.jobs.pop_front() {
g.busy = true;
break j;
}
if dc.strong_count() == 0 {
return;
}
g = match cv.wait_timeout(g, std::time::Duration::from_millis(500)) {
Ok((g, _)) => g,
Err(_) => return,
};
}
};
let Some(cache) = dc.upgrade() else { return };
match job {
Job::Write(kind, key, payload) => cache.write_now(kind, key, &payload),
}
drop(cache);
if let Ok(mut g) = m.lock() {
g.busy = false;
}
cv.notify_all();
}
}
/// Check magic, kind and checksum; returns the payload.
fn verify(d: &[u8], kind: Kind) -> Option<&[u8]> {
if d.len() < 16 || &d[..4] != MAGIC || d[4] != kind.tag() {
return None;
}
let (body, sum) = d.split_at(d.len() - 8);
let mut s = [0u8; 8];
s.copy_from_slice(sum);
(checksum(body) == u64::from_le_bytes(s)).then_some(&body[8..])
}
fn encode_frame(w: u32, h: u32, rgba: &[u8]) -> Vec<u8> {
let mut out = Vec::with_capacity(rgba.len() / 2 + 16);
out.extend_from_slice(&w.to_le_bytes());
out.extend_from_slice(&h.to_le_bytes());
out.extend_from_slice(&lz4_flex::compress_prepend_size(rgba));
out
}
fn decode_frame(p: &[u8]) -> Option<Frame8> {
if p.len() < 8 {
return None;
}
let w = u32::from_le_bytes(p[0..4].try_into().ok()?);
let h = u32::from_le_bytes(p[4..8].try_into().ok()?);
let rgba = lz4_flex::decompress_size_prepended(&p[8..]).ok()?;
(rgba.len() == w as usize * h as usize * 4).then_some(Frame8 { width: w, height: h, rgba })
}
fn encode_layer(b: &Buf) -> Vec<u8> {
let (w, h) = (b.img.width, b.img.height);
let n = b.img.data.len();
// Byte planes: all first bytes of every float, then all second bytes… (compresses well).
let mut planes = vec![0u8; n * 16];
for (i, px) in b.img.data.iter().enumerate() {
for c in 0..4 {
let bytes = px[c].to_le_bytes();
for (k, byte) in bytes.iter().enumerate() {
planes[(k * 4 + c) * n + i] = *byte;
}
}
}
let mut out = Vec::with_capacity(planes.len() / 4 + 40);
out.extend_from_slice(&w.to_le_bytes());
out.extend_from_slice(&h.to_le_bytes());
out.extend_from_slice(&b.offset[0].to_le_bytes());
out.extend_from_slice(&b.offset[1].to_le_bytes());
out.extend_from_slice(&b.scale.to_le_bytes());
out.extend_from_slice(&lz4_flex::compress_prepend_size(&planes));
out
}
fn decode_layer(p: &[u8]) -> Option<Buf> {
if p.len() < 32 {
return None;
}
let w = u32::from_le_bytes(p[0..4].try_into().ok()?);
let h = u32::from_le_bytes(p[4..8].try_into().ok()?);
let f = |i: usize| -> Option<f64> { Some(f64::from_le_bytes(p[i..i + 8].try_into().ok()?)) };
let offset = [f(8)?, f(16)?];
let scale = f(24)?;
let planes = lz4_flex::decompress_size_prepended(&p[32..]).ok()?;
let n = w as usize * h as usize;
if planes.len() != n * 16 {
return None;
}
let mut img = Image::new(w, h);
for (i, px) in img.data.iter_mut().enumerate() {
for c in 0..4 {
let mut bytes = [0u8; 4];
for (k, byte) in bytes.iter_mut().enumerate() {
*byte = planes[(k * 4 + c) * n + i];
}
px[c] = f32::from_le_bytes(bytes);
}
}
Some(Buf { img, offset, scale })
}
// ---------------------------------------------------------------- content keys
/// Bump when rendering changes in a way that makes old disk entries wrong.
const RENDER_VERSION: &str = "effectcraft-render-1";
fn hash_debug(h: &mut Hash128, v: &impl std::fmt::Debug) {
h.write(format!("{v:?}").as_bytes());
h.write(&[0xff]);
}
/// A file's size and modification time (so edited footage gets new keys).
fn file_stamp(h: &mut Hash128, path: &str) {
if path.is_empty() {
return;
}
if let Ok(md) = std::fs::metadata(path) {
h.write_u64(md.len());
if let Ok(t) = md.modified()
&& let Ok(d) = t.duration_since(std::time::UNIX_EPOCH)
{
h.write_u64(d.as_nanos() as u64);
}
}
}
fn footage_stamp(h: &mut Hash128, f: &effectcraft_project::Footage) {
file_stamp(h, &f.path);
for p in &f.sequence {
file_stamp(h, p);
}
}
/// Salt for layer cache keys on disk: the renderer version and every footage file's identity
/// (layer keys name footage items, not file contents).
pub fn footage_salt(project: &effectcraft_project::Project) -> u64 {
let mut h = Hash128::default();
h.write(RENDER_VERSION.as_bytes());
for (id, it) in &project.items {
if let effectcraft_project::ItemKind::Footage(f) = &it.kind {
h.write_u64(id.0);
hash_debug(&mut h, f);
footage_stamp(&mut h, f);
}
}
let v = h.finish();
(v as u64) ^ ((v >> 64) as u64)
}
/// Content key of a composition's pixels: project render settings, the comp and everything it
/// uses (nested comps, footage, solids, footage files' size and time). With expressions in play
/// (which can read any comp) the whole project counts.
pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcraft_project::ItemId) -> u128 {
use effectcraft_project::{ItemKind, LayerSource};
let mut h = Hash128::default();
h.write(RENDER_VERSION.as_bytes());
hash_debug(&mut h, &project.settings);
let mut seen = std::collections::BTreeSet::new();
let mut stack = vec![comp];
let mut expressions = false;
while let Some(id) = stack.pop() {
if !seen.insert(id) {
continue;
}
let Some(it) = project.item(id) else { continue };
if let ItemKind::Comp(c) = &it.kind {
for l in &c.layers {
if let LayerSource::Comp { item } | LayerSource::Footage { item } | LayerSource::Solid { item } = l.source {
stack.push(item);
}
if !expressions {
l.props.walk("", &mut |_, p| expressions |= p.expr.is_some());
}
}
}
}
let ids: Vec<effectcraft_project::ItemId> = if expressions { project.items.keys().copied().collect() } else { seen.into_iter().collect() };
for id in ids {
let Some(it) = project.item(id) else { continue };
h.write_u64(id.0);
h.write(it.name.as_bytes());
hash_debug(&mut h, &it.kind);
if let ItemKind::Footage(f) = &it.kind {
footage_stamp(&mut h, f);
}
}
h.write_u64(comp.0);
h.finish()
}
/// Disk key of one viewer frame.
pub fn frame_key(content: u128, frame: i64, scale_milli: u32, view: u64) -> u128 {
let mut h = Hash128::default();
h.write(&content.to_le_bytes());
h.write_u64(frame as u64);
h.write_u64(scale_milli as u64);
h.write_u64(view);
h.finish()
}
#[cfg(test)]
mod tests {
use super::*;
fn tmpdir(name: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("effectcraft-disk-cache-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
d
}
fn buf(seed: u32, w: u32, h: u32) -> Buf {
let mut img = Image::new(w, h);
for (i, p) in img.data.iter_mut().enumerate() {
let v = ((i as u32).wrapping_mul(2654435761u32).wrapping_add(seed) % 1000) as f32 / 999.0;
*p = [v, v * 0.5, 1.0 - v, if i % 7 == 0 { 0.0 } else { 1.0 }];
}
Buf { img, offset: [1.5, -2.25], scale: 0.5 }
}
#[test]
fn hit_miss_and_exact_round_trip() {
let dir = tmpdir("hit");
let dc = DiskCache::open(&dir, 1 << 30).unwrap();
assert!(dc.get_layer(42).is_none());
let b = buf(1, 33, 17);
dc.put_layer(42, &b);
dc.flush();
let back = dc.get_layer(42).unwrap();
assert_eq!(back.img, b.img);
assert_eq!((back.offset, back.scale), (b.offset, b.scale));
let rgba: Vec<u8> = (0..20 * 10 * 4).map(|i| (i % 251) as u8).collect();
dc.put_frame(7, 20, 10, &rgba);
dc.flush();
assert_eq!(dc.get_frame(7).unwrap().rgba, rgba);
let st = dc.stats();
assert_eq!((st.entries, st.hits, st.misses, st.writes), (2, 2, 1, 2));
// Wrong kind is a miss.
assert!(dc.get_frame(42).is_none());
}
#[test]
fn persists_across_restarts_and_clears() {
let dir = tmpdir("persist");
{
let dc = DiskCache::open(&dir, 1 << 30).unwrap();
dc.put_layer_now(1, &buf(1, 8, 8));
dc.put_frame_now(2, 2, 2, &[9; 16]);
}
let dc = DiskCache::open(&dir, 1 << 30).unwrap();
assert_eq!(dc.stats().entries, 2);
assert!(dc.contains(Kind::Layer, 1) && dc.contains(Kind::Frame, 2));
assert_eq!(dc.get_frame(2).unwrap().rgba, vec![9; 16]);
dc.clear();
assert_eq!(dc.stats().entries, 0);
assert!(dc.get_frame(2).is_none());
let again = DiskCache::open(&dir, 1 << 30).unwrap();
assert_eq!(again.stats().entries, 0);
}
#[test]
fn lru_eviction_respects_the_limit() {
let dir = tmpdir("lru");
let dc = DiskCache::open(&dir, u64::MAX).unwrap();
for k in 0..6u128 {
dc.put_layer_now(k, &buf(k as u32, 64, 64));
}
let per = dc.stats().bytes / 6;
// Touch 0 so it is the most recently used.
assert!(dc.get_layer(0).is_some());
dc.set_max_bytes(per * 3 + per / 2);
let st = dc.stats();
assert!(st.bytes <= per * 3 + per / 2, "{st:?}");
assert_eq!(st.entries, 3);
assert!(dc.contains(Kind::Layer, 0), "recently used survives");
assert!(!dc.contains(Kind::Layer, 1) && !dc.contains(Kind::Layer, 2) && !dc.contains(Kind::Layer, 3));
assert_eq!(st.evictions, 3);
// Files are really gone.
let files = walk(&dir);
assert_eq!(files, 3);
// New writes keep evicting.
dc.put_layer_now(99, &buf(9, 64, 64));
assert!(dc.stats().bytes <= per * 3 + per / 2);
}
fn walk(d: &Path) -> usize {
let mut n = 0;
if let Ok(rd) = std::fs::read_dir(d) {
for e in rd.flatten() {
let p = e.path();
if p.is_dir() {
n += walk(&p);
} else if p.extension().is_some_and(|x| x == "ecc") {
n += 1;
}
}
}
n
}
#[test]
fn damaged_files_are_misses() {
let dir = tmpdir("damaged");
let dc = DiskCache::open(&dir, 1 << 30).unwrap();
dc.put_frame_now(5, 4, 4, &[1; 64]);
let p = dc.path(Kind::Frame, 5);
let mut d = std::fs::read(&p).unwrap();
let n = d.len();
d[n / 2] ^= 0xff;
std::fs::write(&p, d).unwrap();
assert!(dc.get_frame(5).is_none());
assert!(!dc.contains(Kind::Frame, 5));
assert!(!p.exists());
}
#[test]
fn concurrent_readers_and_writers() {
let dir = tmpdir("concurrent");
let a = DiskCache::open(&dir, 1 << 30).unwrap();
// A second instance on the same folder (another process, in effect).
let b = DiskCache::open(&dir, 1 << 30).unwrap();
std::thread::scope(|s| {
for t in 0..4u32 {
let (a, b) = (a.clone(), b.clone());
s.spawn(move || {
for i in 0..20u128 {
let k = i % 5;
let w = if t % 2 == 0 { &a } else { &b };
w.put_layer_now(k, &buf(k as u32, 16, 16));
if let Some(x) = w.get_layer(k) {
assert_eq!(x.img, buf(k as u32, 16, 16).img);
}
}
});
}
});
b.rescan();
for k in 0..5u128 {
assert_eq!(b.get_layer(k).unwrap().img, buf(k as u32, 16, 16).img);
}
}
}
+28 -1
View File
@@ -9,6 +9,7 @@
pub mod audio;
pub mod cache;
pub mod color;
pub mod disk_cache;
use color::Region;
pub mod eval;
pub mod masks;
@@ -56,6 +57,16 @@ pub trait FootageSource: Send + Sync {
fn cache_budget(&self) -> Option<usize> {
None
}
/// Vector footage (SVG) rasterised at `scale` × its pixel size, for Continuously Rasterize.
/// `None` when the footage is not vector or cannot be read.
fn vector_frame(&self, _item: ItemId, _footage: &Footage, _scale: f64) -> Option<Arc<Image>> {
None
}
}
/// Footage that can be rasterised at any scale (SVG).
pub fn is_vector_footage(f: &Footage) -> bool {
f.codec == "SVG"
}
/// Layer parameters and audio for effects (see [`effectcraft_effects::EffectHost`]).
@@ -387,7 +398,12 @@ impl<'a> Renderer<'a> {
/// stays sharp when scaled up).
pub fn raster_scale(&self, ctx: &EvalCtx, layer: &Layer) -> f64 {
let s = self.opts.scale;
if !layer.switches.collapse || !matches!(layer.source, LayerSource::Text | LayerSource::Shape) {
let vector = match layer.source {
LayerSource::Text | LayerSource::Shape => true,
LayerSource::Footage { item } => matches!(self.project.item(item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if is_vector_footage(f)),
_ => false,
};
if !layer.switches.collapse || !vector {
return s;
}
let (l2c, _) = ctx.layer_to_comp(layer);
@@ -677,6 +693,17 @@ impl<'a> Renderer<'a> {
if !f.has_video {
return None;
}
// Continuously rasterised vector footage: drawn at the on-screen scale.
if layer.switches.collapse && is_vector_footage(f) {
let k = self.raster_scale(ctx, layer);
if let Some(img) = self.footage.vector_frame(*item, f, k) {
let mut buf = Buf { img: (*img).clone(), offset: [0.0; 2], scale: k };
if let Some(c) = self.pipe.media_in(f.color_profile) {
color::convert(&mut buf.img, &c);
}
return Some(buf);
}
}
let lt = ctx.source_time(layer);
let img = self.footage_frame(ctx, layer, *item, f, lt)?;
let k = if img.width > 0 { f.width.max(1) as f64 / img.width as f64 } else { 1.0 };
+1
View File
@@ -51,6 +51,7 @@ fn audio_footage() -> Footage {
missing: false,
sequence: vec![],
color_profile: None,
layer: None,
}
}
+1
View File
@@ -172,6 +172,7 @@ fn footage(profile: Option<ColorSpace>) -> Footage {
missing: false,
sequence: vec![],
color_profile: profile,
layer: None,
}
}
+1
View File
@@ -80,6 +80,7 @@ fn setup(mode: FrameBlend, size: (u32, u32)) -> (Project, ItemId) {
missing: false,
sequence: vec![],
color_profile: None,
layer: None,
};
let fid = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(f));
let mut l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, size, None);
+18
View File
@@ -0,0 +1,18 @@
[package]
name = "effectcraft-svg"
description = "SVG import: parses SVG 1.1 / SVG 2 static vector content (paths, basic shapes, groups, transforms, use, CSS styles, solid and gradient fills/strokes, opacity) and rasterises it at any scale"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
[lints]
workspace = true
[dependencies]
effectcraft-geom = { workspace = true }
effectcraft-raster = { workspace = true }
effectcraft-path = { workspace = true }
kurbo = { workspace = true }
roxmltree = "0.21"
rayon = { workspace = true }
+26
View File
@@ -0,0 +1,26 @@
# effectcraft-svg
SVG import for EffectCraft: `.svg` files import as footage (rasterised at any scale, so
Continuously Rasterize stays sharp) and Layer ▸ Create ▸ Create Shapes from Vector Layer turns
them into editable shape layers.
Implemented from the W3C **Scalable Vector Graphics (SVG) 1.1 (Second Edition)** recommendation
(16 August 2011) and the **SVG 2** candidate recommendation (4 October 2018) for the parts listed
here, plus **CSS Color Module Level 3** for colour keywords and functions:
- document: `svg` (width/height, `viewBox`, `preserveAspectRatio`), nested `svg`, `g`, `defs`,
`use` (x/y, `href` / `xlink:href`), `symbol` (through `use`), `title`/`desc`/`metadata` ignored;
- shapes: `path` (all path data commands incl. elliptical arcs), `rect` (rx/ry), `circle`,
`ellipse`, `line`, `polyline`, `polygon`;
- `transform` lists (matrix, translate, scale, rotate with centre, skewX, skewY);
- painting: `fill`, `fill-rule`, `fill-opacity`, `stroke`, `stroke-width`, `stroke-linecap`,
`stroke-linejoin`, `stroke-miterlimit`, `stroke-dasharray`, `stroke-dashoffset`,
`stroke-opacity`, `opacity`, `display`, `visibility`, `color`/`currentColor`;
- `linearGradient` and `radialGradient` (with `fx`/`fy` focal point), `gradientUnits`,
`gradientTransform`, `spreadMethod`, `href` inheritance of stops and attributes;
- styling through presentation attributes, the `style` attribute and `<style>` sheets with
type, class, id and universal selectors (simple selectors and selector lists).
Not supported (ignored): text, images, filters, clip paths, masks, patterns, markers, animation.
XML parsing uses `roxmltree` (MIT/Apache-2.0). Test fixtures are original work (see
`tests/fixtures/*.attribution`).
+254
View File
@@ -0,0 +1,254 @@
//! CSS colour values (CSS Color Module Level 3): keywords, `#rgb[a]`, `#rrggbb[aa]`, `rgb[a]()`,
//! `hsl[a]()`.
/// The CSS colour keywords (CSS Color Module Level 3, §4.3 extended colour keywords).
const NAMED: &[(&str, u32)] = &[
("aliceblue", 0xF0F8FF),
("antiquewhite", 0xFAEBD7),
("aqua", 0x00FFFF),
("aquamarine", 0x7FFFD4),
("azure", 0xF0FFFF),
("beige", 0xF5F5DC),
("bisque", 0xFFE4C4),
("black", 0x000000),
("blanchedalmond", 0xFFEBCD),
("blue", 0x0000FF),
("blueviolet", 0x8A2BE2),
("brown", 0xA52A2A),
("burlywood", 0xDEB887),
("cadetblue", 0x5F9EA0),
("chartreuse", 0x7FFF00),
("chocolate", 0xD2691E),
("coral", 0xFF7F50),
("cornflowerblue", 0x6495ED),
("cornsilk", 0xFFF8DC),
("crimson", 0xDC143C),
("cyan", 0x00FFFF),
("darkblue", 0x00008B),
("darkcyan", 0x008B8B),
("darkgoldenrod", 0xB8860B),
("darkgray", 0xA9A9A9),
("darkgreen", 0x006400),
("darkgrey", 0xA9A9A9),
("darkkhaki", 0xBDB76B),
("darkmagenta", 0x8B008B),
("darkolivegreen", 0x556B2F),
("darkorange", 0xFF8C00),
("darkorchid", 0x9932CC),
("darkred", 0x8B0000),
("darksalmon", 0xE9967A),
("darkseagreen", 0x8FBC8F),
("darkslateblue", 0x483D8B),
("darkslategray", 0x2F4F4F),
("darkslategrey", 0x2F4F4F),
("darkturquoise", 0x00CED1),
("darkviolet", 0x9400D3),
("deeppink", 0xFF1493),
("deepskyblue", 0x00BFFF),
("dimgray", 0x696969),
("dimgrey", 0x696969),
("dodgerblue", 0x1E90FF),
("firebrick", 0xB22222),
("floralwhite", 0xFFFAF0),
("forestgreen", 0x228B22),
("fuchsia", 0xFF00FF),
("gainsboro", 0xDCDCDC),
("ghostwhite", 0xF8F8FF),
("gold", 0xFFD700),
("goldenrod", 0xDAA520),
("gray", 0x808080),
("grey", 0x808080),
("green", 0x008000),
("greenyellow", 0xADFF2F),
("honeydew", 0xF0FFF0),
("hotpink", 0xFF69B4),
("indianred", 0xCD5C5C),
("indigo", 0x4B0082),
("ivory", 0xFFFFF0),
("khaki", 0xF0E68C),
("lavender", 0xE6E6FA),
("lavenderblush", 0xFFF0F5),
("lawngreen", 0x7CFC00),
("lemonchiffon", 0xFFFACD),
("lightblue", 0xADD8E6),
("lightcoral", 0xF08080),
("lightcyan", 0xE0FFFF),
("lightgoldenrodyellow", 0xFAFAD2),
("lightgray", 0xD3D3D3),
("lightgreen", 0x90EE90),
("lightgrey", 0xD3D3D3),
("lightpink", 0xFFB6C1),
("lightsalmon", 0xFFA07A),
("lightseagreen", 0x20B2AA),
("lightskyblue", 0x87CEFA),
("lightslategray", 0x778899),
("lightslategrey", 0x778899),
("lightsteelblue", 0xB0C4DE),
("lightyellow", 0xFFFFE0),
("lime", 0x00FF00),
("limegreen", 0x32CD32),
("linen", 0xFAF0E6),
("magenta", 0xFF00FF),
("maroon", 0x800000),
("mediumaquamarine", 0x66CDAA),
("mediumblue", 0x0000CD),
("mediumorchid", 0xBA55D3),
("mediumpurple", 0x9370DB),
("mediumseagreen", 0x3CB371),
("mediumslateblue", 0x7B68EE),
("mediumspringgreen", 0x00FA9A),
("mediumturquoise", 0x48D1CC),
("mediumvioletred", 0xC71585),
("midnightblue", 0x191970),
("mintcream", 0xF5FFFA),
("mistyrose", 0xFFE4E1),
("moccasin", 0xFFE4B5),
("navajowhite", 0xFFDEAD),
("navy", 0x000080),
("oldlace", 0xFDF5E6),
("olive", 0x808000),
("olivedrab", 0x6B8E23),
("orange", 0xFFA500),
("orangered", 0xFF4500),
("orchid", 0xDA70D6),
("palegoldenrod", 0xEEE8AA),
("palegreen", 0x98FB98),
("paleturquoise", 0xAFEEEE),
("palevioletred", 0xDB7093),
("papayawhip", 0xFFEFD5),
("peachpuff", 0xFFDAB9),
("peru", 0xCD853F),
("pink", 0xFFC0CB),
("plum", 0xDDA0DD),
("powderblue", 0xB0E0E6),
("purple", 0x800080),
("rebeccapurple", 0x663399),
("red", 0xFF0000),
("rosybrown", 0xBC8F8F),
("royalblue", 0x4169E1),
("saddlebrown", 0x8B4513),
("salmon", 0xFA8072),
("sandybrown", 0xF4A460),
("seagreen", 0x2E8B57),
("seashell", 0xFFF5EE),
("sienna", 0xA0522D),
("silver", 0xC0C0C0),
("skyblue", 0x87CEEB),
("slateblue", 0x6A5ACD),
("slategray", 0x708090),
("slategrey", 0x708090),
("snow", 0xFFFAFA),
("springgreen", 0x00FF7F),
("steelblue", 0x4682B4),
("tan", 0xD2B48C),
("teal", 0x008080),
("thistle", 0xD8BFD8),
("tomato", 0xFF6347),
("turquoise", 0x40E0D0),
("violet", 0xEE82EE),
("wheat", 0xF5DEB3),
("white", 0xFFFFFF),
("whitesmoke", 0xF5F5F5),
("yellow", 0xFFFF00),
("yellowgreen", 0x9ACD32),
];
/// Parse a CSS colour into RGBA 0..1 (`transparent` is RGBA 0). `None` for anything else
/// (`none`, `currentColor` and `url()` are handled by the caller).
pub fn parse_color(s: &str) -> Option<[f64; 4]> {
let s = s.trim();
let lower = s.to_ascii_lowercase();
if lower == "transparent" {
return Some([0.0; 4]);
}
if let Some(h) = lower.strip_prefix('#') {
let d: Vec<u32> = h.chars().map(|c| c.to_digit(16)).collect::<Option<_>>()?;
let v = |i: usize| d[i] as f64 / 15.0;
let w = |i: usize| (d[i] * 16 + d[i + 1]) as f64 / 255.0;
return Some(match d.len() {
3 => [v(0), v(1), v(2), 1.0],
4 => [v(0), v(1), v(2), v(3)],
6 => [w(0), w(2), w(4), 1.0],
8 => [w(0), w(2), w(4), w(6)],
_ => return None,
});
}
if let Some(args) = func(&lower, "rgba").or_else(|| func(&lower, "rgb")) {
let a: Vec<&str> = split_args(args);
if a.len() < 3 {
return None;
}
let comp = |t: &str| -> Option<f64> {
Some(if let Some(p) = t.strip_suffix('%') { p.trim().parse::<f64>().ok()? / 100.0 } else { t.parse::<f64>().ok()? / 255.0 })
};
let alpha = a.get(3).map_or(Some(1.0), |t| alpha(t))?;
return Some([comp(a[0])?.clamp(0.0, 1.0), comp(a[1])?.clamp(0.0, 1.0), comp(a[2])?.clamp(0.0, 1.0), alpha]);
}
if let Some(args) = func(&lower, "hsla").or_else(|| func(&lower, "hsl")) {
let a: Vec<&str> = split_args(args);
if a.len() < 3 {
return None;
}
let h = a[0].trim_end_matches("deg").parse::<f64>().ok()?.rem_euclid(360.0) / 360.0;
let s = a[1].trim_end_matches('%').parse::<f64>().ok()?.clamp(0.0, 100.0) / 100.0;
let l = a[2].trim_end_matches('%').parse::<f64>().ok()?.clamp(0.0, 100.0) / 100.0;
let al = a.get(3).map_or(Some(1.0), |t| alpha(t))?;
let [r, g, b] = hsl(h, s, l);
return Some([r, g, b, al]);
}
NAMED.iter().find(|(n, _)| *n == lower).map(|(_, v)| [((v >> 16) & 255) as f64 / 255.0, ((v >> 8) & 255) as f64 / 255.0, (v & 255) as f64 / 255.0, 1.0])
}
fn func<'a>(s: &'a str, name: &str) -> Option<&'a str> {
s.strip_prefix(name)?.trim_start().strip_prefix('(')?.strip_suffix(')')
}
fn split_args(s: &str) -> Vec<&str> {
s.split(|c: char| c == ',' || c == '/' || c.is_whitespace()).filter(|t| !t.is_empty()).collect()
}
fn alpha(t: &str) -> Option<f64> {
Some(if let Some(p) = t.strip_suffix('%') { p.parse::<f64>().ok()? / 100.0 } else { t.parse::<f64>().ok()? }.clamp(0.0, 1.0))
}
/// CSS HSL → RGB (CSS Color 3 §4.2.4).
fn hsl(h: f64, s: f64, l: f64) -> [f64; 3] {
let m2 = if l <= 0.5 { l * (s + 1.0) } else { l + s - l * s };
let m1 = l * 2.0 - m2;
let hue = |mut h: f64| {
if h < 0.0 {
h += 1.0;
}
if h > 1.0 {
h -= 1.0;
}
if h * 6.0 < 1.0 {
m1 + (m2 - m1) * h * 6.0
} else if h * 2.0 < 1.0 {
m2
} else if h * 3.0 < 2.0 {
m1 + (m2 - m1) * (2.0 / 3.0 - h) * 6.0
} else {
m1
}
};
[hue(h + 1.0 / 3.0), hue(h), hue(h - 1.0 / 3.0)]
}
#[cfg(test)]
mod tests {
use super::parse_color;
#[test]
fn colors() {
assert_eq!(parse_color("#f00"), Some([1.0, 0.0, 0.0, 1.0]));
assert_eq!(parse_color("#00ff0080").map(|c| (c[1], (c[3] * 255.0).round())), Some((1.0, 128.0)));
assert_eq!(parse_color("rgb(255, 0, 0)"), Some([1.0, 0.0, 0.0, 1.0]));
assert_eq!(parse_color("rgba(0,0,255,0.5)"), Some([0.0, 0.0, 1.0, 0.5]));
assert_eq!(parse_color("rgb(100%, 50%, 0%)"), Some([1.0, 0.5, 0.0, 1.0]));
assert_eq!(parse_color("hsl(120, 100%, 50%)"), Some([0.0, 1.0, 0.0, 1.0]));
assert_eq!(parse_color("CornflowerBlue"), Some([100.0 / 255.0, 149.0 / 255.0, 237.0 / 255.0, 1.0]));
assert_eq!(parse_color("transparent"), Some([0.0; 4]));
assert_eq!(parse_color("nonsense"), None);
}
}
+949
View File
@@ -0,0 +1,949 @@
//! SVG import (see README.md for the supported subset and the specifications used).
//!
//! [`parse`] resolves styles, `use` references and gradients into a small render tree
//! ([`Doc`]): groups with transforms and opacity, and shapes with fills and strokes in their own
//! user space. [`rasterize`] draws it at any scale; the engine converts it to shape layers.
mod color;
mod pathdata;
mod raster;
use std::collections::HashMap;
pub use color::parse_color;
pub use kurbo::{Affine, BezPath};
pub use pathdata::parse_path_data;
pub use raster::{rasterize, rasterize_with};
#[derive(Debug, Clone, PartialEq)]
pub enum Error {
Xml(String),
NotSvg,
}
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Error::Xml(e) => write!(f, "invalid SVG: {e}"),
Error::NotSvg => write!(f, "not an SVG document"),
}
}
}
impl std::error::Error for Error {}
/// Quick sniff: an XML document whose root looks like `<svg`.
pub fn looks_like_svg(bytes: &[u8]) -> bool {
let head = &bytes[..bytes.len().min(2048)];
let s = String::from_utf8_lossy(head);
s.contains("<svg") && (s.trim_start().starts_with('<') || s.starts_with('\u{feff}'))
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum FillRule {
#[default]
NonZero,
EvenOdd,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum Cap {
#[default]
Butt,
Round,
Square,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum Join {
#[default]
Miter,
Round,
Bevel,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum Spread {
#[default]
Pad,
Reflect,
Repeat,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum GradientKind {
Linear { x1: f64, y1: f64, x2: f64, y2: f64 },
Radial { cx: f64, cy: f64, r: f64, fx: f64, fy: f64 },
}
#[derive(Clone, Debug, PartialEq)]
pub struct Gradient {
pub kind: GradientKind,
/// (offset 0..1, straight RGBA 0..1), sorted.
pub stops: Vec<(f64, [f64; 4])>,
/// `gradientUnits="objectBoundingBox"` (the default): coordinates are fractions of the
/// shape's bounding box.
pub bbox_units: bool,
pub transform: Affine,
pub spread: Spread,
}
#[derive(Clone, Debug, PartialEq)]
pub enum Paint {
Color([f64; 3]),
Gradient(Gradient),
}
#[derive(Clone, Debug, PartialEq)]
pub struct Stroke {
pub paint: Paint,
pub opacity: f64,
pub width: f64,
pub cap: Cap,
pub join: Join,
pub miter: f64,
/// Dash array and offset.
pub dash: Option<(Vec<f64>, f64)>,
}
/// Shape geometry in its user space (basic shapes are kept so they can become parametric shape
/// layer items).
#[derive(Clone, Debug, PartialEq)]
pub enum Geom {
Rect { x: f64, y: f64, w: f64, h: f64, rx: f64, ry: f64 },
Ellipse { cx: f64, cy: f64, rx: f64, ry: f64 },
Path(BezPath),
}
impl Geom {
pub fn to_path(&self) -> BezPath {
use kurbo::Shape as _;
match self {
Geom::Rect { x, y, w, h, rx, ry } => {
let r = kurbo::Rect::new(*x, *y, x + w, y + h);
if *rx > 0.0 || *ry > 0.0 {
if (rx - ry).abs() < 1e-9 { kurbo::RoundedRect::from_rect(r, *rx).to_path(0.01) } else { elliptic_round_rect(*x, *y, *w, *h, *rx, *ry) }
} else {
r.to_path(0.01)
}
}
Geom::Ellipse { cx, cy, rx, ry } => kurbo::Ellipse::new((*cx, *cy), (*rx, *ry), 0.0).to_path(0.01),
Geom::Path(p) => p.clone(),
}
}
}
fn elliptic_round_rect(x: f64, y: f64, w: f64, h: f64, rx: f64, ry: f64) -> BezPath {
const K: f64 = 0.552_284_749_830_793_6;
let (kx, ky) = (rx * K, ry * K);
let mut p = BezPath::new();
p.move_to((x + rx, y));
p.line_to((x + w - rx, y));
p.curve_to((x + w - rx + kx, y), (x + w, y + ry - ky), (x + w, y + ry));
p.line_to((x + w, y + h - ry));
p.curve_to((x + w, y + h - ry + ky), (x + w - rx + kx, y + h), (x + w - rx, y + h));
p.line_to((x + rx, y + h));
p.curve_to((x + rx - kx, y + h), (x, y + h - ry + ky), (x, y + h - ry));
p.line_to((x, y + ry));
p.curve_to((x, y + ry - ky), (x + rx - kx, y), (x + rx, y));
p.close_path();
p
}
#[derive(Clone, Debug, PartialEq)]
pub struct Shape {
/// Element id, or the element name (`rect`, `path`…).
pub name: String,
pub geom: Geom,
/// User space → parent space.
pub transform: Affine,
pub fill: Option<Paint>,
pub fill_opacity: f64,
pub fill_rule: FillRule,
pub stroke: Option<Stroke>,
pub opacity: f64,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Group {
pub name: String,
pub transform: Affine,
pub opacity: f64,
pub children: Vec<Node>,
}
#[derive(Clone, Debug, PartialEq)]
#[allow(clippy::large_enum_variant)]
pub enum Node {
Group(Group),
Shape(Shape),
}
/// A parsed SVG document. `root.transform` maps the viewBox to `width`×`height` pixels.
#[derive(Clone, Debug, PartialEq)]
pub struct Doc {
pub width: f64,
pub height: f64,
pub root: Group,
/// Elements that were skipped (text, image, filter…), by name.
pub skipped: Vec<String>,
}
impl Doc {
/// Pixel size (rounded up, at least 1×1).
pub fn pixel_size(&self) -> (u32, u32) {
(self.width.ceil().clamp(1.0, 30000.0) as u32, self.height.ceil().clamp(1.0, 30000.0) as u32)
}
/// Every shape with its accumulated transform (to document pixels) and opacity, in paint
/// order.
pub fn flatten(&self) -> Vec<(Affine, f64, &Shape)> {
fn walk<'a>(g: &'a Group, m: Affine, o: f64, out: &mut Vec<(Affine, f64, &'a Shape)>) {
let m = m * g.transform;
let o = o * g.opacity;
for c in &g.children {
match c {
Node::Group(sub) => walk(sub, m, o, out),
Node::Shape(s) => out.push((m * s.transform, o * s.opacity, s)),
}
}
}
let mut out = vec![];
walk(&self.root, Affine::IDENTITY, 1.0, &mut out);
out
}
}
// ---------------------------------------------------------------- styles
/// Inherited style state.
#[derive(Clone, Debug)]
struct Style {
fill: Option<PaintRef>,
fill_opacity: f64,
fill_rule: FillRule,
stroke: Option<PaintRef>,
stroke_opacity: f64,
stroke_width: f64,
cap: Cap,
join: Join,
miter: f64,
dash: Option<Vec<f64>>,
dash_offset: f64,
color: [f64; 4],
visible: bool,
font_size: f64,
}
impl Default for Style {
fn default() -> Self {
Style {
fill: Some(PaintRef::Color([0.0, 0.0, 0.0, 1.0])),
fill_opacity: 1.0,
fill_rule: FillRule::NonZero,
stroke: None,
stroke_opacity: 1.0,
stroke_width: 1.0,
cap: Cap::Butt,
join: Join::Miter,
miter: 4.0,
dash: None,
dash_offset: 0.0,
color: [0.0, 0.0, 0.0, 1.0],
visible: true,
font_size: 16.0,
}
}
}
#[derive(Clone, Debug)]
enum PaintRef {
Color([f64; 4]),
Url(String, Option<[f64; 4]>),
Current,
}
/// A CSS rule: selector (type, class, id; `*`) with its specificity and declarations.
#[derive(Clone, Debug)]
struct Rule {
tag: Option<String>,
classes: Vec<String>,
id: Option<String>,
specificity: u32,
order: usize,
decls: Vec<(String, String)>,
}
fn parse_decls(s: &str) -> Vec<(String, String)> {
s.split(';')
.filter_map(|d| {
let (k, v) = d.split_once(':')?;
let v = v.trim().trim_end_matches("!important").trim();
Some((k.trim().to_ascii_lowercase(), v.to_string()))
})
.filter(|(k, _)| !k.is_empty())
.collect()
}
fn strip_comments(s: &str) -> String {
let mut out = String::new();
let mut rest = s;
while let Some(i) = rest.find("/*") {
out.push_str(&rest[..i]);
match rest[i + 2..].find("*/") {
Some(j) => rest = &rest[i + 2 + j + 2..],
None => return out,
}
}
out.push_str(rest);
out
}
fn parse_css(css: &str, rules: &mut Vec<Rule>) {
let css = strip_comments(css);
let mut rest = css.as_str();
while let Some(open) = rest.find('{') {
let sel = rest[..open].trim();
let Some(close) = rest[open..].find('}') else { break };
let body = &rest[open + 1..open + close];
rest = &rest[open + close + 1..];
if sel.starts_with('@') {
continue;
}
let decls = parse_decls(body);
for one in sel.split(',') {
let one = one.trim();
// Only simple selectors (no combinators / pseudo-classes / attributes).
if one.is_empty() || one.contains([' ', '>', '+', '~', ':', '[']) {
continue;
}
let mut tag = None;
let mut classes = vec![];
let mut id = None;
let mut cur = String::new();
let mut kind = 't';
let flush = |kind: char, cur: &mut String, tag: &mut Option<String>, classes: &mut Vec<String>, id: &mut Option<String>| {
if !cur.is_empty() {
match kind {
'.' => classes.push(std::mem::take(cur)),
'#' => *id = Some(std::mem::take(cur)),
_ => {
if cur != "*" {
*tag = Some(std::mem::take(cur));
}
cur.clear();
}
}
}
};
for c in one.chars() {
if c == '.' || c == '#' {
flush(kind, &mut cur, &mut tag, &mut classes, &mut id);
kind = c;
} else {
cur.push(c);
}
}
flush(kind, &mut cur, &mut tag, &mut classes, &mut id);
let specificity = id.is_some() as u32 * 100 + classes.len() as u32 * 10 + tag.is_some() as u32;
let order = rules.len();
rules.push(Rule { tag, classes, id, specificity, order, decls: decls.clone() });
}
}
}
const PROPS: &[&str] = &[
"fill",
"fill-opacity",
"fill-rule",
"stroke",
"stroke-width",
"stroke-opacity",
"stroke-linecap",
"stroke-linejoin",
"stroke-miterlimit",
"stroke-dasharray",
"stroke-dashoffset",
"opacity",
"display",
"visibility",
"color",
"font-size",
"transform",
"stop-color",
"stop-opacity",
];
struct Parser<'a, 'i> {
ids: HashMap<String, roxmltree::Node<'a, 'i>>,
rules: Vec<Rule>,
vw: f64,
vh: f64,
skipped: Vec<String>,
}
fn is_svg_ns(n: &roxmltree::Node) -> bool {
n.is_element()
}
impl<'a, 'i> Parser<'a, 'i> {
/// The declared properties of an element: presentation attributes, then CSS rules (by
/// specificity), then the `style` attribute.
fn declared(&self, n: roxmltree::Node) -> HashMap<String, String> {
let mut out = HashMap::new();
for a in n.attributes() {
if PROPS.contains(&a.name()) && a.name() != "transform" {
out.insert(a.name().to_string(), a.value().to_string());
}
}
let tag = n.tag_name().name();
let classes: Vec<&str> = n.attribute("class").map(|c| c.split_whitespace().collect()).unwrap_or_default();
let id = n.attribute("id");
let mut matching: Vec<&Rule> = self
.rules
.iter()
.filter(|r| {
r.tag.as_deref().is_none_or(|t| t == tag)
&& r.id.as_deref().is_none_or(|i| Some(i) == id)
&& r.classes.iter().all(|c| classes.contains(&c.as_str()))
})
.collect();
matching.sort_by_key(|r| (r.specificity, r.order));
for r in matching {
for (k, v) in &r.decls {
out.insert(k.clone(), v.clone());
}
}
if let Some(st) = n.attribute("style") {
for (k, v) in parse_decls(st) {
out.insert(k, v);
}
}
out
}
fn length(&self, s: &str, axis: char, font: f64) -> Option<f64> {
let s = s.trim();
let num_end = s.find(|c: char| !(c.is_ascii_digit() || matches!(c, '.' | '-' | '+' | 'e' | 'E'))).unwrap_or(s.len());
// "e" also starts "em"/"ex": back off if the number does not parse.
let (mut num, mut unit) = (&s[..num_end], &s[num_end..]);
if num.parse::<f64>().is_err() && (num.ends_with('e') || num.ends_with('E')) {
num = &s[..num_end - 1];
unit = &s[num_end - 1..];
}
let v: f64 = num.parse().ok()?;
Some(match unit.trim() {
"" | "px" => v,
"pt" => v * 4.0 / 3.0,
"pc" => v * 16.0,
"mm" => v * 96.0 / 25.4,
"cm" => v * 96.0 / 2.54,
"in" => v * 96.0,
"em" => v * font,
"ex" => v * font / 2.0,
"%" => {
let base = match axis {
'x' => self.vw,
'y' => self.vh,
_ => ((self.vw * self.vw + self.vh * self.vh) / 2.0).sqrt(),
};
v / 100.0 * base
}
_ => v,
})
}
fn attr_len(&self, n: roxmltree::Node, name: &str, axis: char, st: &Style) -> f64 {
n.attribute(name).and_then(|v| self.length(v, axis, st.font_size)).unwrap_or(0.0)
}
fn paint_ref(&self, v: &str) -> Option<Option<PaintRef>> {
let v = v.trim();
if v == "none" {
return Some(None);
}
if v == "currentColor" || v == "currentcolor" {
return Some(Some(PaintRef::Current));
}
if let Some(rest) = v.strip_prefix("url(") {
let (url, fallback) = rest.split_once(')').unwrap_or((rest, ""));
let id = url.trim().trim_matches(['"', '\'']).trim_start_matches('#').to_string();
let fb = fallback.trim();
let fb = if fb.is_empty() || fb == "none" { None } else { parse_color(fb) };
return Some(Some(PaintRef::Url(id, fb)));
}
if v == "inherit" {
return None;
}
parse_color(v).map(|c| Some(PaintRef::Color(c)))
}
/// Apply an element's declared inherited properties to `st`. Returns (opacity, display).
fn cascade(&self, n: roxmltree::Node, st: &mut Style) -> (f64, bool) {
let d = self.declared(n);
let num =
|k: &str| d.get(k).and_then(|v| v.trim().trim_end_matches('%').parse::<f64>().ok().map(|x| if v.trim().ends_with('%') { x / 100.0 } else { x }));
if let Some(c) = d.get("color").and_then(|v| parse_color(v)) {
st.color = c;
}
if let Some(v) = d.get("font-size").and_then(|v| self.length(v, 'o', st.font_size)) {
st.font_size = v;
}
if let Some(p) = d.get("fill").and_then(|v| self.paint_ref(v)) {
st.fill = p;
}
if let Some(p) = d.get("stroke").and_then(|v| self.paint_ref(v)) {
st.stroke = p;
}
if let Some(v) = num("fill-opacity") {
st.fill_opacity = v.clamp(0.0, 1.0);
}
if let Some(v) = num("stroke-opacity") {
st.stroke_opacity = v.clamp(0.0, 1.0);
}
if let Some(v) = d.get("stroke-width").and_then(|v| self.length(v, 'o', st.font_size)) {
st.stroke_width = v.max(0.0);
}
if let Some(v) = num("stroke-miterlimit") {
st.miter = v.max(1.0);
}
if let Some(v) = d.get("fill-rule") {
st.fill_rule = if v.trim() == "evenodd" { FillRule::EvenOdd } else { FillRule::NonZero };
}
if let Some(v) = d.get("stroke-linecap") {
st.cap = match v.trim() {
"round" => Cap::Round,
"square" => Cap::Square,
_ => Cap::Butt,
};
}
if let Some(v) = d.get("stroke-linejoin") {
st.join = match v.trim() {
"round" => Join::Round,
"bevel" => Join::Bevel,
_ => Join::Miter,
};
}
if let Some(v) = d.get("stroke-dasharray") {
st.dash = if v.trim() == "none" {
None
} else {
let a: Vec<f64> =
v.split(|c: char| c == ',' || c.is_whitespace()).filter(|t| !t.is_empty()).filter_map(|t| self.length(t, 'o', st.font_size)).collect();
if a.iter().all(|x| *x >= 0.0) && a.iter().any(|x| *x > 0.0) { Some(a) } else { None }
};
}
if let Some(v) = d.get("stroke-dashoffset").and_then(|v| self.length(v, 'o', st.font_size)) {
st.dash_offset = v;
}
if let Some(v) = d.get("visibility") {
st.visible = !matches!(v.trim(), "hidden" | "collapse");
}
let display = d.get("display").is_none_or(|v| v.trim() != "none");
(num("opacity").unwrap_or(1.0).clamp(0.0, 1.0), display)
}
fn resolve_paint(&self, p: &PaintRef, st: &Style) -> Option<(Paint, f64)> {
match p {
PaintRef::Color(c) => Some((Paint::Color([c[0], c[1], c[2]]), c[3])),
PaintRef::Current => Some((Paint::Color([st.color[0], st.color[1], st.color[2]]), st.color[3])),
PaintRef::Url(id, fb) => match self.gradient(id, 0) {
Some(g) if g.stops.len() == 1 => {
let c = g.stops[0].1;
Some((Paint::Color([c[0], c[1], c[2]]), c[3]))
}
Some(g) if !g.stops.is_empty() => Some((Paint::Gradient(g), 1.0)),
Some(_) => None,
None => fb.map(|c| (Paint::Color([c[0], c[1], c[2]]), c[3])),
},
}
}
fn href(&self, n: roxmltree::Node<'a, 'i>) -> Option<roxmltree::Node<'a, 'i>> {
let h = n.attributes().find(|a| a.name() == "href").map(|a| a.value())?;
self.ids.get(h.trim().trim_start_matches('#')).copied()
}
/// A gradient by id, with `href` inheritance of attributes and stops.
fn gradient(&self, id: &str, depth: usize) -> Option<Gradient> {
let n = *self.ids.get(id)?;
let tag = n.tag_name().name();
if !matches!(tag, "linearGradient" | "radialGradient") || depth > 16 {
return None;
}
// Chain: this element, then what it references.
let mut chain = vec![n];
let mut cur = n;
while let Some(next) = self.href(cur) {
if chain.len() > 16 || chain.contains(&next) {
break;
}
chain.push(next);
cur = next;
}
let attr = |name: &str| chain.iter().find_map(|e| e.attribute(name));
let bbox_units = attr("gradientUnits") != Some("userSpaceOnUse");
let st = Style::default();
let len = |name: &str, axis: char, def: &str| -> f64 {
let v = attr(name).unwrap_or(def);
if bbox_units {
let v = v.trim();
if let Some(p) = v.strip_suffix('%') { p.parse::<f64>().unwrap_or(0.0) / 100.0 } else { v.parse::<f64>().unwrap_or(0.0) }
} else {
self.length(v, axis, st.font_size).unwrap_or(0.0)
}
};
let kind = if tag == "linearGradient" {
GradientKind::Linear { x1: len("x1", 'x', "0%"), y1: len("y1", 'y', "0%"), x2: len("x2", 'x', "100%"), y2: len("y2", 'y', "0%") }
} else {
let cx = len("cx", 'x', "50%");
let cy = len("cy", 'y', "50%");
let r = len("r", 'o', "50%");
let fx = if attr("fx").is_some() { len("fx", 'x', "50%") } else { cx };
let fy = if attr("fy").is_some() { len("fy", 'y', "50%") } else { cy };
GradientKind::Radial { cx, cy, r, fx, fy }
};
let transform = attr("gradientTransform").map(parse_transform).unwrap_or(Affine::IDENTITY);
let spread = match attr("spreadMethod") {
Some("reflect") => Spread::Reflect,
Some("repeat") => Spread::Repeat,
_ => Spread::Pad,
};
// Stops: from the first element in the chain that has any.
let mut stops = vec![];
if let Some(src) = chain.iter().find(|e| e.children().any(|c| c.is_element() && c.tag_name().name() == "stop")) {
let mut last = 0.0f64;
for s in src.children().filter(|c| c.is_element() && c.tag_name().name() == "stop") {
let d = self.declared(s);
let off = s
.attribute("offset")
.map(|o| {
let o = o.trim();
if let Some(p) = o.strip_suffix('%') { p.parse::<f64>().unwrap_or(0.0) / 100.0 } else { o.parse::<f64>().unwrap_or(0.0) }
})
.unwrap_or(0.0)
.clamp(0.0, 1.0)
.max(last);
last = off;
let c = d
.get("stop-color")
.and_then(|c| if c.trim() == "currentColor" { d.get("color").and_then(|x| parse_color(x)) } else { parse_color(c) })
.unwrap_or([0.0, 0.0, 0.0, 1.0]);
let o = d.get("stop-opacity").and_then(|v| v.trim().parse::<f64>().ok()).unwrap_or(1.0).clamp(0.0, 1.0);
stops.push((off, [c[0], c[1], c[2], c[3] * o]));
}
}
Some(Gradient { kind, stops, bbox_units, transform, spread })
}
fn element_name(n: roxmltree::Node) -> String {
n.attribute("id").map(str::to_string).unwrap_or_else(|| n.tag_name().name().to_string())
}
/// Convert an element (and its subtree) into nodes appended to `out`.
fn node(&mut self, n: roxmltree::Node<'a, 'i>, parent: &Style, out: &mut Vec<Node>, depth: usize) {
if depth > 64 || !is_svg_ns(&n) {
return;
}
let tag = n.tag_name().name();
match tag {
"defs" | "title" | "desc" | "metadata" | "style" | "linearGradient" | "radialGradient" | "symbol" | "clipPath" | "mask" | "pattern" | "marker"
| "script" => return,
"text" | "image" | "filter" | "foreignObject" | "switch" => {
self.skipped.push(tag.to_string());
return;
}
_ => {}
}
let mut st = parent.clone();
let (opacity, display) = self.cascade(n, &mut st);
if !display {
return;
}
let mut transform = n.attribute("transform").map(parse_transform).unwrap_or(Affine::IDENTITY);
match tag {
"g" | "a" => {
let mut children = vec![];
for c in n.children() {
self.node(c, &st, &mut children, depth + 1);
}
out.push(Node::Group(Group { name: Self::element_name(n), transform, opacity, children }));
}
"svg" => {
// Nested viewport.
let x = self.attr_len(n, "x", 'x', &st);
let y = self.attr_len(n, "y", 'y', &st);
let w = n.attribute("width").and_then(|v| self.length(v, 'x', st.font_size)).unwrap_or(self.vw);
let h = n.attribute("height").and_then(|v| self.length(v, 'y', st.font_size)).unwrap_or(self.vh);
let vb = viewbox_transform(n, w, h);
let mut children = vec![];
let saved = (self.vw, self.vh);
if let Some((_, vw, vh)) = viewbox(n) {
self.vw = vw;
self.vh = vh;
}
for c in n.children() {
self.node(c, &st, &mut children, depth + 1);
}
(self.vw, self.vh) = saved;
out.push(Node::Group(Group { name: Self::element_name(n), transform: Affine::translate((x, y)) * vb, opacity, children }));
}
"use" => {
let Some(target) = self.href(n) else { return };
if target == n || n.ancestors().any(|a| a == target) {
return;
}
let x = self.attr_len(n, "x", 'x', &st);
let y = self.attr_len(n, "y", 'y', &st);
transform *= Affine::translate((x, y));
let mut children = vec![];
if target.tag_name().name() == "symbol" {
let w = n.attribute("width").and_then(|v| self.length(v, 'x', st.font_size)).unwrap_or(self.vw);
let h = n.attribute("height").and_then(|v| self.length(v, 'y', st.font_size)).unwrap_or(self.vh);
let vb = viewbox_transform(target, w, h);
let mut inner = vec![];
let mut sst = st.clone();
let (o, _) = self.cascade(target, &mut sst);
for c in target.children() {
self.node(c, &sst, &mut inner, depth + 1);
}
children.push(Node::Group(Group { name: Self::element_name(target), transform: vb, opacity: o, children: inner }));
} else {
self.node(target, &st, &mut children, depth + 1);
}
out.push(Node::Group(Group { name: Self::element_name(n), transform, opacity, children }));
}
"path" | "rect" | "circle" | "ellipse" | "line" | "polyline" | "polygon" => {
if !st.visible {
return;
}
let Some(geom) = self.geometry(n, &st) else { return };
let fill = if matches!(tag, "line") { None } else { st.fill.as_ref().and_then(|p| self.resolve_paint(p, &st)) };
let (fill, fill_alpha) = match fill {
Some((p, a)) => (Some(p), a),
None => (None, 1.0),
};
let stroke = st.stroke.as_ref().and_then(|p| self.resolve_paint(p, &st)).filter(|_| st.stroke_width > 0.0).map(|(paint, a)| Stroke {
paint,
opacity: st.stroke_opacity * a,
width: st.stroke_width,
cap: st.cap,
join: st.join,
miter: st.miter,
dash: st.dash.clone().map(|d| (d, st.dash_offset)),
});
out.push(Node::Shape(Shape {
name: Self::element_name(n),
geom,
transform,
fill,
fill_opacity: st.fill_opacity * fill_alpha,
fill_rule: st.fill_rule,
stroke,
opacity,
}));
}
_ => {
// Unknown container-ish elements: render their children.
if n.has_children() {
let mut children = vec![];
for c in n.children() {
self.node(c, &st, &mut children, depth + 1);
}
if !children.is_empty() {
out.push(Node::Group(Group { name: Self::element_name(n), transform, opacity, children }));
}
}
}
}
}
fn geometry(&self, n: roxmltree::Node, st: &Style) -> Option<Geom> {
let l = |name: &str, axis: char| self.attr_len(n, name, axis, st);
Some(match n.tag_name().name() {
"path" => {
let p = parse_path_data(n.attribute("d")?);
if p.elements().is_empty() {
return None;
}
Geom::Path(p)
}
"rect" => {
let (w, h) = (l("width", 'x'), l("height", 'y'));
if w <= 0.0 || h <= 0.0 {
return None;
}
let rx_a = n.attribute("rx").and_then(|v| self.length(v, 'x', st.font_size));
let ry_a = n.attribute("ry").and_then(|v| self.length(v, 'y', st.font_size));
let (rx, ry) = match (rx_a, ry_a) {
(Some(x), Some(y)) => (x, y),
(Some(x), None) => (x, x),
(None, Some(y)) => (y, y),
(None, None) => (0.0, 0.0),
};
Geom::Rect { x: l("x", 'x'), y: l("y", 'y'), w, h, rx: rx.clamp(0.0, w / 2.0), ry: ry.clamp(0.0, h / 2.0) }
}
"circle" => {
let r = l("r", 'o');
if r <= 0.0 {
return None;
}
Geom::Ellipse { cx: l("cx", 'x'), cy: l("cy", 'y'), rx: r, ry: r }
}
"ellipse" => {
let (rx, ry) = (l("rx", 'x'), l("ry", 'y'));
if rx <= 0.0 || ry <= 0.0 {
return None;
}
Geom::Ellipse { cx: l("cx", 'x'), cy: l("cy", 'y'), rx, ry }
}
"line" => {
let mut p = BezPath::new();
p.move_to((l("x1", 'x'), l("y1", 'y')));
p.line_to((l("x2", 'x'), l("y2", 'y')));
Geom::Path(p)
}
"polyline" | "polygon" => {
let nums = pathdata::numbers(n.attribute("points")?);
if nums.len() < 4 {
return None;
}
let mut p = BezPath::new();
p.move_to((nums[0], nums[1]));
for c in nums[2..].chunks_exact(2) {
p.line_to((c[0], c[1]));
}
if n.tag_name().name() == "polygon" {
p.close_path();
}
Geom::Path(p)
}
_ => return None,
})
}
}
fn viewbox(n: roxmltree::Node) -> Option<([f64; 2], f64, f64)> {
let v = pathdata::numbers(n.attribute("viewBox")?);
(v.len() == 4 && v[2] > 0.0 && v[3] > 0.0).then(|| ([v[0], v[1]], v[2], v[3]))
}
/// The viewBox → viewport transform (`preserveAspectRatio`, default `xMidYMid meet`).
fn viewbox_transform(n: roxmltree::Node, w: f64, h: f64) -> Affine {
let Some(([x, y], vw, vh)) = viewbox(n) else { return Affine::IDENTITY };
let par = n.attribute("preserveAspectRatio").unwrap_or("xMidYMid meet");
let mut parts = par.split_whitespace();
let mut align = parts.next().unwrap_or("xMidYMid");
if align == "defer" {
align = parts.next().unwrap_or("xMidYMid");
}
let slice = parts.next() == Some("slice");
let (sx, sy) = (w / vw, h / vh);
if align == "none" {
return Affine::scale_non_uniform(sx, sy) * Affine::translate((-x, -y));
}
let s = if slice { sx.max(sy) } else { sx.min(sy) };
let fx = if align.contains("xMid") {
0.5
} else if align.contains("xMax") {
1.0
} else {
0.0
};
let fy = if align.contains("YMid") {
0.5
} else if align.contains("YMax") {
1.0
} else {
0.0
};
let tx = (w - vw * s) * fx;
let ty = (h - vh * s) * fy;
Affine::translate((tx, ty)) * Affine::scale(s) * Affine::translate((-x, -y))
}
/// Parse a `transform` list.
pub fn parse_transform(s: &str) -> Affine {
let mut m = Affine::IDENTITY;
let mut rest = s;
while let Some(open) = rest.find('(') {
let name = rest[..open].trim().trim_start_matches(',').trim();
let Some(close) = rest[open..].find(')') else { break };
let a = pathdata::numbers(&rest[open + 1..open + close]);
rest = &rest[open + close + 1..];
let t = match (name, a.len()) {
("matrix", 6) => Affine::new([a[0], a[1], a[2], a[3], a[4], a[5]]),
("translate", 1) => Affine::translate((a[0], 0.0)),
("translate", 2) => Affine::translate((a[0], a[1])),
("scale", 1) => Affine::scale(a[0]),
("scale", 2) => Affine::scale_non_uniform(a[0], a[1]),
("rotate", 1) => Affine::rotate(a[0].to_radians()),
("rotate", 3) => Affine::translate((a[1], a[2])) * Affine::rotate(a[0].to_radians()) * Affine::translate((-a[1], -a[2])),
("skewX", 1) => Affine::new([1.0, 0.0, a[0].to_radians().tan(), 1.0, 0.0, 0.0]),
("skewY", 1) => Affine::new([1.0, a[0].to_radians().tan(), 0.0, 1.0, 0.0, 0.0]),
_ => Affine::IDENTITY,
};
m *= t;
}
m
}
/// Parse an SVG document.
pub fn parse(bytes: &[u8]) -> Result<Doc, Error> {
let text = String::from_utf8_lossy(bytes);
let text = text.trim_start_matches('\u{feff}');
let opts = roxmltree::ParsingOptions { allow_dtd: true, ..Default::default() };
let xml = roxmltree::Document::parse_with_options(text, opts).map_err(|e| Error::Xml(e.to_string()))?;
let root = xml.root_element();
if root.tag_name().name() != "svg" {
return Err(Error::NotSvg);
}
let mut ids = HashMap::new();
let mut rules = vec![];
for n in root.descendants().filter(|n| n.is_element()) {
if let Some(id) = n.attribute("id") {
ids.entry(id.to_string()).or_insert(n);
}
if n.tag_name().name() == "style" {
let css: String = n.children().filter_map(|c| c.text()).collect();
parse_css(&css, &mut rules);
}
}
let vb = viewbox(root);
let mut p = Parser { ids, rules, vw: vb.map_or(300.0, |v| v.1), vh: vb.map_or(150.0, |v| v.2), skipped: vec![] };
let st = Style::default();
let pct_or = |name: &str, axis: char, def: f64| -> f64 {
match root.attribute(name) {
Some(v) if v.trim().ends_with('%') => {
let f = v.trim().trim_end_matches('%').parse::<f64>().unwrap_or(100.0) / 100.0;
def * f
}
Some(v) => p.length(v, axis, st.font_size).filter(|x| *x > 0.0).unwrap_or(def),
None => def,
}
};
// Without width/height the viewBox size is used (or 300×150).
let (dw, dh) = match vb {
Some((_, w, h)) => {
let w2 = pct_or("width", 'x', w);
let h2 = match (root.attribute("width"), root.attribute("height")) {
// Only width given: keep the aspect ratio.
(Some(_), None) => w2 * h / w,
_ => pct_or("height", 'y', h),
};
let w2 = if root.attribute("width").is_none() && root.attribute("height").is_some() { h2 * w / h } else { w2 };
(w2, h2)
}
None => (pct_or("width", 'x', 300.0), pct_or("height", 'y', 150.0)),
};
let mut rst = st.clone();
let (opacity, _) = p.cascade(root, &mut rst);
let mut children = vec![];
for c in root.children() {
p.node(c, &rst, &mut children, 1);
}
let transform = viewbox_transform(root, dw, dh);
Ok(Doc { width: dw, height: dh, root: Group { name: "svg".into(), transform, opacity, children }, skipped: std::mem::take(&mut p.skipped) })
}
#[cfg(test)]
mod tests;
+293
View File
@@ -0,0 +1,293 @@
//! SVG path data (SVG 1.1 §8.3 / SVG 2 §9.3): every command, implicit repeats, relative
//! coordinates, smooth curves and elliptical arcs (converted to cubics through kurbo's `SvgArc`).
//! Parsing stops at the first error, keeping what was read (as the specification asks).
use kurbo::{BezPath, Point, SvgArc, Vec2};
/// Tokenizer over numbers and command letters.
struct Lex<'a> {
b: &'a [u8],
i: usize,
}
impl Lex<'_> {
fn skip_sep(&mut self) {
while self.i < self.b.len() && (self.b[self.i].is_ascii_whitespace() || self.b[self.i] == b',') {
self.i += 1;
}
}
fn command(&mut self) -> Option<u8> {
self.skip_sep();
let c = *self.b.get(self.i)?;
if c.is_ascii_alphabetic() && c != b'e' && c != b'E' {
self.i += 1;
Some(c)
} else {
None
}
}
fn at_number(&mut self) -> bool {
self.skip_sep();
self.b.get(self.i).is_some_and(|c| c.is_ascii_digit() || matches!(c, b'-' | b'+' | b'.'))
}
fn number(&mut self) -> Option<f64> {
self.skip_sep();
let start = self.i;
let b = self.b;
let mut i = self.i;
if i < b.len() && matches!(b[i], b'-' | b'+') {
i += 1;
}
let mut digits = false;
while i < b.len() && b[i].is_ascii_digit() {
i += 1;
digits = true;
}
if i < b.len() && b[i] == b'.' {
i += 1;
while i < b.len() && b[i].is_ascii_digit() {
i += 1;
digits = true;
}
}
if !digits {
return None;
}
if i < b.len() && matches!(b[i], b'e' | b'E') {
let mut j = i + 1;
if j < b.len() && matches!(b[j], b'-' | b'+') {
j += 1;
}
if j < b.len() && b[j].is_ascii_digit() {
while j < b.len() && b[j].is_ascii_digit() {
j += 1;
}
i = j;
}
}
self.i = i;
std::str::from_utf8(&b[start..i]).ok()?.parse().ok()
}
/// Arc flags may be written without separators ("a1 1 0 00 1 1").
fn flag(&mut self) -> Option<bool> {
self.skip_sep();
let c = *self.b.get(self.i)?;
self.i += 1;
match c {
b'0' => Some(false),
b'1' => Some(true),
_ => None,
}
}
}
/// All numbers in a list (`points`, `viewBox`, transform arguments).
pub fn numbers(s: &str) -> Vec<f64> {
let mut l = Lex { b: s.as_bytes(), i: 0 };
let mut v = vec![];
while l.at_number() {
match l.number() {
Some(x) => v.push(x),
None => break,
}
}
v
}
/// Parse path data into a kurbo path.
pub fn parse_path_data(d: &str) -> BezPath {
let mut l = Lex { b: d.as_bytes(), i: 0 };
let mut p = BezPath::new();
let mut cur = Point::ZERO;
let mut start = Point::ZERO;
// Last control point of the previous C/S (cubic) or Q/T (quad) command.
let mut last_cubic: Option<Point> = None;
let mut last_quad: Option<Point> = None;
let mut cmd: Option<u8> = None;
let mut open = false;
loop {
let c = match l.command() {
Some(c) => c,
None => match cmd {
// Implicit repeat (a moveto repeats as lineto).
Some(c) if l.at_number() => match c {
b'M' => b'L',
b'm' => b'l',
other => other,
},
_ => break,
},
};
let rel = c.is_ascii_lowercase();
let base = if rel { cur.to_vec2() } else { Vec2::ZERO };
let pt = |l: &mut Lex| -> Option<Point> { Some(Point::new(l.number()?, l.number()?) + base) };
let ok = (|| -> Option<()> {
match c.to_ascii_uppercase() {
b'M' => {
cur = pt(&mut l)?;
p.move_to(cur);
start = cur;
open = true;
last_cubic = None;
last_quad = None;
}
b'Z' => {
if open {
p.close_path();
}
cur = start;
last_cubic = None;
last_quad = None;
}
_ if !open => {
// A segment without a preceding moveto starts at the current point.
p.move_to(cur);
open = true;
return segment(&mut l, c, base, &mut p, &mut cur, &mut last_cubic, &mut last_quad);
}
_ => return segment(&mut l, c, base, &mut p, &mut cur, &mut last_cubic, &mut last_quad),
}
Some(())
})();
if ok.is_none() {
break;
}
cmd = Some(c);
if c.eq_ignore_ascii_case(&b'z') {
// After Z a new segment starts from the subpath start (an unused move is dropped
// below).
p.move_to(start);
open = true;
}
}
strip_trailing_moves(p)
}
fn segment(l: &mut Lex, c: u8, base: Vec2, p: &mut BezPath, cur: &mut Point, last_cubic: &mut Option<Point>, last_quad: &mut Option<Point>) -> Option<()> {
let num = |l: &mut Lex| l.number();
let pt = |l: &mut Lex| -> Option<Point> { Some(Point::new(l.number()?, l.number()?) + base) };
match c.to_ascii_uppercase() {
b'L' => {
*cur = pt(l)?;
p.line_to(*cur);
*last_cubic = None;
*last_quad = None;
}
b'H' => {
let x = num(l)? + base.x;
*cur = Point::new(x, cur.y);
p.line_to(*cur);
*last_cubic = None;
*last_quad = None;
}
b'V' => {
let y = num(l)? + base.y;
*cur = Point::new(cur.x, y);
p.line_to(*cur);
*last_cubic = None;
*last_quad = None;
}
b'C' => {
let (c1, c2, e) = (pt(l)?, pt(l)?, pt(l)?);
p.curve_to(c1, c2, e);
*last_cubic = Some(c2);
*last_quad = None;
*cur = e;
}
b'S' => {
let c1 = last_cubic.map_or(*cur, |c| *cur + (*cur - c));
let (c2, e) = (pt(l)?, pt(l)?);
p.curve_to(c1, c2, e);
*last_cubic = Some(c2);
*last_quad = None;
*cur = e;
}
b'Q' => {
let (q, e) = (pt(l)?, pt(l)?);
p.quad_to(q, e);
*last_quad = Some(q);
*last_cubic = None;
*cur = e;
}
b'T' => {
let q = last_quad.map_or(*cur, |c| *cur + (*cur - c));
let e = pt(l)?;
p.quad_to(q, e);
*last_quad = Some(q);
*last_cubic = None;
*cur = e;
}
b'A' => {
let rx = num(l)?.abs();
let ry = num(l)?.abs();
let rot = num(l)?;
let large = l.flag()?;
let sweep = l.flag()?;
let e = pt(l)?;
let arc = SvgArc { from: *cur, to: e, radii: Vec2::new(rx, ry), x_rotation: rot.to_radians(), large_arc: large, sweep };
match kurbo::Arc::from_svg_arc(&arc) {
Some(a) => a.to_cubic_beziers(0.1, |c1, c2, end| p.curve_to(c1, c2, end)),
None => p.line_to(e),
}
*cur = e;
*last_cubic = None;
*last_quad = None;
}
_ => return None,
}
Some(())
}
/// Drop move-tos that start no segment (from closepath bookkeeping).
fn strip_trailing_moves(p: BezPath) -> BezPath {
use kurbo::PathEl;
let els = p.elements();
let mut out = BezPath::new();
for (i, e) in els.iter().enumerate() {
if let PathEl::MoveTo(_) = e {
let next_is_seg = els.get(i + 1).is_some_and(|n| !matches!(n, PathEl::MoveTo(_)));
if !next_is_seg {
continue;
}
}
out.push(*e);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
use kurbo::PathEl;
#[test]
fn commands() {
let p = parse_path_data("M10,10 L20,10 h10 v10 H10 z m5 5 l1-1.5.5.5e1 C 0 0 1 1 2 2 s3 3 4 4 Q1 1 2 2 t3 3 A5 5 0 0 1 20 20 a5 5 0 1010 0");
let els = p.elements();
assert_eq!(els[0], PathEl::MoveTo(Point::new(10.0, 10.0)));
assert_eq!(els[2], PathEl::LineTo(Point::new(30.0, 10.0)));
assert_eq!(els[3], PathEl::LineTo(Point::new(30.0, 20.0)));
assert!(els.iter().any(|e| matches!(e, PathEl::ClosePath)));
// Relative move after Z is relative to the subpath start (10,10).
assert!(els.contains(&PathEl::MoveTo(Point::new(15.0, 15.0))));
// "l1-1.5.5.5e1": two implicit linetos (1,-1.5) then (.5,5).
assert!(els.contains(&PathEl::LineTo(Point::new(16.0, 13.5))));
assert!(els.contains(&PathEl::LineTo(Point::new(16.5, 18.5))));
assert!(els.iter().filter(|e| matches!(e, PathEl::CurveTo(..))).count() >= 4);
}
#[test]
fn errors_keep_prefix() {
let p = parse_path_data("M0 0 L10 0 L10 x10 L0 10");
assert_eq!(p.elements().len(), 2);
assert!(parse_path_data("garbage").elements().is_empty());
assert!(parse_path_data("").elements().is_empty());
}
#[test]
fn number_lists() {
assert_eq!(numbers("0 0,100 50"), vec![0.0, 0.0, 100.0, 50.0]);
assert_eq!(numbers("-1-2.5.5"), vec![-1.0, -2.5, 0.5]);
assert_eq!(numbers("1e2,2E-1"), vec![100.0, 0.2]);
}
}
+220
View File
@@ -0,0 +1,220 @@
//! Rasterisation of a parsed SVG into a premultiplied `f32` image (anti-aliased coverage from
//! `effectcraft-path`; group opacity composites the group as a whole, as SVG specifies).
use effectcraft_geom::Mat3;
use effectcraft_raster::{Image, Mask};
use kurbo::{Affine, Point, Shape as _};
use rayon::prelude::*;
use crate::{Doc, FillRule, Gradient, GradientKind, Group, Node, Paint, Shape, Spread};
fn mat3(a: Affine) -> Mat3 {
let c = a.as_coeffs();
Mat3([[c[0], c[2], c[4]], [c[1], c[3], c[5]], [0.0, 0.0, 1.0]])
}
/// Rasterise `doc` into a `w`×`h` image, scaled by `scale` (1 = the document's pixel size).
pub fn rasterize(doc: &Doc, w: u32, h: u32, scale: f64) -> Image {
rasterize_with(doc, w, h, Affine::scale(scale))
}
/// Rasterise `doc` with `m` mapping document pixels to image pixels.
pub fn rasterize_with(doc: &Doc, w: u32, h: u32, m: Affine) -> Image {
let mut img = Image::new(w, h);
draw_group(&doc.root, m, 1.0, &mut img);
img
}
fn draw_group(g: &Group, m: Affine, opacity: f64, dst: &mut Image) {
let m = m * g.transform;
let o = opacity * g.opacity;
if o <= 0.0 {
return;
}
if g.opacity < 1.0 && g.children.len() > 1 {
let mut tmp = Image::new(dst.width, dst.height);
for c in &g.children {
draw_node(c, m, 1.0, &mut tmp);
}
over(dst, &tmp, o as f32);
} else {
for c in &g.children {
draw_node(c, m, o, dst);
}
}
}
fn draw_node(n: &Node, m: Affine, opacity: f64, dst: &mut Image) {
match n {
Node::Group(g) => draw_group(g, m, opacity, dst),
Node::Shape(s) => {
if s.opacity < 1.0 && s.fill.is_some() && s.stroke.is_some() {
let mut tmp = Image::new(dst.width, dst.height);
draw_shape(s, m, 1.0, &mut tmp);
over(dst, &tmp, (opacity * s.opacity) as f32);
} else {
draw_shape(s, m, opacity * s.opacity, dst);
}
}
}
}
/// `dst = src·k over dst` (premultiplied).
fn over(dst: &mut Image, src: &Image, k: f32) {
dst.data.par_iter_mut().zip(src.data.par_iter()).for_each(|(d, s)| {
let a = s[3] * k;
if a > 0.0 {
for c in 0..3 {
d[c] = s[c] * k + d[c] * (1.0 - a);
}
d[3] = a + d[3] * (1.0 - a);
}
});
}
fn draw_shape(s: &Shape, m: Affine, opacity: f64, dst: &mut Image) {
let m = m * s.transform;
let path = s.geom.to_path();
let bbox = path.bounding_box();
let (w, h) = (dst.width, dst.height);
if let Some(fill) = &s.fill {
let rule = match s.fill_rule {
FillRule::NonZero => effectcraft_path::FillRule::NonZero,
FillRule::EvenOdd => effectcraft_path::FillRule::EvenOdd,
};
let cov = effectcraft_path::fill_coverage(std::slice::from_ref(&path), &mat3(m), w, h, rule);
paint(dst, &cov, fill, (opacity * s.fill_opacity) as f32, m, bbox);
}
if let Some(st) = &s.stroke {
let style = effectcraft_path::StrokeStyle {
width: st.width,
cap: match st.cap {
crate::Cap::Butt => effectcraft_path::Cap::Butt,
crate::Cap::Round => effectcraft_path::Cap::Round,
crate::Cap::Square => effectcraft_path::Cap::Square,
},
join: match st.join {
crate::Join::Miter => effectcraft_path::Join::Miter,
crate::Join::Round => effectcraft_path::Join::Round,
crate::Join::Bevel => effectcraft_path::Join::Bevel,
},
miter: st.miter,
dash: st.dash.clone(),
};
let cov = effectcraft_path::stroke_coverage(std::slice::from_ref(&path), &style, &mat3(m), w, h);
paint(dst, &cov, &st.paint, (opacity * st.opacity) as f32, m, bbox);
}
}
/// Composite paint through coverage.
fn paint(dst: &mut Image, cov: &Mask, p: &Paint, opacity: f32, m: Affine, bbox: kurbo::Rect) {
let w = dst.width as usize;
if w == 0 {
return;
}
let shader = match p {
Paint::Color(c) => Shader::Solid([c[0] as f32, c[1] as f32, c[2] as f32, 1.0]),
Paint::Gradient(g) => {
let unit = if g.bbox_units { Affine::new([bbox.width(), 0.0, 0.0, bbox.height(), bbox.x0, bbox.y0]) } else { Affine::IDENTITY };
let to_grad = (m * unit * g.transform).inverse();
if !to_grad.as_coeffs().iter().all(|v| v.is_finite()) {
return;
}
Shader::Gradient(g, to_grad)
}
};
dst.data.par_chunks_mut(w).enumerate().for_each(|(y, row)| {
for (x, d) in row.iter_mut().enumerate() {
let c = cov.data[y * w + x];
if c <= 0.0 {
continue;
}
let col = shader.at(x as f64 + 0.5, y as f64 + 0.5);
let a = col[3] * c * opacity;
if a <= 0.0 {
continue;
}
for k in 0..3 {
d[k] = col[k] * a + d[k] * (1.0 - a);
}
d[3] = a + d[3] * (1.0 - a);
}
});
}
enum Shader<'a> {
Solid([f32; 4]),
Gradient(&'a Gradient, Affine),
}
impl Shader<'_> {
/// Straight RGBA at a pixel centre.
fn at(&self, x: f64, y: f64) -> [f32; 4] {
match self {
Shader::Solid(c) => *c,
Shader::Gradient(g, inv) => {
let p = *inv * Point::new(x, y);
let t = match g.kind {
GradientKind::Linear { x1, y1, x2, y2 } => {
let (dx, dy) = (x2 - x1, y2 - y1);
let l2 = dx * dx + dy * dy;
if l2 <= 0.0 { 1.0 } else { ((p.x - x1) * dx + (p.y - y1) * dy) / l2 }
}
GradientKind::Radial { cx, cy, r, fx, fy } => radial_t(p, cx, cy, r, fx, fy),
};
eval_stops(&g.stops, spread(t, g.spread))
}
}
}
}
/// Radial gradient parameter with a focal point: the circle through `p` from the focus.
pub(crate) fn radial_t(p: Point, cx: f64, cy: f64, r: f64, fx: f64, fy: f64) -> f64 {
if r <= 0.0 {
return 1.0;
}
let (dx, dy) = (p.x - fx, p.y - fy);
let (ex, ey) = (fx - cx, fy - cy);
if ex * ex + ey * ey < 1e-12 {
return (dx * dx + dy * dy).sqrt() / r;
}
// Solve |e + u·d|² = r² for u > 0; t = 1/u.
let a = dx * dx + dy * dy;
if a < 1e-18 {
return 0.0;
}
let b = ex * dx + ey * dy;
let c = ex * ex + ey * ey - r * r;
let disc = b * b - a * c;
if disc < 0.0 {
return 1.0;
}
let u = (-b + disc.sqrt()) / a;
if u <= 0.0 { 1.0 } else { 1.0 / u }
}
fn spread(t: f64, s: Spread) -> f64 {
match s {
Spread::Pad => t.clamp(0.0, 1.0),
Spread::Repeat => t.rem_euclid(1.0),
Spread::Reflect => {
let r = t.rem_euclid(2.0);
if r > 1.0 { 2.0 - r } else { r }
}
}
}
pub(crate) fn eval_stops(stops: &[(f64, [f64; 4])], t: f64) -> [f32; 4] {
let Some(first) = stops.first() else { return [0.0; 4] };
if t <= first.0 {
return first.1.map(|v| v as f32);
}
for w in stops.windows(2) {
let (a, b) = (w[0], w[1]);
if t <= b.0 {
let k = if b.0 > a.0 { (t - a.0) / (b.0 - a.0) } else { 1.0 };
return [0, 1, 2, 3].map(|i| (a.1[i] + (b.1[i] - a.1[i]) * k) as f32);
}
}
stops.last().map_or([0.0; 4], |s| s.1.map(|v| v as f32))
}
+108
View File
@@ -0,0 +1,108 @@
use super::*;
const BASIC: &[u8] = include_bytes!("../tests/fixtures/basic-shapes.svg");
const PATHS: &[u8] = include_bytes!("../tests/fixtures/paths-gradients.svg");
fn px(img: &effectcraft_raster::Image, x: u32, y: u32) -> [f32; 4] {
img.data[img.idx(x, y)]
}
fn near(a: [f32; 4], b: [f32; 4], tol: f32) -> bool {
(0..4).all(|i| (a[i] - b[i]).abs() <= tol)
}
#[test]
fn basic_shapes_structure() {
let d = parse(BASIC).unwrap();
assert_eq!((d.width, d.height), (200.0, 100.0));
assert_eq!(d.pixel_size(), (200, 100));
// viewBox 400×200 into 200×100.
assert!((d.root.transform.as_coeffs()[0] - 0.5).abs() < 1e-12);
let shapes = d.flatten();
assert_eq!(shapes.len(), 6);
let (m, o, s) = shapes[1];
assert_eq!(s.name, "box");
assert!(matches!(s.geom, Geom::Rect { w, h, rx, .. } if w == 120.0 && h == 80.0 && rx == 12.0));
assert!((o - 0.8).abs() < 1e-12);
// translate(20 20) then the viewBox scale.
let p = m * kurbo::Point::new(10.0, 10.0);
assert!((p.x - 15.0).abs() < 1e-9 && (p.y - 15.0).abs() < 1e-9);
let st = s.stroke.as_ref().unwrap();
assert_eq!(st.width, 6.0);
assert_eq!(st.paint, Paint::Color([1.0, 1.0, 1.0]));
assert!(matches!(shapes[2].2.geom, Geom::Ellipse { rx, ry, .. } if rx == 30.0 && ry == 30.0));
assert_eq!(shapes[3].2.fill_opacity, 0.75);
assert!(shapes[4].2.fill.is_none());
assert_eq!(shapes[4].2.stroke.as_ref().unwrap().join, Join::Round);
assert_eq!(shapes[5].2.stroke.as_ref().unwrap().cap, Cap::Round);
}
#[test]
fn basic_shapes_raster() {
let d = parse(BASIC).unwrap();
let img = rasterize(&d, 200, 100, 1.0);
let bg = [0x20 as f32 / 255.0, 0x28 as f32 / 255.0, 0x30 as f32 / 255.0, 1.0];
assert!(near(px(&img, 2, 2), bg, 1e-3));
// Inside the box: tomato at group opacity 0.8 over the background.
let tomato = [1.0, 99.0 / 255.0, 71.0 / 255.0];
let want = [0, 1, 2].map(|i| tomato[i] * 0.8 + bg[i] * 0.2);
let got = px(&img, 45, 35);
assert!(near(got, [want[0], want[1], want[2], 1.0], 2e-3), "{got:?} vs {want:?}");
// Scaled ×2: the same point lands at double the coordinates, edges stay sharp.
let big = rasterize(&d, 400, 200, 2.0);
assert!(near(px(&big, 90, 70), got, 2e-3));
}
#[test]
fn paths_gradients_css_use() {
let d = parse(PATHS).unwrap();
assert_eq!(d.pixel_size(), (160, 120));
let shapes = d.flatten();
assert_eq!(shapes.len(), 6, "{:?}", shapes.iter().map(|s| &s.2.name).collect::<Vec<_>>());
// CSS class → gradient fill.
let Some(Paint::Gradient(g)) = &shapes[0].2.fill else { panic!("gradient expected") };
assert!(g.bbox_units);
assert_eq!(g.stops.len(), 2);
assert_eq!(shapes[1].2.fill_rule, FillRule::EvenOdd);
// #wave: id selector beats nothing; stroke from CSS.
assert!(shapes[2].2.fill.is_none());
assert_eq!(shapes[2].2.stroke.as_ref().unwrap().width, 5.0);
// use → the referenced path with the use's fill, translated.
assert_eq!(shapes[4].2.fill, Some(Paint::Color([128.0 / 255.0, 0.0, 128.0 / 255.0])));
let p = shapes[4].0 * kurbo::Point::new(0.0, 0.0);
assert!((p.x - 100.0).abs() < 1e-9 && (p.y - 110.0).abs() < 1e-9);
assert_eq!(shapes[5].2.stroke.as_ref().unwrap().dash, Some((vec![4.0, 2.0], 0.0)));
let img = rasterize(&d, 160, 120, 1.0);
// Vertical gradient: yellow at the top, red at the bottom.
let top = px(&img, 40, 11);
let bottom = px(&img, 40, 48);
assert!(top[1] > 0.7 && bottom[1] < 0.3, "{top:?} {bottom:?}");
// Even-odd hole.
assert_eq!(px(&img, 120, 35)[3], 0.0);
assert!(px(&img, 95, 15)[3] > 0.99);
// Radial gradient: white centre.
let c = px(&img, 40, 90);
assert!(c[0] > 0.9 && c[3] > 0.9, "{c:?}");
}
#[test]
fn malformed_input() {
assert!(parse(b"<html/>").is_err());
assert!(parse(b"not xml").is_err());
let d = parse(b"<svg xmlns='http://www.w3.org/2000/svg'><path d='M0 0 L 10'/><rect width='-5' height='5'/><text>hi</text></svg>").unwrap();
assert_eq!(d.pixel_size(), (300, 150));
assert_eq!(d.skipped, vec!["text".to_string()]);
assert!(looks_like_svg(BASIC));
assert!(!looks_like_svg(b"\x89PNG"));
}
#[test]
fn transforms() {
let m = parse_transform("translate(10,20) rotate(90) scale(2 3)");
let p = m * kurbo::Point::new(1.0, 1.0);
assert!((p.x - 7.0).abs() < 1e-9 && (p.y - 22.0).abs() < 1e-9, "{p:?}");
let m = parse_transform("matrix(1 0 0 1 5 6) skewX(45)");
let p = m * kurbo::Point::new(0.0, 1.0);
assert!((p.x - 6.0).abs() < 1e-9 && (p.y - 7.0).abs() < 1e-9);
}
+15
View File
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- EffectCraft test fixture (original work, MIT OR Apache-2.0): basic shapes, groups, transforms. -->
<svg xmlns="http://www.w3.org/2000/svg" width="200" height="100" viewBox="0 0 400 200">
<title>Basic shapes</title>
<rect id="bg" x="0" y="0" width="400" height="200" fill="#202830"/>
<g id="left" transform="translate(20 20)" opacity="0.8">
<rect id="box" x="10" y="10" width="120" height="80" rx="12" fill="tomato" stroke="white" stroke-width="6"/>
<circle id="dot" cx="70" cy="130" r="30" fill="rgb(40, 180, 255)"/>
</g>
<g id="right" transform="translate(220 0) rotate(15 80 100)">
<ellipse cx="80" cy="60" rx="70" ry="30" fill="gold" fill-opacity="0.75"/>
<polygon points="20,180 80,110 140,180" fill="none" stroke="#7CFC00" stroke-width="8" stroke-linejoin="round"/>
<line x1="10" y1="100" x2="150" y2="100" stroke="white" stroke-width="4" stroke-linecap="round"/>
</g>
</svg>

After

Width:  |  Height:  |  Size: 957 B

@@ -0,0 +1,7 @@
asset: basic-shapes.svg
title: SVG test fixture: basic shapes, groups and transforms
author: EffectCraft contributors
source: original work
license: MIT OR Apache-2.0
added: 2026-10-02 by EffectCraft contributors (Claude agent, SVG import)
notes: Hand-written for the effectcraft-svg parser and Create Shapes from Vector Layer tests.
+26
View File
@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- EffectCraft test fixture (original work, MIT OR Apache-2.0): path data, gradients, CSS, use. -->
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="160" height="120">
<defs>
<style>
.warm { fill: url(#sunset); }
#wave { stroke: #3366ff; stroke-width: 5; fill: none; }
path.hole { fill-rule: evenodd; }
</style>
<linearGradient id="sunset" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#ffcc00"/>
<stop offset="100%" stop-color="#ff3300"/>
</linearGradient>
<radialGradient id="glow" cx="40" cy="90" r="25" gradientUnits="userSpaceOnUse">
<stop offset="0%" style="stop-color: white; stop-opacity: 1"/>
<stop offset="100%" stop-color="#00aaff" stop-opacity="0.2"/>
</radialGradient>
<path id="tri" d="M0 0 L20 0 L10 -16 Z"/>
</defs>
<path class="warm" d="M10 10 h60 v40 h-60 z"/>
<path class="hole" fill="#44aa66" d="M90,10 L150,10 L150,60 L90,60 Z M105,25 L135,25 L135,45 L105,45 Z"/>
<path id="wave" d="M10 75 C 30 55, 50 95, 70 75 S 110 55, 130 75 Q 140 85 150 75"/>
<circle cx="40" cy="90" r="25" fill="url(#glow)"/>
<use xlink:href="#tri" x="100" y="110" fill="purple"/>
<path d="M120 100 a 15 10 0 1 0 30 0 a15 10 0 1 0 -30 0" fill="none" stroke="black" stroke-width="2" stroke-dasharray="4 2"/>
</svg>

After

Width:  |  Height:  |  Size: 1.4 KiB

@@ -0,0 +1,7 @@
asset: paths-gradients.svg
title: SVG test fixture: path data, gradients, CSS styles and use
author: EffectCraft contributors
source: original work
license: MIT OR Apache-2.0
added: 2026-10-02 by EffectCraft contributors (Claude agent, SVG import)
notes: Hand-written for the effectcraft-svg parser and Create Shapes from Vector Layer tests.
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "effectcraft-track"
description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization and Roto Brush segmentation (graph cut, matting)"
description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization, Roto Brush segmentation (graph cut, matting) and 3D camera solving (structure from motion)"
version.workspace = true
edition.workspace = true
license.workspace = true
@@ -14,3 +14,4 @@ effectcraft-raster = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
web-time = { workspace = true }
+470
View File
@@ -0,0 +1,470 @@
//! Sparse bundle adjustment: Levenberg–Marquardt over cameras and points with the **Schur
//! complement** eliminating the points (Triggs, McLauchlan, Hartley & Fitzgibbon, "Bundle
//! Adjustment — A Modern Synthesis", 2000, §6; H&Z Appendix 6). Residuals are reprojection
//! errors in pixels with a **Huber** robust loss (applied by iteratively reweighted least
//! squares, Triggs et al. §3.3).
//!
//! Cameras are `(R, C, f)`; rotations are updated multiplicatively (`R ← exp([δ]×) R`), focal
//! lengths in log space. The focal length is held fixed, shared by all cameras, or per camera
//! (with a smoothness prior between consecutive cameras). Tripod solves hold the centres fixed.
use rayon::prelude::*;
use super::geometry::Pose;
use super::linalg::*;
/// A camera in the adjustment.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct BaCam {
pub pose: Pose,
/// Focal length (pixels).
pub f: f64,
}
/// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal
/// point).
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct BaObs {
pub cam: u32,
pub pt: u32,
pub uv: [f64; 2],
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum FocalMode {
Fixed,
Shared,
/// Per camera, with prior weight `k` on the log-focal difference of consecutive cameras.
PerCamera(f64),
}
#[derive(Clone, Debug)]
pub struct BaOpts {
pub focal: FocalMode,
/// Motion only: the points stay fixed.
pub fix_points: bool,
/// The camera centres stay fixed (tripod).
pub fix_centers: bool,
/// Cameras that stay fixed (gauge), by index; missing entries are free.
pub fixed: Vec<bool>,
/// Huber threshold (pixels).
pub huber: f64,
pub max_iter: usize,
}
impl Default for BaOpts {
fn default() -> Self {
BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 }
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub struct BaReport {
pub initial_cost: f64,
pub cost: f64,
pub iterations: usize,
}
const BEHIND: f64 = 1e6;
fn huber(e2: f64, k: f64) -> f64 {
if e2 <= k * k { e2 } else { 2.0 * k * e2.sqrt() - k * k }
}
/// Residual (pixels) of one observation, `None` behind the camera.
pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> {
let p = c.pose.to_cam(x);
if p[2] <= 1e-9 {
return None;
}
Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]])
}
/// Total robust cost.
pub fn cost(cams: &[BaCam], pts: &[V3], obs: &[BaObs], o: &BaOpts) -> f64 {
let c: f64 = obs
.par_iter()
.map(|ob| match residual(&cams[ob.cam as usize], pts[ob.pt as usize], ob.uv) {
Some(r) => huber(r[0] * r[0] + r[1] * r[1], o.huber),
None => BEHIND,
})
.sum();
c + prior_cost(cams, o)
}
fn prior_cost(cams: &[BaCam], o: &BaOpts) -> f64 {
match o.focal {
FocalMode::PerCamera(k) => cams.windows(2).map(|w| (k * (w[0].f.ln() - w[1].f.ln())).powi(2)).sum(),
_ => 0.0,
}
}
/// Parameter layout.
struct Layout {
/// First pose parameter of each camera (`usize::MAX` when its pose is fixed).
start: Vec<usize>,
/// Per-camera focal parameter (`usize::MAX` when none). Gauge-fixed cameras keep a free
/// focal length.
fidx: Vec<usize>,
/// Index of the shared focal parameter.
shared: Option<usize>,
n: usize,
rot_only: bool,
}
fn layout(ncam: usize, o: &BaOpts) -> Layout {
let per_cam_f = matches!(o.focal, FocalMode::PerCamera(_));
let d = if o.fix_centers { 3 } else { 6 };
let mut start = vec![usize::MAX; ncam];
let mut fidx = vec![usize::MAX; ncam];
let mut n = 0;
for i in 0..ncam {
if !o.fixed.get(i).copied().unwrap_or(false) {
start[i] = n;
n += d;
}
if per_cam_f {
fidx[i] = n;
n += 1;
}
}
let shared = matches!(o.focal, FocalMode::Shared).then(|| {
n += 1;
n - 1
});
Layout { start, fidx, shared, n, rot_only: o.fix_centers }
}
/// Camera-side Jacobian entries of one observation: (parameter index, d r / d param).
fn cam_jac(l: &Layout, ci: usize, c: &BaCam, xc: V3, a: &[[f64; 3]; 2], p: [f64; 2], out: &mut Vec<(usize, [f64; 2])>) {
out.clear();
let s = l.start[ci];
if s != usize::MAX {
// dXc/dδ = −[Xc]×
let sk = skew(xc);
for k in 0..3 {
let g = [-sk[0][k], -sk[1][k], -sk[2][k]];
out.push((s + k, [dot(a[0], g), dot(a[1], g)]));
}
if !l.rot_only {
// dXc/dC = −R
for k in 0..3 {
let g = [-c.pose.r[0][k], -c.pose.r[1][k], -c.pose.r[2][k]];
out.push((s + 3 + k, [dot(a[0], g), dot(a[1], g)]));
}
}
}
if l.fidx[ci] != usize::MAX {
out.push((l.fidx[ci], p));
}
if let Some(g) = l.shared {
out.push((g, p));
}
}
/// Run Levenberg–Marquardt. `cams` and `pts` are updated in place.
pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> BaReport {
let l = layout(cams.len(), o);
let npts = pts.len();
// Observations by point.
let mut by_pt: Vec<Vec<u32>> = vec![vec![]; npts];
for (i, ob) in obs.iter().enumerate() {
by_pt[ob.pt as usize].push(i as u32);
}
let mut cur = cost(cams, pts, obs, o);
let initial = cur;
let t0 = web_time::Instant::now();
let mut tries = 0usize;
let dbg = std::env::var_os("EC_CAMTRACK_DEBUG").is_some() && l.n > 200;
let mut lambda = 1e-4;
let mut iters = 0;
if l.n == 0 && o.fix_points {
return BaReport { initial_cost: initial, cost: cur, iterations: 0 };
}
let n = l.n;
let parallel = n <= 700;
for _ in 0..o.max_iter {
iters += 1;
// ---- Normal equations (Gauss–Newton with IRLS weights).
// Per point: V (3×3), bp (3), W entries (cam-side index → 3).
struct PtSys {
v: [[f64; 3]; 3],
bp: V3,
w: Vec<(usize, V3)>,
}
let build_point = |j: usize, s_acc: &mut Vec<f64>, bc: &mut Vec<f64>| -> PtSys {
let mut ps = PtSys { v: [[0.0; 3]; 3], bp: [0.0; 3], w: vec![] };
let mut jc: Vec<(usize, [f64; 2])> = Vec::with_capacity(8);
for &oi in &by_pt[j] {
let ob = &obs[oi as usize];
let ci = ob.cam as usize;
let c = &cams[ci];
let xc = c.pose.to_cam(pts[j]);
if xc[2] <= 1e-9 {
continue;
}
let iz = 1.0 / xc[2];
let p = [c.f * xc[0] * iz, c.f * xc[1] * iz];
let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]];
let e = (r[0] * r[0] + r[1] * r[1]).sqrt();
let w = if e <= o.huber { 1.0 } else { o.huber / e };
let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]];
cam_jac(&l, ci, c, xc, &a, p, &mut jc);
// Point Jacobian: A R.
let jx = if o.fix_points {
[[0.0; 3]; 2]
} else {
let r0 = mtv(&c.pose.r, a[0]);
let r1 = mtv(&c.pose.r, a[1]);
[r0, r1]
};
for (ia, ja) in &jc {
bc[*ia] -= w * (ja[0] * r[0] + ja[1] * r[1]);
for (ib, jb) in &jc {
s_acc[ia * n + ib] += w * (ja[0] * jb[0] + ja[1] * jb[1]);
}
}
if !o.fix_points {
for a1 in 0..3 {
ps.bp[a1] -= w * (jx[0][a1] * r[0] + jx[1][a1] * r[1]);
for b1 in 0..3 {
ps.v[a1][b1] += w * (jx[0][a1] * jx[0][b1] + jx[1][a1] * jx[1][b1]);
}
}
for (ia, ja) in &jc {
let wv =
[w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])];
// Each observation of a point is on another camera: only the shared focal
// length repeats.
let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None };
match found {
Some(e) => e.1 = add(e.1, wv),
None => ps.w.push((*ia, wv)),
}
}
}
}
ps
};
let (mut s_mat, mut bc, systems): (Vec<f64>, Vec<f64>, Vec<PtSys>) = if parallel && npts > 64 {
let chunks = rayon::current_num_threads().clamp(1, 16);
let per = npts.div_ceil(chunks);
let parts: Vec<(Vec<f64>, Vec<f64>, Vec<PtSys>)> = (0..chunks)
.into_par_iter()
.map(|t| {
let (mut s, mut b) = (vec![0.0; n * n], vec![0.0; n]);
let v = (t * per..((t + 1) * per).min(npts)).map(|j| build_point(j, &mut s, &mut b)).collect();
(s, b, v)
})
.collect();
let mut s = vec![0.0; n * n];
let mut b = vec![0.0; n];
let mut sys = Vec::with_capacity(npts);
for (ps, pb, pv) in parts {
for (x, y) in s.iter_mut().zip(ps) {
*x += y;
}
for (x, y) in b.iter_mut().zip(pb) {
*x += y;
}
sys.extend(pv);
}
(s, b, sys)
} else {
let mut s = vec![0.0; n * n];
let mut b = vec![0.0; n];
let sys = (0..npts).map(|j| build_point(j, &mut s, &mut b)).collect();
(s, b, sys)
};
// Focal prior.
if let FocalMode::PerCamera(k) = o.focal {
for i in 0..cams.len().saturating_sub(1) {
let r = k * (cams[i].f.ln() - cams[i + 1].f.ln());
let (a, b) = (l.fidx[i], l.fidx[i + 1]);
let ia = (a != usize::MAX).then_some(a);
let ib = (b != usize::MAX).then_some(b);
for (idx, g) in [(ia, k), (ib, -k)] {
let Some(x) = idx else { continue };
bc[x] -= g * r;
for (idy, h) in [(ia, k), (ib, -k)] {
if let Some(y) = idy {
s_mat[x * n + y] += g * h;
}
}
}
}
}
let base_s = s_mat.clone();
let base_bc = bc.clone();
let mean_diag = ((0..n).map(|i| base_s[i * n + i]).sum::<f64>() / n.max(1) as f64).max(1e-12);
// ---- Try steps with increasing damping.
let mut accepted = false;
for _try in 0..12 {
tries += 1;
let mut s = base_s.clone();
let mut rhs = base_bc.clone();
for i in 0..n {
let d = s[i * n + i];
s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag);
}
// Schur complement over the points: S −= W V⁻¹ Wᵀ, rhs −= W V⁻¹ bp (in parallel
// chunks of points when the reduced system is small enough to copy per thread).
let vinv: Vec<Option<M3>> = if o.fix_points {
vec![None; npts]
} else {
systems
.par_iter()
.map(|ps| {
let mut v = ps.v;
let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0;
for k in 0..3 {
v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12;
}
inv3(&v)
})
.collect()
};
if !o.fix_points {
let reduce = |range: std::ops::Range<usize>, s: &mut [f64], rhs: &mut [f64]| {
for j in range {
let (Some(vi), ps) = (&vinv[j], &systems[j]) else { continue };
let vb = mv(vi, ps.bp);
let wv: Vec<V3> = ps.w.iter().map(|(_, w)| mv(vi, *w)).collect();
for (a, (ia, wa)) in ps.w.iter().enumerate() {
rhs[*ia] -= dot(*wa, vb);
for (ib, wb) in &ps.w[a..] {
let d = dot(wv[a], *wb);
s[ia * n + ib] -= d;
if ia != ib {
s[ib * n + ia] -= d;
}
}
}
}
};
let chunks = rayon::current_num_threads().clamp(1, 8);
if n <= 1200 && npts >= 64 && chunks > 1 {
let per = npts.div_ceil(chunks);
let parts: Vec<(Vec<f64>, Vec<f64>)> = (0..chunks)
.into_par_iter()
.map(|t| {
let (mut ds, mut dr) = (vec![0.0; n * n], vec![0.0; n]);
reduce(t * per..((t + 1) * per).min(npts), &mut ds, &mut dr);
(ds, dr)
})
.collect();
for (ds, dr) in parts {
s.par_iter_mut().zip(ds.par_iter()).for_each(|(x, y)| *x += y);
for (x, y) in rhs.iter_mut().zip(dr) {
*x += y;
}
}
} else {
reduce(0..npts, &mut s, &mut rhs);
}
}
let mut dc = rhs;
if n > 0 && !cholesky_solve(&mut s, n, &mut dc) {
lambda *= 10.0;
continue;
}
// Apply.
let mut nc: Vec<BaCam> = cams.to_vec();
for (i, c) in nc.iter_mut().enumerate() {
let st = l.start[i];
if st != usize::MAX {
let dr = [dc[st], dc[st + 1], dc[st + 2]];
c.pose.r = mm(&rodrigues(dr), &c.pose.r);
if !l.rot_only {
c.pose.c = add(c.pose.c, [dc[st + 3], dc[st + 4], dc[st + 5]]);
}
}
if l.fidx[i] != usize::MAX {
c.f *= dc[l.fidx[i]].clamp(-0.5, 0.5).exp();
}
if let Some(g) = l.shared {
c.f *= dc[g].clamp(-0.5, 0.5).exp();
}
}
let mut np: Vec<V3> = pts.to_vec();
if !o.fix_points {
np.par_iter_mut().enumerate().for_each(|(j, x)| {
let Some(vi) = &vinv[j] else { return };
let ps = &systems[j];
let mut b = ps.bp;
for (ia, w) in &ps.w {
b = sub(b, scale(*w, dc[*ia]));
}
*x = add(*x, mv(vi, b));
});
}
let c2 = cost(&nc, &np, obs, o);
if c2.is_finite() && c2 < cur {
cams.copy_from_slice(&nc);
pts.copy_from_slice(&np);
let gain = cur - c2;
cur = c2;
lambda = (lambda / 3.0).max(1e-9);
accepted = gain > 1e-6 * cur.max(1e-12);
if !accepted {
// Converged (tiny improvement).
if dbg {
eprintln!(
"ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}",
l.n,
npts,
obs.len(),
t0.elapsed().as_secs_f64()
);
}
return BaReport { initial_cost: initial, cost: cur, iterations: iters };
}
break;
}
lambda *= 4.0;
}
if !accepted {
break;
}
}
if dbg {
eprintln!("ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", l.n, npts, obs.len(), t0.elapsed().as_secs_f64());
}
BaReport { initial_cost: initial, cost: cur, iterations: iters }
}
#[cfg(test)]
mod tests {
use super::*;
use crate::camtrack::geometry::Rng;
#[test]
fn bundle_adjustment_converges_from_noise() {
let mut rng = Rng(99);
let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5;
let truth: Vec<V3> = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect();
let f = 900.0;
let cams_true: Vec<BaCam> = (0..6)
.map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f })
.collect();
let mut obs = vec![];
for (ci, c) in cams_true.iter().enumerate() {
for (pi, x) in truth.iter().enumerate() {
let r = residual(c, *x, [0.0, 0.0]).unwrap();
obs.push(BaObs { cam: ci as u32, pt: pi as u32, uv: r });
}
}
let mut cams = cams_true.clone();
for (i, c) in cams.iter_mut().enumerate().skip(2) {
c.pose.r = mm(&rodrigues([0.01, -0.01 * i as f64 * 0.3, 0.005]), &c.pose.r);
c.pose.c = add(c.pose.c, [0.02, -0.03, 0.01]);
}
let mut pts: Vec<V3> = truth.iter().map(|x| add(*x, [u() * 0.1, u() * 0.1, u() * 0.2])).collect();
let mut fixed = vec![false; 6];
fixed[0] = true;
fixed[1] = true;
let rep = adjust(&mut cams, &mut pts, &obs, &BaOpts { fixed, max_iter: 50, ..Default::default() });
let rms = (rep.cost / obs.len() as f64).sqrt();
assert!(rms < 1e-4, "{rep:?}");
}
}
+115
View File
@@ -0,0 +1,115 @@
//! Step 1 of the camera analysis: long feature tracks through the clip.
//!
//! Every frame, the active features are followed from the previous frame with pyramidal
//! Lucas–Kanade and a forward–backward check ([`crate::klt::track`]); a feature that fails ends
//! its track. When fewer than ~85 % of the wanted features remain, new Shi–Tomasi corners are
//! detected away from the active ones and start new tracks. Tracks shorter than three frames are
//! dropped at the end.
use crate::Frame;
use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track};
use super::{CameraTracks, Track2D};
/// Analysis options.
#[derive(Clone, Copy, Debug, Default)]
pub struct AnalyzeOpts {
/// Detailed Analysis: twice the resolution and more features.
pub detailed: bool,
}
/// Incremental feature tracking: push the clip's frames in order.
pub struct TrackAnalyzer {
opts: AnalyzeOpts,
prev: Option<GrayPyramid>,
/// (track index, level-0 plane position) of the live features.
active: Vec<(usize, [f64; 2])>,
tracks: Vec<Track2D>,
frame: u32,
size: [f64; 2],
factor: f64,
}
impl TrackAnalyzer {
pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> TrackAnalyzer {
TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0 }
}
pub fn frames(&self) -> u32 {
self.frame
}
/// Features currently followed.
pub fn active(&self) -> usize {
self.active.len()
}
fn params(&self) -> (u32, usize) {
if self.opts.detailed { (1280, 900) } else { (720, 500) }
}
/// Analyse the next frame.
pub fn push(&mut self, frame: &Frame) {
let (max_side, want) = self.params();
let factor = analysis_factor(frame.img.width, frame.img.height, max_side);
self.factor = factor as f64;
let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 4);
let k = self.frame;
match &self.prev {
Some(prev) if prev.width() == pyr.width() && prev.height() == pyr.height() && prev.offset == pyr.offset => {
let pts: Vec<[f64; 2]> = self.active.iter().map(|a| a.1).collect();
let q = track(prev, &pyr, &pts, None, &LkOpts { radius: 6, iterations: 30, epsilon: 0.005, fb_max: 0.5, ..Default::default() });
let mut next = Vec::with_capacity(self.active.len());
for ((ti, _), q) in self.active.iter().zip(q) {
if let Some(q) = q {
let l = pyr.to_layer(q);
self.tracks[*ti].pts.push([l[0] as f32, l[1] as f32]);
next.push((*ti, q));
}
}
self.active = next;
}
_ => self.active.clear(),
}
// Re-detect where there are no features.
if (self.active.len() as f64) < want as f64 * 0.85 {
let md = (pyr.width().max(pyr.height()) as f64 / (want as f64).sqrt() * 0.6).clamp(5.0, 40.0);
let cell = md;
let (gw, gh) = ((pyr.width() as f64 / cell).ceil() as usize + 1, (pyr.height() as f64 / cell).ceil() as usize + 1);
let mut occ = vec![false; gw * gh];
for (_, p) in &self.active {
let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize);
for yy in cy.saturating_sub(1)..=(cy + 1).min(gh - 1) {
for xx in cx.saturating_sub(1)..=(cx + 1).min(gw - 1) {
occ[yy * gw + xx] = true;
}
}
}
let free = |p: [f64; 2]| {
let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize);
!occ.get(cy * gw + cx).copied().unwrap_or(true)
};
let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.001, window: 2, border: 8 };
for p in good_features(&pyr, &opts, Some(&free)) {
let l = pyr.to_layer(p);
let id = self.tracks.len() as u32;
self.tracks.push(Track2D { id, start: k, pts: vec![[l[0] as f32, l[1] as f32]] });
self.active.push((id as usize, p));
}
}
self.prev = Some(pyr);
self.frame += 1;
}
pub fn finish(self, start: f64, frame_duration: f64) -> CameraTracks {
let mut tracks: Vec<Track2D> = self.tracks.into_iter().filter(|t| t.pts.len() >= 3).collect();
// Round to 1/100 pixel (smaller JSON) and renumber densely.
for (i, t) in tracks.iter_mut().enumerate() {
t.id = i as u32;
for p in t.pts.iter_mut() {
*p = [(p[0] * 100.0).round() / 100.0, (p[1] * 100.0).round() / 100.0];
}
}
CameraTracks { version: 1, size: self.size, start, frame_duration, frames: self.frame, factor: self.factor, detailed: self.opts.detailed, tracks }
}
}
+432
View File
@@ -0,0 +1,432 @@
//! Two-view and multi-view geometry for the camera solver, from Hartley & Zisserman (H&Z),
//! *Multiple View Geometry in Computer Vision* (2nd ed.) and Ma et al., *An Invitation to 3-D
//! Vision* (2004).
//!
//! All image points here are **normalised** (calibrated) coordinates `x = (u − c) / f`.
//! A camera is `(R, C)` with `X_cam = R (X − C)`.
use super::linalg::*;
/// A deterministic xorshift generator for RANSAC samples.
pub struct Rng(pub u64);
impl Rng {
pub fn next_u64(&mut self) -> u64 {
let mut x = self.0;
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
self.0 = x;
x
}
pub fn below(&mut self, n: usize) -> usize {
(self.next_u64() % n.max(1) as u64) as usize
}
/// `k` distinct indices below `n`.
pub fn sample(&mut self, n: usize, k: usize, out: &mut Vec<usize>) {
out.clear();
let mut guard = 0;
while out.len() < k && guard < 1000 {
guard += 1;
let c = self.below(n);
if !out.contains(&c) {
out.push(c);
}
}
}
}
/// Hartley normalisation (H&Z §4.4.4): centroid to the origin, mean distance √2.
fn normalizer(p: &[[f64; 2]]) -> M3 {
let n = p.len().max(1) as f64;
let c = [p.iter().map(|q| q[0]).sum::<f64>() / n, p.iter().map(|q| q[1]).sum::<f64>() / n];
let d = p.iter().map(|q| (q[0] - c[0]).hypot(q[1] - c[1])).sum::<f64>() / n;
let s = if d > 1e-12 { std::f64::consts::SQRT_2 / d } else { 1.0 };
[[s, 0.0, -s * c[0]], [0.0, s, -s * c[1]], [0.0, 0.0, 1.0]]
}
fn h2(m: &M3, p: [f64; 2]) -> V3 {
mv(m, [p[0], p[1], 1.0])
}
/// The essential matrix from ≥ 8 correspondences (`x2ᵀ E x1 = 0`) by the normalised 8-point
/// algorithm (H&Z Alg. 11.1), with the essential constraint (two equal singular values, one
/// zero) enforced (H&Z §9.6.1 / Result 9.18).
pub fn essential_8pt(x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option<M3> {
let n = x1.len().min(x2.len());
if n < 8 {
return None;
}
let (t1, t2) = (normalizer(&x1[..n]), normalizer(&x2[..n]));
let mut a = Vec::with_capacity(n * 9);
for i in 0..n {
let p = h2(&t1, x1[i]);
let q = h2(&t2, x2[i]);
a.extend_from_slice(&[q[0] * p[0], q[0] * p[1], q[0], q[1] * p[0], q[1] * p[1], q[1], p[0], p[1], 1.0]);
}
let e = null_vector(&a, n, 9);
let en = [[e[0], e[1], e[2]], [e[3], e[4], e[5]], [e[6], e[7], e[8]]];
// Denormalise: E = T2ᵀ Ê T1.
let e = mm(&mt(&t2), &mm(&en, &t1));
let (u, s, v) = svd3(&e);
let m = (s[0] + s[1]) * 0.5;
if m <= 0.0 || !m.is_finite() {
return None;
}
let d = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]];
Some(mm(&u, &mm(&d, &mt(&v))))
}
/// Squared Sampson distance of a correspondence to `x2ᵀ E x1 = 0` (H&Z §11.4.3), in the units of
/// the coordinates.
pub fn sampson2(e: &M3, p: [f64; 2], q: [f64; 2]) -> f64 {
let x1 = [p[0], p[1], 1.0];
let x2 = [q[0], q[1], 1.0];
let ex1 = mv(e, x1);
let etx2 = mtv(e, x2);
let num = dot(x2, ex1);
let den = ex1[0] * ex1[0] + ex1[1] * ex1[1] + etx2[0] * etx2[0] + etx2[1] * etx2[1];
if den < 1e-300 { f64::INFINITY } else { num * num / den }
}
/// RANSAC essential matrix (8-point samples, Sampson distance below `thr`), refitted on the
/// consensus set. Returns the matrix and the inlier mask.
pub fn ransac_essential(x1: &[[f64; 2]], x2: &[[f64; 2]], thr: f64, iterations: usize, seed: u64) -> Option<(M3, Vec<bool>)> {
let n = x1.len().min(x2.len());
if n < 8 {
return None;
}
let t2 = thr * thr;
let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1);
let mut best: Option<(usize, f64, M3)> = None;
let mut idx = vec![];
let mut max_iter = iterations;
let mut it = 0;
let (mut s1, mut s2) = (vec![[0.0; 2]; 8], vec![[0.0; 2]; 8]);
while it < max_iter {
it += 1;
rng.sample(n, 8, &mut idx);
if idx.len() < 8 {
break;
}
for (k, &i) in idx.iter().enumerate() {
s1[k] = x1[i];
s2[k] = x2[i];
}
let Some(e) = essential_8pt(&s1, &s2) else { continue };
let (mut c, mut err) = (0, 0.0);
for i in 0..n {
let d = sampson2(&e, x1[i], x2[i]);
if d < t2 {
c += 1;
err += d;
}
}
if best.as_ref().is_none_or(|b| c > b.0 || (c == b.0 && err < b.1)) {
best = Some((c, err, e));
// Adaptive iteration count for 99 % confidence.
let w = c as f64 / n as f64;
let p_good = w.powi(8);
if p_good > 1e-9 {
let need = ((1.0 - 0.99f64).ln() / (1.0 - p_good).max(1e-12).ln()).ceil();
if need.is_finite() {
max_iter = max_iter.min((need as usize).max(30));
}
}
}
}
let (_, _, mut e) = best?;
// Refit on the inliers (twice: the consensus may grow).
let mut inl = vec![false; n];
for _ in 0..2 {
for i in 0..n {
inl[i] = sampson2(&e, x1[i], x2[i]) < t2;
}
let (a, b): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..n).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip();
match essential_8pt(&a, &b) {
Some(e2) => e = e2,
None => break,
}
}
for i in 0..n {
inl[i] = sampson2(&e, x1[i], x2[i]) < t2;
}
Some((e, inl))
}
/// The four `(R, t)` with `x2 ~ R x1 + t` (|t| = 1) of an essential matrix (H&Z Result 9.19).
pub fn decompose_essential(e: &M3) -> [(M3, V3); 4] {
let (mut u, _, mut v) = svd3(e);
if det3(&u) < 0.0 {
for row in u.iter_mut() {
row[2] = -row[2];
}
}
if det3(&v) < 0.0 {
for row in v.iter_mut() {
row[2] = -row[2];
}
}
let w = [[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]];
let r1 = mm(&u, &mm(&w, &mt(&v)));
let r2 = mm(&u, &mm(&mt(&w), &mt(&v)));
let t = col(&u, 2);
[(r1, t), (r1, scale(t, -1.0)), (r2, t), (r2, scale(t, -1.0))]
}
/// A camera `(R, C)`.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Pose {
pub r: M3,
pub c: V3,
}
impl Pose {
pub const IDENTITY: Pose = Pose { r: I3, c: [0.0; 3] };
/// From `x_cam = R X + t`.
pub fn from_rt(r: M3, t: V3) -> Pose {
Pose { r, c: scale(mtv(&r, t), -1.0) }
}
pub fn t(&self) -> V3 {
scale(mv(&self.r, self.c), -1.0)
}
pub fn to_cam(&self, x: V3) -> V3 {
mv(&self.r, sub(x, self.c))
}
}
/// Linear (DLT) triangulation from normalised observations (H&Z §12.2), minimising the
/// algebraic error over all views.
pub fn triangulate(views: &[(Pose, [f64; 2])]) -> Option<V3> {
if views.len() < 2 {
return None;
}
let mut ata = [0.0f64; 16];
for (p, x) in views {
let t = p.t();
let rows = [[p.r[0][0], p.r[0][1], p.r[0][2], t[0]], [p.r[1][0], p.r[1][1], p.r[1][2], t[1]], [p.r[2][0], p.r[2][1], p.r[2][2], t[2]]];
let a0: Vec<f64> = (0..4).map(|j| x[0] * rows[2][j] - rows[0][j]).collect();
let a1: Vec<f64> = (0..4).map(|j| x[1] * rows[2][j] - rows[1][j]).collect();
for a in [a0, a1] {
let nrm = a.iter().map(|v| v * v).sum::<f64>().sqrt().max(1e-300);
for i in 0..4 {
for j in 0..4 {
ata[i * 4 + j] += a[i] * a[j] / (nrm * nrm);
}
}
}
}
let (_, vecs) = sym_eigen(&ata, 4);
let h = [vecs[0], vecs[4], vecs[8], vecs[12]];
if h[3].abs() < 1e-12 {
return None;
}
let x = [h[0] / h[3], h[1] / h[3], h[2] / h[3]];
x.iter().all(|v| v.is_finite()).then_some(x)
}
/// Angle (radians) between the rays from two camera centres to `x`.
pub fn ray_angle(a: V3, b: V3, x: V3) -> f64 {
let (u, v) = (normalize(sub(x, a)), normalize(sub(x, b)));
dot(u, v).clamp(-1.0, 1.0).acos()
}
/// Squared reprojection error (normalised units) of `x` in `pose` against observation `o`
/// (`None` behind the camera).
pub fn reproj2(pose: &Pose, x: V3, o: [f64; 2]) -> Option<f64> {
let c = pose.to_cam(x);
if c[2] <= 1e-9 {
return None;
}
let (u, v) = (c[0] / c[2] - o[0], c[1] / c[2] - o[1]);
Some(u * u + v * v)
}
/// Pick the essential decomposition with the most points in front of both cameras; returns
/// the second camera (the first is the identity) and that count.
pub fn select_pose(cands: &[(M3, V3)], x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option<(Pose, usize)> {
let mut best: Option<(Pose, usize)> = None;
for (r, t) in cands {
let p2 = Pose::from_rt(*r, *t);
let mut c = 0;
for i in 0..x1.len() {
if let Some(x) = triangulate(&[(Pose::IDENTITY, x1[i]), (p2, x2[i])])
&& x[2] > 0.0
&& p2.to_cam(x)[2] > 0.0
{
c += 1;
}
}
if best.as_ref().is_none_or(|b| c > b.1) {
best = Some((p2, c));
}
}
best
}
/// Decompose a calibrated planar homography `H ~ R + t nᵀ / d` (normalised coordinates,
/// `x2 ~ H x1`) into its motions `(R, t/d, n)` (Ma et al. §5.3.3). Up to four solutions; those
/// that put the plane behind the first camera are dropped.
pub fn decompose_homography(h: &M3, x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Vec<(M3, V3, V3)> {
let (_, s, _) = svd3(h);
if s[1] <= 1e-12 {
return vec![];
}
let mut hn = mscale(h, 1.0 / s[1]);
// Sign: x2ᵀ H x1 > 0 for the correspondences (positive depth).
let pos = x1.iter().zip(x2).filter(|(p, q)| dot([q[0], q[1], 1.0], mv(&hn, [p[0], p[1], 1.0])) > 0.0).count();
if pos * 2 < x1.len() {
hn = mscale(&hn, -1.0);
}
let hth = mm(&mt(&hn), &hn);
let a: Vec<f64> = hth.iter().flatten().copied().collect();
let (vals, vecs) = sym_eigen(&a, 3);
// Descending: σ1² ≥ σ2² = 1 ≥ σ3².
let v1 = [vecs[2], vecs[5], vecs[8]];
let v2 = [vecs[1], vecs[4], vecs[7]];
let v3 = [vecs[0], vecs[3], vecs[6]];
let (s1, s3) = (vals[2].max(1.0), vals[0].clamp(0.0, 1.0));
let mut out = vec![];
if s1 - s3 < 1e-10 {
// H is a rotation: no translation (pure rotation or plane at infinity).
return vec![(nearest_rotation(&hn), [0.0; 3], [0.0, 0.0, 1.0])];
}
let k = (s1 - s3).sqrt();
let (a1, a3) = ((1.0 - s3).sqrt(), (s1 - 1.0).sqrt());
let u1 = scale(add(scale(v1, a1), scale(v3, a3)), 1.0 / k);
let u2 = scale(sub(scale(v1, a1), scale(v3, a3)), 1.0 / k);
for u in [u1, u2] {
let uu = from_cols(v2, u, cross(v2, u));
let hv2 = mv(&hn, v2);
let hu = mv(&hn, u);
let ww = from_cols(hv2, hu, cross(hv2, hu));
let r = mm(&ww, &mt(&uu));
let n = cross(v2, u);
let t = mv(&madd(&hn, &r, -1.0), n);
for sgn in [1.0, -1.0] {
let (n2, t2) = (scale(n, sgn), scale(t, sgn));
// The plane must be in front of the first camera: nᵀ x1 > 0 (with n pointing away).
let front = x1.iter().filter(|p| dot(n2, [p[0], p[1], 1.0]) > 0.0).count();
if front * 2 >= x1.len() {
out.push((r, t2, n2));
}
}
}
out
}
/// Least-squares plane through points: (centroid, unit normal, RMS in-plane spread).
pub fn fit_plane(p: &[V3]) -> Option<(V3, V3, f64)> {
if p.len() < 3 {
return None;
}
let k = 1.0 / p.len() as f64;
let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k);
let mut cov = [0.0; 9];
for q in p {
let d = sub(*q, c);
for i in 0..3 {
for j in 0..3 {
cov[i * 3 + j] += d[i] * d[j] * k;
}
}
}
let (vals, vecs) = sym_eigen(&cov, 3);
if vals[1] <= 1e-18 {
return None;
}
let n = normalize([vecs[0], vecs[3], vecs[6]]);
Some((c, n, (vals[1] + vals[2]).max(0.0).sqrt()))
}
/// The rotation `R` minimising `Σ |b − R a|²` over unit rays (orthogonal Procrustes / Kabsch).
pub fn procrustes(a: &[V3], b: &[V3]) -> Option<M3> {
if a.len() < 2 {
return None;
}
let mut m = [[0.0; 3]; 3];
for (x, y) in a.iter().zip(b) {
m = madd(&m, &outer(*y, *x), 1.0);
}
let (u, _, v) = svd3(&m);
let mut r = mm(&u, &mt(&v));
if det3(&r) < 0.0 {
let mut u2 = u;
for row in u2.iter_mut() {
row[2] = -row[2];
}
r = mm(&u2, &mt(&v));
}
Some(r)
}
#[cfg(test)]
mod tests {
use super::*;
fn scene() -> (Vec<V3>, Pose) {
let mut rng = Rng(12345);
let pts: Vec<V3> = (0..60)
.map(|_| {
let f = |r: &mut Rng| (r.next_u64() % 10_000) as f64 / 10_000.0 - 0.5;
[f(&mut rng) * 4.0, f(&mut rng) * 3.0, 5.0 + f(&mut rng) * 3.0]
})
.collect();
let p2 = Pose { r: rodrigues([0.02, -0.15, 0.03]), c: [0.6, 0.1, -0.2] };
(pts, p2)
}
fn proj(p: &Pose, x: V3) -> [f64; 2] {
let c = p.to_cam(x);
[c[0] / c[2], c[1] / c[2]]
}
#[test]
fn essential_recovers_relative_pose() {
let (pts, p2) = scene();
let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect();
let mut x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect();
// Outliers.
x2[3] = [0.3, -0.2];
x2[17] = [-0.1, 0.25];
let (e, inl) = ransac_essential(&x1, &x2, 1e-3, 500, 7).unwrap();
assert!(!inl[3] && !inl[17]);
assert!(inl.iter().filter(|b| **b).count() >= 57);
let (pose, n) = select_pose(&decompose_essential(&e), &x1, &x2).unwrap();
assert!(n >= 57);
let r_err = norm(log_rot(&mm(&pose.r, &mt(&p2.r))));
assert!(r_err < 1e-6, "{r_err}");
let dir = normalize(p2.c);
assert!(norm(sub(normalize(pose.c), dir)) < 1e-5, "{:?} {:?}", pose.c, dir);
}
#[test]
fn homography_decomposition_contains_the_motion() {
// Plane z = 4 (normal (0,0,1) in camera 1), seen after a motion.
let p2 = Pose { r: rodrigues([0.05, 0.2, -0.04]), c: [0.8, -0.3, 0.4] };
let pts: Vec<V3> = (0..30).map(|i| [((i * 37) % 17) as f64 / 8.0 - 1.0, ((i * 11) % 13) as f64 / 6.0 - 1.0, 4.0]).collect();
let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect();
let x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect();
// H = R + t nᵀ / d with t = −R C, n = (0,0,1), d = 4.
let t = p2.t();
let h = madd(&p2.r, &outer(t, [0.0, 0.0, 0.25]), 1.0);
let sols = decompose_homography(&mscale(&h, 3.7), &x1, &x2);
assert!(!sols.is_empty());
let ok = sols
.iter()
.any(|(r, td, n)| norm(log_rot(&mm(r, &mt(&p2.r)))) < 1e-6 && norm(sub(*td, scale(t, 0.25))) < 1e-6 && norm(sub(*n, [0.0, 0.0, 1.0])) < 1e-6);
assert!(ok, "{sols:?}");
}
#[test]
fn triangulation_and_plane() {
let (pts, p2) = scene();
for x in &pts[..5] {
let y = triangulate(&[(Pose::IDENTITY, proj(&Pose::IDENTITY, *x)), (p2, proj(&p2, *x))]).unwrap();
assert!(norm(sub(*x, y)) < 1e-8);
}
let plane: Vec<V3> = (0..10).map(|i| [i as f64, (i * i % 7) as f64, 2.0 * i as f64 + 1.0]).collect();
let (_, n, _) = fit_plane(&plane).unwrap();
// All points satisfy z = 2x + 1: normal ∝ (2, 0, −1).
assert!(norm(cross(n, normalize([2.0, 0.0, -1.0]))) < 1e-9);
}
}
+486
View File
@@ -0,0 +1,486 @@
//! Small dense linear algebra for the camera solver, written from the textbook methods:
//!
//! - 3-vectors / 3 × 3 matrices and rotations (Rodrigues' formula and its inverse);
//! - the **one-sided Jacobi SVD** (Hestenes 1958; Golub & Van Loan, *Matrix Computations*,
//! §8.6.4), accurate for the small, badly scaled systems of the 8-point algorithm;
//! - the **cyclic Jacobi eigenvalue method** for symmetric matrices (Golub & Van Loan §8.5);
//! - **Cholesky** factorisation for the bundle adjustment's normal equations;
//! - the closed-form **similarity alignment** of two point sets (Umeyama, "Least-squares
//! estimation of transformation parameters between two point patterns", PAMI 1991).
pub type V3 = [f64; 3];
pub type M3 = [[f64; 3]; 3];
pub const I3: M3 = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]];
pub fn add(a: V3, b: V3) -> V3 {
[a[0] + b[0], a[1] + b[1], a[2] + b[2]]
}
pub fn sub(a: V3, b: V3) -> V3 {
[a[0] - b[0], a[1] - b[1], a[2] - b[2]]
}
pub fn scale(a: V3, k: f64) -> V3 {
[a[0] * k, a[1] * k, a[2] * k]
}
pub fn dot(a: V3, b: V3) -> f64 {
a[0] * b[0] + a[1] * b[1] + a[2] * b[2]
}
pub fn cross(a: V3, b: V3) -> V3 {
[a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]]
}
pub fn norm(a: V3) -> f64 {
dot(a, a).sqrt()
}
pub fn normalize(a: V3) -> V3 {
let n = norm(a);
if n > 0.0 { scale(a, 1.0 / n) } else { a }
}
pub fn mv(m: &M3, v: V3) -> V3 {
[dot(m[0], v), dot(m[1], v), dot(m[2], v)]
}
/// `mᵀ v`.
pub fn mtv(m: &M3, v: V3) -> V3 {
[m[0][0] * v[0] + m[1][0] * v[1] + m[2][0] * v[2], m[0][1] * v[0] + m[1][1] * v[1] + m[2][1] * v[2], m[0][2] * v[0] + m[1][2] * v[1] + m[2][2] * v[2]]
}
pub fn mm(a: &M3, b: &M3) -> M3 {
let mut o = [[0.0; 3]; 3];
for (i, row) in o.iter_mut().enumerate() {
for (j, v) in row.iter_mut().enumerate() {
*v = a[i][0] * b[0][j] + a[i][1] * b[1][j] + a[i][2] * b[2][j];
}
}
o
}
pub fn mt(a: &M3) -> M3 {
[[a[0][0], a[1][0], a[2][0]], [a[0][1], a[1][1], a[2][1]], [a[0][2], a[1][2], a[2][2]]]
}
pub fn madd(a: &M3, b: &M3, k: f64) -> M3 {
let mut o = *a;
for i in 0..3 {
for j in 0..3 {
o[i][j] += k * b[i][j];
}
}
o
}
pub fn mscale(a: &M3, k: f64) -> M3 {
madd(&[[0.0; 3]; 3], a, k)
}
pub fn det3(m: &M3) -> f64 {
m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0])
}
pub fn skew(v: V3) -> M3 {
[[0.0, -v[2], v[1]], [v[2], 0.0, -v[0]], [-v[1], v[0], 0.0]]
}
pub fn outer(a: V3, b: V3) -> M3 {
[[a[0] * b[0], a[0] * b[1], a[0] * b[2]], [a[1] * b[0], a[1] * b[1], a[1] * b[2]], [a[2] * b[0], a[2] * b[1], a[2] * b[2]]]
}
pub fn col(m: &M3, j: usize) -> V3 {
[m[0][j], m[1][j], m[2][j]]
}
pub fn from_cols(a: V3, b: V3, c: V3) -> M3 {
[[a[0], b[0], c[0]], [a[1], b[1], c[1]], [a[2], b[2], c[2]]]
}
pub fn inv3(m: &M3) -> Option<M3> {
let d = det3(m);
if d.abs() < 1e-300 {
return None;
}
let c = |r0: usize, c0: usize, r1: usize, c1: usize| m[r0][c0] * m[r1][c1] - m[r0][c1] * m[r1][c0];
let adj = [[c(1, 1, 2, 2), -c(0, 1, 2, 2), c(0, 1, 1, 2)], [-c(1, 0, 2, 2), c(0, 0, 2, 2), -c(0, 0, 1, 2)], [c(1, 0, 2, 1), -c(0, 0, 2, 1), c(0, 0, 1, 1)]];
Some(mscale(&adj, 1.0 / d))
}
/// Rotation matrix of an axis-angle vector (Rodrigues' formula).
pub fn rodrigues(w: V3) -> M3 {
let th = norm(w);
let k = skew(w);
if th < 1e-12 {
return madd(&I3, &k, 1.0);
}
let a = th.sin() / th;
let b = (1.0 - th.cos()) / (th * th);
madd(&madd(&I3, &k, a), &mm(&k, &k), b)
}
/// Axis-angle vector of a rotation matrix (inverse of [`rodrigues`]).
pub fn log_rot(r: &M3) -> V3 {
let c = ((r[0][0] + r[1][1] + r[2][2] - 1.0) * 0.5).clamp(-1.0, 1.0);
let th = c.acos();
let v = [r[2][1] - r[1][2], r[0][2] - r[2][0], r[1][0] - r[0][1]];
if th < 1e-9 {
return scale(v, 0.5);
}
if std::f64::consts::PI - th < 1e-6 {
// Near π: the axis is the dominant column of R + I.
let b = madd(r, &I3, 1.0);
let j = (0..3).max_by(|&a, &bb| b[a][a].total_cmp(&b[bb][bb])).unwrap_or(0);
let axis = normalize(col(&b, j));
return scale(axis, th);
}
scale(v, th / (2.0 * th.sin()))
}
/// The rotation closest to `m` (Frobenius norm), via the SVD.
pub fn nearest_rotation(m: &M3) -> M3 {
let (u, _, v) = svd3(m);
let mut r = mm(&u, &mt(&v));
if det3(&r) < 0.0 {
let mut u2 = u;
for row in u2.iter_mut() {
row[2] = -row[2];
}
r = mm(&u2, &mt(&v));
}
r
}
/// Spherical interpolation between two rotations.
pub fn slerp(a: &M3, b: &M3, t: f64) -> M3 {
let d = log_rot(&mm(b, &mt(a)));
mm(&rodrigues(scale(d, t)), a)
}
/// Thin SVD `A = U diag(s) Vᵀ` of an `m × n` row-major matrix (one-sided Jacobi). `U` is
/// `max(m, n) × n` (rows beyond `m` belong to zero padding), `s` is sorted descending and `V`
/// is `n × n` (columns are the right singular vectors).
pub fn svd(a: &[f64], m: usize, n: usize) -> (Vec<f64>, Vec<f64>, Vec<f64>) {
let rows = m.max(n);
let mut u = vec![0.0; rows * n];
u[..m * n].copy_from_slice(&a[..m * n]);
let mut v = vec![0.0; n * n];
for i in 0..n {
v[i * n + i] = 1.0;
}
for _sweep in 0..60 {
let mut off = 0.0f64;
for p in 0..n {
for q in p + 1..n {
let (mut alpha, mut beta, mut gamma) = (0.0, 0.0, 0.0);
for r in 0..rows {
let (x, y) = (u[r * n + p], u[r * n + q]);
alpha += x * x;
beta += y * y;
gamma += x * y;
}
if gamma == 0.0 {
continue;
}
let c0 = gamma.abs() / (alpha * beta).sqrt().max(1e-300);
off = off.max(c0);
if c0 < 1e-15 {
continue;
}
let zeta = (beta - alpha) / (2.0 * gamma);
let t = zeta.signum() / (zeta.abs() + (1.0 + zeta * zeta).sqrt());
let t = if zeta == 0.0 { 1.0 } else { t };
let c = 1.0 / (1.0 + t * t).sqrt();
let s = c * t;
for r in 0..rows {
let (x, y) = (u[r * n + p], u[r * n + q]);
u[r * n + p] = c * x - s * y;
u[r * n + q] = s * x + c * y;
}
for r in 0..n {
let (x, y) = (v[r * n + p], v[r * n + q]);
v[r * n + p] = c * x - s * y;
v[r * n + q] = s * x + c * y;
}
}
}
if off < 1e-15 {
break;
}
}
let mut s: Vec<f64> = (0..n).map(|j| (0..rows).map(|r| u[r * n + j] * u[r * n + j]).sum::<f64>().sqrt()).collect();
for (j, sj) in s.iter().enumerate() {
if *sj > 1e-300 {
for r in 0..rows {
u[r * n + j] /= sj;
}
}
}
// Sort descending.
let mut order: Vec<usize> = (0..n).collect();
order.sort_by(|&a, &b| s[b].total_cmp(&s[a]));
let (mut u2, mut v2) = (vec![0.0; rows * n], vec![0.0; n * n]);
for (k, &j) in order.iter().enumerate() {
for r in 0..rows {
u2[r * n + k] = u[r * n + j];
}
for r in 0..n {
v2[r * n + k] = v[r * n + j];
}
}
s = order.iter().map(|&j| s[j]).collect();
(u2, s, v2)
}
/// SVD of a 3 × 3 matrix: (U, singular values descending, V) with `m = U diag(s) Vᵀ`.
pub fn svd3(m: &M3) -> (M3, V3, M3) {
let a: Vec<f64> = m.iter().flatten().copied().collect();
let (u, s, v) = svd(&a, 3, 3);
let g = |x: &[f64]| [[x[0], x[1], x[2]], [x[3], x[4], x[5]], [x[6], x[7], x[8]]];
let mut um = g(&u);
// Columns of U for zero singular values are undefined: complete them to an orthonormal basis.
if s[2] < 1e-300 {
let c = cross(col(&um, 0), col(&um, 1));
for (r, row) in um.iter_mut().enumerate() {
row[2] = c[r];
}
}
(um, [s[0], s[1], s[2]], g(&v))
}
/// The unit vector `x` minimising `|A x|` (`m × n` row-major): the right singular vector of
/// the smallest singular value.
pub fn null_vector(a: &[f64], m: usize, n: usize) -> Vec<f64> {
let (_, _, v) = svd(a, m, n);
(0..n).map(|r| v[r * n + n - 1]).collect()
}
/// Eigen-decomposition of a symmetric `n × n` matrix (cyclic Jacobi): eigenvalues ascending
/// and the matching eigenvectors as the columns of the returned row-major matrix.
pub fn sym_eigen(a: &[f64], n: usize) -> (Vec<f64>, Vec<f64>) {
let mut a = a.to_vec();
let mut v = vec![0.0; n * n];
for i in 0..n {
v[i * n + i] = 1.0;
}
for _ in 0..100 {
let mut off = 0.0;
for p in 0..n {
for q in p + 1..n {
off += a[p * n + q] * a[p * n + q];
}
}
if off < 1e-30 {
break;
}
for p in 0..n {
for q in p + 1..n {
let apq = a[p * n + q];
if apq.abs() < 1e-300 {
continue;
}
let theta = (a[q * n + q] - a[p * n + p]) / (2.0 * apq);
let t = theta.signum() / (theta.abs() + (theta * theta + 1.0).sqrt());
let t = if theta == 0.0 { 1.0 } else { t };
let c = 1.0 / (t * t + 1.0).sqrt();
let s = t * c;
for k in 0..n {
let (akp, akq) = (a[k * n + p], a[k * n + q]);
a[k * n + p] = c * akp - s * akq;
a[k * n + q] = s * akp + c * akq;
}
for k in 0..n {
let (apk, aqk) = (a[p * n + k], a[q * n + k]);
a[p * n + k] = c * apk - s * aqk;
a[q * n + k] = s * apk + c * aqk;
}
for k in 0..n {
let (vkp, vkq) = (v[k * n + p], v[k * n + q]);
v[k * n + p] = c * vkp - s * vkq;
v[k * n + q] = s * vkp + c * vkq;
}
}
}
}
let mut order: Vec<usize> = (0..n).collect();
order.sort_by(|&x, &y| a[x * n + x].total_cmp(&a[y * n + y]));
let vals = order.iter().map(|&j| a[j * n + j]).collect();
let mut vecs = vec![0.0; n * n];
for (k, &j) in order.iter().enumerate() {
for r in 0..n {
vecs[r * n + k] = v[r * n + j];
}
}
(vals, vecs)
}
/// Dot product, four lanes (lets the compiler vectorise).
fn dot_n(a: &[f64], b: &[f64]) -> f64 {
let mut acc = [0.0f64; 4];
let (ca, cb) = (a.chunks_exact(4), b.chunks_exact(4));
let (ra, rb) = (ca.remainder(), cb.remainder());
for (x, y) in ca.zip(cb) {
for k in 0..4 {
acc[k] += x[k] * y[k];
}
}
let mut s = acc[0] + acc[1] + acc[2] + acc[3];
for (x, y) in ra.iter().zip(rb) {
s += x * y;
}
s
}
/// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by
/// Cholesky factorisation. Returns false when `A` is not positive definite.
pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool {
for j in 0..n {
let mut d = a[j * n + j];
for k in 0..j {
d -= a[j * n + k] * a[j * n + k];
}
if d <= 0.0 || !d.is_finite() {
return false;
}
let d = d.sqrt();
a[j * n + j] = d;
let (head, tail) = a.split_at_mut((j + 1) * n);
let rowj = &head[j * n..j * n + j];
// Rows below j (sequential: per-column parallel dispatch costs more than it saves).
for row in tail.chunks_mut(n) {
row[j] = (row[j] - dot_n(&row[..j], rowj)) / d;
}
}
// Forward: L y = b.
for i in 0..n {
let mut s = b[i];
for k in 0..i {
s -= a[i * n + k] * b[k];
}
b[i] = s / a[i * n + i];
}
// Back: Lᵀ x = y.
for i in (0..n).rev() {
let mut s = b[i];
for k in i + 1..n {
s -= a[k * n + i] * b[k];
}
b[i] = s / a[i * n + i];
}
b.iter().all(|v| v.is_finite())
}
/// A similarity transform `x ↦ s R x + t`.
#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct Similarity {
pub s: f64,
pub r: M3,
pub t: V3,
}
impl Default for Similarity {
fn default() -> Self {
Similarity { s: 1.0, r: I3, t: [0.0; 3] }
}
}
impl Similarity {
pub fn apply(&self, x: V3) -> V3 {
add(scale(mv(&self.r, x), self.s), self.t)
}
}
/// The similarity minimising `Σ |dst − (s R src + t)|²` (Umeyama 1991).
pub fn umeyama(src: &[V3], dst: &[V3]) -> Option<Similarity> {
let n = src.len().min(dst.len());
if n < 3 {
return None;
}
let k = 1.0 / n as f64;
let ms = scale(src[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k);
let md = scale(dst[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k);
let mut cov = [[0.0; 3]; 3];
let mut var = 0.0;
for i in 0..n {
let (a, b) = (sub(src[i], ms), sub(dst[i], md));
cov = madd(&cov, &outer(b, a), k);
var += dot(a, a) * k;
}
if var < 1e-300 {
return None;
}
let (u, d, v) = svd3(&cov);
let mut sgn = [1.0, 1.0, 1.0];
if det3(&u) * det3(&v) < 0.0 {
sgn[2] = -1.0;
}
let ud = from_cols(scale(col(&u, 0), sgn[0]), scale(col(&u, 1), sgn[1]), scale(col(&u, 2), sgn[2]));
let r = mm(&ud, &mt(&v));
let s = (d[0] * sgn[0] + d[1] * sgn[1] + d[2] * sgn[2]) / var;
let t = sub(md, scale(mv(&r, ms), s));
Some(Similarity { s, r, t })
}
/// Euler angles (degrees) `[x, y, z]` with `m = Rz(z) Ry(y) Rx(x)` (After Effects' Orientation
/// order: X, then Y, then Z applied to the vector).
pub fn euler_xyz(m: &M3) -> V3 {
let sy = (-m[2][0]).clamp(-1.0, 1.0);
let y = sy.asin();
let (x, z) = if sy.abs() < 0.999_999 { (m[2][1].atan2(m[2][2]), m[1][0].atan2(m[0][0])) } else { ((-m[1][2]).atan2(m[1][1]), 0.0) };
[x.to_degrees(), y.to_degrees(), z.to_degrees()]
}
/// `Rz(z) Ry(y) Rx(x)` from degrees.
pub fn from_euler_xyz(e: V3) -> M3 {
let (sx, cx) = e[0].to_radians().sin_cos();
let (sy, cy) = e[1].to_radians().sin_cos();
let (sz, cz) = e[2].to_radians().sin_cos();
let rx = [[1.0, 0.0, 0.0], [0.0, cx, -sx], [0.0, sx, cx]];
let ry = [[cy, 0.0, sy], [0.0, 1.0, 0.0], [-sy, 0.0, cy]];
let rz = [[cz, -sz, 0.0], [sz, cz, 0.0], [0.0, 0.0, 1.0]];
mm(&rz, &mm(&ry, &rx))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn svd_reconstructs() {
let a = [3.0, 1.0, 2.0, -1.0, 4.0, 0.5, 2.0, 2.0, 7.0, 0.1, -3.0, 1.0];
let (u, s, v) = svd(&a, 4, 3);
for r in 0..4 {
for c in 0..3 {
let x: f64 = (0..3).map(|k| u[r * 3 + k] * s[k] * v[c * 3 + k]).sum();
assert!((x - a[r * 3 + c]).abs() < 1e-10);
}
}
assert!(s[0] >= s[1] && s[1] >= s[2]);
}
#[test]
fn rotations_round_trip() {
let w = [0.3, -1.2, 0.7];
let r = rodrigues(w);
let w2 = log_rot(&r);
assert!(norm(sub(w, w2)) < 1e-12);
let e = euler_xyz(&r);
let r2 = from_euler_xyz(e);
for i in 0..3 {
for j in 0..3 {
assert!((r[i][j] - r2[i][j]).abs() < 1e-12);
}
}
assert!((det3(&r) - 1.0).abs() < 1e-12);
}
#[test]
fn eigen_and_cholesky() {
let a = [4.0, 1.0, 0.5, 1.0, 3.0, 0.2, 0.5, 0.2, 2.0];
let (vals, vecs) = sym_eigen(&a, 3);
for k in 0..3 {
let v = [vecs[k], vecs[3 + k], vecs[6 + k]];
let av = [dot([a[0], a[1], a[2]], v), dot([a[3], a[4], a[5]], v), dot([a[6], a[7], a[8]], v)];
assert!(norm(sub(av, scale(v, vals[k]))) < 1e-10);
}
let mut m = a;
let mut b = [1.0, 2.0, 3.0];
assert!(cholesky_solve(&mut m, 3, &mut b));
let r = [dot([a[0], a[1], a[2]], b), dot([a[3], a[4], a[5]], b), dot([a[6], a[7], a[8]], b)];
assert!(norm(sub(r, [1.0, 2.0, 3.0])) < 1e-12);
}
#[test]
fn umeyama_recovers_a_similarity() {
let t = Similarity { s: 2.5, r: rodrigues([0.1, 0.4, -0.3]), t: [1.0, -2.0, 5.0] };
let src = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 2.0, 3.0]];
let dst: Vec<V3> = src.iter().map(|p| t.apply(*p)).collect();
let e = umeyama(&src, &dst).unwrap();
assert!((e.s - 2.5).abs() < 1e-10);
assert!(norm(sub(e.t, t.t)) < 1e-9);
}
}
+380
View File
@@ -0,0 +1,380 @@
//! The 3D Camera Tracker's analysis: structure from motion on a single clip, clean-room from the
//! textbook pipeline (Hartley & Zisserman, *Multiple View Geometry in Computer Vision*, 2nd ed.,
//! 2004; Triggs et al., "Bundle Adjustment — A Modern Synthesis", 2000):
//!
//! 1. **Feature tracks** ([`features::TrackAnalyzer`]): Shi–Tomasi corners followed through the
//! clip with pyramidal Lucas–Kanade and a forward–backward check ([`crate::klt`]); lost
//! features end their track and new ones are detected where the frame has none, so tracks
//! are as long as possible.
//! 2. **Solve** ([`solve`]): keyframes are selected by parallax; a two-view reconstruction
//! initialises the scene from either the **essential matrix** (normalised 8-point algorithm
//! inside RANSAC, H&Z §11.2 and §9.6) or, for mostly flat scenes, a **homography
//! decomposition** (Ma, Soatto, Košecká & Sastry, *An Invitation to 3-D Vision*, 2004, §5.3);
//! the other keyframes are added incrementally (**resection** by robust pose refinement,
//! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse
//! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points
//! ([`bundle`]) refines everything; the frames between keyframes are resected last. Inlier
//! thresholds start from the analysis resolution and are re-estimated from the residuals
//! (median absolute residual) once the keyframes are solved, so features on moving objects,
//! occlusion edges and screen-locked overlays drop out.
//! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it
//! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal
//! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment
//! as one shared parameter; *Variable Zoom* starts from the same estimate and adjusts a focal
//! length per frame, with a weak smoothness prior between neighbouring frames.
//! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame
//! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle
//! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the
//! tripod solution when it explains the tracks as well as the general one (a quick rotation fit
//! at the general solution's focal length skips the tripod search when it clearly fails).
//!
//! The camera model has square pixels, the principal point at the layer centre and no lens
//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera
//! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`.
//!
//! The solve is stored in a canonical frame: the first solved frame's camera is at the origin
//! looking along +z, scaled so the median depth of the points it sees is 1. [`CameraSolve::world`]
//! maps it into composition space for a camera zoom, optionally re-based on a ground plane.
pub mod bundle;
pub mod features;
pub mod geometry;
pub mod linalg;
mod solver;
#[cfg(test)]
mod tests;
use serde::{Deserialize, Serialize};
pub use features::{AnalyzeOpts, TrackAnalyzer};
use linalg::{M3, Similarity, V3};
pub use solver::{SolveError, solve};
/// One feature track: positions (layer pixels) on consecutive frames from `start`.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Track2D {
pub id: u32,
pub start: u32,
pub pts: Vec<[f32; 2]>,
}
impl Track2D {
pub fn end(&self) -> u32 {
self.start + self.pts.len() as u32
}
/// Position on frame `k` when the track covers it.
pub fn at(&self, k: u32) -> Option<[f64; 2]> {
let i = k.checked_sub(self.start)? as usize;
self.pts.get(i).map(|p| [p[0] as f64, p[1] as f64])
}
}
/// The tracked features of a clip (step 1 of the analysis).
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct CameraTracks {
pub version: u32,
/// Layer size (pixels).
pub size: [f64; 2],
/// Layer time (seconds) of frame 0, and the frame duration.
pub start: f64,
pub frame_duration: f64,
pub frames: u32,
/// Analysis downsampling factor (image pixels per analysis pixel): the tracking noise scale.
pub factor: f64,
pub detailed: bool,
pub tracks: Vec<Track2D>,
}
impl CameraTracks {
pub fn to_json(&self) -> String {
serde_json::to_string(self).unwrap_or_default()
}
pub fn from_json(s: &str) -> Option<CameraTracks> {
if s.trim().is_empty() {
return None;
}
serde_json::from_str(s).ok()
}
pub fn is_empty(&self) -> bool {
self.frames == 0
}
pub fn track(&self, id: u32) -> Option<&Track2D> {
self.tracks.binary_search_by_key(&id, |t| t.id).ok().map(|i| &self.tracks[i])
}
}
/// 3D Camera Tracker ▸ Shot Type.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum ShotType {
#[default]
FixedAngle,
VariableZoom,
SpecifyAngle,
}
impl ShotType {
pub const ALL: [ShotType; 3] = [ShotType::FixedAngle, ShotType::VariableZoom, ShotType::SpecifyAngle];
pub fn label(self) -> &'static str {
match self {
ShotType::FixedAngle => "Fixed Angle of View",
ShotType::VariableZoom => "Variable Zoom",
ShotType::SpecifyAngle => "Specify Angle of View",
}
}
pub fn from_name(s: &str) -> Option<ShotType> {
let k = s.to_ascii_lowercase().replace([' ', '-', '_'], "");
match k.as_str() {
"fixed" | "fixedangle" | "fixedangleofview" => Some(ShotType::FixedAngle),
"variable" | "variablezoom" | "zoom" => Some(ShotType::VariableZoom),
"specify" | "specifyangle" | "specifyangleofview" => Some(ShotType::SpecifyAngle),
_ => None,
}
}
}
/// 3D Camera Tracker ▸ Advanced ▸ Solve Method (and Method Used).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum SolveMethod {
#[default]
Auto,
Typical,
MostlyFlat,
TripodPan,
}
impl SolveMethod {
pub const ALL: [SolveMethod; 4] = [SolveMethod::Auto, SolveMethod::Typical, SolveMethod::MostlyFlat, SolveMethod::TripodPan];
pub fn label(self) -> &'static str {
match self {
SolveMethod::Auto => "Auto Detect",
SolveMethod::Typical => "Typical",
SolveMethod::MostlyFlat => "Mostly Flat Scene",
SolveMethod::TripodPan => "Tripod Pan",
}
}
pub fn from_name(s: &str) -> Option<SolveMethod> {
let k = s.to_ascii_lowercase().replace([' ', '-', '_'], "");
match k.as_str() {
"auto" | "autodetect" => Some(SolveMethod::Auto),
"typical" => Some(SolveMethod::Typical),
"flat" | "mostlyflat" | "mostlyflatscene" => Some(SolveMethod::MostlyFlat),
"tripod" | "tripodpan" => Some(SolveMethod::TripodPan),
_ => None,
}
}
}
/// What a solve is made with.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct SolveSettings {
pub shot: ShotType,
/// Horizontal angle of view (degrees) for [`ShotType::SpecifyAngle`].
pub hfov: f64,
pub method: SolveMethod,
/// Track ids the user deleted.
pub deleted: Vec<u32>,
}
/// The solved camera of one frame.
#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct SolvedFrame {
/// World → camera rotation (axis-angle).
pub rot: V3,
/// Camera centre.
pub center: V3,
/// Focal length (layer pixels).
pub focal: f64,
pub solved: bool,
}
impl SolvedFrame {
pub fn r(&self) -> M3 {
linalg::rodrigues(self.rot)
}
/// Camera-space position of a world point.
pub fn to_cam(&self, x: V3) -> V3 {
linalg::mv(&self.r(), linalg::sub(x, self.center))
}
/// Project a world point to layer pixels (`None` behind the camera).
pub fn project(&self, size: [f64; 2], x: V3) -> Option<[f64; 2]> {
let c = self.to_cam(x);
(c[2] > 1e-9).then(|| [self.focal * c[0] / c[2] + size[0] * 0.5, self.focal * c[1] / c[2] + size[1] * 0.5])
}
}
/// A solved 3D track point.
#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct SolvedPoint {
/// The feature track it was made from.
pub id: u32,
pub pos: V3,
/// Average reprojection error (pixels).
pub error: f32,
/// Frames `[first, last]` the point was tracked on.
pub first: u32,
pub last: u32,
}
/// A ground plane and origin (canonical solve coordinates).
#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct Ground {
pub origin: V3,
/// Unit normal on the cameras' side ("up").
pub normal: V3,
/// Unit in-plane direction that becomes world +x.
pub x_axis: V3,
}
/// A plane through solved points: After Effects' target.
#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct Target {
pub center: V3,
/// Unit normal facing the camera.
pub normal: V3,
/// Diameter in canonical units.
pub size: f64,
}
/// The camera solve (step 2 of the analysis).
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct CameraSolve {
pub version: u32,
pub size: [f64; 2],
pub start: f64,
pub frame_duration: f64,
pub shot: ShotType,
/// The model that was used (Auto Detect resolves to one of the others).
pub method_used: SolveMethod,
/// Average reprojection error (pixels) over all solved observations.
pub average_error: f64,
pub frames: Vec<SolvedFrame>,
pub points: Vec<SolvedPoint>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ground: Option<Ground>,
}
impl CameraSolve {
pub fn to_json(&self) -> String {
serde_json::to_string(self).unwrap_or_default()
}
pub fn from_json(s: &str) -> Option<CameraSolve> {
if s.trim().is_empty() {
return None;
}
serde_json::from_str(s).ok()
}
pub fn is_empty(&self) -> bool {
self.frames.is_empty()
}
/// Frame index at a layer time (held at the ends).
pub fn frame_at(&self, layer_time: f64) -> Option<usize> {
if self.frames.is_empty() || self.frame_duration <= 0.0 {
return None;
}
let i = ((layer_time - self.start) / self.frame_duration).round();
Some(i.clamp(0.0, (self.frames.len() - 1) as f64) as usize)
}
pub fn point(&self, id: u32) -> Option<&SolvedPoint> {
self.points.iter().find(|p| p.id == id)
}
/// Points tracked on frame `k`.
pub fn visible(&self, k: usize) -> impl Iterator<Item = &SolvedPoint> {
let k = k as u32;
self.points.iter().filter(move |p| p.first <= k && k <= p.last)
}
/// Median focal length of the solved frames (layer pixels).
pub fn focal(&self) -> f64 {
let mut f: Vec<f64> = self.frames.iter().filter(|c| c.solved).map(|c| c.focal).collect();
if f.is_empty() {
return 0.0;
}
f.sort_by(f64::total_cmp);
f[f.len() / 2]
}
/// Horizontal angle of view (degrees) of focal length `f`.
pub fn hfov(&self, f: f64) -> f64 {
2.0 * (self.size[0] * 0.5 / f.max(1e-9)).atan().to_degrees()
}
/// The plane through points `ids` seen from frame `k` (needs three points; with more the
/// plane is a least-squares fit).
pub fn target(&self, ids: &[u32], k: usize) -> Option<Target> {
let pts: Vec<V3> = ids.iter().filter_map(|i| self.point(*i)).map(|p| p.pos).collect();
let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?;
geometry::fit_plane(&pts).map(|(c, mut n, spread)| {
if linalg::dot(n, linalg::sub(cam.center, c)) < 0.0 {
n = linalg::scale(n, -1.0);
}
// After Effects sizes the target from the camera distance when the points are close.
let depth = linalg::norm(linalg::sub(cam.center, c));
Target { center: c, normal: n, size: (spread * 2.0).max(depth * 0.12) }
})
}
/// A ground plane through `ids` (origin at their centroid, normal towards frame `k`'s
/// camera, +x along that camera's right vector projected on the plane).
pub fn ground_from(&self, ids: &[u32], k: usize) -> Option<Ground> {
let t = self.target(ids, k)?;
let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?;
let right = linalg::mtv(&cam.r(), [1.0, 0.0, 0.0]);
let mut x = linalg::sub(right, linalg::scale(t.normal, linalg::dot(right, t.normal)));
if linalg::norm(x) < 1e-6 {
let fwd = linalg::mtv(&cam.r(), [0.0, 0.0, 1.0]);
x = linalg::cross(fwd, t.normal);
}
Some(Ground { origin: t.center, normal: t.normal, x_axis: linalg::normalize(x) })
}
/// Canonical solve coordinates → composition world, for a camera whose zoom on the first
/// solved frame is `zoom0` (comp pixels) in a `comp` sized composition. Without a ground
/// plane, the first solved frame's camera becomes After Effects' default camera (at
/// `(w/2, h/2, −zoom)` looking down +z, so points at the median depth land near z = 0). With
/// one, the ground plane becomes the X-Z plane through the origin, normal along −y (up).
pub fn world(&self, comp: [f64; 2], zoom0: f64) -> Similarity {
match &self.ground {
Some(g) => {
// Rows: canonical directions that map to world x, y, z.
let y = linalg::scale(g.normal, -1.0);
let x = g.x_axis;
let z = linalg::cross(x, y);
let r = [x, y, z];
let t = linalg::scale(linalg::mv(&r, g.origin), -zoom0);
Similarity { s: zoom0, r, t }
}
None => Similarity { s: zoom0, r: linalg::I3, t: [comp[0] * 0.5, comp[1] * 0.5, -zoom0] },
}
}
/// The first solved frame.
pub fn first_solved(&self) -> Option<usize> {
self.frames.iter().position(|c| c.solved)
}
}
/// Deterministic colour of a track point (After Effects draws each target in its own colour).
pub fn point_color(id: u32) -> [f32; 3] {
let mut h = id.wrapping_mul(0x9E37_79B9) ^ 0x5bd1_e995;
h ^= h >> 15;
let hue = (h % 360) as f32 / 60.0;
let x = 1.0 - (hue % 2.0 - 1.0).abs();
let (r, g, b) = match hue as u32 {
0 => (1.0, x, 0.0),
1 => (x, 1.0, 0.0),
2 => (0.0, 1.0, x),
3 => (0.0, x, 1.0),
4 => (x, 0.0, 1.0),
_ => (1.0, 0.0, x),
};
// Bright, slightly desaturated.
[0.25 + 0.75 * r, 0.25 + 0.75 * g, 0.25 + 0.75 * b]
}
+953
View File
@@ -0,0 +1,953 @@
//! The camera solve pipeline (see the module docs of [`super`]).
use std::collections::{BTreeMap, HashMap, HashSet};
use std::sync::atomic::{AtomicBool, Ordering};
use rayon::prelude::*;
use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode};
use super::geometry::{self, Pose};
use super::linalg::*;
use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint};
use crate::fit::{Model, ransac};
fn debug() -> bool {
std::env::var_os("EC_CAMTRACK_DEBUG").is_some()
}
/// Why a solve failed.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SolveError(pub String);
impl std::fmt::Display for SolveError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
fn err<T>(m: &str) -> Result<T, SolveError> {
Err(SolveError(m.to_string()))
}
/// Tracks in solver form: pixels relative to the principal point.
struct Data {
n: usize,
w: f64,
/// (track id, first frame, positions).
tracks: Vec<(u32, usize, Vec<[f64; 2]>)>,
/// Per frame: (track index, position).
by_frame: Vec<Vec<(u32, [f64; 2])>>,
/// Tracking noise scale (pixels): starts from the analysis resolution and is re-estimated
/// from the residuals once the keyframes are solved.
noise_bits: std::sync::atomic::AtomicU64,
}
impl Data {
fn new(t: &CameraTracks, deleted: &[u32]) -> Data {
let n = t.frames as usize;
let c = [t.size[0] * 0.5, t.size[1] * 0.5];
let del: HashSet<u32> = deleted.iter().copied().collect();
let mut tracks = vec![];
let mut by_frame = vec![vec![]; n];
for tr in t.tracks.iter().filter(|tr| !del.contains(&tr.id) && tr.pts.len() >= 3) {
let ti = tracks.len() as u32;
let pts: Vec<[f64; 2]> = tr.pts.iter().map(|p| [p[0] as f64 - c[0], p[1] as f64 - c[1]]).collect();
for (k, p) in pts.iter().enumerate() {
if let Some(v) = by_frame.get_mut(tr.start as usize + k) {
v.push((ti, *p));
}
}
tracks.push((tr.id, tr.start as usize, pts));
}
let noise = (0.5 * t.factor).max(0.5);
Data { n, w: t.size[0], tracks, by_frame, noise_bits: std::sync::atomic::AtomicU64::new(noise.to_bits()) }
}
fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> {
let (_, s, p) = &self.tracks[ti as usize];
k.checked_sub(*s).and_then(|i| p.get(i).copied())
}
fn noise(&self) -> f64 {
f64::from_bits(self.noise_bits.load(Ordering::Relaxed))
}
fn set_noise(&self, v: f64) {
self.noise_bits.store(v.to_bits(), Ordering::Relaxed);
}
fn inlier_thr(&self) -> f64 {
4.0 * self.noise()
}
fn huber(&self) -> f64 {
self.noise()
}
}
/// Parallax-based keyframes: a new keyframe when the median feature motion since the last
/// exceeds 1.5 % of the width, when fewer than 70 % of its features survive, after 12 frames, or
/// before the shared features drop below 24. The first and last frames with features are
/// keyframes.
fn keyframes(d: &Data) -> Vec<usize> {
let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] };
let mut kf = vec![first];
let mut last = first;
let mut j = first + 1;
while j < d.n {
let base = &d.by_frame[last];
let mut disp = vec![];
for (ti, p) in base {
if let Some(q) = d.at(*ti, j) {
disp.push((q[0] - p[0]).hypot(q[1] - p[1]));
}
}
if disp.len() < 24 && j > last + 1 {
// Too few shared features: the previous frame becomes a keyframe.
last = j - 1;
kf.push(last);
continue;
}
disp.sort_by(f64::total_cmp);
let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0);
let ratio = disp.len() as f64 / base.len().max(1) as f64;
if med > 0.015 * d.w || ratio < 0.7 || disp.len() < 24 || j - last >= 12 {
last = j;
kf.push(j);
}
j += 1;
}
if let Some(lastf) = (0..d.n).rev().find(|k| d.by_frame[*k].len() >= 8)
&& *kf.last().expect("one") < lastf
{
kf.push(lastf);
}
kf
}
/// A (partial) reconstruction.
#[derive(Clone, Default)]
struct Recon {
cams: BTreeMap<usize, BaCam>,
pts: HashMap<u32, V3>,
/// Rejected observations (track index, frame).
outliers: HashSet<(u32, usize)>,
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum Kind {
General,
Tripod,
}
#[derive(Clone, Copy, PartialEq, Debug)]
struct Cfg {
kind: Kind,
/// Focal mode of the global adjustments.
focal: FocalMode,
/// Registration estimates a focal length per frame.
per_frame_f: bool,
}
const F_PRIOR: f64 = 30.0;
impl Recon {
/// Observations among `frames` (inliers, with a 3D point).
fn observations(&self, d: &Data, frames: &[usize]) -> (Vec<BaCam>, Vec<V3>, Vec<BaObs>, Vec<u32>) {
let cams: Vec<BaCam> = frames.iter().map(|k| self.cams[k]).collect();
let mut pidx: HashMap<u32, u32> = HashMap::new();
let mut pts = vec![];
let mut ids = vec![];
let mut obs = vec![];
for (ci, k) in frames.iter().enumerate() {
for (ti, uv) in &d.by_frame[*k] {
let Some(x) = self.pts.get(ti) else { continue };
if self.outliers.contains(&(*ti, *k)) {
continue;
}
let pi = *pidx.entry(*ti).or_insert_with(|| {
pts.push(*x);
ids.push(*ti);
(pts.len() - 1) as u32
});
obs.push(BaObs { cam: ci as u32, pt: pi, uv: *uv });
}
}
(cams, pts, obs, ids)
}
/// Bundle-adjust the cameras of `frames` (the first stays fixed) and their points.
fn adjust(&mut self, d: &Data, frames: &[usize], cfg: &Cfg, focal: FocalMode, iters: usize) -> f64 {
let (mut cams, mut pts, obs, ids) = self.observations(d, frames);
if obs.is_empty() {
return f64::INFINITY;
}
let mut fixed = vec![false; cams.len()];
fixed[0] = true;
let rep = bundle::adjust(
&mut cams,
&mut pts,
&obs,
&BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters },
);
for (k, c) in frames.iter().zip(&cams) {
self.cams.insert(*k, *c);
}
for (ti, x) in ids.iter().zip(&pts) {
self.pts.insert(*ti, *x);
}
rep.cost
}
/// Mark observations above the inlier threshold as outliers; drop points left with fewer
/// than two observations among the registered frames.
fn reject(&mut self, d: &Data) {
let thr2 = d.inlier_thr().powi(2);
let mut count: HashMap<u32, usize> = HashMap::new();
for (k, c) in &self.cams {
for (ti, uv) in &d.by_frame[*k] {
let Some(x) = self.pts.get(ti) else { continue };
if self.outliers.contains(&(*ti, *k)) {
continue;
}
match bundle::residual(c, *x, *uv) {
Some(r) if r[0] * r[0] + r[1] * r[1] <= thr2 => *count.entry(*ti).or_default() += 1,
_ => {
self.outliers.insert((*ti, *k));
}
}
}
}
self.pts.retain(|ti, _| count.get(ti).copied().unwrap_or(0) >= 2);
}
/// Triangulate tracks seen on ≥ 2 registered frames that have no point yet.
fn triangulate(&mut self, d: &Data, kind: Kind, min_angle_deg: f64, check: f64) {
let mut seen: HashMap<u32, Vec<usize>> = HashMap::new();
for k in self.cams.keys() {
for (ti, _) in &d.by_frame[*k] {
if !self.pts.contains_key(ti) {
seen.entry(*ti).or_default().push(*k);
}
}
}
let thr2 = (d.inlier_thr() * check).powi(2);
let new: Vec<(u32, V3)> = seen
.par_iter()
.filter(|(_, ks)| ks.len() >= 2)
.filter_map(|(ti, ks)| {
let views: Vec<(Pose, [f64; 2], &BaCam)> = ks
.iter()
.map(|k| {
let c = &self.cams[k];
let uv = d.at(*ti, *k).expect("seen");
(c.pose, [uv[0] / c.f, uv[1] / c.f], c)
})
.collect();
let x = match kind {
Kind::Tripod => normalize(views.iter().fold([0.0; 3], |a, (p, o, _)| add(a, normalize(mtv(&p.r, [o[0], o[1], 1.0]))))),
Kind::General => {
let v: Vec<(Pose, [f64; 2])> = views.iter().map(|(p, o, _)| (*p, *o)).collect();
let x = geometry::triangulate(&v)?;
// Enough parallax between the most separated views.
let (a, b) = (views[0].0.c, views[views.len() - 1].0.c);
if geometry::ray_angle(a, b, x).to_degrees() < min_angle_deg {
return None;
}
x
}
};
for (k, (_, _, c)) in ks.iter().zip(&views) {
let uv = d.at(*ti, *k).expect("seen");
let r = bundle::residual(c, x, uv)?;
if r[0] * r[0] + r[1] * r[1] > thr2 {
return None;
}
}
Some((*ti, x))
})
.collect();
self.pts.extend(new);
}
/// Resect frame `k` from the existing points starting at `init` (pose, and the focal
/// length when `per_frame_f`). Returns false when too few points agree.
fn register(&mut self, d: &Data, k: usize, init: BaCam, cfg: &Cfg) -> bool {
let mut pts = vec![];
let mut obs = vec![];
let mut ids = vec![];
for (ti, uv) in &d.by_frame[k] {
if let Some(x) = self.pts.get(ti) {
obs.push(BaObs { cam: 0, pt: pts.len() as u32, uv: *uv });
pts.push(*x);
ids.push(*ti);
}
}
if obs.len() < 6 {
return false;
}
let thr2 = d.inlier_thr().powi(2);
let opts = |huber: f64| BaOpts {
focal: if cfg.per_frame_f { FocalMode::PerCamera(0.0) } else { FocalMode::Fixed },
fix_points: true,
fix_centers: cfg.kind == Kind::Tripod,
fixed: vec![false],
huber,
max_iter: 40,
};
let mut best: Option<(usize, BaCam)> = None;
// Robust start (large Huber threshold), then tighten; from the prediction and, failing
// that, from the nearest registered camera.
let nearest = self.cams.iter().min_by_key(|(c, _)| c.abs_diff(k)).map(|(_, c)| *c);
for start in std::iter::once(init).chain(nearest) {
if best.as_ref().is_some_and(|b| b.0 as f64 >= 0.8 * obs.len() as f64) {
break;
}
let mut cams = vec![start];
bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.inlier_thr() * 4.0));
bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.huber()));
let inl = obs.iter().filter(|o| bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).count();
if best.as_ref().is_none_or(|b| inl > b.0) {
best = Some((inl, cams[0]));
}
}
let Some((inl, cam)) = best else { return false };
if inl < 6 || (inl as f64) < 0.5 * obs.len() as f64 {
return false;
}
// Refine on the inliers only.
let keep: Vec<BaObs> =
obs.iter().filter(|o| bundle::residual(&cam, pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).copied().collect();
let mut cams = vec![cam];
bundle::adjust(&mut cams, &mut pts.clone(), &keep, &opts(d.huber()));
for o in &obs {
let ok = bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2);
if !ok {
self.outliers.insert((ids[o.pt as usize], k));
}
}
self.cams.insert(k, cams[0]);
true
}
/// How well the reconstruction explains every observation on its frames: the mean of the
/// reprojection error truncated at the inlier threshold, an observation without a point
/// counting as the threshold.
fn score(&self, d: &Data) -> f64 {
let thr = d.inlier_thr();
let mut s = 0.0;
let mut n = 0usize;
for (k, c) in &self.cams {
for (ti, uv) in &d.by_frame[*k] {
n += 1;
s += match self.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) {
Some(r) => r[0].hypot(r[1]).min(thr),
None => thr,
};
}
}
if n > 0 { s / n as f64 } else { f64::INFINITY }
}
/// Average reprojection error (pixels) and observation count over the inliers.
fn error(&self, d: &Data) -> (f64, usize) {
let mut s = 0.0;
let mut n = 0;
for (k, c) in &self.cams {
for (ti, uv) in &d.by_frame[*k] {
let Some(x) = self.pts.get(ti) else { continue };
if self.outliers.contains(&(*ti, *k)) {
continue;
}
if let Some(r) = bundle::residual(c, *x, *uv) {
s += r[0].hypot(r[1]);
n += 1;
}
}
}
(if n > 0 { s / n as f64 } else { f64::INFINITY }, n)
}
}
/// Interpolated / extrapolated starting camera for frame `k` from the registered ones.
fn predict(r: &Recon, k: usize) -> Option<BaCam> {
let before = r.cams.range(..k).next_back().map(|(a, b)| (*a, *b));
let after = r.cams.range(k + 1..).next().map(|(a, b)| (*a, *b));
match (before, after) {
(Some((ka, a)), Some((kb, b))) => {
let t = (k - ka) as f64 / (kb - ka) as f64;
Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) })
}
(Some((ka, a)), None) => {
// Constant velocity from the two previous cameras.
match r.cams.range(..ka).next_back() {
Some((kp, p)) => {
let t = (k - ka) as f64 / (ka - kp) as f64;
let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r)));
let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r);
let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0)));
Some(BaCam { pose: Pose { r: rr, c }, f: a.f })
}
None => Some(a),
}
}
(None, Some((kb, b))) => match r.cams.range(kb + 1..).next() {
Some((kn, nx)) => {
let t = (kb - k) as f64 / (kn - kb) as f64;
let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r)));
let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r);
let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0)));
Some(BaCam { pose: Pose { r: rr, c }, f: b.f })
}
None => Some(b),
},
(None, None) => None,
}
}
/// Shared tracks of frames `a` and `b`: (track index, pixels in a, pixels in b).
fn shared(d: &Data, a: usize, b: usize) -> Vec<(u32, [f64; 2], [f64; 2])> {
d.by_frame[a].iter().filter_map(|(ti, p)| d.at(*ti, b).map(|q| (*ti, *p, q))).collect()
}
/// Two-view initialisations of frames `a`, `b` at focal length `f`: the second camera and the
/// triangulated points (the first camera is the identity).
fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> Vec<(Pose, HashMap<u32, V3>)> {
let sh = shared(d, a, b);
if sh.len() < 12 {
return vec![];
}
let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect();
let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect();
let thr = 2.0 * d.noise() / f;
let mut cands: Vec<Pose> = vec![];
if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) {
let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip();
if let Some((pose, _)) = geometry::select_pose(&geometry::decompose_essential(&e), &p1, &p2) {
cands.push(pose);
}
}
if use_h {
let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip();
if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise(), 500, (a * 31 + b) as u64) {
let h = fit.h.0;
// Normalised: K⁻¹ H K with K = diag(f, f, 1).
let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]];
let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| fit.inliers[*i]).map(|i| (x1[i], x2[i])).unzip();
for (r, td, _) in geometry::decompose_homography(&hn, &p1, &p2) {
if norm(td) > 1e-6 {
cands.push(Pose::from_rt(r, normalize(td)));
}
}
}
}
let thr2 = (d.inlier_thr() / f).powi(2);
cands
.into_iter()
.filter_map(|pose| {
let mut pts = HashMap::new();
for (i, s) in sh.iter().enumerate() {
let Some(x) = geometry::triangulate(&[(Pose::IDENTITY, x1[i]), (pose, x2[i])]) else { continue };
let ok = geometry::reproj2(&Pose::IDENTITY, x, x1[i]).is_some_and(|e| e < thr2)
&& geometry::reproj2(&pose, x, x2[i]).is_some_and(|e| e < thr2)
&& geometry::ray_angle([0.0; 3], pose.c, x).to_degrees() > 0.3;
if ok {
pts.insert(s.0, x);
}
}
(pts.len() >= 10 && pts.len() * 3 >= sh.len()).then_some((pose, pts))
})
.collect()
}
/// Grow a reconstruction over keyframes `kfs` from an initial pair (`kfs[0]`, `kfs[j]`).
fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap<u32, V3>), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option<Recon> {
let (a, b) = (kfs[0], kfs[j]);
let mut r = Recon::default();
r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f });
r.cams.insert(b, BaCam { pose: init.0, f });
r.pts = init.1.clone();
let inner = |r: &mut Recon, frames: &[usize]| {
let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed };
r.adjust(d, frames, cfg, focal, 10);
r.reject(d);
};
inner(&mut r, &[a, b]);
let order: Vec<usize> = kfs[1..j].iter().chain(&kfs[j + 1..]).copied().collect();
let mut done = 0;
for k in order {
if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) {
return None;
}
let start = predict(&r, k)?;
if !r.register(d, k, start, cfg) {
// A failed keyframe ends the reconstruction (a later frame is too far from the known
// points); the caller resects the rest from what is known.
break;
}
r.triangulate(d, cfg.kind, 1.0, 1.0);
done += 1;
let frames: Vec<usize> = r.cams.keys().copied().collect();
if done % 3 == 0 || frames.len() <= 6 {
inner(&mut r, &frames);
}
}
let frames: Vec<usize> = r.cams.keys().copied().collect();
inner(&mut r, &frames);
(r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r)
}
/// Tripod reconstruction over keyframes `kfs` at focal length `f`.
fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option<Recon> {
let mut r = Recon::default();
r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f });
let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]);
for w in kfs.windows(2) {
if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) {
return None;
}
let (a, b) = (w[0], w[1]);
let sh = shared(d, a, b);
if sh.len() < 6 {
break;
}
let ra: Vec<V3> = sh.iter().map(|s| ray(s.1, f)).collect();
let rb: Vec<V3> = sh.iter().map(|s| ray(s.2, f)).collect();
let mut rel = geometry::procrustes(&ra, &rb)?;
// One round of outlier trimming.
let errs: Vec<f64> = ra.iter().zip(&rb).map(|(x, y)| norm(sub(*y, mv(&rel, *x)))).collect();
let mut sorted = errs.clone();
sorted.sort_by(f64::total_cmp);
let lim = (sorted[sorted.len() / 2] * 3.0).max(d.inlier_thr() / f);
let (ia, ib): (Vec<V3>, Vec<V3>) = ra.iter().zip(&rb).zip(&errs).filter(|(_, e)| **e <= lim).map(|((x, y), _)| (*x, *y)).unzip();
if ia.len() >= 4 {
rel = geometry::procrustes(&ia, &ib)?;
}
let pa = r.cams[&a].pose.r;
r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f });
}
r.triangulate(d, Kind::Tripod, 0.0, 20.0);
let frames: Vec<usize> = r.cams.keys().copied().collect();
let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed };
r.adjust(d, &frames, cfg, focal, 15);
r.reject(d);
r.triangulate(d, Kind::Tripod, 0.0, 1.0);
r.adjust(d, &frames, cfg, focal, 15);
(r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r)
}
/// Focal length (pixels) of a horizontal angle of view.
fn focal_of(w: f64, hfov: f64) -> f64 {
w * 0.5 / (hfov.clamp(1.0, 170.0).to_radians() * 0.5).tan()
}
struct Attempt {
recon: Recon,
err: f64,
method: SolveMethod,
}
/// Choose the initial pair: the first keyframe and the later keyframe (among the next few)
/// that keeps the most points while having enough parallax.
fn best_init(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Option<(usize, (Pose, HashMap<u32, V3>), bool)> {
let (use_e, use_h) = match method {
SolveMethod::Typical => (true, false),
SolveMethod::MostlyFlat => (false, true),
_ => (true, true),
};
let mut best: Option<(f64, usize, (Pose, HashMap<u32, V3>), bool)> = None;
for j in 1..kfs.len().min(6) {
let cands = init_pair(d, kfs[0], kfs[j], f, use_e, false);
let hc = if use_h { init_pair(d, kfs[0], kfs[j], f, false, true) } else { vec![] };
for (c, is_h) in cands.into_iter().map(|c| (c, false)).chain(hc.into_iter().map(|c| (c, true))) {
// Score: points × median triangulation angle (capped).
let mut ang: Vec<f64> = c.1.values().map(|x| geometry::ray_angle([0.0; 3], c.0.c, *x).to_degrees()).collect();
ang.sort_by(f64::total_cmp);
let med = ang.get(ang.len() / 2).copied().unwrap_or(0.0);
let score = c.1.len() as f64 * med.min(4.0);
if best.as_ref().is_none_or(|b| score > b.0) {
best = Some((score, j, c, is_h));
}
}
}
best.map(|b| (b.1, b.2, b.3))
}
/// All initial candidates for the pair chosen by [`best_init`] (E and H decompositions).
fn init_candidates(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Vec<(usize, (Pose, HashMap<u32, V3>), bool)> {
let Some((j, first, is_h)) = best_init(d, kfs, f, method) else { return vec![] };
let mut out = vec![(j, first, is_h)];
if matches!(method, SolveMethod::Auto | SolveMethod::MostlyFlat) {
for c in init_pair(d, kfs[0], kfs[j], f, false, true) {
if out.iter().all(|o| norm(log_rot(&mm(&o.1.0.r, &mt(&c.0.r)))) > 1e-3 || norm(sub(o.1.0.c, c.0.c)) > 1e-3) {
out.push((j, c, true));
}
}
}
out
}
/// A general (translating camera) reconstruction of keyframes `kfs` at focal length `f`.
fn general(d: &Data, kfs: &[usize], f: f64, method: SolveMethod, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option<Attempt> {
let cands = init_candidates(d, kfs, f, method);
let res: Vec<Attempt> = cands
.par_iter()
.filter_map(|(j, init, is_h)| {
let r = grow(d, kfs, *j, init, f, cfg, cancel)?;
let (_, n) = r.error(d);
// Prefer reconstructions covering more keyframes.
let cover = r.cams.len() as f64 / kfs.len() as f64;
(n > 0).then(|| Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: if *is_h { SolveMethod::MostlyFlat } else { SolveMethod::Typical } })
})
.collect();
res.into_iter().min_by(|a, b| a.err.total_cmp(&b.err))
}
fn tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option<Attempt> {
let r = grow_tripod(d, kfs, f, cfg, cancel)?;
let cover = r.cams.len() as f64 / kfs.len() as f64;
Some(Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: SolveMethod::TripodPan })
}
/// Search the focal length minimising `eval(f)` (a reprojection error): a log-spaced scan of
/// horizontal angles of view, then golden-section refinement around the best.
fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 {
let fovs = [8.0, 12.0, 16.0, 20.0, 25.0, 30.0, 36.0, 43.0, 51.0, 60.0, 70.0, 82.0, 96.0, 112.0];
let fs: Vec<f64> = fovs.iter().map(|a| focal_of(w, *a)).collect();
let errs: Vec<f64> = fs.par_iter().map(|f| eval(*f)).collect();
if debug() {
eprintln!("focal scan: {:?}", fs.iter().zip(&errs).map(|(f, e)| format!("{f:.0}:{e:.3}")).collect::<Vec<_>>());
}
let i = (0..fs.len()).min_by(|a, b| errs[*a].total_cmp(&errs[*b])).unwrap_or(5);
let (mut lo, mut hi) = (fs[(i + 1).min(fs.len() - 1)].ln(), fs[i.saturating_sub(1)].ln());
if lo > hi {
std::mem::swap(&mut lo, &mut hi);
}
let g = 0.618_033_988_75;
let (mut x1, mut x2) = (hi - g * (hi - lo), lo + g * (hi - lo));
let (mut e1, mut e2) = rayon::join(|| eval(x1.exp()), || eval(x2.exp()));
for _ in 0..7 {
if e1 < e2 {
hi = x2;
x2 = x1;
e2 = e1;
x1 = hi - g * (hi - lo);
e1 = eval(x1.exp());
} else {
lo = x1;
x1 = x2;
e1 = e2;
x2 = lo + g * (hi - lo);
e2 = eval(x2.exp());
}
}
let best = if e1 < e2 { (x1, e1) } else { (x2, e2) };
if best.1 <= errs[i] { best.0.exp() } else { fs[i] }
}
/// Solve the camera of a tracked clip.
pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result<CameraSolve, SolveError> {
let d = Data::new(tracks, &s.deleted);
if d.n < 2 {
return err("the clip must be at least two frames long");
}
let kfs = keyframes(&d);
if kfs.len() < 2 {
return err("not enough features were tracked to solve the camera");
}
let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed));
let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 }));
let variable = s.shot == ShotType::VariableZoom;
let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false };
let tcfg = Cfg { kind: Kind::Tripod, ..gcfg };
// The focal-length search runs on the first keyframes only.
let mini: Vec<usize> = kfs.iter().copied().take(10).collect();
let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat);
let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan);
let t0 = web_time::Instant::now();
let lap = |what: &str| {
if debug() {
eprintln!("{what}: {:.2} s", t0.elapsed().as_secs_f64());
}
};
let mut results: Vec<(Attempt, f64)> = vec![];
if try_general {
let f = match fixed_f {
Some(f) => f,
None => {
search_focal(d.w, &|f| general(&d, &mini, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY))
}
};
if cancelled() {
return err("cancelled");
}
if let Some(a) = general(&d, &kfs, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel) {
results.push((a, f));
}
}
lap("general");
// Auto Detect: a quick tripod check at the general solve's focal length; when a rotation
// clearly cannot explain the tracks, the tripod search is skipped.
let tripod_plausible = match (s.method, results.first()) {
(SolveMethod::Auto, Some((g, f))) => {
let quick = tripod(&d, &mini, *f, &tcfg, cancel).map(|a| a.err).unwrap_or(f64::INFINITY);
if debug() {
eprintln!("quick tripod check: {quick:.3} vs general {:.3}", g.err);
}
quick <= 3.0 * g.err + d.noise()
}
_ => true,
};
if try_tripod && tripod_plausible && !cancelled() {
let f = match fixed_f {
Some(f) => f,
None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)),
};
if let Some(a) = tripod(&d, &kfs, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel) {
results.push((a, f));
}
}
if cancelled() {
return err("cancelled");
}
// Auto Detect keeps the tripod model when it explains the tracks about as well.
let pick = {
let g = results.iter().position(|r| r.0.method != SolveMethod::TripodPan);
let t = results.iter().position(|r| r.0.method == SolveMethod::TripodPan);
match (g, t) {
(Some(g), Some(t)) => {
let (eg, et) = (results[g].0.err, results[t].0.err);
// A general solve whose camera hardly moves relative to the scene depth is a pan.
let rg = &results[g].0.recon;
let cs: Vec<V3> = rg.cams.values().map(|c| c.pose.c).collect();
let base = cs.iter().flat_map(|a| cs.iter().map(move |b| norm(sub(*a, *b)))).fold(0.0, f64::max);
let c0 = rg.cams.values().next().expect("camera");
let mut depth: Vec<f64> = rg.pts.values().map(|x| norm(sub(*x, c0.pose.c))).collect();
depth.sort_by(f64::total_cmp);
let med = depth.get(depth.len() / 2).copied().unwrap_or(1.0);
if debug() {
eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med);
}
if et <= (eg * 1.25).max(eg + 0.05 * d.noise()) || base / med < 0.01 { t } else { g }
}
(Some(g), None) => g,
(None, Some(t)) => t,
(None, None) => return err("the camera could not be solved (not enough parallax or features)"),
}
};
if debug() {
for (a, f) in &results {
eprintln!("candidate {:?}: err {:.4} f {f:.1} cams {} pts {}", a.method, a.err, a.recon.cams.len(), a.recon.pts.len());
}
}
let (att, _) = results.swap_remove(pick);
let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General };
let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable };
let mut r = att.recon;
let focal = match (fixed_f, variable) {
(Some(_), _) => FocalMode::Fixed,
(None, true) => FocalMode::PerCamera(F_PRIOR),
(None, false) => FocalMode::Shared,
};
lap("tripod");
// Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again.
let kf_frames: Vec<usize> = r.cams.keys().copied().collect();
r.adjust(&d, &kf_frames, &cfg, focal, 40);
r.reject(&d);
// Re-estimate the tracking noise from the inlier residuals (robustly, from their median:
// for 2D Gaussian noise of deviation σ the median distance is 1.1774 σ), then tighten the
// thresholds: observations of moving objects and occlusion edges become outliers.
let mut res: Vec<f64> = vec![];
for (k, c) in &r.cams {
for (ti, uv) in &d.by_frame[*k] {
if r.outliers.contains(&(*ti, *k)) {
continue;
}
if let Some(e) = r.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) {
res.push(e[0].hypot(e[1]));
}
}
}
if res.len() > 20 {
res.sort_by(f64::total_cmp);
let sigma = res[res.len() / 2] / 1.1774;
let floor = 0.05 * tracks.factor.max(1.0);
d.set_noise((1.5 * sigma).clamp(floor, d.noise()));
if debug() {
eprintln!("noise: σ {sigma:.3} → {:.3}", d.noise());
}
r.outliers.clear();
r.reject(&d);
}
r.triangulate(&d, kind, 1.0, 1.0);
r.adjust(&d, &kf_frames, &cfg, focal, 40);
r.reject(&d);
r.adjust(&d, &kf_frames, &cfg, focal, 20);
if cancelled() {
return err("cancelled");
}
lap("keyframe adjustment");
// Every other frame: resect from the points (nearest registered camera first).
let mut todo: Vec<usize> = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect();
todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX));
let base = r.clone();
let resected: Vec<(usize, Option<BaCam>)> = todo
.par_iter()
.map(|k| {
let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() };
let ok = predict(&local, *k).is_some_and(|start| local.register(&d, *k, start, &cfg));
(*k, if ok { local.cams.get(k).copied() } else { None })
})
.collect();
for (k, c) in resected {
if let Some(c) = c {
r.cams.insert(k, c);
}
}
if cancelled() {
return err("cancelled");
}
lap("resection");
// Points for the tracks that only lived between keyframes, then the final adjustment over
// every frame (when that stays affordable).
r.triangulate(&d, kind, 1.0, 1.0);
let all: Vec<usize> = r.cams.keys().copied().collect();
r.reject(&d);
if all.len() <= 160 {
r.adjust(&d, &all, &cfg, focal, 25);
r.reject(&d);
} else {
// Points only (cameras fixed) would be cheap but the keyframe solution is already
// adjusted; refine each non-keyframe once more against the final points.
let base = r.clone();
let refined: Vec<(usize, Option<BaCam>)> = all
.par_iter()
.filter(|k| !kf_frames.contains(k))
.map(|k| {
let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() };
let start = local.cams[k];
let ok = local.register(&d, *k, start, &cfg);
(*k, if ok { local.cams.get(k).copied() } else { None })
})
.collect();
for (k, c) in refined.into_iter() {
if let Some(c) = c {
r.cams.insert(k, c);
}
}
r.reject(&d);
}
// Method Used: the chosen model; Auto Detect calls a general solve "Mostly Flat Scene" when
// its points are close to one plane.
let method = match (s.method, att.method) {
(_, SolveMethod::TripodPan) => SolveMethod::TripodPan,
(SolveMethod::Auto, _) => {
let p: Vec<V3> = r.pts.values().copied().collect();
let flat = cov_ratio(&p) < 0.08;
if flat { SolveMethod::MostlyFlat } else { SolveMethod::Typical }
}
(m, _) => m,
};
lap("final adjustment");
Ok(finish(&d, tracks, s, r, method, kind))
}
/// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance).
fn cov_ratio(p: &[V3]) -> f64 {
if p.len() < 3 {
return 1.0;
}
let k = 1.0 / p.len() as f64;
let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k);
let mut cov = [0.0; 9];
for q in p {
let d = sub(*q, c);
for i in 0..3 {
for j in 0..3 {
cov[i * 3 + j] += d[i] * d[j] * k;
}
}
}
let (vals, _) = sym_eigen(&cov, 3);
(vals[0].max(0.0) / vals[1].max(1e-300)).sqrt()
}
/// Canonical frame, per-frame output (unsolved frames interpolated), points and errors.
fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: SolveMethod, kind: Kind) -> CameraSolve {
let (k0, c0) = r.cams.iter().next().map(|(k, c)| (*k, *c)).expect("solved");
// Scale: median depth of the points seen by the first solved camera is 1.
let mut depths: Vec<f64> = d.by_frame[k0].iter().filter_map(|(ti, _)| r.pts.get(ti)).map(|x| c0.pose.to_cam(*x)[2]).filter(|z| *z > 0.0).collect();
depths.sort_by(f64::total_cmp);
let sc = match kind {
Kind::Tripod => 1.0,
Kind::General => depths.get(depths.len() / 2).copied().filter(|v| *v > 1e-12).unwrap_or(1.0),
};
let r0 = c0.pose.r;
let cc = c0.pose.c;
let to_world = |x: V3| scale(mv(&r0, sub(x, cc)), 1.0 / sc);
let mut frames = vec![SolvedFrame::default(); d.n];
for (k, c) in &r.cams {
let rr = mm(&c.pose.r, &mt(&r0));
frames[*k] = SolvedFrame { rot: log_rot(&rr), center: to_world(c.pose.c), focal: c.f, solved: true };
}
// Hold / interpolate unsolved frames so the camera path stays continuous.
let solved: Vec<usize> = (0..d.n).filter(|k| frames[*k].solved).collect();
for k in 0..d.n {
if frames[k].solved {
continue;
}
let a = solved.iter().rev().find(|x| **x < k).copied();
let b = solved.iter().find(|x| **x > k).copied();
frames[k] = match (a, b) {
(Some(a), Some(b)) => {
let t = (k - a) as f64 / (b - a) as f64;
let (fa, fb) = (frames[a], frames[b]);
let rr = slerp(&fa.r(), &fb.r(), t);
SolvedFrame {
rot: log_rot(&rr),
center: add(scale(fa.center, 1.0 - t), scale(fb.center, t)),
focal: fa.focal + (fb.focal - fa.focal) * t,
solved: false,
}
}
(Some(a), None) => SolvedFrame { solved: false, ..frames[a] },
(None, Some(b)) => SolvedFrame { solved: false, ..frames[b] },
(None, None) => frames[k],
};
}
let mut points = vec![];
let (mut esum, mut en) = (0.0, 0usize);
let mut ids: Vec<&u32> = r.pts.keys().collect();
ids.sort();
for ti in ids {
let x = r.pts[ti];
let (id, start, pts) = &d.tracks[*ti as usize];
let (mut s1, mut n1) = (0.0, 0);
for (i, uv) in pts.iter().enumerate() {
let k = start + i;
if r.outliers.contains(&(*ti, k)) {
continue;
}
if let Some(c) = r.cams.get(&k)
&& let Some(res) = bundle::residual(c, x, *uv)
{
s1 += res[0].hypot(res[1]);
n1 += 1;
}
}
if n1 < 2 {
continue;
}
esum += s1;
en += n1;
points.push(SolvedPoint { id: *id, pos: to_world(x), error: (s1 / n1 as f64) as f32, first: *start as u32, last: (start + pts.len() - 1) as u32 });
}
CameraSolve {
version: 1,
size: tracks.size,
start: tracks.start,
frame_duration: tracks.frame_duration,
shot: s.shot,
method_used: method,
average_error: if en > 0 { esum / en as f64 } else { 0.0 },
frames,
points,
ground: None,
}
}
+438
View File
@@ -0,0 +1,438 @@
//! Synthetic scenes: random 3D points (or a plane) seen by a known moving camera with a known
//! focal length, projected directly (with sub-pixel noise) into feature tracks.
use super::geometry::Rng;
use super::linalg::*;
use super::*;
const W: f64 = 1280.0;
const H: f64 = 720.0;
struct Truth {
/// Per frame: (R world→cam, centre, focal).
cams: Vec<(M3, V3, f64)>,
}
fn noise(rng: &mut Rng) -> f64 {
// Sum of uniforms ≈ Gaussian, σ ≈ 0.1 px.
let u: f64 = (0..4).map(|_| (rng.next_u64() % 10_000) as f64 / 10_000.0 - 0.5).sum();
u * 0.17
}
/// Project `pts` through the cameras into tracks (a track ends when its point leaves the frame;
/// re-entering starts a new track).
fn tracks_of(pts: &[V3], t: &Truth, seed: u64, noise_on: bool) -> CameraTracks {
let mut rng = Rng(seed);
let mut tracks: Vec<Track2D> = vec![];
for x in pts {
let mut cur: Option<Track2D> = None;
for (k, (r, c, f)) in t.cams.iter().enumerate() {
let p = mv(r, sub(*x, *c));
let uv = (p[2] > 0.1).then(|| [f * p[0] / p[2] + W / 2.0, f * p[1] / p[2] + H / 2.0]);
match uv.filter(|u| u[0] >= 2.0 && u[1] >= 2.0 && u[0] < W - 2.0 && u[1] < H - 2.0) {
Some(u) => {
let (nx, ny) = if noise_on { (noise(&mut rng), noise(&mut rng)) } else { (0.0, 0.0) };
let q = [(u[0] + nx) as f32, (u[1] + ny) as f32];
match &mut cur {
Some(tr) => tr.pts.push(q),
None => cur = Some(Track2D { id: 0, start: k as u32, pts: vec![q] }),
}
}
None => {
if let Some(tr) = cur.take() {
tracks.push(tr);
}
}
}
}
if let Some(tr) = cur.take() {
tracks.push(tr);
}
}
tracks.retain(|t| t.pts.len() >= 3);
for (i, tr) in tracks.iter_mut().enumerate() {
tr.id = i as u32;
}
CameraTracks { version: 1, size: [W, H], start: 0.0, frame_duration: 1.0 / 30.0, frames: t.cams.len() as u32, factor: 1.0, detailed: false, tracks }
}
fn random_points(n: usize, seed: u64, center: V3, ext: V3) -> Vec<V3> {
let mut rng = Rng(seed);
(0..n)
.map(|_| {
let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5;
[center[0] + u() * ext[0], center[1] + u() * ext[1], center[2] + u() * ext[2]]
})
.collect()
}
/// A camera dollying sideways and forward while panning to keep the scene in view.
fn moving_camera(frames: usize, f: impl Fn(usize) -> f64) -> Truth {
let cams = (0..frames)
.map(|k| {
let t = k as f64 / (frames - 1) as f64;
let c = [-3.0 + 6.0 * t, -0.6 + 0.8 * t + 0.5 * (std::f64::consts::PI * t).sin(), -1.0 + 2.0 * t - 1.5 * (t - 0.5) * (t - 0.5)];
// Look at the scene centre (0, 0, 10).
let fwd = normalize(sub([0.3 * t, 0.2, 10.0], c));
let right = normalize(cross([0.0, 1.0, 0.0], fwd));
let down = cross(fwd, right);
let r = [right, down, fwd];
// A little roll.
let roll = rodrigues([0.0, 0.0, 0.05 * (t * 6.0).sin()]);
(mm(&roll, &r), c, f(k))
})
.collect();
Truth { cams }
}
/// Align the solved centres to the true ones (similarity) and return the RMS path error
/// relative to the path length, plus the worst rotation error (degrees).
fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) {
let solved: Vec<usize> = (0..s.frames.len()).filter(|k| s.frames[*k].solved).collect();
let src: Vec<V3> = solved.iter().map(|k| s.frames[*k].center).collect();
let dst: Vec<V3> = solved.iter().map(|k| t.cams[*k].1).collect();
let sim = umeyama(&src, &dst).expect("alignment");
let len = norm(sub(dst[0], dst[dst.len() - 1])).max(1e-9);
let rms = (src.iter().zip(&dst).map(|(a, b)| dot(sub(sim.apply(*a), *b), sub(sim.apply(*a), *b))).sum::<f64>() / src.len() as f64).sqrt();
// Rotation: R_true ≈ R_solved · simᵀ.
let mut worst: f64 = 0.0;
for k in &solved {
let rs = mm(&s.frames[*k].r(), &mt(&sim.r));
let e = norm(log_rot(&mm(&rs, &mt(&t.cams[*k].0)))).to_degrees();
worst = worst.max(e);
}
(rms / len, worst)
}
fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings {
SolveSettings { shot, hfov: 0.0, method, deleted: vec![] }
}
#[test]
fn fixed_angle_solve_recovers_path_and_focal() {
let f = 1100.0;
let t = moving_camera(40, |_| f);
let pts = random_points(400, 1, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]);
let tr = tracks_of(&pts, &t, 2, true);
let t0 = std::time::Instant::now();
let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("solve");
eprintln!("fixed: {:?} err {:.3} px focal {:.1} in {:?}", s.method_used, s.average_error, s.focal(), t0.elapsed());
assert!(s.frames.iter().all(|c| c.solved));
assert!(s.average_error < 0.5, "{}", s.average_error);
assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal());
let (pe, re) = path_error(&s, &t);
assert!(pe < 0.01, "path {pe}");
assert!(re < 0.5, "rotation {re}");
// Canonical frame: first camera at the origin.
assert!(norm(s.frames[0].center) < 1e-9 && norm(s.frames[0].rot) < 1e-9);
// Serde round trip.
let j = s.to_json();
assert_eq!(CameraSolve::from_json(&j).unwrap().points.len(), s.points.len());
}
#[test]
fn auto_detect_picks_a_general_model_for_a_translating_camera() {
let f = 900.0;
let t = moving_camera(30, |_| f);
let pts = random_points(300, 3, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]);
let tr = tracks_of(&pts, &t, 4, true);
let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve");
assert_eq!(s.method_used, SolveMethod::Typical);
assert!(s.average_error < 0.5);
assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal());
}
#[test]
fn specified_angle_of_view_is_kept() {
let f = 1000.0;
let t = moving_camera(25, |_| f);
let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]);
let tr = tracks_of(&pts, &t, 6, true);
let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees();
let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap();
assert!((s.focal() - f).abs() < 1e-6);
assert!(s.average_error < 0.5);
let (pe, _) = path_error(&s, &t);
assert!(pe < 0.01, "{pe}");
}
#[test]
fn variable_zoom_recovers_per_frame_focal() {
let t = moving_camera(40, |k| 950.0 + 6.0 * k as f64);
let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]);
let tr = tracks_of(&pts, &t, 8, true);
let s = solve(&tr, &settings(ShotType::VariableZoom, SolveMethod::Typical), None).expect("solve");
eprintln!("variable: err {:.3}", s.average_error);
assert!(s.average_error < 0.5);
for (k, c) in s.frames.iter().enumerate() {
let f = t.cams[k].2;
assert!((c.focal - f).abs() / f < 0.02, "frame {k}: {} vs {f}", c.focal);
}
}
#[test]
fn tripod_pan_solve() {
let f = 1200.0;
let cams = (0..40)
.map(|k| {
let a = -0.4 + 0.8 * k as f64 / 39.0;
(mm(&rodrigues([0.0, 0.0, 0.02]), &mm(&rodrigues([0.05 * a, 0.0, 0.0]), &rodrigues([0.0, a, 0.0]))), [0.0; 3], f)
})
.collect();
let t = Truth { cams };
// Points all around at varying distances (rotation only: depth is unobservable).
let pts = random_points(600, 9, [0.0, 0.0, 0.0], [60.0, 20.0, 60.0]).into_iter().filter(|p| p[2] > 3.0).collect::<Vec<_>>();
let tr = tracks_of(&pts, &t, 10, true);
let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve");
eprintln!("tripod: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal());
assert_eq!(s.method_used, SolveMethod::TripodPan);
assert!(s.average_error < 0.5);
assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal());
for (k, c) in s.frames.iter().enumerate() {
assert!(norm(c.center) < 1e-9);
// Relative rotation to frame 0 matches.
let rel_s = mm(&c.r(), &mt(&s.frames[0].r()));
let rel_t = mm(&t.cams[k].0, &mt(&t.cams[0].0));
assert!(norm(log_rot(&mm(&rel_s, &mt(&rel_t)))).to_degrees() < 0.2);
}
}
#[test]
fn mostly_flat_scene_and_ground_plane() {
let f = 1000.0;
// A textured floor y = 2 (below the camera), seen by a camera moving forward and sideways.
let cams = (0..40)
.map(|k| {
let t = k as f64 / 39.0;
let c = [-2.0 + 4.0 * t, -0.3 * (std::f64::consts::PI * t).sin(), 2.0 * t];
let fwd = normalize([0.1 - 0.2 * t, 0.45, 1.0]);
let right = normalize(cross([0.0, 1.0, 0.0], fwd));
let down = cross(fwd, right);
([right, down, fwd], c, f)
})
.collect();
let t = Truth { cams };
let mut rng = Rng(11);
let pts: Vec<V3> = (0..500).map(|_| [((rng.next_u64() % 2000) as f64 / 100.0) - 10.0, 2.0, 2.0 + (rng.next_u64() % 2000) as f64 / 100.0]).collect();
let tr = tracks_of(&pts, &t, 12, true);
for method in [SolveMethod::MostlyFlat, SolveMethod::Auto] {
let s = solve(&tr, &settings(ShotType::FixedAngle, method), None).expect("solve");
eprintln!("flat {method:?}: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal());
assert_eq!(s.method_used, SolveMethod::MostlyFlat);
assert!(s.average_error < 0.5, "{}", s.average_error);
assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal());
let (pe, _) = path_error(&s, &t);
assert!(pe < 0.01, "{pe}");
}
// Ground plane from the solved (coplanar) points: every point lies on it.
let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees();
let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve");
let ids: Vec<u32> = s.visible(0).map(|p| p.id).take(12).collect();
let g = s.ground_from(&ids, 0).expect("ground");
let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max);
let depth = norm(sub(s.frames[0].center, g.origin));
assert!(spread < 0.01 * depth, "{spread}");
// Up points towards the camera.
assert!(dot(sub(s.frames[0].center, g.origin), g.normal) > 0.0);
s.ground = Some(g);
let w = s.world([W, H], 1000.0);
// In the world, ground points have y ≈ 0 and the cameras are above (y < 0).
for p in s.points.iter().take(20) {
assert!(w.apply(p.pos)[1].abs() < 0.01 * 1000.0 * depth, "{:?}", w.apply(p.pos));
}
assert!(w.apply(s.frames[0].center)[1] < 0.0);
assert!(norm(w.apply(g.origin)) < 1e-6);
}
#[test]
fn deleted_tracks_are_ignored_and_cancel_stops() {
let t = moving_camera(20, |_| 1000.0);
let pts = random_points(250, 13, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]);
let tr = tracks_of(&pts, &t, 14, true);
let deleted: Vec<u32> = (0..40).collect();
let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees();
let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap();
assert!(s.points.iter().all(|p| !deleted.contains(&p.id)));
let stop = std::sync::atomic::AtomicBool::new(true);
let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop));
assert!(r.is_err());
}
/// End to end: rendered frames of a textured scene through the KLT feature tracker.
#[test]
fn rendered_scene_end_to_end() {
use effectcraft_raster::Image;
let (w, h) = (480u32, 270u32);
let f = 420.0;
// Three textured planes at different depths (a floor, a back wall and a box face).
fn tex(u: f64, v: f64) -> f32 {
fn hsh(i: i64, j: i64) -> f64 {
let mut x = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663)) as u64;
x ^= x >> 13;
x = x.wrapping_mul(0x5bd1e995);
x ^= x >> 15;
(x % 1000) as f64 / 1000.0
}
fn noise(x: f64, y: f64) -> f64 {
let (i, j) = (x.floor(), y.floor());
let (tx, ty) = (x - i, y - j);
let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty));
let (i, j) = (i as i64, j as i64);
let a = hsh(i, j) + (hsh(i + 1, j) - hsh(i, j)) * sx;
let b = hsh(i, j + 1) + (hsh(i + 1, j + 1) - hsh(i, j + 1)) * sx;
a + (b - a) * sy
}
(0.15 + 0.5 * noise(u * 4.0, v * 4.0) + 0.3 * noise(u * 11.0 + 7.0, v * 11.0 + 3.0)) as f32
}
let frames = 24;
let t = Truth {
cams: (0..frames)
.map(|k| {
let s = k as f64 / (frames - 1) as f64;
let c = [-1.0 + 2.0 * s, -0.2, 0.5 * s];
let fwd = normalize(sub([0.0, 0.3, 8.0], c));
let right = normalize(cross([0.0, 1.0, 0.0], fwd));
([right, cross(fwd, right), fwd], c, f)
})
.collect(),
};
// Planes: (point, normal, u axis, v axis).
let planes: [(V3, V3, V3, V3); 3] = [
([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]),
([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]),
([-1.0, 0.0, 6.0], [0.3, 0.0, -1.0], [1.0, 0.0, 0.3], [0.0, 1.0, 0.0]),
];
let render = |k: usize| {
let (r, c, f) = t.cams[k];
let mut img = Image::new(w, h);
for y in 0..h {
for x in 0..w {
let d = mtv(&r, [(x as f64 + 0.5 - w as f64 / 2.0) / f, (y as f64 + 0.5 - h as f64 / 2.0) / f, 1.0]);
let mut best = (f64::INFINITY, 0.0f32);
for (i, (p0, n, ua, va)) in planes.iter().enumerate() {
let den = dot(*n, d);
if den.abs() < 1e-9 {
continue;
}
let tt = dot(*n, sub(*p0, c)) / den;
let hit = add(c, scale(d, tt));
// The box face is limited in extent.
if i == 2 && (dot(sub(hit, *p0), *ua).abs() > 1.2 || dot(sub(hit, *p0), *va).abs() > 1.0) {
continue;
}
if tt > 0.1 && tt < best.0 {
best = (tt, tex(dot(sub(hit, *p0), *ua) + 10.0 * i as f64, dot(sub(hit, *p0), *va)));
}
}
let v = best.1;
img.set(x, y, [v, v, v, 1.0]);
}
}
img
};
let mut an = TrackAnalyzer::new([w as f64, h as f64], AnalyzeOpts::default());
for k in 0..frames {
let img = render(k);
an.push(&crate::Frame::new(&img));
}
let tr = an.finish(0.0, 1.0 / 24.0);
assert!(tr.tracks.len() > 100, "{}", tr.tracks.len());
let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve");
eprintln!("rendered: {:?} err {:.3} focal {:.1} points {}", s.method_used, s.average_error, s.focal(), s.points.len());
assert!(s.average_error < 0.5, "{}", s.average_error);
assert!((s.focal() - f).abs() / f < 0.05, "{}", s.focal());
let (pe, _) = path_error(&s, &t);
assert!(pe < 0.03, "{pe}");
}
/// A long clip (300 frames, ~900 points): `cargo test -p effectcraft-track --release -- --ignored`.
#[test]
#[ignore]
fn perf_long_clip() {
let f = 1100.0;
let t = moving_camera(300, |_| f);
let pts = random_points(900, 21, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]);
let tr = tracks_of(&pts, &t, 22, true);
let t0 = std::time::Instant::now();
let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve");
eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed());
assert!(s.average_error < 0.5);
}
/// Solve tracks saved from the app (`EC_CAMTRACK_FILE=tracks.json cargo test -p effectcraft-track
/// --release solve_saved_tracks -- --ignored --nocapture`): a debugging aid.
#[test]
#[ignore]
fn solve_saved_tracks() {
let Some(path) = std::env::var_os("EC_CAMTRACK_FILE") else { return };
let mut tr = CameraTracks::from_json(&std::fs::read_to_string(path).unwrap()).unwrap();
if let Some(maxy) = std::env::var("EC_CAMTRACK_MAXY").ok().and_then(|v| v.parse::<f32>().ok()) {
tr.tracks.retain(|t| t.pts.iter().all(|p| p[1] < maxy));
}
let shot = if std::env::var_os("EC_CAMTRACK_VARIABLE").is_some() { ShotType::VariableZoom } else { ShotType::FixedAngle };
let t0 = std::time::Instant::now();
// With the true cameras (`[{pos, poi, zoom}]` per frame, two-node), how consistent are the
// tracks themselves?
if let Some(tp) = std::env::var_os("EC_CAMTRACK_TRUTH") {
let v: Vec<serde_json::Value> = serde_json::from_str(&std::fs::read_to_string(tp).unwrap()).unwrap();
let g = |x: &serde_json::Value| -> V3 { [x[0].as_f64().unwrap(), x[1].as_f64().unwrap(), x[2].as_f64().unwrap()] };
let cams: Vec<(super::geometry::Pose, f64)> = v
.iter()
.map(|c| {
let (pos, poi) = (g(&c["pos"]), g(&c["poi"]));
let fwd = normalize(sub(poi, pos));
let d0 = [0.0, 1.0, 0.0];
let down = normalize(sub(d0, scale(fwd, dot(d0, fwd))));
let right = cross(down, fwd);
(super::geometry::Pose { r: [right, down, fwd], c: pos }, c["zoom"].as_f64().unwrap())
})
.collect();
let (w, h) = (tr.size[0], tr.size[1]);
let mut means = vec![];
let mut drop_ids = vec![];
for t in &tr.tracks {
let views: Vec<(super::geometry::Pose, [f64; 2])> = t
.pts
.iter()
.enumerate()
.map(|(i, p)| {
let (pose, z) = cams[t.start as usize + i];
(pose, [(p[0] as f64 - w / 2.0) / z, (p[1] as f64 - h / 2.0) / z])
})
.collect();
let Some(x) = super::geometry::triangulate(&views) else { continue };
let e: Vec<f64> = views.iter().filter_map(|(p, o)| super::geometry::reproj2(p, x, *o).map(|e| e.sqrt() * cams[0].1)).collect();
means.push((e.iter().sum::<f64>() / e.len().max(1) as f64, t.pts[0]));
if std::env::var_os("EC_CAMTRACK_TRUTH_FILTER").is_some() && means.last().unwrap().0 > 1.0 {
drop_ids.push(t.id);
}
}
tr.tracks.retain(|t| !drop_ids.contains(&t.id));
means.sort_by(|a, b| a.0.total_cmp(&b.0));
eprintln!(
"true-camera track errors (px): median {:.2}; quartiles {:?}",
means[means.len() / 2].0,
[means[means.len() / 4].0, means[means.len() * 3 / 4].0]
);
for m in means.iter().step_by(6) {
eprintln!(" {:.2} at {:?}", m.0, m.1);
}
}
let mut st = settings(shot, SolveMethod::Auto);
if let Some(a) = std::env::var("EC_CAMTRACK_AOV").ok().and_then(|v| v.parse::<f64>().ok()) {
st.shot = ShotType::SpecifyAngle;
st.hfov = a;
}
let s = solve(&tr, &st, None).expect("solve");
eprintln!(
"{} frames, {} tracks: {:?} err {:.3} focal {:.1} (hfov {:.1}) points {} in {:?}",
tr.frames,
tr.tracks.len(),
s.method_used,
s.average_error,
s.focal(),
s.hfov(s.focal()),
s.points.len(),
t0.elapsed()
);
}
+1
View File
@@ -18,6 +18,7 @@
//! Coordinates are *layer* pixels: the frame image's pixel `(i, j)` covers `[i, i+1) × [j, j+1)`
//! after subtracting the frame's `offset` (see [`Frame`]).
pub mod camtrack;
pub mod fit;
pub mod klt;
pub mod mask;
+93 -15
View File
@@ -7,6 +7,11 @@
//! wasm32 has no threads: jobs wait in the same priority queue and [`Frames::pump`] renders them
//! on the UI thread between egui frames, within a time budget.
//!
//! With the disk cache on (Settings ▸ Media & Disk Cache), every CPU-rendered frame is also
//! written to disk under a content key (the comp's content hash, frame, scale, view and render
//! options), and a job first looks there: frames survive restarts and project reopenings. The
//! timeline shows disk-only frames as a blue cache bar.
//!
//! With Mercury GPU Acceleration (a [`Gpu`] on egui-wgpu's device and the project's renderer set
//! to the GPU) frames stay on the GPU: the compositor writes an RGBA8 texture that the viewer
//! registers with egui-wgpu and draws directly; pixels are read back only when something needs
@@ -16,6 +21,7 @@ use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::{Arc, Mutex};
use effectcraft_engine::project::{ItemId, Project};
use effectcraft_engine::render::disk_cache::{self, DiskCache};
use effectcraft_engine::render::{ExprHost, FootageSource, LayerCache, RenderOpts, Renderer};
use effectcraft_engine::time::Tick;
use effectcraft_gpu::{DisplayFrame, Gpu};
@@ -124,6 +130,37 @@ pub struct RenderSource {
/// while the viewer draws plainly, because there frames can't be read back synchronously
/// (`EffectcraftApp::gpu_display`).
pub gpu_display: bool,
/// The persistent disk cache, when enabled.
pub disk: Option<Arc<DiskCache>>,
}
/// Content keys of (project revision, comp), computed once per revision.
type ContentKeys = Arc<Mutex<HashMap<(u64, u64), u128>>>;
fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64) -> u128 {
if let Some(k) = keys.lock().ok().and_then(|m| m.get(&(revision, comp)).copied()) {
return k;
}
let k = disk_cache::comp_content_key(project, ItemId(comp));
if let Ok(mut m) = keys.lock() {
if m.len() > 256 {
m.clear();
}
m.insert((revision, comp), k);
}
k
}
fn opts_hash(o: &RenderOpts) -> u64 {
let mut h = disk_cache::Hash128::default();
h.write(format!("{o:?}").as_bytes());
h.finish() as u64
}
/// Disk key of a viewer frame (render options included: region of interest, draft…).
fn frame_disk_key(keys: &ContentKeys, project: &Project, key: &FrameKey, opts_hash: u64) -> u128 {
let content = content_key(keys, project, key.revision, key.comp);
disk_cache::frame_key(content, key.frame, key.scale, key.view ^ opts_hash)
}
pub struct Frames {
@@ -137,6 +174,7 @@ pub struct Frames {
ctx: Option<egui::Context>,
/// Render time of the last viewer (urgent) frame in ms, for the Info panel / perf readout.
pub last_ms: Arc<Mutex<f64>>,
content_keys: ContentKeys,
}
impl Default for Frames {
@@ -151,6 +189,7 @@ impl Default for Frames {
pool: rayon::ThreadPoolBuilder::new().num_threads(threads).thread_name(|i| format!("ec-frame-{i}")).build().expect("frame pool"),
ctx: None,
last_ms: Arc::new(Mutex::new(0.0)),
content_keys: Arc::default(),
}
}
}
@@ -202,6 +241,21 @@ impl Frames {
v
}
/// Frames of (revision, comp, scale) in the disk cache but not in RAM — the blue cache bar.
/// `opts` are the viewer's render options (part of the disk key).
pub fn disk_frames(&self, src: &RenderSource, revision: u64, comp: u64, scale: u32, view: u64, frames: i64, opts: &RenderOpts) -> Vec<i64> {
let Some(dc) = &src.disk else { return vec![] };
let ram: HashSet<i64> = self.cached_frames(revision, comp, scale).into_iter().collect();
let oh = opts_hash(opts);
(0..frames.clamp(0, 100_000))
.filter(|f| !ram.contains(f))
.filter(|f| {
let key = FrameKey { revision, comp, frame: *f, scale, view };
dc.contains(disk_cache::Kind::Frame, frame_disk_key(&self.content_keys, &src.project, &key, oh))
})
.collect()
}
/// Change the RAM preview cache budget (Settings ▸ Memory & CPU), evicting the oldest
/// frames when it shrinks.
pub fn set_budget(&self, bytes: usize) {
@@ -285,7 +339,14 @@ impl Frames {
}
fn worker(&self) -> Worker {
Worker { queue: self.queue.clone(), cache: self.cache.clone(), inflight: self.inflight.clone(), ctx: self.ctx.clone(), last: self.last_ms.clone() }
Worker {
queue: self.queue.clone(),
cache: self.cache.clone(),
inflight: self.inflight.clone(),
ctx: self.ctx.clone(),
last: self.last_ms.clone(),
content_keys: self.content_keys.clone(),
}
}
/// Render queued frames on this thread, most important first, until the queue is empty or
@@ -313,12 +374,13 @@ struct Worker {
inflight: Arc<Mutex<HashSet<FrameKey>>>,
ctx: Option<egui::Context>,
last: Arc<Mutex<f64>>,
content_keys: ContentKeys,
}
impl Worker {
/// Render the queued job that matters most right now. `false` when the queue was empty.
fn run_next(&self) -> bool {
let Worker { queue, cache, inflight, ctx, last } = self;
let Worker { queue, cache, inflight, ctx, last, content_keys } = self;
let job = {
let Ok(mut q) = queue.lock() else { return false };
// Urgent first (newest), then prefetch in request order.
@@ -332,19 +394,18 @@ impl Worker {
job
};
let t0 = web_time::Instant::now();
let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts);
r.expr = job.src.expr.as_deref();
r.cache = Some(&job.src.layer_cache);
r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator);
// The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no
// adapter, or a frame the GPU cannot finish here) the CPU renders it.
let gpu_frame = match (&job.src.gpu, r.active_accel()) {
(Some(g), Some(_)) if job.src.gpu_display => g.render_display(&r, job.comp, job.t),
_ => None,
};
let ci = match gpu_frame {
Some(f) => FrameImage::Gpu(Arc::new(f)),
None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))),
let disk = job.src.disk.as_ref().map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts))));
let from_disk = disk.as_ref().and_then(|(dc, k)| dc.get_frame(*k));
let ci = match from_disk {
Some(f) => FrameImage::Cpu(Arc::new(egui::ColorImage::from_rgba_premultiplied([f.width as usize, f.height as usize], &f.rgba))),
None => {
let ci = self.render_job(&job);
if let (Some((dc, k)), FrameImage::Cpu(img)) = (&disk, &ci) {
let rgba: Vec<u8> = img.pixels.iter().flat_map(|c| c.to_array()).collect();
dc.put_frame(*k, img.size[0] as u32, img.size[1] as u32, &rgba);
}
ci
}
};
if job.urgent
&& let Ok(mut l) = last.lock()
@@ -368,6 +429,23 @@ impl Worker {
}
true
}
fn render_job(&self, job: &Job) -> FrameImage {
let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts);
r.expr = job.src.expr.as_deref();
r.cache = Some(&job.src.layer_cache);
r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator);
// The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no
// adapter, or a frame the GPU cannot finish here) the CPU renders it.
let gpu_frame = match (&job.src.gpu, r.active_accel()) {
(Some(g), Some(_)) if job.src.gpu_display => g.render_display(&r, job.comp, job.t),
_ => None,
};
match gpu_frame {
Some(f) => FrameImage::Gpu(Arc::new(f)),
None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))),
}
}
}
struct Job {
+17 -6
View File
@@ -327,6 +327,7 @@ impl EffectcraftApp {
layer_cache: self.session.layer_cache.clone(),
gpu: self.gpu.clone(),
gpu_display: self.gpu_display(),
disk: self.session.disk_cache.clone(),
}
}
@@ -460,13 +461,11 @@ impl EffectcraftApp {
self.request_frame_with(comp, frame, scale, true);
}
fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) {
let Some(c) = self.session.project.comp(comp) else { return };
let key = self.frame_key(comp, frame, scale);
let t = c.frame_rate.tick_of(frame);
/// Render options of viewer frames of `comp` at `scale`.
pub fn frame_opts(&self, comp: ItemId, scale: f64) -> RenderOpts {
let (draft, _) = self.session.state.viewer.fast_previews.render(self.ui.viewer.interacting);
let roi = self.session.state.region_of_interest.filter(|_| self.session.active_comp_id() == Some(comp));
let opts = RenderOpts {
RenderOpts {
scale,
motion_blur: true,
guides: true,
@@ -474,7 +473,14 @@ impl EffectcraftApp {
view: self.session.view_camera(comp),
roi,
backend: effectcraft_engine::render::Backend::Auto,
};
}
}
fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) {
let Some(c) = self.session.project.comp(comp) else { return };
let key = self.frame_key(comp, frame, scale);
let t = c.frame_rate.tick_of(frame);
let opts = self.frame_opts(comp, scale);
if urgent {
self.frames.request_urgent(&self.render_source(), key, comp, t, opts);
} else {
@@ -789,6 +795,11 @@ impl EffectcraftApp {
self.session.poll_roto(true);
ctx.request_repaint_after(std::time::Duration::from_millis(100));
}
// 3D Camera Tracker: likewise (tracking, then solving).
if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() {
self.session.poll_camera(true);
ctx.request_repaint_after(std::time::Duration::from_millis(100));
}
self.apply_prefs(&ctx);
self.handle_events(&ctx);
self.tick_autosave(&ctx);
+5 -1
View File
@@ -802,7 +802,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option<Res
let Some(f) = app.hooks.pick_files.as_ref() else { return Some(Err("no file dialog available (pass `paths`)".into())) };
let paths = f(&[
"mp4", "mov", "m4v", "mkv", "webm", "png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr", "wav", "aif", "aiff", "mp3", "flac",
"ogg", "opus", "svg", "gltf", "glb", "obj",
"ogg", "opus", "svg", "psd", "psb", "gltf", "glb", "obj",
]);
match (paths.is_empty(), key) {
(true, _) => None,
@@ -825,6 +825,10 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option<Res
};
let Some(v) = picked else { return Some(Ok(Value::Null)) };
p.insert(key.to_string(), v);
// Photoshop files ask how to import them first.
if id == "file.import" && crate::panels::dialogs::open_form(app, id, &Value::Object(p.clone())) {
return Some(Ok(json!({"dialog": id})));
}
// Save (untitled) becomes Save As.
let id = if id == "file.save" { "file.saveAs" } else { id };
let r = app.session.execute(id, Value::Object(p)).map_err(|e| e.to_string());
@@ -0,0 +1,438 @@
//! 3D Camera Tracker in the Composition viewer and Effect Controls:
//!
//! - while the effect is selected, the solved 3D track points are drawn as coloured targets
//! sized by depth (or the 2D source tracks with Show Track Points ▸ 2D Source);
//! - hovering between three points shows the target plane they span (a disc in perspective);
//! click a point (Shift adds / toggles) or drag a marquee to select points, click inside a
//! triangle to select its three points; the selection's target is drawn;
//! - right-click: Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher …
//! and Camera, Create Multiple …, Delete Selected Points; Delete removes the selection;
//! - the "Analyzing in background (step 1 of 2)" / "Solving camera" banner;
//! - the Effect Controls block with Analyze / Cancel / Create Camera, Method Used and Average
//! Error.
//!
//! Everything is dispatched to the `camera.*` commands.
use std::sync::Arc;
use effectcraft_engine::Session;
use effectcraft_engine::effects::camera_tracker as ct;
use effectcraft_engine::geom::{Mat3, vec2 as gv2};
use effectcraft_engine::keyframe::Value;
use effectcraft_engine::project::{Comp, Layer, LayerId, PropGroup, Uid};
use effectcraft_engine::render::EvalCtx;
use effectcraft_engine::track::camtrack::{CameraSolve, CameraTracks, Target, point_color};
use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, pos2, vec2};
use serde_json::json;
use super::viewer::ViewerMap;
use crate::theme::Tokens;
use crate::{EffectcraftApp, widgets};
/// The tracker whose points the viewer shows.
struct Active {
layer: LayerId,
uid: Uid,
solve: Option<Arc<CameraSolve>>,
tracks: Option<Arc<CameraTracks>>,
show_3d: bool,
point_size: f64,
target_size: f64,
/// Layer pixels → comp pixels.
l2c: Mat3,
frame: usize,
}
/// A drawn point: (id, screen position, screen radius).
type Hit = (u32, Pos2, f32);
fn tracker_group(l: &Layer, uid: Uid) -> Option<&PropGroup> {
l.props.find_group(uid).filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID))
}
/// The selected 3D Camera Tracker effect on an active layer (After Effects shows the points
/// while the effect is selected).
fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) -> Option<Active> {
let time = ectx.time;
for (lid, uid) in &s.state.selected_props {
let Some(l) = comp.layer(*lid).filter(|l| l.is_active_at(time)) else { continue };
let Some(g) = tracker_group(l, *uid) else { continue };
let params = effectcraft_engine::camera_track::static_params(g);
let val = |id: &str| effectcraft_engine::effects::warp_stab::param(&params, id).cloned();
let solve = ct::solve(&params);
let tracks = ct::tracks(&params);
let lt = l.layer_time(time).seconds();
let frame = solve.as_ref().and_then(|sv| sv.frame_at(lt)).or_else(|| {
let t = tracks.as_ref()?;
(t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize)
});
return Some(Active {
layer: *lid,
uid: *uid,
solve,
tracks,
show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1,
point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0),
target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0),
l2c: l2c(ectx, l),
frame: frame.unwrap_or(0),
});
}
None
}
fn to_screen(a: &Active, map: &ViewerMap, q: [f64; 2]) -> Pos2 {
let c = a.l2c.apply(gv2(q[0], q[1]));
map.to_screen([c.x, c.y])
}
fn col(id: u32) -> Color32 {
let c = point_color(id);
Color32::from_rgb((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8)
}
/// Comp-pixel scale of the layer → comp mapping.
fn l2c_scale(m: &Mat3) -> f32 {
m.0[0][0].hypot(m.0[1][0]) as f32
}
/// Draw the target disc (concentric rings in perspective) of `t` on frame `k`.
fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) {
let Some(cam) = sv.frames.get(a.frame) else { return };
let r = t.size * 0.5 * a.target_size / 100.0;
for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() {
let ring = ct::target_circle(t.center, t.normal, r * frac, 48);
let pts: Vec<Pos2> = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect();
if pts.len() < 3 {
return;
}
let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) };
p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40))));
}
if let Some(c) = cam.project(sv.size, t.center) {
let c = to_screen(a, map, c);
p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40));
}
}
/// The three drawn points forming a triangle around `hp` (nearest first).
fn triangle_at(hits: &[Hit], hp: Pos2) -> Option<[u32; 3]> {
let mut near: Vec<&Hit> = hits.iter().collect();
near.sort_by(|x, y| x.1.distance_sq(hp).total_cmp(&y.1.distance_sq(hp)));
let near: Vec<&Hit> = near.into_iter().take(8).collect();
let inside = |a: Pos2, b: Pos2, c: Pos2| {
let s = |p: Pos2, q: Pos2| (q.x - p.x) * (hp.y - p.y) - (q.y - p.y) * (hp.x - p.x);
let (d1, d2, d3) = (s(a, b), s(b, c), s(c, a));
!((d1 < 0.0 || d2 < 0.0 || d3 < 0.0) && (d1 > 0.0 || d2 > 0.0 || d3 > 0.0))
};
for i in 0..near.len() {
for j in i + 1..near.len() {
for k in j + 1..near.len() {
let (a, b, c) = (near[i], near[j], near[k]);
// Skip slivers.
let area = ((b.1 - a.1).x * (c.1 - a.1).y - (b.1 - a.1).y * (c.1 - a.1).x).abs();
if area > 30.0 && inside(a.1, b.1, c.1) {
return Some([a.0, b.0, c.0]);
}
}
}
}
None
}
fn run(app: &mut EffectcraftApp, ctx: &egui::Context, cmd: &str, p: serde_json::Value) {
if let Err(e) = crate::menus::invoke(app, ctx, cmd, p) {
app.ui.status = e;
}
}
/// The viewer overlay and its interaction (call after the viewer's own gestures so it takes
/// the clicks while a 3D Camera Tracker is selected).
pub fn viewer_hook(
app: &mut EffectcraftApp,
ui: &mut egui::Ui,
painter: &egui::Painter,
map: &ViewerMap,
ectx: &EvalCtx,
l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3,
) {
let ctx = ui.ctx().clone();
let comp = ectx.comp;
banner(app, painter, map, ectx, l2c);
if !app.ui.viewer.show_layer_controls {
return;
}
let Some(a) = active(&app.session, comp, ectx, l2c) else { return };
let selected = app.session.state.camera_points.clone();
let lscale = l2c_scale(&a.l2c) * map.zoom;
let mut hits: Vec<Hit> = vec![];
match (&a.solve, a.show_3d) {
(Some(sv), true) => {
let pts = ct::projected(sv, a.frame);
let med = ct::median_depth(&pts);
for (id, q, z) in pts {
let sp = to_screen(&a, map, q);
if !map.area.expand(20.0).contains(sp) {
continue;
}
let r = (ct::target_radius(a.point_size, z, med) as f32 * lscale).clamp(2.0, 30.0);
let c = col(id);
let sel = selected.contains(&id);
if sel {
painter.circle_filled(sp, r, c);
painter.circle_stroke(sp, r + 1.5, Stroke::new(1.5, Color32::WHITE));
} else {
painter.circle_stroke(sp, r, Stroke::new((r * 0.3).clamp(1.0, 3.0), c));
painter.circle_filled(sp, (r * 0.25).max(1.0), c);
}
hits.push((id, sp, r));
}
}
_ => {
// 2D source tracks (also shown before the solve finishes).
if let Some(tr) = &a.tracks {
for t in &tr.tracks {
let Some(q) = t.at(a.frame as u32) else { continue };
let sp = to_screen(&a, map, q);
let c = col(t.id);
let r = (3.0 * a.point_size as f32 / 100.0).clamp(1.5, 12.0);
painter.line_segment([sp - vec2(r, r), sp + vec2(r, r)], Stroke::new(1.2, c));
painter.line_segment([sp - vec2(r, -r), sp + vec2(r, -r)], Stroke::new(1.2, c));
hits.push((t.id, sp, r));
}
}
}
}
let Some(sv) = a.solve.clone() else { return };
if app.ui.tool != crate::state::Tool::Selection {
return;
}
// Interaction over the image.
let resp = ui.interact(map.area, egui::Id::new("viewer-camera-tracker"), Sense::click_and_drag());
app.auto.add("viewer.cameraTracker", map.area, "3D Camera Tracker points");
for (id, sp, r) in hits.iter().take(400) {
app.auto.add(&format!("viewer.cameraTracker.point.{id}"), Rect::from_center_size(*sp, vec2(r * 2.0 + 4.0, r * 2.0 + 4.0)), "track point");
}
let hover = resp.hover_pos();
let point_at = |p: Pos2| hits.iter().filter(|h| h.1.distance(p) <= h.2 + 3.0).min_by(|x, y| x.1.distance(p).total_cmp(&y.1.distance(p))).map(|h| h.0);
let tri = hover.filter(|p| point_at(*p).is_none()).and_then(|p| triangle_at(&hits, p));
let mid = egui::Id::new("viewer-camera-tracker-marquee");
let marquee: Option<Pos2> = ui.data(|d| d.get_temp(mid));
// Hover target / the selection's target.
if marquee.is_none() {
if let Some(t3) = tri
&& let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &t3, a.frame)
{
let ps: Vec<Pos2> = t3.iter().filter_map(|i| hits.iter().find(|h| h.0 == *i)).map(|h| h.1).collect();
if ps.len() == 3 {
painter.add(egui::Shape::closed_line(ps, Stroke::new(1.0, Color32::from_white_alpha(200))));
}
draw_target(painter, &a, map, &sv, &t);
} else if !selected.is_empty()
&& let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &selected, a.frame)
{
draw_target(painter, &a, map, &sv, &t);
}
}
let mods = ui.input(|i| i.modifiers);
if resp.drag_started()
&& let Some(p) = ui.input(|i| i.pointer.press_origin())
{
ui.data_mut(|d| d.insert_temp(mid, p));
}
if let (Some(start), Some(p)) = (marquee, resp.interact_pointer_pos()) {
let r = Rect::from_two_pos(start, p);
let t = app.tokens;
painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12));
painter.rect_stroke(r, 0.0, Stroke::new(1.0, t.accent), egui::StrokeKind::Middle);
if resp.drag_stopped() {
let ids: Vec<u32> = hits.iter().filter(|h| r.contains(h.1)).map(|h| h.0).collect();
ui.data_mut(|d| d.remove::<Pos2>(mid));
run(app, &ctx, "camera.selectPoints", json!({"points": ids, "add": mods.shift}));
}
} else if resp.drag_stopped() {
ui.data_mut(|d| d.remove::<Pos2>(mid));
}
if resp.clicked()
&& let Some(p) = resp.interact_pointer_pos()
{
match (point_at(p), triangle_at(&hits, p)) {
(Some(id), _) => run(app, &ctx, "camera.selectPoints", json!({"points": [id], "toggle": mods.shift})),
(None, Some(t3)) => run(app, &ctx, "camera.selectPoints", json!({"points": t3, "add": mods.shift})),
(None, None) => run(app, &ctx, "camera.selectPoints", json!({"points": []})),
}
}
// Right-click: the points under the cursor (selection, else the hovered triangle).
let cid = egui::Id::new("viewer-camera-tracker-menu-points");
if resp.secondary_clicked() {
let pts: Vec<u32> = if !selected.is_empty() {
selected.clone()
} else if let Some(p) = resp.interact_pointer_pos() {
point_at(p).map(|i| vec![i]).or_else(|| triangle_at(&hits, p).map(|t| t.to_vec())).unwrap_or_default()
} else {
vec![]
};
ui.data_mut(|d| d.insert_temp(cid, pts));
}
let base = json!({"layer": a.layer.0, "effect": a.uid});
resp.context_menu(|ui| {
let pts: Vec<u32> = ui.data(|d| d.get_temp(cid)).unwrap_or_default();
let with = |kind: Option<&str>, multiple: bool| {
let mut q = base.clone();
q["points"] = json!(pts);
if let Some(k) = kind {
q["kind"] = json!(k);
}
if multiple {
q["multiple"] = json!(true);
}
q
};
let some = !pts.is_empty();
let many = pts.len() > 1;
let mut pick: Option<(&str, serde_json::Value)> = None;
if ui.add_enabled(some, egui::Button::new("Set Ground Plane and Origin")).clicked() {
pick = Some(("camera.setGroundPlane", with(None, false)));
}
ui.separator();
for (label, kind) in [
("Create Text and Camera", "text"),
("Create Solid and Camera", "solid"),
("Create Null and Camera", "null"),
("Create Shadow Catcher, Camera and Light", "shadowCatcher"),
] {
if ui.add_enabled(some, egui::Button::new(label)).clicked() {
pick = Some(("camera.createFromSolve", with(Some(kind), false)));
}
}
ui.separator();
for (label, kind) in
[("Create Multiple Text Layers and Camera", "text"), ("Create Multiple Solids and Camera", "solid"), ("Create Multiple Nulls and Camera", "null")]
{
if ui.add_enabled(many, egui::Button::new(label)).clicked() {
pick = Some(("camera.createFromSolve", with(Some(kind), true)));
}
}
ui.separator();
if ui.add_enabled(some, egui::Button::new("Delete Selected Points")).clicked() {
pick = Some(("camera.deletePoints", with(None, false)));
}
if let Some((cmd, q)) = pick {
ui.close();
run(app, &ctx, cmd, q);
}
});
if resp.hovered()
&& !selected.is_empty()
&& ui.input(|i| i.key_pressed(egui::Key::Delete) || i.key_pressed(egui::Key::Backspace))
&& !ctx.egui_wants_keyboard_input()
{
run(app, &ctx, "camera.deletePoints", json!({"layer": a.layer.0, "effect": a.uid, "points": selected}));
}
}
/// "Analyzing in background (step 1 of 2)" / "Solving camera" over the analysed layer, and a
/// warning when the solve failed (unless Hide Warning Banner).
fn banner(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) {
let comp = ectx.comp;
let time = ectx.time;
let cid = ectx.comp_id;
let draw = |app: &mut EffectcraftApp, layer: &Layer, text: String, color: Color32, id: &str| {
let m = l2c(ectx, layer);
let (w, h) = effectcraft_engine::render::source_size(&app.session.project, layer);
let c = m.apply(gv2(w as f64 / 2.0, h as f64 / 2.0));
let at = map.to_screen([c.x, c.y]);
let galley = painter.layout_no_wrap(text.clone(), Tokens::medium(13.0), Color32::WHITE);
let r = Rect::from_center_size(at, galley.size() + vec2(28.0, 14.0)).intersect(map.area);
painter.rect_filled(r, 3.0, color);
painter.galley(pos2(r.center().x - galley.size().x / 2.0, r.center().y - galley.size().y / 2.0), galley, Color32::WHITE);
app.auto.add(id, r, &text);
};
if let Some((cc, cl, _)) = app.session.camera_target()
&& cc == cid
&& let Some(pr) = app.session.camera_progress()
&& let Some(layer) = comp.layer(cl)
&& layer.is_active_at(time)
{
draw(app, layer, pr.banner(), Color32::from_rgba_unmultiplied(0x1d, 0x4f, 0x9c, 0xe0), "viewer.cameraBanner");
return;
}
// A failed solve: tracks without a solve while nothing is running or queued.
for l in comp.layers.iter().filter(|l| l.is_active_at(time)) {
let Some(fx) = l.effects() else { continue };
for g in fx.groups().filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) {
let params = effectcraft_engine::camera_track::static_params(g);
let hide = effectcraft_engine::effects::warp_stab::param(&params, "advanced/hideWarningBanner").is_some_and(Value::as_bool);
let solve_key = matches!(g.prop(ct::SOLVE_KEY).map(|p| &p.value), Some(Value::Str(s)) if !s.is_empty());
let failed = solve_key && ct::tracks(&params).is_some() && ct::solve(&params).is_none();
if failed && !hide && !app.session.camera_pending.iter().any(|(_, ll, u)| *ll == l.id && *u == g.uid) {
draw(app, l, "Unable to solve camera".into(), Color32::from_rgba_unmultiplied(0x9c, 0x2a, 0x1d, 0xe0), "viewer.cameraWarning");
return;
}
}
}
}
/// Effect Controls: Analyze / Cancel, Create Camera and the solve report.
#[allow(clippy::too_many_arguments)]
pub fn editor(
app: &mut EffectcraftApp,
ui: &mut egui::Ui,
p: &egui::Painter,
layer: &Layer,
g: &PropGroup,
r: Rect,
actions: &mut Vec<(String, serde_json::Value)>,
) {
let t = app.tokens;
let running = app.session.camera_target().is_some_and(|(_, l, u)| l == layer.id && u == g.uid) && app.session.is_camera_analyzing();
let busy = app.session.is_camera_analyzing();
let me = json!({"layer": layer.id.0, "effect": g.uid});
let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0));
let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0));
let b3 = Rect::from_min_size(pos2(b2.max.x + 8.0, b1.min.y), vec2(110.0, 22.0));
if widgets::text_button(ui, b1, "Analyze", false, &t, egui::Id::new(("ct-analyze", g.uid))).clicked() && !busy {
actions.push(("camera.analyze".into(), me.clone()));
}
app.auto.add(&format!("effectControls.cameraTracker.{}.analyze", g.uid), b1, "Analyze");
if running {
if widgets::text_button(ui, b2, "Cancel", false, &t, egui::Id::new(("ct-cancel", g.uid))).clicked() {
actions.push(("camera.cancel".into(), json!({})));
}
app.auto.add(&format!("effectControls.cameraTracker.{}.cancel", g.uid), b2, "Cancel");
}
let params = effectcraft_engine::camera_track::static_params(g);
let solve = ct::solve(&params);
if solve.is_some() {
if widgets::text_button(ui, b3, "Create Camera", false, &t, egui::Id::new(("ct-camera", g.uid))).clicked() {
actions.push(("camera.create".into(), me.clone()));
}
app.auto.add(&format!("effectControls.cameraTracker.{}.createCamera", g.uid), b3, "Create Camera");
}
let status = match app.session.camera_progress() {
Some(pr) if running => pr.banner(),
_ => match &solve {
Some(sv) => format!("Method Used: {} Average Error: {:.2} pixels", sv.method_used.label(), sv.average_error),
None if ct::tracks(&params).is_some() && app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => {
"Waiting to solve".into()
}
None if ct::tracks(&params).is_some() => "Unable to solve camera: try another Solve Method or Shot Type".into(),
None if app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to analyze".into(),
None => "Not analyzed: click Analyze".into(),
},
};
p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim);
if let Some(sv) = &solve {
let f = sv.focal();
p.text(
pos2(r.min.x + 24.0, r.min.y + 56.0),
Align2::LEFT_CENTER,
format!("{} solved points · Horizontal Angle of View {:.1}°", sv.points.len(), sv.hfov(f)),
Tokens::ui(11.5),
t.text_faint,
);
}
}
/// Height of [`editor`].
pub const EDITOR_HEIGHT: f32 = 68.0;
@@ -574,6 +574,7 @@ fn editor_height(effect: &str, width: f32) -> f32 {
match effect {
"ec.color.curves" => 34.0 + curves_size(width) + 26.0,
effectcraft_engine::effects::warp_stab::ID => 50.0,
effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT,
effectcraft_engine::effects::roto::ID => 50.0,
"ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0,
"ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0,
@@ -950,6 +951,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
match effect.as_str() {
"ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions),
effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions),
effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions),
effectcraft_engine::effects::roto::ID => roto_editor(app, ui, &bp, &layer, g, er, &mut actions),
"ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions),
_ => histogram_only(app, ui, &bp, g, er),
+20
View File
@@ -108,6 +108,16 @@ fn has(p: &Value, keys: &[&str]) -> bool {
}
/// If `id` is a dialog command invoked without its parameters, open its form and return true.
/// The import includes a Photoshop document.
fn is_layered(p: &Value) -> bool {
let psd = |v: &Value| v.as_str().is_some_and(|s| matches!(s.rsplit('.').next().map(str::to_ascii_lowercase).as_deref(), Some("psd" | "psb")));
match p.get("paths").or(p.get("path")) {
Some(Value::Array(a)) => a.iter().any(psd),
Some(v) => psd(v),
None => false,
}
}
pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool {
let s = &app.session;
let comp = s.active_comp();
@@ -116,6 +126,16 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool {
let lt = layer.map(|l| l.layer_time(t)).unwrap_or(t);
let base = p.clone();
let (title, fields): (String, Vec<Field>) = match id {
// Photoshop files: Import Kind (Footage / Composition / – Retain Layer Sizes).
"file.import" if !has(p, &["importAs"]) && is_layered(p) => (
"Import Photoshop File".into(),
vec![Field::choice(
"importAs",
"Import Kind",
&[("Footage", json!("footage")), ("Composition", json!("composition")), ("Composition - Retain Layer Sizes", json!("compositionLayerSizes"))],
1,
)],
),
"layer.setTransform" if !has(p, &["value"]) => {
let prop = p.get("prop").and_then(Value::as_str).unwrap_or("position");
let cur = layer.and_then(|l| l.transform()).and_then(|tr| tr.get(if prop == "anchorPoint" { "anchor" } else { prop })).map(|pr| pr.value_at(lt));
+1
View File
@@ -1,6 +1,7 @@
//! Panel bodies.
pub mod anim_tools;
pub mod camera_tracker_ui;
pub mod comp_settings;
pub mod dialogs;
pub mod dialogs_3d;

Some files were not shown because too many files have changed in this diff Show More