Files
effectcraft/crates/engine/src/tests_track.rs
T
Brandon ThomasandClaude Opus 5.5 243098a144 M12.3: tracker core
New L2 crate effectcraft-track: per-point feature/search region tracking with
pyramid normalized cross-correlation and Lucas-Kanade (similarity + gain/bias)
sub-pixel refinement, confidence, adapt/stop/extrapolate/continue actions,
RGB/luminance/saturation channels, blur/enhance pre-processing, and
homography/affine/similarity solves.

Project: Motion Trackers > Tracker n > Track Point n property tree
(GroupKind::Tracker with type, target and options). Engine: track.* commands
(new/motion/stabilize/property/select/setType/setTarget/options/setPoint/
analyze/stop/apply/reset/delete/status), background or blocking analysis
job with progress and cancel (one undo step), apply to Transform, Stabilize
and Corner Pin. Renderer::layer_source for the tracker's frames.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 08:28:44 -04:00

357 lines
16 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Motion tracking through the commands: synthetic footage generated in-test (a textured patch on
//! a known path, or a whole frame under a known homography), analysis (blocking and background),
//! apply (Transform, Stabilize, Corner Pin), undo/redo and serde.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use effectcraft_color::Label;
use effectcraft_project::build;
use effectcraft_project::tracking::TrackKind;
use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, LayerId, LayerSource};
use effectcraft_raster::Image;
use effectcraft_render::{EvalCtx, FootageSource};
use effectcraft_time::{FrameRate, Tick};
use effectcraft_track::Homography;
use serde_json::json;
use crate::Session;
const W: u32 = 320;
const H: u32 = 240;
const FPS: u32 = 25;
fn texture(x: f64, y: f64, seed: u32) -> f32 {
let mut v = 0.0f64;
let mut s = seed.wrapping_mul(2654435761).wrapping_add(12345);
let mut rnd = || {
s ^= s << 13;
s ^= s >> 17;
s ^= s << 5;
(s % 10_000) as f64 / 10_000.0
};
for _ in 0..6 {
let ang = rnd() * std::f64::consts::TAU;
let period = 9.0 + rnd() * 25.0;
let phase = rnd() * std::f64::consts::TAU;
let k = std::f64::consts::TAU / period;
v += (k * (x * ang.cos() + y * ang.sin()) + phase).sin() * (0.4 + rnd()) * 0.1;
}
(0.5 + v).clamp(0.0, 1.0) as f32
}
/// Footage computed per frame (cached).
struct Synth {
make: Box<dyn Fn(u32) -> Image + Send + Sync>,
cache: Mutex<HashMap<u32, Arc<Image>>>,
}
impl FootageSource for Synth {
fn frame(&self, _: ItemId, _: &Footage, t: Tick) -> Option<Arc<Image>> {
let f = (t.seconds() * FPS as f64).round().max(0.0) as u32;
let mut c = self.cache.lock().unwrap();
Some(c.entry(f).or_insert_with(|| Arc::new((self.make)(f))).clone())
}
}
/// Patch pose on frame `f`: centre, rotation (degrees), scale.
type Pose = fn(u32) -> ([f64; 2], f64, f64);
fn patch_frames(pose: Pose) -> impl Fn(u32) -> Image + Send + Sync {
move |f| {
let (c, rot, sc) = pose(f);
let (sn, cs) = rot.to_radians().sin_cos();
let mut img = Image::new(W, H);
for y in 0..H {
for x in 0..W {
let p = [x as f64 + 0.5, y as f64 + 0.5];
let d = [(p[0] - c[0]) / sc, (p[1] - c[1]) / sc];
let l = [cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]];
let v = if l[0].abs() <= 50.0 && l[1].abs() <= 50.0 { texture(l[0], l[1], 7) } else { 0.6 * texture(p[0] * 0.7, p[1] * 0.7, 3) };
img.set(x, y, [v, v, v, 1.0]);
}
}
img
}
}
/// A comp with the synthetic clip (layer `clip`) and a 100 × 100 solid above it (layer `solid`).
fn setup(make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Session, LayerId, LayerId) {
let mut s = Session::default();
s.execute("comp.new", json!({"name": "Track", "width": W, "height": H, "frameRate": FPS, "duration": 1})).unwrap();
let cid = s.active_comp_id().unwrap();
let clip = s
.edit("clip", None, |p, _| {
let fr = FrameRate::new(FPS as i64, 1);
let footage = Footage {
path: "synthetic.mov".into(),
kind: FootageKind::Video,
width: W,
height: H,
pixel_aspect: 1.0,
frame_rate: fr,
native_rate: None,
duration: Tick::from_seconds_f64(1.0),
has_video: true,
has_audio: false,
alpha: Default::default(),
premul_color: [0.0; 3],
loop_count: 1,
codec: String::new(),
missing: false,
sequence: vec![],
};
let item = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(footage));
let comp = p.comp(cid).unwrap().clone();
let l = build::layer(p, &comp, "Clip", LayerSource::Footage { item }, (W, H), None);
let id = l.id;
p.comp_mut(cid).unwrap().layers.insert(0, l);
Ok(id)
})
.unwrap();
s.footage = Arc::new(Synth { make: Box::new(make), cache: Mutex::default() });
let solid = s.execute("layer.newSolid", json!({"width": 100, "height": 100, "color": "#ff0000"})).unwrap()["layer"].as_u64().unwrap();
s.execute("layer.select", json!({"layers": [clip.0]})).unwrap();
(s, clip, LayerId(solid))
}
fn tracker_points(s: &Session, layer: LayerId) -> Vec<effectcraft_project::PropGroup> {
let l = s.active_comp().unwrap().layer(layer).unwrap();
l.trackers().next().unwrap().0.track_points().cloned().collect()
}
fn frame_time(f: u32) -> Tick {
FrameRate::new(FPS as i64, 1).tick_of(f as i64)
}
fn dist(a: [f64; 2], b: [f64; 2]) -> f64 {
((a[0] - b[0]).powi(2) + (a[1] - b[1]).powi(2)).sqrt()
}
fn translate_pose(f: u32) -> ([f64; 2], f64, f64) {
let t = f as f64;
([150.0 + 2.37 * t + 4.0 * (t * 0.4).sin(), 110.0 + 1.13 * t], 0.0, 1.0)
}
#[test]
fn track_motion_analyze_apply_transform_undo_redo() {
let (mut s, clip, solid) = setup(patch_frames(translate_pose));
// The menu entry and the Tracker panel's Track Motion button.
assert!(s.is_enabled("track.motion"));
let r = s.execute("track.motion", json!({})).unwrap();
assert_eq!(r["points"], 1);
let st = s.execute("track.status", json!({})).unwrap();
assert_eq!(st["tracker"]["settings"]["target"], json!(solid.0), "{st}");
// Put the track point on the patch and analyze to the end (blocking).
s.execute("track.setPoint", json!({"point": 1, "center": translate_pose(0).0, "featureSize": [36, 36], "searchSize": [80, 80]})).unwrap();
let r = s.execute("track.analyze", json!({"direction": "forward", "wait": true})).unwrap();
assert_eq!(r["frames"], 24);
s.poll_track();
assert!(s.track_job.is_none());
let tp = &tracker_points(&s, clip)[0];
let fc = tp.get("featureCenter").unwrap();
assert_eq!(fc.keys.len(), 25);
let conf = tp.get("confidence").unwrap();
let mut worst: f64 = 0.0;
for f in 0..25 {
let t = frame_time(f);
worst = worst.max(dist(fc.value_at(t).as_vec2(), translate_pose(f).0));
assert!(conf.value_at(t).as_f64() > 90.0);
}
assert!(worst < 0.25, "worst {worst}");
// CTI moved to the last analysed frame.
assert_eq!(s.time(), frame_time(24));
// Apply: the solid's position follows the attach point.
s.execute("track.apply", json!({})).unwrap();
let pos = s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().clone();
assert_eq!(pos.keys.len(), 25);
for f in [0, 7, 24] {
let v = pos.value_at(frame_time(f)).as_vec2();
assert!(dist(v, translate_pose(f).0) < 0.25, "frame {f}: {v:?}");
}
// Undo the apply, then the analysis; redo both.
assert!(s.undo());
assert!(s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().keys.is_empty());
assert!(s.undo());
assert!(tracker_points(&s, clip)[0].get("featureCenter").unwrap().keys.is_empty());
assert!(s.redo());
assert_eq!(tracker_points(&s, clip)[0].get("featureCenter").unwrap().keys.len(), 25);
assert!(s.redo());
assert_eq!(s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().keys.len(), 25);
// X only keeps the solid's Y.
s.undo();
s.execute("track.apply", json!({"dimensions": "x"})).unwrap();
let pos = s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().clone();
let v = pos.value_at(frame_time(10)).as_vec2();
assert!((v[0] - translate_pose(10).0[0]).abs() < 0.25 && (v[1] - 120.0).abs() < 1e-9, "{v:?}");
// Serde round trip keeps the tracker.
let json = s.project.to_json();
let back = effectcraft_project::Project::from_json(&json).unwrap();
let l = back.comp(s.active_comp_id().unwrap()).unwrap().layer(clip).unwrap();
let (g, st) = l.trackers().next().unwrap();
assert_eq!(st.kind, TrackKind::Transform);
assert_eq!(g.track_points().next().unwrap().get("featureCenter").unwrap().keys.len(), 25);
assert_eq!(l.props.children[0].match_id(), "motionTrackers");
}
#[test]
fn stabilize_keeps_the_feature_fixed() {
let (mut s, clip, _) = setup(patch_frames(translate_pose));
s.execute("track.stabilize", json!({})).unwrap();
s.execute("track.setPoint", json!({"point": 1, "center": translate_pose(0).0, "featureSize": [36, 36], "searchSize": [80, 80]})).unwrap();
s.execute("time.set", json!({"time": 0})).unwrap();
s.execute("track.analyze", json!({"wait": true})).unwrap();
s.execute("track.apply", json!({})).unwrap();
let cid = s.active_comp_id().unwrap();
let comp = s.project.comp(cid).unwrap().clone();
let l = comp.layer(clip).unwrap();
let at = |f: u32| {
let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(f));
let (m, _) = ctx.layer_to_comp(l);
let c = translate_pose(f).0;
let q = m.apply(effectcraft_geom::vec2(c[0], c[1]));
[q.x, q.y]
};
let p0 = at(0);
// The layer itself didn't move on the first frame.
assert!(dist(p0, translate_pose(0).0) < 1e-6, "{p0:?}");
let worst = (1..25).map(|f| dist(at(f), p0)).fold(0.0, f64::max);
assert!(worst < 0.25, "feature drifts by {worst} px");
}
fn rotating_pose(f: u32) -> ([f64; 2], f64, f64) {
let t = f as f64;
([160.0 + 1.2 * t, 120.0 - 0.7 * t], 1.5 * t, 1.0 + 0.008 * t)
}
#[test]
fn transform_rotation_scale_and_background_job() {
let (mut s, clip, solid) = setup(patch_frames(rotating_pose));
s.execute("track.new", json!({"kind": "transform", "rotation": true, "scale": true})).unwrap();
assert_eq!(tracker_points(&s, clip).len(), 2);
let (c, _, _) = rotating_pose(0);
s.execute("track.setPoint", json!({"point": 1, "center": [c[0] - 26.0, c[1]], "featureSize": [26, 26], "searchSize": [56, 56]})).unwrap();
s.execute("track.setPoint", json!({"point": 2, "center": [c[0] + 26.0, c[1]], "featureSize": [26, 26], "searchSize": [56, 56]})).unwrap();
// Background analysis with progress, polled like the UI does.
let r = s.execute("track.analyze", json!({"direction": "forward", "end": 0.6})).unwrap();
assert_eq!(r["frames"], 15);
let t0 = std::time::Instant::now();
while s.is_tracking() {
assert!(t0.elapsed().as_secs() < 120);
std::thread::sleep(std::time::Duration::from_millis(5));
s.poll_track();
}
s.poll_track();
let st = s.execute("track.status", json!({})).unwrap();
assert_eq!(st["running"], false);
assert_eq!(st["tracker"]["points"][0]["keys"], 16);
s.execute("track.apply", json!({})).unwrap();
let tr = s.active_comp().unwrap().layer(solid).unwrap().props.sub("transform").unwrap().clone();
let rot = tr.get("rotation").unwrap();
let sc = tr.get("scale").unwrap();
for f in [5, 10, 15] {
let (_, r, k) = rotating_pose(f);
let got = rot.value_at(frame_time(f)).as_f64();
assert!((got - r).abs() < 0.3, "frame {f}: rotation {got} vs {r}");
let gs = sc.value_at(frame_time(f)).as_vec3()[0];
assert!((gs - 100.0 * k).abs() < 0.5, "frame {f}: scale {gs} vs {}", 100.0 * k);
}
}
#[test]
fn perspective_corner_pin_follows_a_known_homography() {
// Frame f shows the texture under H_f (identity → `end` over 12 frames).
let end = Homography([[1.04, 0.05, 6.0], [-0.03, 0.97, 4.0], [0.00012, -0.00008, 1.0]]);
let hf = move |f: u32| {
let k = f as f64 / 12.0;
let mut m = [[0.0; 3]; 3];
for i in 0..3 {
for j in 0..3 {
let id = if i == j { 1.0 } else { 0.0 };
m[i][j] = id + (end.0[i][j] - id) * k;
}
}
Homography(m)
};
let inv = move |f: u32, p: [f64; 2]| {
let h = hf(f);
// Solve H x = p by mapping four points back (exact for a homography).
let src = [[0.0, 0.0], [W as f64, 0.0], [0.0, H as f64], [W as f64, H as f64]];
let dst = src.map(|q| h.apply(q));
Homography::from_points(dst, src).unwrap().apply(p)
};
let make = move |f: u32| {
let h = hf(f);
let src = [[0.0, 0.0], [W as f64, 0.0], [0.0, H as f64], [W as f64, H as f64]];
let back = Homography::from_points(src.map(|q| h.apply(q)), src).unwrap();
let _ = inv;
let mut img = Image::new(W, H);
for y in 0..H {
for x in 0..W {
let q = back.apply([x as f64 + 0.5, y as f64 + 0.5]);
let v = texture(q[0], q[1], 11);
img.set(x, y, [v, v, v, 1.0]);
}
}
img
};
let (mut s, clip, solid) = setup(make);
s.execute("track.new", json!({"kind": "perspective"})).unwrap();
let corners = [[90.0, 70.0], [230.0, 70.0], [90.0, 170.0], [230.0, 170.0]];
for (i, c) in corners.iter().enumerate() {
s.execute("track.setPoint", json!({"point": i + 1, "center": c, "featureSize": [30, 30], "searchSize": [60, 60]})).unwrap();
}
s.execute("track.analyze", json!({"wait": true, "end": 12.0 / FPS as f64})).unwrap();
let pts = tracker_points(&s, clip);
for (i, tp) in pts.iter().enumerate() {
let got = tp.get("featureCenter").unwrap().value_at(frame_time(12)).as_vec2();
assert!(dist(got, end.apply(corners[i])) < 0.3, "point {i}: {got:?} vs {:?}", end.apply(corners[i]));
}
s.execute("track.apply", json!({})).unwrap();
let l = s.active_comp().unwrap().layer(solid).unwrap().clone();
let fx = l.effects().unwrap().groups().next().unwrap().clone();
assert_eq!(fx.match_id, "ec.distort.cornerpin");
// The solid is 100 × 100, centred: comp = layer + (110, 70).
for (m, i) in [("ul", 0), ("ur", 1), ("ll", 2), ("lr", 3)] {
let p = fx.get(m).unwrap();
assert_eq!(p.keys.len(), 13);
for f in [6, 12] {
let want = hf(f).apply(corners[i]);
let got = p.value_at(frame_time(f)).as_vec2();
assert!(dist([got[0] + 110.0, got[1] + 70.0], want) < 0.35, "{m} frame {f}: {got:?} vs {want:?}");
}
}
}
#[test]
fn backward_frame_steps_options_and_reset() {
let (mut s, clip, _) = setup(patch_frames(translate_pose));
s.execute("track.motion", json!({})).unwrap();
s.execute("time.set", json!({"time": 10.0 / FPS as f64})).unwrap();
s.execute("track.setPoint", json!({"point": 1, "center": translate_pose(10).0, "featureSize": [36, 36], "searchSize": [80, 80]})).unwrap();
s.execute("track.options", json!({"channel": "rgb", "blur": 2, "threshold": 70, "action": "extrapolate", "name": "Patch"})).unwrap();
let st = s.execute("track.status", json!({})).unwrap();
assert_eq!(st["tracker"]["name"], "Patch");
assert_eq!(st["tracker"]["settings"]["options"]["channel"], "rgb");
s.execute("track.analyze", json!({"direction": "frameBackward", "wait": true})).unwrap();
s.execute("track.analyze", json!({"direction": "backward", "wait": true})).unwrap();
assert_eq!(s.time(), Tick::ZERO);
s.execute("track.analyze", json!({"direction": "frameForward", "wait": true, "start": 10.0 / FPS as f64})).unwrap();
let fc = tracker_points(&s, clip)[0].get("featureCenter").unwrap().clone();
assert_eq!(fc.keys.len(), 12);
for f in 0..12 {
assert!(dist(fc.value_at(frame_time(f)).as_vec2(), translate_pose(f).0) < 0.25, "frame {f}");
}
// Unknown params are rejected for agents.
assert!(s.execute_checked("track.analyze", json!({"dir": "forward"})).is_err());
s.execute("track.setType", json!({"kind": "perspective"})).unwrap();
assert_eq!(tracker_points(&s, clip).len(), 4);
s.execute("track.setType", json!({"kind": "transform", "rotation": false})).unwrap();
assert_eq!(tracker_points(&s, clip).len(), 1);
s.execute("track.reset", json!({})).unwrap();
assert!(tracker_points(&s, clip)[0].get("featureCenter").unwrap().keys.is_empty());
s.execute("track.delete", json!({})).unwrap();
assert!(s.active_comp().unwrap().layer(clip).unwrap().motion_trackers().is_none());
assert!(s.execute("track.analyze", json!({})).is_err());
}