From 0f8685b49e170fc50ebb8d246bbc1f185185930d Mon Sep 17 00:00:00 2001 From: type-two Date: Fri, 17 Jul 2026 17:48:31 +1000 Subject: [PATCH 1/4] =?UTF-8?q?M10:=20audio=20features=20=E2=80=94=20beats?= =?UTF-8?q?=20&=20onsets=20from=20the=20reference=20ingest=20WAV?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_features() fills the foundation2 stub: librosa beat/tempo tracking + onset-strength detection at a 10 ms hop (librosa's default 32 ms grid at 16 kHz quantizes past the +/-50 ms acceptance bound), tempo from the median inter-beat interval, times mapped to t_global via the frozen timebase helpers. Degradation per house style: missing videos/WAV -> summary with note, nothing written; silent/short/beatless audio -> valid-empty beats.json, never a crash. Acceptance (test_audio_features.py, 7 tests): synthetic 120 BPM grid recovered — tempo 120.0 (+/-2 required), worst beat 30 ms off grid (+/-50 required); the three ground-truth bangs appear as strong onsets; silence/short/missing all degrade cleanly; CLI dispatch end-to-end. CR-4: minimal stub-era assertion updates in test_phase5_api.py (CLI stub loop shrinks to capsule) per the approved CR-1 precedent — see plan/CHANGE_REQUESTS.md. Co-Authored-By: Claude Fable 5 --- backend/festival4d/audio_features.py | 166 +++++++++++++++++++++++++- backend/tests/test_audio_features.py | 171 +++++++++++++++++++++++++++ backend/tests/test_phase5_api.py | 20 ++-- plan/CHANGE_REQUESTS.md | 21 ++++ 4 files changed, 368 insertions(+), 10 deletions(-) create mode 100644 backend/tests/test_audio_features.py diff --git a/backend/festival4d/audio_features.py b/backend/festival4d/audio_features.py index 809e59c..9d7c2bb 100644 --- a/backend/festival4d/audio_features.py +++ b/backend/festival4d/audio_features.py @@ -1,4 +1,4 @@ -"""Audio features: beats & onsets (spec M10, lane E). STUB — lane E fills the body. +"""Audio features: beats & onsets (spec M10, lane E). Contract (frozen at foundation2 merge — phase-5 contract #1): @@ -15,11 +15,173 @@ and onset-strength detection (librosa is already a dependency), and writes All times are ``t_global`` seconds. Quiet / short / beatless audio must produce a *valid, empty* result (``tempo_bpm: null``, empty lists) — never a crash. Returns a small summary dict for the CLI log. ``GET /api/beats`` serves the written file. + +Implementation notes +-------------------- +- Analysis runs on the reference's local timeline and maps to ``t_global`` via the frozen + :func:`config.t_global_from_video` (identity for a true reference: offset 0, drift 0; + correct either way if we ever fall back to a non-zero-offset "reference"). +- ``HOP_LENGTH = 160`` (10 ms at 16 kHz): librosa's default 512-sample hop is a 32 ms + grid at this rate, which quantizes beats past the ±50 ms acceptance bound. 10 ms frames + recover the synthetic 120 BPM pulse grid to ~30 ms / ±0.001 BPM (see test_audio_features). +- Reported ``tempo_bpm`` is the median inter-beat interval of the tracked beats (more + accurate than the tracker's coarse tempo prior) and ``null`` when < 2 beats are found. +- Missing prerequisites (no videos ingested / WAV absent) degrade to a summary dict with a + ``note`` and write nothing — mirroring ``events_ai.run_events`` house style; degenerate + *content* (silence, too short, beatless) writes the valid-empty JSON above. """ from __future__ import annotations +import json +import logging +import wave +from pathlib import Path + +import numpy as np + +from festival4d import config, db + +log = logging.getLogger("festival4d.audio_features") + +GENERATED_BY = "festival4d.audio_features" +HOP_LENGTH = 160 # analysis hop in samples: 10 ms at the 16 kHz ingest rate +MIN_DURATION_S = 1.0 # anything shorter is "no usable audio" -> valid-empty result +SILENCE_PEAK = 1e-4 # |peak| below this is treated as silence -> valid-empty result + + +def _empty_result() -> dict: + """The valid-empty beats.json body (contract #1) for silent/short/beatless audio.""" + return { + "tempo_bpm": None, + "beats_s": [], + "onsets": [], + "generated_by": GENERATED_BY, + } + + +def _select_reference(videos: list) -> "object | None": + """The reference video (spec §2: offset 0). Fall back to the smallest |offset|, else first.""" + if not videos: + return None + exact = [v for v in videos if v.offset_ms == 0] + if exact: + return exact[0] + with_offset = [v for v in videos if v.offset_ms is not None] + if with_offset: + return min(with_offset, key=lambda v: abs(v.offset_ms)) + return videos[0] + + +def _read_wav_mono(path: Path) -> tuple[np.ndarray, int]: + """Read a 16-bit PCM WAV (the ingest format) as float64 mono in [-1, 1].""" + with wave.open(str(path), "rb") as w: + sr = w.getframerate() + channels = w.getnchannels() + frames = w.readframes(w.getnframes()) + pcm = np.frombuffer(frames, dtype=" 1: + pcm = pcm.reshape(-1, channels).mean(axis=1) + return pcm, sr + + +def analyze(y: np.ndarray, sr: int) -> dict: + """Beat/tempo + onset-strength analysis of a mono signal on its *local* timeline. + + Returns the contract #1 dict with times in local (signal) seconds — the caller maps + them to ``t_global``. Degenerate input (short/silent/beatless) returns the valid-empty + shape; this function never raises on quiet or short audio. + """ + import librosa # deferred: keeps `festival4d` import light for the API/CLI + + y = np.asarray(y, dtype=np.float32).reshape(-1) + if len(y) < MIN_DURATION_S * sr or float(np.max(np.abs(y), initial=0.0)) < SILENCE_PEAK: + return _empty_result() + + try: + onset_env = librosa.onset.onset_strength(y=y, sr=sr, hop_length=HOP_LENGTH) + _tempo, beat_frames = librosa.beat.beat_track( + onset_envelope=onset_env, sr=sr, hop_length=HOP_LENGTH + ) + beat_times = librosa.frames_to_time(beat_frames, sr=sr, hop_length=HOP_LENGTH) + + onset_frames = librosa.onset.onset_detect( + onset_envelope=onset_env, sr=sr, hop_length=HOP_LENGTH + ) + onset_times = librosa.frames_to_time(onset_frames, sr=sr, hop_length=HOP_LENGTH) + except Exception as exc: # librosa internals on weird signals: degrade, never crash + log.warning("audio_features: analysis failed (%s) — returning empty result", exc) + return _empty_result() + + beats = [float(t) for t in np.atleast_1d(beat_times)] + if len(beats) >= 2: + tempo_bpm: float | None = float(60.0 / np.median(np.diff(beats))) + elif beats: + tempo_bpm = float(np.atleast_1d(_tempo)[0]) or None + else: + tempo_bpm = None + + env_max = float(onset_env.max()) if len(onset_env) else 0.0 + onsets = [] + if env_max > 0: + for frame, t in zip(np.atleast_1d(onset_frames), np.atleast_1d(onset_times)): + strength = float(onset_env[int(frame)]) / env_max + onsets.append({"t_global_s": float(t), "strength": min(1.0, max(0.0, strength))}) + + return { + "tempo_bpm": tempo_bpm, + "beats_s": beats, + "onsets": onsets, + "generated_by": GENERATED_BY, + } + def run_features() -> dict: """Analyze the reference soundtrack and write ``data/work/beats.json`` (see module doc).""" - raise NotImplementedError("audio features (lane E / M10)") + videos = db.get_videos() + reference = _select_reference(videos) + if reference is None: + log.warning("audio_features: no videos in the project — run ingest first") + return {"status": "no_videos", "note": "no videos in the project (run ingest first)"} + + wav_path = config.AUDIO_DIR / f"{reference.id}.wav" # ingest.audio_wav_path convention + if not wav_path.exists(): + log.warning("audio_features: reference WAV missing at %s — run ingest first", wav_path) + return { + "status": "no_audio", + "note": f"reference audio not found ({wav_path.name}) — run `python -m festival4d ingest`", + } + + y, sr = _read_wav_mono(wav_path) + result = analyze(y, sr) + + # Map local (reference) times onto the master timeline via the frozen timebase helpers. + # For a true reference (offset 0, drift 0) this is the identity. + offset_ms = reference.offset_ms or 0.0 + drift_ppm = reference.drift_ppm or 0.0 + if offset_ms != 0.0 or drift_ppm != 0.0: + result["beats_s"] = [ + float(config.t_global_from_video(t, offset_ms, drift_ppm)) for t in result["beats_s"] + ] + for onset in result["onsets"]: + onset["t_global_s"] = float( + config.t_global_from_video(onset["t_global_s"], offset_ms, drift_ppm) + ) + + config.BEATS_JSON.parent.mkdir(parents=True, exist_ok=True) + config.BEATS_JSON.write_text(json.dumps(result, indent=2)) + log.info( + "audio_features: %s beats (tempo %s BPM), %s onsets -> %s", + len(result["beats_s"]), + f"{result['tempo_bpm']:.1f}" if result["tempo_bpm"] else "null", + len(result["onsets"]), + config.BEATS_JSON, + ) + return { + "status": "ok", + "reference_video_id": reference.id, + "tempo_bpm": result["tempo_bpm"], + "beats": len(result["beats_s"]), + "onsets": len(result["onsets"]), + "beats_json": str(config.BEATS_JSON), + } diff --git a/backend/tests/test_audio_features.py b/backend/tests/test_audio_features.py new file mode 100644 index 0000000..5ffa630 --- /dev/null +++ b/backend/tests/test_audio_features.py @@ -0,0 +1,171 @@ +"""M10 acceptance tests (lane E): beat/onset analysis of the reference ingest WAV. + +Built on the synthetic fixture + a real ingest run (per the phase-5 test pattern): the +fixture's master audio carries a known 120 BPM click grid (``synthetic.BEAT_INTERVAL_S`` += 0.5 s), so acceptance is objective — beats within ±50 ms of the grid, tempo within +±2 BPM. Silence / short / missing audio must degrade per the house style, never crash. +""" + +from __future__ import annotations + +import json +import shutil + +import numpy as np +import pytest + +pytestmark = pytest.mark.skipif( + shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None, + reason="ffmpeg/ffprobe required to build the fixture + run ingest", +) + +BEAT_GRID_S = 0.5 # synthetic.BEAT_INTERVAL_S — the fixture's known pulse grid +TRUE_TEMPO_BPM = 60.0 / BEAT_GRID_S # 120 + + +@pytest.fixture(scope="module") +def project(): + """Full synthetic project + ingest (produces the 16 kHz WAVs in config.AUDIO_DIR).""" + from festival4d import config, ingest, synthetic + + synthetic.build(run_ffmpeg=True) # default 20 s — enough signal for tempo tracking + ingest.run_ingest() + config.BEATS_JSON.unlink(missing_ok=True) + yield config + config.BEATS_JSON.unlink(missing_ok=True) # don't leak state into later test modules + + +def _grid_error_s(t: float) -> float: + """Distance from t to the nearest 0.5 s grid line.""" + return abs(t - round(t / BEAT_GRID_S) * BEAT_GRID_S) + + +# --------------------------------------------------------------------------- +# The real thing: known pulse grid recovered from the ingest WAV. +# --------------------------------------------------------------------------- +def test_run_features_recovers_synthetic_pulse_grid(project): + from festival4d import audio_features + + summary = audio_features.run_features() + assert summary["status"] == "ok" + assert project.BEATS_JSON.exists() + + data = json.loads(project.BEATS_JSON.read_text()) + # Contract #1 structure lock: exactly these keys. + assert set(data) == {"tempo_bpm", "beats_s", "onsets", "generated_by"} + assert data["generated_by"] == "festival4d.audio_features" + + # Tempo within ±2 BPM of the known 120 BPM grid. + assert data["tempo_bpm"] == pytest.approx(TRUE_TEMPO_BPM, abs=2.0) + + # Beats: a healthy number over 20 s @ 120 BPM, each within ±50 ms of the grid, + # strictly increasing, inside the reference window (t_global == reference local time). + beats = data["beats_s"] + assert len(beats) >= 20 + assert beats == sorted(beats) + # AAC encode/decode pads the clip tail slightly, so allow a small margin past 20 s. + assert all(0.0 <= b <= 20.5 for b in beats) + worst = max(_grid_error_s(b) for b in beats) + assert worst <= 0.050, f"worst beat off grid by {worst * 1000:.1f} ms" + + # Onsets: present, valid strengths, times on the master timeline. + onsets = data["onsets"] + assert len(onsets) > 0 + assert all(set(o) == {"t_global_s", "strength"} for o in onsets) + assert all(0.0 <= o["strength"] <= 1.0 for o in onsets) + assert any(o["strength"] >= 0.9 for o in onsets) # normalized: the peak onset is ~1.0 + assert all(0.0 <= o["t_global_s"] <= 20.5 for o in onsets) + + # The summary mirrors the file. + assert summary["beats"] == len(beats) + assert summary["onsets"] == len(onsets) + assert summary["reference_video_id"] == 1 # cam0 (offset 0) is the reference + + +def test_onsets_catch_the_ground_truth_bangs(project): + """The three loud synthetic bangs (3, 10, 17 s) must appear as strong onsets.""" + from festival4d import audio_features, synthetic + + audio_features.run_features() + data = json.loads(project.BEATS_JSON.read_text()) + onset_times = [o["t_global_s"] for o in data["onsets"]] + for bang in synthetic.BANG_TIMES_S: + assert any(abs(t - bang) <= 0.075 for t in onset_times), f"bang at {bang}s not detected" + + +# --------------------------------------------------------------------------- +# Degradation: silence / short / missing input — valid output or clear note, never a crash. +# --------------------------------------------------------------------------- +def test_silence_produces_valid_empty_json(project): + from festival4d import audio_features, synthetic + + wav = project.AUDIO_DIR / "1.wav" + original = wav.read_bytes() + try: + synthetic.write_wav(wav, np.zeros(16_000 * 5, dtype=np.float32), 16_000) + summary = audio_features.run_features() + assert summary["status"] == "ok" + data = json.loads(project.BEATS_JSON.read_text()) + assert data == { + "tempo_bpm": None, + "beats_s": [], + "onsets": [], + "generated_by": "festival4d.audio_features", + } + finally: + wav.write_bytes(original) + project.BEATS_JSON.unlink(missing_ok=True) + + +def test_too_short_audio_produces_valid_empty_json(project): + from festival4d import audio_features, synthetic + + wav = project.AUDIO_DIR / "1.wav" + original = wav.read_bytes() + try: + rng = np.random.default_rng(3) + synthetic.write_wav(wav, rng.standard_normal(4_000).astype(np.float32) * 0.5, 16_000) + audio_features.run_features() + data = json.loads(project.BEATS_JSON.read_text()) + assert data["tempo_bpm"] is None and data["beats_s"] == [] and data["onsets"] == [] + finally: + wav.write_bytes(original) + project.BEATS_JSON.unlink(missing_ok=True) + + +def test_missing_wav_degrades_with_note(project): + from festival4d import audio_features + + wav = project.AUDIO_DIR / "1.wav" + moved = wav.with_suffix(".wav.bak") + wav.rename(moved) + try: + summary = audio_features.run_features() # must not raise + assert summary["status"] == "no_audio" + assert "ingest" in summary["note"] + assert not project.BEATS_JSON.exists() # nothing stale written + finally: + moved.rename(wav) + + +def test_analyze_is_pure_and_beatless_noise_is_empty(): + """Unit-level: analyze() on beatless noise returns the valid-empty shape.""" + from festival4d import audio_features + + rng = np.random.default_rng(11) + y = (rng.standard_normal(16_000 * 6) * 0.2).astype(np.float32) # 6 s of plain noise + data = audio_features.analyze(y, 16_000) + assert set(data) == {"tempo_bpm", "beats_s", "onsets", "generated_by"} + # librosa may or may not hallucinate a weak pulse in noise; the hard requirement is a + # valid, well-typed result — not a crash. + assert isinstance(data["beats_s"], list) + assert all(isinstance(b, float) for b in data["beats_s"]) + + +def test_cli_features_runs_end_to_end(project): + """`python -m festival4d features` dispatches into the real implementation now.""" + from festival4d import cli + + assert cli.main(["features"]) == 0 + assert project.BEATS_JSON.exists() + project.BEATS_JSON.unlink() diff --git a/backend/tests/test_phase5_api.py b/backend/tests/test_phase5_api.py index 92f630b..8cbfafe 100644 --- a/backend/tests/test_phase5_api.py +++ b/backend/tests/test_phase5_api.py @@ -52,13 +52,15 @@ def test_beats_404_then_serves_file(client): config.BEATS_JSON.unlink() -def test_director_degrades_to_valid_empty_campath(client): - # While lane E is stubbed the response must still round-trip through camPath.fromJSON: - # a valid version-1 path with zero keyframes, plus a "note" marking the degradation. +def test_director_returns_valid_campath(client): + # CR-4: lane E's M11 landed, so on the synthetic fixture (events + poses present) the + # response is a REAL path — assert the frozen camPath shape (holds stubbed or implemented). + # The empty+note degradation (no events / no poses) is covered in test_director.py. data = client.post("/api/director", json={}).json() assert data["version"] == 1 - assert data["keyframes"] == [] - assert "note" in data + assert isinstance(data["keyframes"], list) + for kf in data["keyframes"]: + assert set(kf) == {"t_global", "pos", "quat", "fov"} assert client.post("/api/director", json={"top_n": 3, "lead_s": 1.0}).status_code == 200 @@ -99,9 +101,11 @@ def test_anchor_patch_label_and_color(client): def test_cli_dispatches_stubs_gracefully(): - # The three new subcommands exist and degrade per the house pattern: exit code 2, - # no traceback, while their lane entrypoints are stubs. + # The still-stubbed subcommands degrade per the house pattern: exit code 2, no traceback. + # CR-4: `features` and `direct` left this list when lane E landed M10/M11 (their real + # dispatch is covered in test_audio_features.py / test_director.py); lane G drops + # `capsule` when M17 lands. from festival4d import cli - for cmd in (["features"], ["direct"], ["capsule"]): + for cmd in (["capsule"],): assert cli.main(cmd) == 2 diff --git a/plan/CHANGE_REQUESTS.md b/plan/CHANGE_REQUESTS.md index 2f9ff9f..ca6b0ad 100644 --- a/plan/CHANGE_REQUESTS.md +++ b/plan/CHANGE_REQUESTS.md @@ -72,3 +72,24 @@ Instead append an entry here and work around it locally; the integration agent a pre-existing route/shape/column and the timebase + pose contracts are unchanged; the annotation response only gained a field. Suite 101 → **103** (added supersede + delete tests). - **Decision:** **APPLIED** (phase4, 2026-07-16). Fixes the sole Round-3 wart; strictly additive. + +### CR-4 — lane E — 2026-07-17 — Update stub-era assertions in test_phase5_api.py (M10/M11 landed) +- **File:** `backend/tests/test_phase5_api.py::test_cli_dispatches_stubs_gracefully` and + `::test_director_degrades_to_valid_empty_campath` +- **Problem:** Both tests assert the *stub* contracts of lane E's entrypoints, correct only while + M10/M11 were unimplemented. With `run_features`/`generate_path` landed: (a) + `cli.main(["features"])` / `["direct"]` no longer raise `NotImplementedError`, so they exit 0, + not 2; (b) `POST /api/director` on the synthetic fixture (which has events + poses) returns a + real non-empty camPath, so `keyframes == []` / `"note" in data` fail. Identical in kind to + **CR-1** (lane D, approved Round 1: "a test file, not a frozen contract file"). +- **Proposed change:** (a) shrink the CLI stub loop to `["capsule"]` (lane G, still stubbed — + lane G drops it when M17 lands); (b) rename the director test to + `test_director_returns_valid_campath` asserting the frozen camPath *shape* (version 1, + keyframe key-set), which holds in both stub and implemented states; the empty+note degradation + itself is now covered by lane E's own `test_director.py` (no-events / no-poses cases). +- **Workaround in place:** applied the minimal edit directly per the CR-1 precedent (test file, + not a frozen contract file; `api.py`/`cli.py` byte-for-byte unchanged). Suite green + (127 baseline + lane E tests, zero failures). If the coordinator prefers the stricter reading + of the freeze, revert the test edit and re-apply at integration2 — lane E's modules are + unaffected either way. +- **Decision:** (pending — coordinator/integration2) From 52e891204d996b78d5913cee69695e96a384f455 Mon Sep 17 00:00:00 2001 From: type-two Date: Fri, 17 Jul 2026 17:51:52 +1000 Subject: [PATCH 2/4] =?UTF-8?q?M11:=20auto-director=20=E2=80=94=20top-N=20?= =?UTF-8?q?events=20->=20beat-snapped=20camPath=20(backend=20+=20UI)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit generate_path() fills the foundation2 stub with the deterministic v1 rules: top-N events by confidence (ties -> earlier, then shot chronologically); per event, the registered camera whose pose track is nearest the event time; keyframes at t_event - lead_s (clamped >= 0) and t_event + duration; nearest-beat snapping when beats.json exists (missing/corrupt file -> no snapping); FOV = 2*atan(h/(2*fy)) like scene3d.snapTo. Every pose goes through the frozen geometry.colmap_to_threejs and the camPath quaternion is emitted in Three.js [x,y,z,w] order (pitfall #1 — locked by test). Degradation: no events / no registered poses -> valid empty camPath with a note; the response always round-trips through camPath.fromJSON. frontend/src/director.js wires the pre-wired hidden 🎬 button: POST /api/director, camPath.fromJSON(response), auto-play via the existing #btn-path flow; empty path or fetch failure surfaces on the button and recovers — never a crash. Acceptance (test_director.py, 14 tests): structure-locked against the frozen camPath shape (exact key sets, unit quats, monotone times); top-3 grid == [1,4,8,11,14] incl. shared-keyframe dedupe; conversion matches colmap_to_threejs + mat_to_quat exactly; snapping puts every key on the beat grid; unregistered poses excluded; API route + CLI dispatch real output. Suite 148 passed; frontend build clean. Co-Authored-By: Claude Fable 5 --- backend/festival4d/director.py | 131 +++++++++++++++++- backend/tests/test_director.py | 242 +++++++++++++++++++++++++++++++++ frontend/src/director.js | 78 +++++++++-- 3 files changed, 437 insertions(+), 14 deletions(-) create mode 100644 backend/tests/test_director.py diff --git a/backend/festival4d/director.py b/backend/festival4d/director.py index afb7a60..4e6753a 100644 --- a/backend/festival4d/director.py +++ b/backend/festival4d/director.py @@ -1,4 +1,4 @@ -"""Auto-director: events -> keyframed camera path (spec M11, lane E). STUB — lane E fills. +"""Auto-director: events -> keyframed camera path (spec M11, lane E). Contract (frozen at foundation2 merge — phase-5 contract #2): :func:`generate_path` returns a **camPath-compatible** dict:: @@ -15,11 +15,138 @@ for each, choose the registered camera with a pose nearest the event time; keyfr keyframe times to the nearest beat (no beats file -> no snapping — degrade, don't block); FOV from that camera's intrinsics (``2 * atan(height / (2 * fy))`` in degrees, the same formula as ``scene3d.snapTo``). + +Degradation (house style): no events -> valid empty path; no registered poses -> valid +empty path with a clear ``note`` (the response always round-trips through +``camPath.fromJSON``); no beats.json -> same path, just unsnapped. Never a crash. """ from __future__ import annotations +import json +import logging +import math + +from festival4d import config, db +from festival4d.geometry import colmap_to_threejs, mat_to_quat + +log = logging.getLogger("festival4d.director") + +DEFAULT_EVENT_DURATION_S = 1.0 # used when an event has no duration_s + + +def _empty(note: str | None = None) -> dict: + path: dict = {"version": 1, "keyframes": []} + if note: + path["note"] = note + log.info("director: %s -> empty path", note) + return path + + +def _load_beats() -> list[float]: + """The beat grid from beats.json, if valid; otherwise [] (no snapping — pitfall #5).""" + if not config.BEATS_JSON.exists(): + return [] + try: + beats = json.loads(config.BEATS_JSON.read_text()).get("beats_s", []) + return sorted(float(b) for b in beats) + except (ValueError, TypeError, AttributeError) as exc: + log.warning("director: unreadable beats.json (%s) — skipping beat snap", exc) + return [] + + +def _snap_to_beat(t: float, beats: list[float]) -> float: + """Nearest beat to t (beats sorted ascending); t unchanged when the grid is empty.""" + if not beats: + return t + return min(beats, key=lambda b: abs(b - t)) + + +def _nearest_pose(poses: list, t_video: float): + """The registered pose nearest a local time, and its |dt|. Poses are t_video_s-sorted.""" + best = min(poses, key=lambda p: abs(p.t_video_s - t_video)) + return best, abs(best.t_video_s - t_video) + + +def _keyframe(pose, video, t_global: float) -> dict: + """Build one frozen-shape keyframe from a COLMAP pose row (contract #2). + + The COLMAP [w,x,y,z] world->camera pose goes through the frozen + ``geometry.colmap_to_threejs`` (never inline — pitfall #1); the resulting Three.js + world rotation matrix becomes the camPath quaternion, reordered to Three.js [x,y,z,w]. + """ + position, rotation = colmap_to_threejs( + [pose.qw, pose.qx, pose.qy, pose.qz], [pose.tx, pose.ty, pose.tz] + ) + qw, qx, qy, qz = mat_to_quat(rotation) # [w,x,y,z] of the *Three.js* world rotation + fov_deg = math.degrees(2.0 * math.atan(video.height / (2.0 * pose.fy))) # scene3d.snapTo + return { + "t_global": float(t_global), + "pos": [float(v) for v in position], + "quat": [float(qx), float(qy), float(qz), float(qw)], # Three.js order [x,y,z,w] + "fov": float(fov_deg), + } + def generate_path(top_n: int = 8, lead_s: float = 2.0) -> dict: """Build a camPath JSON dict from the detected events (see module doc).""" - raise NotImplementedError("auto-director (lane E / M11)") + events = db.get_events() + if not events: + return _empty("no events to direct (run `python -m festival4d events`)") + + videos = db.get_videos() + # Registered poses per video (interpolated rows are excluded: the director only cuts + # to cameras that were actually solved at that moment). + tracks: dict[int, list] = {} + for video in videos: + registered = [p for p in db.get_poses(video.id) if p.registered] + if registered: + tracks[video.id] = registered + if not tracks: + return _empty("no registered camera poses (run `python -m festival4d reconstruct`)") + video_by_id = {v.id: v for v in videos} + + # Top-N events by confidence, ties -> earlier; then shoot them in chronological order. + ranked = sorted(events, key=lambda e: (-(e.confidence or 0.0), e.t_global_s, e.id)) + chosen = sorted(ranked[: max(0, int(top_n))], key=lambda e: (e.t_global_s, e.id)) + + beats = _load_beats() + + keyframes: list[dict] = [] + for event in chosen: + # The registered camera whose pose track comes closest to the event time. + best_id, best_dt = None, None + for vid, poses in sorted(tracks.items()): # sorted -> deterministic tie-break + video = video_by_id[vid] + tv_event = config.t_video_from_global( + event.t_global_s, video.offset_ms or 0.0, video.drift_ppm or 0.0 + ) + _, dt = _nearest_pose(poses, tv_event) + if best_dt is None or dt < best_dt: + best_id, best_dt = vid, dt + video = video_by_id[best_id] + poses = tracks[best_id] + + duration = event.duration_s if event.duration_s is not None else DEFAULT_EVENT_DURATION_S + for t_key in (event.t_global_s - lead_s, event.t_global_s + duration): + t_key = _snap_to_beat(max(0.0, t_key), beats) + tv_key = config.t_video_from_global( + t_key, video.offset_ms or 0.0, video.drift_ppm or 0.0 + ) + pose, _ = _nearest_pose(poses, tv_key) + keyframes.append(_keyframe(pose, video, t_key)) + + # Chronological, and collapse keyframes that landed on the same instant (adjacent events + # snapping to a shared beat) — camPath needs a monotone timeline. + keyframes.sort(key=lambda k: k["t_global"]) + deduped: list[dict] = [] + for kf in keyframes: + if deduped and abs(kf["t_global"] - deduped[-1]["t_global"]) < 1e-6: + continue + deduped.append(kf) + + log.info( + "director: %d event(s) -> %d keyframe(s)%s", + len(chosen), len(deduped), " (beat-snapped)" if beats else "", + ) + return {"version": 1, "keyframes": deduped} diff --git a/backend/tests/test_director.py b/backend/tests/test_director.py new file mode 100644 index 0000000..61ed877 --- /dev/null +++ b/backend/tests/test_director.py @@ -0,0 +1,242 @@ +"""M11 acceptance tests (lane E): deterministic auto-director -> frozen camPath JSON. + +Runs on the synthetic fixture WITHOUT ffmpeg (poses + events + DB only): generate_path +never touches media files. The synthetic seed events have known times/confidences, so +event selection, keyframe placement, beat snapping, and the pose conversion are all +checked against ground truth. The quaternion test locks the [x,y,z,w] Three.js order via +the frozen geometry.colmap_to_threejs — this repo's #1 historical bug source. +""" + +from __future__ import annotations + +import json +import math + +import numpy as np +import pytest + +# Synthetic seed events (synthetic.SEED_EVENTS): (t_global, type, confidence) +# 3.0 bass_drop .94 | 6.0 crowd_wave .72 | 8.0 quiet .61 | 10.0 pyro .88 +# 13.0 light_show .90 | 17.0 confetti .81 | 19.0 artist .77 +TOP3_TIMES = [3.0, 10.0, 13.0] # confidences .94, .88, .90 -> top 3 +KEYFRAME_KEYS = {"t_global", "pos", "quat", "fov"} + + +@pytest.fixture() +def project(): + """Fresh synthetic project (no ffmpeg) per test — several tests mutate the DB.""" + from festival4d import config, synthetic + + synthetic.build(run_ffmpeg=False) + config.BEATS_JSON.unlink(missing_ok=True) # unsnapped by default; tests opt in + yield config + config.BEATS_JSON.unlink(missing_ok=True) + + +# --------------------------------------------------------------------------- +# Structure lock: the frozen camPath shape (contract #2). +# --------------------------------------------------------------------------- +def test_output_matches_frozen_campath_shape(project): + from festival4d import director + + path = director.generate_path() + assert set(path) == {"version", "keyframes"} # exact-set lock (no stray fields on success) + assert path["version"] == 1 + assert len(path["keyframes"]) > 0 + for kf in path["keyframes"]: + assert set(kf) == KEYFRAME_KEYS # exact-set lock per keyframe + assert isinstance(kf["t_global"], float) + assert len(kf["pos"]) == 3 and all(isinstance(v, float) for v in kf["pos"]) + assert len(kf["quat"]) == 4 and all(isinstance(v, float) for v in kf["quat"]) + assert abs(math.hypot(*kf["quat"][:3], kf["quat"][3]) - 1.0) < 1e-6 # unit quaternion + assert 10.0 < kf["fov"] < 120.0 + times = [kf["t_global"] for kf in path["keyframes"]] + assert times == sorted(times) + assert len(set(times)) == len(times) # strictly monotone (dedup after snapping) + # JSON-serializable end to end (the API returns this dict verbatim). + json.dumps(path) + + +def test_covers_the_top_n_events_with_lead_and_duration(project): + from festival4d import director + + path = director.generate_path(top_n=3, lead_s=2.0) + times = [kf["t_global"] for kf in path["keyframes"]] + # Each top-3 event contributes keys at t-2.0 and t+duration (1.0 in the fixture): + # 3.0 -> {1.0, 4.0}; 10.0 -> {8.0, 11.0}; 13.0 -> {11.0, 14.0}. The shared 11.0 + # collapses to one keyframe -> exactly 5, and every event is bracketed. + assert times == [1.0, 4.0, 8.0, 11.0, 14.0] + for t_event in TOP3_TIMES: + assert any(t <= t_event for t in times) and any(t >= t_event for t in times) + + +def test_full_top_n_covers_every_event(project): + from festival4d import director + + path = director.generate_path(top_n=8, lead_s=2.0) # fixture has 7 events + times = [kf["t_global"] for kf in path["keyframes"]] + for t_event in [3.0, 6.0, 8.0, 10.0, 13.0, 17.0, 19.0]: + assert min(abs(t - (t_event - 2.0)) for t in times) < 1e-9 + assert min(abs(t - (t_event + 1.0)) for t in times) < 1e-9 + + +def test_confidence_ties_break_to_the_earlier_event(project): + from festival4d import db, director + + db.clear_events() + db.add_event(t_global_s=15.0, event_type="pyro", source="ai", duration_s=1.0, confidence=0.9) + db.add_event(t_global_s=5.0, event_type="pyro", source="ai", duration_s=1.0, confidence=0.9) + path = director.generate_path(top_n=1, lead_s=2.0) + assert [kf["t_global"] for kf in path["keyframes"]] == [3.0, 6.0] # the t=5 event won + + +def test_lead_time_clamps_at_zero(project): + from festival4d import db, director + + db.clear_events() + db.add_event(t_global_s=0.5, event_type="pyro", source="ai", duration_s=1.0, confidence=0.9) + path = director.generate_path(top_n=1, lead_s=2.0) + assert [kf["t_global"] for kf in path["keyframes"]] == [0.0, 1.5] + + +# --------------------------------------------------------------------------- +# Pose conversion: through the frozen geometry.colmap_to_threejs, quat order [x,y,z,w]. +# --------------------------------------------------------------------------- +def test_keyframe_pose_matches_frozen_conversion(project): + from festival4d import db, director + from festival4d.geometry import colmap_to_threejs, mat_to_quat, quat_to_mat + + path = director.generate_path(top_n=1, lead_s=2.0) # top event: bass_drop @ 3.0 + kf = path["keyframes"][0] # t_global = 1.0 + + # Every synthetic event time is an exact cam0 pose time (offset 0), so the chosen + # camera is video 1 and the keyframe pose is cam0's registered row at t_video_s=1.0. + pose = next(p for p in db.get_poses(1) if abs(p.t_video_s - 1.0) < 1e-9) + expected_pos, expected_rot = colmap_to_threejs( + [pose.qw, pose.qx, pose.qy, pose.qz], [pose.tx, pose.ty, pose.tz] + ) + assert np.allclose(kf["pos"], expected_pos, atol=1e-9) + + # Quaternion order lock: reordering [x,y,z,w] -> [w,x,y,z] must reproduce the Three.js + # world rotation matrix (up to the quaternion double cover). + x, y, z, w = kf["quat"] + assert np.allclose(quat_to_mat([w, x, y, z]), expected_rot, atol=1e-9) + expected_q = mat_to_quat(expected_rot) # [w,x,y,z], canonical w >= 0 + q_out = np.array([w, x, y, z]) + if np.dot(q_out, expected_q) < 0: + q_out = -q_out + assert np.allclose(q_out, expected_q, atol=1e-9) + + # FOV from the chosen camera's intrinsics: 2*atan(height / (2*fy)) in degrees. + video = db.get_video(1) + expected_fov = math.degrees(2.0 * math.atan(video.height / (2.0 * pose.fy))) + assert kf["fov"] == pytest.approx(expected_fov, abs=1e-9) + + +def test_unregistered_poses_are_excluded(project): + from festival4d import db, director + + # Mark every cam0 pose unregistered: the director must cut to cam1/cam2 instead. + poses = db.get_poses(1) + db.set_poses(1, [ + {"frame_idx": p.frame_idx, "t_video_s": p.t_video_s, + "qw": p.qw, "qx": p.qx, "qy": p.qy, "qz": p.qz, + "tx": p.tx, "ty": p.ty, "tz": p.tz, + "fx": p.fx, "fy": p.fy, "cx": p.cx, "cy": p.cy, "registered": False} + for p in poses + ]) + path = director.generate_path(top_n=1, lead_s=2.0) + assert len(path["keyframes"]) == 2 + cam0_centers = { + tuple(np.round(np.asarray(_center(p)), 6)) for p in poses + } + for kf in path["keyframes"]: + assert tuple(np.round(kf["pos"], 6)) not in cam0_centers + + +def _center(pose): + from festival4d.geometry import colmap_to_threejs + + pos, _ = colmap_to_threejs([pose.qw, pose.qx, pose.qy, pose.qz], [pose.tx, pose.ty, pose.tz]) + return pos + + +# --------------------------------------------------------------------------- +# Beat snapping (M10 -> M11), and degradation without it. +# --------------------------------------------------------------------------- +def test_keyframes_snap_to_beats_when_beats_exist(project): + from festival4d import director + + grid = [round(k * 0.75, 4) for k in range(0, 28)] # 0.75 s grid != the keyframe times + project.BEATS_JSON.parent.mkdir(parents=True, exist_ok=True) + project.BEATS_JSON.write_text(json.dumps({ + "tempo_bpm": 80.0, "beats_s": grid, "onsets": [], + "generated_by": "festival4d.audio_features", + })) + path = director.generate_path(top_n=3, lead_s=2.0) + times = [kf["t_global"] for kf in path["keyframes"]] + assert len(times) > 0 + assert all(any(abs(t - b) < 1e-9 for b in grid) for t in times) # every key ON a beat + assert times != [1.0, 4.0, 8.0, 11.0, 14.0] # actually moved vs the unsnapped grid + + +def test_no_beats_file_means_no_snapping(project): + from festival4d import director + + assert not project.BEATS_JSON.exists() + times = [kf["t_global"] for kf in director.generate_path(top_n=3, lead_s=2.0)["keyframes"]] + assert times == [1.0, 4.0, 8.0, 11.0, 14.0] + + +def test_corrupt_beats_file_degrades_to_unsnapped(project): + from festival4d import director + + project.BEATS_JSON.parent.mkdir(parents=True, exist_ok=True) + project.BEATS_JSON.write_text("{ not json") + times = [kf["t_global"] for kf in director.generate_path(top_n=3, lead_s=2.0)["keyframes"]] + assert times == [1.0, 4.0, 8.0, 11.0, 14.0] + + +# --------------------------------------------------------------------------- +# Degradation: empty inputs -> valid empty path (still camPath.fromJSON-safe). +# --------------------------------------------------------------------------- +def test_no_events_yields_valid_empty_path(project): + from festival4d import db, director + + db.clear_events() + path = director.generate_path() + assert path["version"] == 1 and path["keyframes"] == [] + assert "events" in path["note"] + + +def test_no_registered_poses_yields_valid_empty_path_with_note(project): + from festival4d import db, director + + for video in db.get_videos(): + db.set_poses(video.id, []) + path = director.generate_path() + assert path["version"] == 1 and path["keyframes"] == [] + assert "poses" in path["note"] + + +# --------------------------------------------------------------------------- +# Dispatch: the API route and CLI now run the real implementation. +# --------------------------------------------------------------------------- +def test_api_director_route_returns_real_path(project): + from fastapi.testclient import TestClient + + from festival4d import api + + with TestClient(api.app) as client: + data = client.post("/api/director", json={"top_n": 3, "lead_s": 2.0}).json() + assert data["version"] == 1 + assert [kf["t_global"] for kf in data["keyframes"]] == [1.0, 4.0, 8.0, 11.0, 14.0] + assert all(set(kf) == KEYFRAME_KEYS for kf in data["keyframes"]) + + +def test_cli_direct_prints_campath_json(project, capsys): + from festival4d import cli + + assert cli.main(["direct", "--top-n", "2", "--lead-s", "1.0"]) == 0 + printed = json.loads(capsys.readouterr().out) + assert printed["version"] == 1 and len(printed["keyframes"]) > 0 diff --git a/frontend/src/director.js b/frontend/src/director.js index 0ab6e91..93b9c74 100644 --- a/frontend/src/director.js +++ b/frontend/src/director.js @@ -1,16 +1,70 @@ -// Auto-director UI (spec M11, lane E). STUB — lane E fills THIS FILE ONLY. +// Auto-director UI (spec M11, lane E). // -// Pre-wired by foundation2: the hidden 🎬 button (#btn-director in index.html), and this -// module's import + init() call in main.js. main.js does NOT wire the click — you do, -// inside init(). -// -// When implementing: unhide the button (el.style.display = ""), set `ready = true`; on click -// POST {top_n?, lead_s?} to `${state.apiBase}/api/director` and load the response with -// camPath.fromJSON(...) — never edit camPath.js (frozen). The response is ALWAYS a valid -// camPath JSON; zero keyframes means "no events yet / stubbed" — surface that in the UI, -// don't crash (pitfall #5). Play it via the existing #btn-path flow. +// Pre-wired by foundation2: the hidden 🎬 button (#btn-director in index.html) and this +// module's import + init() call in main.js. This module wires the click: POST /api/director +// -> the response is ALWAYS a valid camPath JSON (frozen contract #2) -> load it with +// camPath.fromJSON and play it via the existing #btn-path flow (never edit camPath.js). +// Zero keyframes means "no events / no poses yet" — surfaced on the button, never a crash +// (pitfall #5). Failures degrade the same way; the button always recovers. + +import { state } from "./state.js"; +import { camPath } from "./camPath.js"; + +const BTN_LABEL = "🎬 Auto-path"; +let flashTimer = null; + +/** Show a transient message on the button, then restore the label. */ +function flash(btn, text, ms = 2200) { + btn.textContent = text; + clearTimeout(flashTimer); + flashTimer = setTimeout(() => (btn.textContent = BTN_LABEL), ms); +} + +async function requestPath(btn) { + btn.disabled = true; + btn.textContent = "🎬 …"; + try { + const resp = await fetch(`${state.apiBase}/api/director`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), // server defaults: top_n=8, lead_s=2.0 + }); + if (!resp.ok) throw new Error(`HTTP ${resp.status}`); + const path = await resp.json(); + + if (!Array.isArray(path.keyframes) || path.keyframes.length === 0) { + // Valid-empty path: no events/poses yet (or still stubbed). Surface, don't crash. + console.info("[director] empty path from /api/director", path.note ?? ""); + flash(btn, "🎬 no events yet"); + return; + } + + camPath.fromJSON(path); // replaces any existing keyframes; updates the ＋ Key counter + flash(btn, `🎬 ${path.keyframes.length} keys`); + + // Play through the existing #btn-path flow so its UI state (⏹ label, transport + // auto-start) stays consistent with a manually loaded path. + const playBtn = document.getElementById("btn-path"); + if (playBtn && !camPath.playing && path.keyframes.length >= 2) playBtn.click(); + } catch (err) { + console.error("[director] /api/director failed", err); + flash(btn, "🎬 error — see console"); + } finally { + btn.disabled = false; + } +} export const director = { - ready: false, // flips to true when the lane implements this module - init() {}, // ponytail: stub — lane E replaces the body + ready: false, + + init() { + const btn = document.getElementById("btn-director"); + if (!btn) return; // markup missing: stay hidden, degrade silently + btn.addEventListener("click", () => { + if (btn.disabled) return; + requestPath(btn); + }); + btn.style.display = ""; // unhide: this lane has landed + this.ready = true; + }, }; From ddb8dea6d889cb5afa2d043f10f5f6d5ee5b3c5f Mon Sep 17 00:00:00 2001 From: type-two Date: Fri, 17 Jul 2026 17:53:34 +1000 Subject: [PATCH 3/4] =?UTF-8?q?M12:=20cinematic=20export=20=E2=80=94=20cam?= =?UTF-8?q?Path=20ride=20->=20festival4d-cut.webm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit frontend/src/exportVideo.js fills the foundation2 stub: the ⏺ button records the scene3d canvas captureStream(30) muxed with the live WebAudio mix via the transport.captureAudioStream() hook (spatial included, silent tap when no track — degrade, don't block), MediaRecorder vp9/opus webm (vp8/webm fallbacks), spanning first -> last camPath keyframe. Playback is driven through the existing #btn-path flow at forced rate 1 (file duration must equal the path span), stops at the last keyframe (or early on a second click / anything that halts the ride), downloads festival4d-cut.webm, then restores ALL prior transport/UI state (playhead, rate, play state, path state, followed camera) and releases the audio tap + canvas tracks. Button disabled under 2 keyframes by chaining main.js's camPath.onChange (frozen files untouched). Requires live-browser verification (focused tab — captureStream delivers no frames hidden): see plan/status/lane-E.md. Frontend build clean. Co-Authored-By: Claude Fable 5 --- frontend/src/exportVideo.js | 215 ++++++++++++++++++++++++++++++++++-- 1 file changed, 204 insertions(+), 11 deletions(-) diff --git a/frontend/src/exportVideo.js b/frontend/src/exportVideo.js index 3530b5e..060e8ce 100644 --- a/frontend/src/exportVideo.js +++ b/frontend/src/exportVideo.js @@ -1,18 +1,211 @@ -// Cinematic export (spec M12, lane E). STUB — lane E fills THIS FILE ONLY. +// Cinematic export (spec M12, lane E). // // Pre-wired by foundation2: the hidden ⏺ button (#btn-export in index.html), this module's -// import + init() call in main.js, and the transport hooks -// `transport.captureAudioStream()` -> MediaStream tapping the live WebAudio mix (post-gain, -// spatial included) and `transport.releaseAudioStream()` when done. +// import + init() call in main.js, and the transport hooks transport.captureAudioStream() +// (a MediaStream tapping the live WebAudio mix — post-gain, spatial included) + +// transport.releaseAudioStream() when done. // -// When implementing: unhide the button, set `ready = true`; on click seek to the first -// camPath keyframe, play the path, mux scene3d canvas `captureStream(30)` + the audio stream -// with MediaRecorder (video/webm;codecs=vp9,opus), stop at the last keyframe, download -// `festival4d-cut.webm`, then RESTORE all prior UI/transport state. The canvas is -// `scene3d.renderer.domElement`. MediaRecorder needs a *focused* tab to capture real frames -// (pitfall #2) — say so in your evidence if you couldn't. +// Flow: seek to the first camPath keyframe, play the path via the existing #btn-path flow +// (so its UI state stays truthful), record scene3d's canvas captureStream(30) muxed with the +// audio tap through MediaRecorder (vp9/opus webm, fallbacks below), stop at the last +// keyframe, download `festival4d-cut.webm`, then restore ALL prior UI/transport state. +// Export runs at rate 1 regardless of the user's speed (the recording is wall-clock, so the +// file duration must equal the path's time span); the prior rate is restored afterwards. +// Degradation: fewer than 2 keyframes -> button disabled; no decoded WebAudio track -> the +// tap is silent but the export still works (pitfall #5); clicking ⏺ mid-export stops early +// and still produces a valid file. NOTE (pitfall #2): canvas.captureStream only delivers +// real frames in a FOCUSED tab — a hidden/backgrounded pane records a frozen image. + +import { state } from "./state.js"; +import { transport } from "./transport.js"; +import { camPath } from "./camPath.js"; +import { scene3d } from "./scene3d.js"; + +const BTN_LABEL = "⏺ Export"; +const MIME_CANDIDATES = [ + "video/webm;codecs=vp9,opus", // the spec'd target + "video/webm;codecs=vp8,opus", + "video/webm", +]; + +function pickMime() { + if (typeof MediaRecorder === "undefined") return null; + return MIME_CANDIDATES.find((m) => MediaRecorder.isTypeSupported(m)) ?? null; +} + +function download(blob, filename) { + const a = document.createElement("a"); + a.href = URL.createObjectURL(blob); + a.download = filename; + a.click(); + URL.revokeObjectURL(a.href); +} + +/** Toggle the shared #btn-path flow (main.js owns its UI state) into the desired state. */ +function setPathPlaying(playing) { + if (camPath.playing === playing) return; + document.getElementById("btn-path")?.click(); +} export const exportVideo = { ready: false, - init() {}, // ponytail: stub — lane E replaces the body + exporting: false, + + _btn: null, + _recorder: null, + _chunks: [], + _canvasStream: null, + _endT: 0, + _prior: null, + _finishing: false, + + init() { + const btn = document.getElementById("btn-export"); + if (!btn) return; // markup missing: stay hidden, degrade silently + this._btn = btn; + + btn.addEventListener("click", () => { + if (this.exporting) this._finish("stopped early"); + else this._start(); + }); + + // Enabled only with a playable path (>= 2 keyframes). main.js assigned camPath.onChange + // before lane modules init, so chain it rather than replace it (camPath.js is frozen). + const chained = camPath.onChange; + camPath.onChange = (n) => { + chained?.(n); + if (!this.exporting) btn.disabled = n < 2; + }; + btn.disabled = camPath.count() < 2; + + btn.style.display = ""; // unhide: this lane has landed + this.ready = true; + }, + + _start() { + if (this.exporting || camPath.keyframes.length < 2) return; + const mime = pickMime(); + if (!mime) { + console.error("[export] MediaRecorder/webm unsupported in this browser"); + this._flash("⏺ unsupported"); + return; + } + + const kf = camPath.keyframes; // sorted by t_global (camPath invariant) + const startT = kf[0].t_global; + this._endT = kf[kf.length - 1].t_global; + + // Snapshot everything we mutate, so _finish can restore it exactly. + this._prior = { + tGlobal: state.tGlobal, + playing: state.playing, + rate: state.rate, + pathPlaying: camPath.playing, + followCameraId: state.followCameraId, + }; + + // Park the world at the first keyframe, real-time rate, path engaged. + setPathPlaying(false); + if (state.playing) transport.pause(); + transport.setRate(1); + transport.seek(startT); + + // Canvas frames + the live WebAudio mix tap (foundation2 hook; silent when no track). + const canvas = scene3d.renderer.domElement; + this._canvasStream = canvas.captureStream(30); + const audioStream = transport.captureAudioStream(); + const mixed = new MediaStream([ + ...this._canvasStream.getVideoTracks(), + ...audioStream.getAudioTracks(), + ]); + + this._chunks = []; + this._recorder = new MediaRecorder(mixed, { + mimeType: mime, + videoBitsPerSecond: 8_000_000, + }); + this._recorder.addEventListener("dataavailable", (e) => { + if (e.data && e.data.size > 0) this._chunks.push(e.data); + }); + this._recorder.addEventListener("stop", () => this._deliver(mime)); + + this.exporting = true; + this._finishing = false; + this._btn.classList.add("active"); + this._btn.textContent = "⏺ REC — click to stop"; + + // Roll: recorder + playback in the same tick so file duration ~= the path span. + setPathPlaying(true); // #btn-path flow: camPath.play() + transport starts the clock + this._recorder.start(250); + this._tick = this._tick.bind(this); + requestAnimationFrame(this._tick); + console.info(`[export] recording ${startT.toFixed(2)}s -> ${this._endT.toFixed(2)}s (${mime})`); + }, + + _tick() { + if (!this.exporting || this._finishing) return; + // Done when the playhead crosses the last keyframe — or when anything halts the ride + // (end of timeline pauses the transport; snap-to-camera exits path mode). + if (state.tGlobal >= this._endT - 1e-3 || !state.playing || !camPath.playing) { + this._finish("end of path"); + return; + } + requestAnimationFrame(this._tick); + }, + + _finish(reason) { + if (!this.exporting || this._finishing) return; + this._finishing = true; + console.info(`[export] finishing (${reason})`); + try { + if (this._recorder && this._recorder.state !== "inactive") this._recorder.stop(); + else this._deliver(null); // recorder never really started: still restore the UI + } catch (err) { + console.error("[export] recorder stop failed", err); + this._deliver(null); + } + }, + + /** MediaRecorder "stop" handler: build + download the file, then restore prior state. */ + _deliver(mime) { + if (this._chunks.length > 0) { + const blob = new Blob(this._chunks, { type: mime ?? "video/webm" }); + download(blob, "festival4d-cut.webm"); + console.info(`[export] festival4d-cut.webm — ${(blob.size / 1e6).toFixed(2)} MB`); + } else { + console.warn("[export] no data recorded (hidden tab? — captureStream needs focus)"); + } + this._chunks = []; + this._recorder = null; + + // Release the shared hooks. + transport.releaseAudioStream(); + this._canvasStream?.getTracks().forEach((t) => t.stop()); + this._canvasStream = null; + + // Restore ALL prior UI/transport state (spec M12). + const prior = this._prior ?? {}; + this._prior = null; + setPathPlaying(false); // resets #btn-path label via main.js's own flow + if (state.playing) transport.pause(); + transport.setRate(prior.rate ?? 1); + transport.seek(prior.tGlobal ?? 0); + if (prior.followCameraId != null) scene3d.snapTo(prior.followCameraId); + if (prior.pathPlaying) setPathPlaying(true); + if (prior.playing && !state.playing) transport.play(); + + this.exporting = false; + this._finishing = false; + this._btn.classList.remove("active"); + this._btn.textContent = BTN_LABEL; + this._btn.disabled = camPath.count() < 2; + }, + + _flash(text, ms = 2200) { + const btn = this._btn; + btn.textContent = text; + setTimeout(() => { + if (!this.exporting) btn.textContent = BTN_LABEL; + }, ms); + }, }; From 369c559b1ec89efb1cf578db5bc585db496a975a Mon Sep 17 00:00:00 2001 From: type-two Date: Fri, 17 Jul 2026 17:54:41 +1000 Subject: [PATCH 4/4] lane E status: M10-M12 ready_to_merge (148 passed; CR-4 filed) Co-Authored-By: Claude Fable 5 --- plan/status/lane-E.md | 62 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 plan/status/lane-E.md diff --git a/plan/status/lane-E.md b/plan/status/lane-E.md new file mode 100644 index 0000000..e08b113 --- /dev/null +++ b/plan/status/lane-E.md @@ -0,0 +1,62 @@ +# Status — lane-E (director & export, phase 5b — M10/M11/M12) + +## Round 1 — 2026-07-17 — STATUS: ready_to_merge + +**Directives acknowledged:** Round 4 of plan/DIRECTIVES.md (phase 5 begins; foundation2 merged; +suite floor 121 — was 127 at lane start after foundation2's tests). + +**Branch:** `lane/e-director2` (own worktree per Round-4 rule). Commits: `0f8685b` M10, +`52e8912` M11, `ddb8dea` M12. Not merged, not pushed — coordinator merges. + +**Acceptance checklist** +- [x] **M10** beats/onsets: synthetic 120 BPM pulse grid recovered — tempo **120.000** (±2 req), + worst beat **30 ms** off grid (±50 req) — `test_run_features_recovers_synthetic_pulse_grid`; + the 3 ground-truth bangs found as strong onsets — `test_onsets_catch_the_ground_truth_bangs`; + silence/short → valid-empty JSON, missing WAV → note + nothing written, beatless noise → + valid typed result, CLI end-to-end — 7 tests in `test_audio_features.py`. Key impl detail: + 10 ms analysis hop (librosa's default 32 ms grid at 16 kHz misses the ±50 ms bound; measured + 60 ms → 30 ms), tempo from median inter-beat interval (117.19 → 120.000). +- [x] **M11** auto-director: structure-locked against the frozen camPath shape (exact key sets, + unit quats, monotone times) — `test_output_matches_frozen_campath_shape`; top-3 grid == + `[1,4,8,11,14]` incl. shared-keyframe dedupe; **quat order [x,y,z,w] locked** via + round-trip against frozen `geometry.colmap_to_threejs` + `quat_to_mat` + (`test_keyframe_pose_matches_frozen_conversion`); beat snap puts every key on the grid, no + beats.json / corrupt beats.json → unsnapped (no block); no events / no poses → valid empty + path + note; confidence ties → earlier; lead clamps at 0; unregistered poses excluded; API + route + CLI dispatch real output — 14 tests in `test_director.py`. `director.js` wires 🎬: + POST → `camPath.fromJSON` → auto-play via the existing #btn-path flow; empty path shows + "no events yet" on the button. +- [x] **M12** export (code-complete; needs live browser, below): canvas captureStream(30) + + `transport.captureAudioStream()` tap → MediaRecorder vp9/opus webm (vp8/webm fallbacks), + first→last keyframe at forced rate 1, downloads `festival4d-cut.webm`, restores playhead/ + rate/play/path/follow state, releases tap + tracks. Button disabled < 2 keyframes (chained + onto main.js's `camPath.onChange` — no frozen edits). +- [x] Suite: `uv run pytest backend/tests -p no:warnings` → **148 passed, 0 failed** + (127 baseline + 21 lane-E). `cd frontend && npm run build` → clean (pre-existing chunk-size + warning only). + +**Change request filed:** **CR-4** (plan/CHANGE_REQUESTS.md) — two stub-era assertions in +`test_phase5_api.py` asserted lane E's *unimplemented* behavior (CLI exit 2; empty director +path) and fail by design once M10/M11 land. Applied the minimal test edit directly per the +approved **CR-1 precedent** (test file ≠ frozen contract file; api.py/cli.py untouched): +CLI stub loop → `["capsule"]` only; director test now asserts the frozen camPath *shape* +(valid stubbed or implemented). Flagging loudly since my brief read the freeze strictly — +if the coordinator prefers, revert `backend/tests/test_phase5_api.py` and re-apply at +integration2; the suite will then show those 2 stub-era failures until adjudication. + +**Coordinator must verify live in a FOCUSED browser** (this session had no browser; rAF/ +captureStream need focus — pitfall #2): +1. M11: `python -m festival4d synthetic && ingest && features && serve` + vite dev → 🎬 visible; + click → path loads (＋ Key counter jumps), camera flies the cut, #btn-path shows ⏹. + With `data/work/beats.json` deleted, 🎬 still works (unsnapped). With events cleared, + button flashes "🎬 no events yet", no console error. +2. M12: with a path loaded, ⏺ → records the ride, auto-stops at the last keyframe, downloads + `festival4d-cut.webm`; `ffprobe` it: duration within ±5% of the path span (top-8 synthetic: + 1.0→20.0 s ⇒ 19 s ±0.95), has an opus audio track; plays in Chrome. Then confirm prior + state restored (playhead back, rate back, path button reset). Clicking ⏺ mid-ride stops + early and still yields a valid file. Note: hidden/backgrounded pane records frozen frames. +3. M12 + 🎧: enable spatial audio before ⏺ — exported track carries the spatial mix. + +**Blockers / questions:** none beyond CR-4 adjudication. + +**Next:** coordinator merge + live checks above; integration2 owns the real-footage pass.