Plan: Phase 5 spec + directives round 4 (M10-M17, lanes E/F/G)

plan/20-phase5.md is the phase-5 canonical spec: M10 beats/onsets,
M11 auto-director, M12 cinematic export (lane E); M13 anchor manager +
friend tags, M14 photo mode, M15 moment FX (lane F); M16 server-side
camera paths, M17 static memory-capsule export (lane G). foundation2
freezes the new contracts (beats.json, camPath JSON, paths table,
PATCH anchors, __F4D_API_BASE__ / manifest.capsule, transport/scene3d
hooks) and pre-wires all UI so parallel lanes never touch shared files.

DIRECTIVES Round 4 records the fresh-eyes upgrades (FFT caching,
WebAudio master clock, spatial audio; suite floor now 121), ratifies
the additive contract amendments, and sets run order:
foundation2 -> E, G, F (own worktrees, mandatory) -> integration2
(real-footage field test, v0.2.0 proposal).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.claude/launch.json

@@ -4,16 +4,42 @@
     {
       "name": "festifun-api",
       "runtimeExecutable": "uv",
-      "runtimeArgs": ["run", "python", "-m", "festival4d", "serve"],
+      "runtimeArgs": [
+        "run",
+        "python",
+        "-m",
+        "festival4d",
+        "serve"
+      ],
       "port": 8000,
       "autoPort": false
     },
     {
       "name": "festifun-web",
       "runtimeExecutable": "npm",
-      "runtimeArgs": ["run", "dev", "--prefix", "frontend"],
+      "runtimeArgs": [
+        "run",
+        "dev",
+        "--prefix",
+        "frontend"
+      ],
       "port": 5173,
       "autoPort": false
+    },
+    {
+      "name": "festifun-web-dist",
+      "runtimeExecutable": "uv",
+      "runtimeArgs": [
+        "run",
+        "python",
+        "-m",
+        "http.server",
+        "5174",
+        "--directory",
+        "/Users/m3ultra/Documents/festifun/frontend/dist"
+      ],
+      "port": 5174,
+      "autoPort": false
     }
   ]
 }

README.md

@@ -23,6 +23,9 @@ fully offline and local; the only optional cloud piece is moment classification.
   correct any misclassification.
 - **Click to locate things in 3D**: draw a box around an object in one or two videos and the
   app triangulates its 3D position, dropping an anchor that appears everywhere at once.
+- **Hear the scene in 3D** 🎧: toggle spatial audio and every camera becomes a positional
+  sound source at its reconstructed pose — fly toward a camera and you hear *that* spot in
+  the crowd get closer.
 
 ---
 
@@ -154,7 +157,8 @@ export GEMINI_API_KEY=...             # default provider (Gemini flash, native v
 | Play / pause | `Space` or the ⏵ button |
 | Nudge time ±1 s (±5 s) | `←` / `→` (hold `Shift`) |
 | Scrub | drag the timeline |
-| Pick the audio you hear | click a video (only one is unmuted at a time) |
+| Pick the audio you hear | click a video (one soundtrack at a time) |
+| Spatial audio | 🎧 **3D audio** (needs poses): every camera plays from its position in the scene; the mix follows the viewer camera as you fly |
 | Enable/disable a video | ◉ on the video |
 | Orbit the 3D scene | drag in the 3D pane |
 | Snap the 3D view to a camera | click its frustum, its ⛶ button, or press `1`–`9` |
@@ -194,9 +198,13 @@ data/raw/ ──ingest──▶ data/work/audio ──sync────▶ videos
   classifiers), `resolve` (annotation→3D), `api` (FastAPI, Range-capable video serving), `db`
   (SQLAlchemy/SQLite), `synthetic` (the test fixture).
 - **Frontend** — vanilla JS + Three.js. A single master clock (`transport.js`) derives each
-  video's local time from the shared timeline and continuously corrects playback; `lib/pose.js`
+  video's local time from the shared timeline and continuously corrects playback. When possible
-  is the one COLMAP→Three.js pose conversion, mirrored from `geometry.py` and locked by a shared
+  the clock is **audio**: the selected soundtrack (`GET /api/audio/{id}`, extracted+cached
-  test.
+  server-side) plays through WebAudio and `t_global` derives from `AudioContext.currentTime` —
+  so every video is muted picture-only and can be seeked/rate-trimmed with zero audible
+  artifacts (falls back to a `performance.now()` clock + `<video>` audio if the track can't
+  load). `lib/pose.js` is the one COLMAP→Three.js pose conversion, mirrored from `geometry.py`
+  and locked by a shared test.
 - **Timebase contract:** `t_video = (t_global − offset_ms/1000) · (1 + drift_ppm·1e−6)`; the
   reference video has offset 0.
 

backend/festival4d/api.py

@@ -194,6 +194,34 @@ def splat() -> FileResponse:
     )
 
 
+@app.get("/api/audio/{video_id}")
+def video_audio(video_id: int) -> FileResponse:
+    """Listening-quality audio track of a video (stereo AAC), for the WebAudio master clock.
+
+    Extracted lazily from the source video via ffmpeg on first request and cached in
+    ``data/work/audio_hq/`` (re-extracted if the source file is newer). The frontend
+    decodes this whole file with ``decodeAudioData`` and derives ``t_global`` from
+    ``AudioContext.currentTime`` — see ``frontend/src/transport.js``.
+    """
+    from festival4d import ingest
+    import subprocess
+
+    v = db.get_video(video_id)
+    if v is None:
+        raise HTTPException(status_code=404, detail=f"no video with id={video_id}")
+    src = config.RAW_DIR / v.filename
+    if not src.exists():
+        raise HTTPException(status_code=404, detail=f"source file missing: {v.filename}")
+    out = config.AUDIO_HQ_DIR / f"{video_id}.m4a"
+    if not out.exists() or out.stat().st_mtime < src.stat().st_mtime:
+        try:
+            ingest.extract_audio_hq(src, out)
+        except (subprocess.CalledProcessError, RuntimeError) as exc:
+            # No audio stream / no ffmpeg — the frontend falls back to <video> audio.
+            raise HTTPException(status_code=404, detail=f"no extractable audio: {exc}") from exc
+    return FileResponse(out, media_type="audio/mp4", filename=out.name)
+
+
 @app.get("/api/anchors")
 def get_anchors() -> list[dict]:
     return [_anchor_dict(a) for a in db.get_anchors()]

backend/festival4d/audio_sync.py

@@ -68,29 +68,15 @@ def _peak_ratio(env: np.ndarray, peak: int, sr: int, guard_s: float = 0.004) ->
     return main / second
 
 
-def gcc_phat(sig: np.ndarray, ref: np.ndarray, sr: int,
+def _whitened_cc(SIG: np.ndarray, REF: np.ndarray, n: int) -> np.ndarray:
-             max_lag_s: float | None = None) -> tuple[float, float]:
+    """PHAT-whitened cross-correlation from precomputed rFFTs (lag 0 at index 0)."""
-    """GCC-PHAT time delay of ``sig`` relative to ``ref`` (spec M1, lane A).
-
-    Returns ``(offset_s, confidence)`` where ``offset_s`` is positive when ``sig`` *lags*
-    ``ref`` (i.e. ``sig[n] ≈ ref[n − offset_s·sr]``), and confidence is the
-    peak-to-second-peak ratio.
-    """
-    sig = np.asarray(sig, dtype=np.float64)
-    ref = np.asarray(ref, dtype=np.float64)
-    # Remove DC so the whitened correlation keys on structure, not a bias term.
-    sig = sig - sig.mean()
-    ref = ref - ref.mean()
-
-    n = len(sig) + len(ref)              # linear (zero-padded) correlation → no circular wrap
-    SIG = np.fft.rfft(sig, n)
-    REF = np.fft.rfft(ref, n)
     R = SIG * np.conj(REF)
     R /= np.abs(R) + _EPS                # PHAT weighting: flatten magnitude, keep phase
-    cc = np.fft.irfft(R, n)              # lag 0 at index 0; negative lags wrap to the tail
+    return np.fft.irfft(R, n)            # lag 0 at index 0; negative lags wrap to the tail
 
-    max_lag = n // 2 if max_lag_s is None else int(round(max_lag_s * sr))
+
-    max_lag = max(1, min(max_lag, n // 2))
+def _peak_offset(cc: np.ndarray, sr: int, max_lag: int) -> tuple[float, float]:
+    """Peak of a whitened correlation → ``(offset_s, confidence)`` over ±``max_lag`` samples."""
     # Re-center to contiguous lags [−max_lag … +max_lag].
     cc = np.concatenate((cc[-max_lag:], cc[:max_lag + 1]))
     env = np.abs(cc)
@@ -107,6 +93,27 @@ def gcc_phat(sig: np.ndarray, ref: np.ndarray, sr: int,
     return offset_s, _peak_ratio(env, peak, sr)
 
 
+def gcc_phat(sig: np.ndarray, ref: np.ndarray, sr: int,
+             max_lag_s: float | None = None) -> tuple[float, float]:
+    """GCC-PHAT time delay of ``sig`` relative to ``ref`` (spec M1, lane A).
+
+    Returns ``(offset_s, confidence)`` where ``offset_s`` is positive when ``sig`` *lags*
+    ``ref`` (i.e. ``sig[n] ≈ ref[n − offset_s·sr]``), and confidence is the
+    peak-to-second-peak ratio.
+    """
+    sig = np.asarray(sig, dtype=np.float64)
+    ref = np.asarray(ref, dtype=np.float64)
+    # Remove DC so the whitened correlation keys on structure, not a bias term.
+    sig = sig - sig.mean()
+    ref = ref - ref.mean()
+
+    n = len(sig) + len(ref)              # linear (zero-padded) correlation → no circular wrap
+    cc = _whitened_cc(np.fft.rfft(sig, n), np.fft.rfft(ref, n), n)
+    max_lag = n // 2 if max_lag_s is None else int(round(max_lag_s * sr))
+    max_lag = max(1, min(max_lag, n // 2))
+    return _peak_offset(cc, sr, max_lag)
+
+
 # ---------------------------------------------------------------------------
 # Pairwise graph
 # ---------------------------------------------------------------------------
@@ -116,13 +123,30 @@ def pairwise_offsets(signals: dict[int, np.ndarray], sr: int) -> list[dict]:
     ``signals`` maps ``video_id -> mono samples``. Returns a list of
     ``{a, b, offset_s, confidence}`` edges, where ``offset_s`` is the *relative start
     offset* ``offset_a − offset_b`` in seconds (see the module's sign conventions).
+
+    Each signal's rFFT is computed **once** at a shared padded length and reused across
+    every pair — O(K) FFTs + O(K²) cheap spectrum products, instead of the O(K²) full
+    FFTs a per-pair :func:`gcc_phat` would cost. The shared length (2·max_len ≥ any
+    pair's la+lb) keeps the correlation linear (no circular wrap), and each pair's lag
+    search stays capped at ±(la+lb)/2 — the same window the per-pair transform used.
     """
     ids = sorted(signals)
+    if len(ids) < 2:
+        return []
+    lengths = {i: len(signals[i]) for i in ids}
+    n = 2 * max(lengths.values())
+    spectra: dict[int, np.ndarray] = {}
+    for i in ids:
+        s = np.asarray(signals[i], dtype=np.float64)
+        s = s - s.mean()                 # same DC removal as gcc_phat
+        spectra[i] = np.fft.rfft(s, n)
     edges: list[dict] = []
     for i in range(len(ids)):
         for j in range(i + 1, len(ids)):
             a, b = ids[i], ids[j]
-            delay_s, conf = gcc_phat(signals[a], signals[b], sr)
+            cc = _whitened_cc(spectra[a], spectra[b], n)
+            max_lag = max(1, min(n // 2, (lengths[a] + lengths[b]) // 2))
+            delay_s, conf = _peak_offset(cc, sr, max_lag)
             edges.append({"a": a, "b": b, "offset_s": -delay_s, "confidence": conf})
     return edges
 

backend/festival4d/config.py

@@ -35,6 +35,7 @@ DATA_DIR = _data_dir()
 RAW_DIR = DATA_DIR / "raw"          # user drops source videos here (gitignored)
 WORK_DIR = DATA_DIR / "work"        # extracted wavs, frames, colmap workspace (gitignored)
 AUDIO_DIR = WORK_DIR / "audio"      # per-video mono 16 kHz wavs (ingest)
+AUDIO_HQ_DIR = WORK_DIR / "audio_hq"  # per-video listening-quality AAC (lazy, /api/audio/{id})
 FRAMES_DIR = WORK_DIR / "frames"    # sampled JPEGs for SfM
 COLMAP_DIR = WORK_DIR / "colmap"    # COLMAP workspace
 DB_PATH = DATA_DIR / "project.db"   # SQLite (gitignored)

backend/festival4d/ingest.py

@@ -95,6 +95,31 @@ def extract_audio(path: Path, out_wav: Path, sample_rate: int = 16_000) -> Path:
     return out_wav
 
 
+def extract_audio_hq(path: Path, out_m4a: Path) -> Path:
+    """Extract a listening-quality stereo AAC track (48 kHz, 192 kbps) from ``path``.
+
+    Feeds the frontend's WebAudio master clock (``GET /api/audio/{id}``). AAC-in-m4a is
+    re-encoded (not stream-copied) so any source codec — including webm/Opus from browser
+    capture — decodes in every browser's ``decodeAudioData``. Written atomically (tmp +
+    rename) so a request racing the first extraction never reads a half-written file.
+    """
+    _require("ffmpeg")
+    out_m4a.parent.mkdir(parents=True, exist_ok=True)
+    tmp = out_m4a.with_suffix(".tmp.m4a")
+    subprocess.run(
+        ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error",
+         "-i", str(path),
+         "-vn",
+         "-ac", "2",
+         "-ar", "48000",
+         "-c:a", "aac", "-b:a", "192k",
+         str(tmp)],
+        check=True,
+    )
+    tmp.replace(out_m4a)
+    return out_m4a
+
+
 def run_ingest() -> list[dict]:
     """Ingest every video in ``config.RAW_DIR``: probe, register in DB, extract audio.
 

backend/tests/test_api.py

@@ -81,6 +81,19 @@ def test_pointcloud_is_ply(client):
     assert resp.content[:3] == b"ply"
 
 
+def test_audio_track_extracted_and_cached(client):
+    """GET /api/audio/{id}: lazily-extracted AAC for the WebAudio master clock."""
+    resp = client.get("/api/audio/1")
+    assert resp.status_code == 200
+    assert resp.headers["content-type"] == "audio/mp4"
+    assert len(resp.content) > 1000
+    # ftyp box near the start marks an MP4/M4A container.
+    assert b"ftyp" in resp.content[:64]
+    # Second request serves the cached file byte-identical (no re-extraction drift).
+    assert client.get("/api/audio/1").content == resp.content
+    assert client.get("/api/audio/999").status_code == 404
+
+
 def test_video_range_returns_206(client):
     """Spec pitfall #3: <video> seeking needs HTTP Range -> 206."""
     resp = client.get("/media/cam0.mp4", headers={"Range": "bytes=0-100"})

backend/tests/test_audio_sync.py

@@ -88,6 +88,28 @@ def test_pairwise_and_solve_recover_synthetic_offsets():
         assert abs(solved[i + 1] - off) <= 10.0, (off, solved[i + 1])
 
 
+def test_pairwise_matches_direct_gcc_phat_with_unequal_lengths():
+    """The cached-FFT pairwise path agrees with a direct per-pair gcc_phat call.
+
+    Signals get different lengths on purpose: the shared padded size (2·max_len) then
+    differs from the per-pair size (la+lb), which is exactly where the two paths could
+    diverge if the lag windowing were wrong.
+    """
+    sr = 16_000
+    master = _broadband(sr * 8, seed=7)
+    signals = {
+        1: master[: sr * 6],
+        2: _delay_samples(master, 900)[: sr * 4],
+        3: _delay_samples(master, -1500)[: sr * 5],
+    }
+    edges = audio_sync.pairwise_offsets(signals, sr)
+    assert len(edges) == 3
+    for e in edges:
+        direct_delay, direct_conf = audio_sync.gcc_phat(signals[e["a"]], signals[e["b"]], sr)
+        assert abs(e["offset_s"] - -direct_delay) < 1.0 / sr, e
+        assert e["confidence"] > 5.0 and direct_conf > 5.0
+
+
 def test_pairwise_edge_sign_is_relative_start_offset():
     """Edge ``offset_s`` = offset_a − offset_b (seconds)."""
     sr = config.AUDIO_SAMPLE_RATE

docs/ideas.md

@@ -3,6 +3,12 @@
 Parking lot for post-v0.1.0 extensions (spec §5 future-extensions). Not committed work — a place
 to capture direction so the prototype's scope stays honest. Roughly ordered by payoff-to-effort.
 
+> **Status 2026-07-17:** splatting ✅ shipped (see `modelbeast-crossover.md`); spatial audio ✅
+> shipped (🎧, WebAudio master clock). Cinematic export → **M12**, multi-modal audio analysis →
+> **M10/M11**, anchor manager → **M13**, path persistence → **M16** — all now specced in
+> `plan/20-phase5.md`. Still parked here: realtime ingest, object/artist tracking,
+> per-project workspaces, sync-graph visualization, per-moment splats (crossover doc §3).
+
 ## Neural rendering — Gaussian Splatting
 Replace the sparse COLMAP point cloud with a **3D Gaussian Splatting** model for photorealistic
 free-roam instead of a dot cloud. COLMAP poses + images already feed splatting trainers

frontend/index.html

@@ -228,6 +228,9 @@
             <button class="btn" id="btn-path-export" title="Export path as JSON">⤓</button>
             <button class="btn" id="btn-path-import" title="Import path JSON">⤒</button>
             <button class="btn" id="btn-path-clear" title="Clear all keyframes">✕</button>
+            <span class="ctl-sep"></span>
+            <button class="btn" id="btn-spatial" style="display:none"
+                    title="Spatial audio: hear every camera from its position in the scene">🎧 3D audio</button>
             <input type="file" id="path-file" accept="application/json,.json" hidden />
           </div>
           <div id="scene-hint">drag to orbit · 1–9 snap to camera · K add keyframe · Esc detach</div>

frontend/src/main.js

@@ -70,9 +70,10 @@ async function boot() {
     startLoop();
     loading.style.display = "none";
 
-    // Dev-only hook so the transport/scene can be driven in tests (e.g. a setTimeout pump when
+    // Debug hook so the transport/scene can be driven in tests (e.g. a setTimeout pump when
-    // a headless pane throttles rAF). Never present in a production build.
+    // a headless pane throttles rAF). Dev builds always; production only when opted in via
-    if (import.meta.env?.DEV) {
+    // localStorage.setItem("f4dDebug", "1").
+    if (import.meta.env?.DEV || localStorage.getItem("f4dDebug")) {
       window.__f4d = {
         state, transport, scene3d, timeline, cells: () => cells,
         poseAt, projectWorldToVideoPx,
@@ -190,6 +191,12 @@ function wireTransportControls() {
     }
     scene3d.freeRoam();
   });
+
+  // 🎧 spatial audio — only offered when there are camera poses to place emitters at.
+  const spatialBtn = el("btn-spatial");
+  if (state.hasPoses) spatialBtn.style.display = "";
+  spatialBtn.addEventListener("click", () => transport.setSpatial(!state.spatialAudio));
+  on("spatial", (enabled) => spatialBtn.classList.toggle("active", enabled));
 }
 
 function wireKeyboard() {

frontend/src/scene3d.js

@@ -10,6 +10,7 @@ import { colmapToThreejs } from "./lib/pose.js";
 import { poseAt } from "./lib/poseTrack.js";
 import { tVideoFromGlobal } from "./lib/timebase.js";
 import { state, emit } from "./state.js";
+import { transport } from "./transport.js";
 
 // Per-video colors (match the frustum, path line, and grid accents).
 export const VIDEO_COLORS = ["#ff5d73", "#4dd0ff", "#c9a2ff", "#ffcf5d", "#7dffb0", "#ff9d5d"];
@@ -25,6 +26,8 @@ const _m4 = new THREE.Matrix4();
 const _pos = new THREE.Vector3();
 const _quat = new THREE.Quaternion();
 const _scale = new THREE.Vector3();
+const _fwd = new THREE.Vector3();
+const _up = new THREE.Vector3();
 
 /** Build a camera-facing text label sprite (used for 3D anchor labels). */
 function makeLabelSprite(text) {
@@ -381,9 +384,28 @@ export class Scene3D {
       this.controls.update();
     }
 
+    this._updateAudio();
     this.renderer.render(this.scene, this.camera);
   }
 
+  // 🎧 Feed the transport's spatial audio: listener = viewer camera, emitters = camera rigs
+  // (rig.group already carries each camera's live pose from the loop above).
+  _updateAudio() {
+    if (!state.spatialAudio) return;
+    this.camera.getWorldPosition(_pos);
+    _fwd.set(0, 0, -1).applyQuaternion(this.camera.quaternion);
+    _up.set(0, 1, 0).applyQuaternion(this.camera.quaternion);
+    const positions = {};
+    for (const v of state.videos) {
+      const rig = this.rigs[v.id];
+      if (rig?.group.visible) positions[v.id] = rig.group.position;
+    }
+    transport.updateSpatial(
+      { px: _pos.x, py: _pos.y, pz: _pos.z, fx: _fwd.x, fy: _fwd.y, fz: _fwd.z, ux: _up.x, uy: _up.y, uz: _up.z },
+      positions
+    );
+  }
+
   _wirePicking() {
     const el = this.renderer.domElement;
     const ray = new THREE.Raycaster();

frontend/src/state.js

@@ -26,7 +26,8 @@ export const state = {
   tGlobal: 0,
   playing: false,
   rate: 1,
-  audioSourceId: null, // video id whose audio is unmuted (exactly one)
+  audioSourceId: null, // video id whose audio you hear (WebAudio soundtrack; fallback: unmuted <video>)
+  spatialAudio: false, // 🎧 every camera is a positional emitter at its 3D pose; listener = viewer cam
   enabled: {}, // id -> bool (per-video enable; disabled videos pause + dim + drop from sync)
   inRange: {}, // id -> bool (t_video within [0, duration] and enabled)
   syncErrorMs: {}, // id -> number | null (currentTime - target, ms; null when out of range)

frontend/src/transport.js

@@ -1,7 +1,14 @@
 // Master clock + per-video synchronization (spec M4 — the heart of the app).
 //
-// The master clock is derived from performance.now(); we NEVER trust a <video> element as the
+// AUDIO IS THE MASTER CLOCK when possible: the selected audio source's soundtrack (fetched from
-// clock (HTML5 video is not frame-accurate — pitfall #2). Correction is continuous:
+// /api/audio/{id}, decoded once) plays through WebAudio, and t_global is derived from
+// AudioContext.currentTime — sample-accurate, immune to rAF throttling, and NEVER corrected.
+// Every <video> element is muted picture-only and is seeked / rate-trimmed freely against that
+// clock with zero audible artifacts (previously the unmuted video's own corrections warbled).
+// If the track can't be fetched/decoded, we fall back per-session to the legacy behavior:
+// performance.now() clock + the selected video's element audio unmuted.
+// We NEVER trust a <video> element as the clock (HTML5 video is not frame-accurate —
+// pitfall #2). Correction is continuous:
 //   |err| > 150 ms          -> hard seek (fastSeek where available)
 //   20 ms < |err| <= 150 ms -> nudge playbackRate within master rate x[0.95, 1.05]
 //   |err| <= 20 ms          -> locked: playbackRate = master rate
@@ -12,9 +19,9 @@
 // better than sampling the frame-quantized currentTime in rAF). Browsers without rVFC fall back
 // to a currentTime measurement inside the rAF tick(). While PAUSED / seeking, videos are parked
 // on their target frame via currentTime. Videos whose target t_video falls outside [0, duration]
-// pause and are marked out-of-range (the grid dims them). Exactly one video's audio is unmuted.
+// pause and are marked out-of-range (the grid dims them).
 
-import { state, emit } from "./state.js";
+import { state, emit, on } from "./state.js";
 import { tVideoFromGlobal } from "./lib/timebase.js";
 
 const HARD_SEEK_S = 0.15; // > this error => hard seek
@@ -27,13 +34,29 @@ export class Transport {
     this._t0 = 0; // master-timeline anchor (seconds)
     this._wall0 = performance.now(); // wall-clock anchor (ms) matching _t0
     this._visWired = false;
+
+    // WebAudio master clock (see header). All lazy: nothing is created until needed.
+    this._actx = null; // AudioContext
+    this._ctx0 = 0; // AudioContext.currentTime anchor (s) matching _t0
+    this._audioActive = false; // audio clock is currently driving t_global
+    this._buffers = new Map(); // video id -> AudioBuffer (decoded) | null (unavailable)
+    this._decoding = new Set(); // video ids with a fetch/decode in flight
+    this._tracks = new Map(); // playing tracks: video id -> { src, gain, panner|null }
   }
 
   /** Register the video cells the transport drives and start their frame loops. */
   register(cells) {
     this.cells = cells;
     this._applyAudio();
+    this._loadTrack(state.audioSourceId); // start fetching the soundtrack immediately
     for (const cell of this.cells) this._startFrameLoop(cell);
+    if (!this._enabledWired) {
+      // In spatial mode a toggled video's emitter must join/leave the running mix.
+      on("enabled", () => {
+        if (state.spatialAudio) this._restartIfPlaying();
+      });
+      this._enabledWired = true;
+    }
     if (!this._visWired) {
       // A throttled/backgrounded tab stalls rAF while decoders free-run; re-lock on return.
       document.addEventListener("visibilitychange", () => {
@@ -46,7 +69,16 @@ export class Transport {
   /** Master-timeline seconds at a given wall-clock (ms) timestamp, clamped to [0, tGlobalMax]. */
   _clockAt(wallMs) {
     if (!state.playing) return this._t0;
-    const t = this._t0 + ((wallMs - this._wall0) / 1000) * state.rate;
+    let elapsed;
+    if (this._audioActive) {
+      // Audio clock: map the wall timestamp onto the AudioContext clock (the two clocks tick
+      // together; only their epochs differ, and this mapping is sub-ms), measure from anchor.
+      const ctxNow = this._actx.currentTime + (wallMs - performance.now()) / 1000;
+      elapsed = ctxNow - this._ctx0;
+    } else {
+      elapsed = (wallMs - this._wall0) / 1000;
+    }
+    const t = this._t0 + elapsed * state.rate;
     if (t <= 0) return 0;
     if (t >= state.tGlobalMax) return state.tGlobalMax;
     return t;
@@ -68,15 +100,17 @@ export class Transport {
     this._wall0 = performance.now();
     state.tGlobal = t;
     state.playing = true;
-    this._applyAudio(); // first play() is a user gesture -> browser lets us unmute
+    this._startAudio(t); // play() is a user gesture -> AudioContext may resume / videos unmute
     emit("play");
   }
 
   pause() {
     if (!state.playing) return;
-    state.tGlobal = this.computeTGlobal();
+    state.tGlobal = this.computeTGlobal(); // BEFORE stopping the audio clock's source node
     this._t0 = state.tGlobal;
     state.playing = false;
+    this._stopTracks();
+    this._audioActive = false;
     for (const { video } of this.cells) this._safePause(video);
     emit("pause");
   }
@@ -92,6 +126,7 @@ export class Transport {
     this._t0 = t;
     this._wall0 = performance.now();
     state.tGlobal = t;
+    if (state.playing) this._startAudio(t); // restart the soundtrack at the new position
     for (const cell of this.cells) {
       const target = this._targetFor(cell.meta, t);
       const inRange =
@@ -121,24 +156,205 @@ export class Transport {
     this._wall0 = performance.now();
     state.rate = r;
     for (const { video } of this.cells) video.playbackRate = r; // re-trimmed by correction
+    if (state.playing) this._startAudio(state.tGlobal); // restart the track at the new rate
     emit("rate", r);
   }
 
-  /** Choose which video's audio is unmuted (all others muted — avoids phasing chaos). */
+  /** Choose whose audio you hear (the WebAudio track when decoded, else that video unmuted). */
   setAudioSource(id) {
     state.audioSourceId = id;
-    this._applyAudio();
+    this._loadTrack(id);
+    this._restartIfPlaying();
     emit("audiosource", id);
   }
 
+  /** 🎧 Toggle spatial audio: every camera becomes a positional emitter at its 3D pose. */
+  setSpatial(enabled) {
+    state.spatialAudio = !!enabled;
+    if (state.spatialAudio) for (const { id } of this.cells) this._loadTrack(id);
+    this._restartIfPlaying();
+    emit("spatial", state.spatialAudio);
+  }
+
+  _restartIfPlaying() {
+    if (state.playing) this._startAudio(this.computeTGlobal());
+    else this._applyAudio();
+  }
+
   _applyAudio() {
+    // With a decoded soundtrack the WebAudio track is the only audible thing: every video is
+    // muted picture-only. Fallback (no track): the selected video's element audio, as before.
+    const trackReady = state.spatialAudio
+      ? [...this._buffers.values()].some((b) => b instanceof AudioBuffer)
+      : this._buffers.get(state.audioSourceId) instanceof AudioBuffer;
     for (const { id, video } of this.cells) {
-      const isSrc = id === state.audioSourceId;
+      const isSrc = !trackReady && id === state.audioSourceId;
       video.muted = !isSrc;
       if (isSrc) video.volume = 1;
     }
   }
 
+  // --- WebAudio master clock ----------------------------------------------------------------
+
+  _ensureCtx() {
+    if (!this._actx) this._actx = new (window.AudioContext || window.webkitAudioContext)();
+    return this._actx;
+  }
+
+  /** Fetch + decode a video's soundtrack once; upgrade the running clock when it lands. */
+  async _loadTrack(id) {
+    if (id == null || this._buffers.has(id) || this._decoding.has(id)) return;
+    if (typeof window === "undefined" || !(window.AudioContext || window.webkitAudioContext)) return;
+    this._decoding.add(id);
+    try {
+      const resp = await fetch(`${state.apiBase}/api/audio/${id}`);
+      if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
+      const buf = await this._ensureCtx().decodeAudioData(await resp.arrayBuffer());
+      // Decoded PCM is big (~23 MB/min stereo 48k): keep only a few tracks around
+      // (spatial mode needs one per camera, so the cap widens to the video count).
+      const cap = state.spatialAudio ? Math.max(this.cells.length, 3) : 3;
+      for (const key of this._buffers.keys()) {
+        if (this._buffers.size < cap) break;
+        if (key !== state.audioSourceId) this._buffers.delete(key);
+      }
+      this._buffers.set(id, buf);
+    } catch (err) {
+      console.warn(`[transport] soundtrack for video ${id} unavailable — using <video> audio`, err);
+      this._buffers.set(id, null);
+    } finally {
+      this._decoding.delete(id);
+    }
+    // A track landing mid-play joins in place: in spatial mode it enters the running mix; as
+    // the selected source it upgrades the fallback clock (both clocks agree on t_global).
+    const decoded = this._buffers.get(id) instanceof AudioBuffer;
+    if (
+      state.playing &&
+      decoded &&
+      (state.spatialAudio || (id === state.audioSourceId && !this._audioActive))
+    ) {
+      this._startAudio(this.computeTGlobal());
+    } else if (!state.playing) {
+      this._applyAudio();
+    }
+  }
+
+  /**
+   * (Re)start the soundtrack at t_global and re-anchor the master clock. Single-track mode
+   * plays the selected source; spatial mode plays every enabled camera's track through a
+   * PannerNode at its 3D pose. Falls back to the performance.now() clock (selected video
+   * unmuted) when no decoded track is available.
+   */
+  _startAudio(tGlobal) {
+    this._stopTracks();
+    this._t0 = tGlobal;
+    this._wall0 = performance.now();
+    const ready = (id) => this._buffers.get(id) instanceof AudioBuffer;
+    const ids = state.spatialAudio
+      ? this.cells.filter((c) => state.enabled[c.id] !== false && ready(c.id)).map((c) => c.id)
+      : ready(state.audioSourceId)
+        ? [state.audioSourceId]
+        : [];
+    if (ids.length === 0) {
+      this._audioActive = false;
+      this._applyAudio(); // fallback: unmute the selected video
+      return;
+    }
+    const ctx = this._ensureCtx();
+    if (ctx.state === "suspended") ctx.resume(); // play()/click gestures allow this
+    // N time-aligned copies of the same sound sum loud — normalize by 1/sqrt(N).
+    const gainValue = 1 / Math.sqrt(ids.length);
+    for (const id of ids) this._startTrack(ctx, id, tGlobal, state.spatialAudio, gainValue);
+    // Even with no live node (source out of range => silence), the audio clock drives time.
+    this._ctx0 = ctx.currentTime;
+    this._audioActive = true;
+    this._applyAudio();
+  }
+
+  _startTrack(ctx, id, tGlobal, spatial, gainValue) {
+    const buf = this._buffers.get(id);
+    const meta =
+      this.cells.find((c) => c.id === id)?.meta ?? state.videos.find((v) => v.id === id);
+    const drift = 1 + (meta?.drift_ppm || 0) * 1e-6;
+    const tv = tVideoFromGlobal(tGlobal, meta?.offset_ms || 0, meta?.drift_ppm || 0);
+    if (tv >= buf.duration) return; // this camera's audio is already over
+    const src = ctx.createBufferSource();
+    src.buffer = buf;
+    // The track's local time must advance at rate·(1+drift) per global second.
+    src.playbackRate.value = state.rate * drift;
+    const gain = ctx.createGain();
+    gain.gain.value = gainValue;
+    src.connect(gain);
+    let panner = null;
+    if (spatial) {
+      panner = ctx.createPanner();
+      panner.panningModel = "HRTF";
+      panner.distanceModel = "inverse";
+      panner.refDistance = 2; // scene units — cameras sit ~5–15 from the stage at COLMAP scale
+      panner.rolloffFactor = 0.6;
+      gain.connect(panner);
+      panner.connect(ctx.destination);
+    } else {
+      gain.connect(ctx.destination);
+    }
+    if (tv >= 0) src.start(0, tv);
+    else src.start(ctx.currentTime + -tv / state.rate, 0); // camera starts later than t_global
+    this._tracks.set(id, { src, gain, panner });
+  }
+
+  /**
+   * 🎧 Per-frame 3D audio update (called by scene3d): the listener follows the viewer camera
+   * and each playing track's panner sits at its camera's current pose. Takes plain numbers in
+   * scene (Three.js) space so the transport stays THREE-free.
+   */
+  updateSpatial(listener, positions) {
+    if (!state.spatialAudio || !this._audioActive || !this._actx) return;
+    const L = this._actx.listener;
+    if (L.positionX) {
+      L.positionX.value = listener.px;
+      L.positionY.value = listener.py;
+      L.positionZ.value = listener.pz;
+      L.forwardX.value = listener.fx;
+      L.forwardY.value = listener.fy;
+      L.forwardZ.value = listener.fz;
+      L.upX.value = listener.ux;
+      L.upY.value = listener.uy;
+      L.upZ.value = listener.uz;
+    } else {
+      // Safari: AudioListener still uses the legacy setters.
+      L.setPosition(listener.px, listener.py, listener.pz);
+      L.setOrientation(listener.fx, listener.fy, listener.fz, listener.ux, listener.uy, listener.uz);
+    }
+    for (const [id, { panner }] of this._tracks) {
+      const pos = positions[id];
+      if (!panner || !pos) continue;
+      if (panner.positionX) {
+        panner.positionX.value = pos.x;
+        panner.positionY.value = pos.y;
+        panner.positionZ.value = pos.z;
+      } else {
+        panner.setPosition(pos.x, pos.y, pos.z);
+      }
+    }
+  }
+
+  _stopTracks() {
+    for (const { src, gain, panner } of this._tracks.values()) {
+      try {
+        src.stop();
+      } catch {
+        /* never started or already ended */
+      }
+      try {
+        src.disconnect();
+        gain.disconnect();
+        panner?.disconnect();
+      } catch {
+        /* ignore */
+      }
+    }
+    this._tracks.clear();
+  }
+
   // Per-presented-frame correction via requestVideoFrameCallback (accurate). Self-sustaining:
   // each callback re-registers the next. No-ops while paused / out of range; rAF tick() handles
   // the fallback path for browsers without rVFC.
@@ -192,7 +408,10 @@ export class Transport {
 
   /** Re-lock every in-range video to the current master time (after a tab-visibility stall). */
   _resync() {
-    const t = state.tGlobal;
+    // Compute fresh: with the audio clock the soundtrack kept playing while rAF (and therefore
+    // state.tGlobal) was frozen, so the live clock may be well past the last stored value.
+    const t = this.computeTGlobal();
+    state.tGlobal = t;
     for (const cell of this.cells) {
       const target = this._targetFor(cell.meta, t);
       if (state.enabled[cell.id] !== false && target >= 0 && target <= cell.meta.duration_s) {

plan/20-phase5.md

@@ -0,0 +1,198 @@
+# Phase 5 — Friends, Director's Cut & Capsules (M10–M17)
+
+Canonical spec for the phase-5 milestones. **M0–M9 live in [`../OPUS_BUILD_INSTRUCTIONS.md`](../OPUS_BUILD_INSTRUCTIONS.md)
+and every contract frozen there still applies** (timebase, DB schema, pose convention,
+classifier contract, synthetic fixture). Standing protocol (status files, directives loop,
+evidence discipline, worktrees) is unchanged from [`README.md`](README.md); the newest round of
+[`DIRECTIVES.md`](DIRECTIVES.md) always wins.
+
+## Context — what shipped since v0.1.0 (read before planning anything)
+
+- **Splat rendering:** `GET /api/splat` + viewer `DropInViewer` with point-cloud fallback
+  (`docs/modelbeast-crossover.md`).
+- **Live capture:** `FESTIVAL4D_CAPTURE=1` mounts `/capture` (README).
+- **WebAudio master clock:** `t_global` derives from `AudioContext.currentTime`;
+  `GET /api/audio/{id}` serves a lazily-extracted stereo AAC per video; all `<video>` elements
+  are muted picture-only, with graceful fallback to the old `performance.now()` clock
+  (`frontend/src/transport.js` header comment is the reference).
+- **Spatial audio (🎧):** per-camera HRTF `PannerNode`s at live poses; listener follows the
+  viewer camera (`transport.updateSpatial`, fed by `scene3d._updateAudio`).
+- **Sync solver:** pairwise GCC-PHAT now reuses one rFFT per signal (`audio_sync.pairwise_offsets`).
+- **Suite floor is 121 passed.** It only goes up.
+
+## Phase map
+
+```
+Phase 5a (serial, ONE agent)      Phase 5b (parallel, one agent per lane)     Phase 5c (serial, ONE agent)
+┌───────────────────────┐         ┌ lane/e-director ── M10 M11 M12 ─┐         ┌───────────────────────────┐
+│ foundation2           │         ├ lane/f-social ──── M13 M14 M15 ─┤         │ integration2              │
+│ contracts + stubs +   │──merge──┤                                 ├──merge──│ full-system pass +        │
+│ UI wiring points      │         └ lane/g-capsule ─── M16 M17 ─────┘         │ real-footage field test   │
+│ branch: foundation2   │         (independent — merge in any order)          │ branch: integration2      │
+└───────────────────────┘         └───────────────────────────────────        └───────────────────────────┘
+```
+
+Lanes depend **only** on foundation2, never on each other. Everything is developed and verified
+against the synthetic fixture first; real footage is integration2's job.
+
+## Milestones
+
+### M10 — Audio features: beats & onsets (lane E, backend)
+`backend/festival4d/audio_features.py`, entrypoint `run_features()`, CLI `python -m festival4d features`.
+Load the **reference** video's ingest WAV (16 kHz mono; the reference timeline *is* `t_global`).
+Beat/tempo tracking + onset-strength detection (librosa is already a dependency). Write
+`data/work/beats.json` (shape frozen below); `GET /api/beats` serves it, 404 when absent.
+Quiet/short/beatless audio must produce a *valid, empty* result — never a crash.
+**Accept:** synthetic master audio's known pulse grid recovered (beats within ±50 ms, tempo ±2 BPM);
+silence → empty-but-valid JSON; tests in `backend/tests/test_audio_features.py`.
+
+### M11 — Auto-director (lane E, backend + thin UI)
+`backend/festival4d/director.py`, entrypoint `generate_path(top_n=8, lead_s=2.0)`, CLI
+`python -m festival4d direct`, `POST /api/director` (stubbed by foundation2) returning a
+**camPath-compatible JSON** (contract below). Deterministic v1 rules — no ML:
+pick the top-N events by confidence (ties → earlier); for each, choose the registered camera
+with a pose nearest the event time (via `db` + `geometry.colmap_to_threejs`); keyframes at
+`t_event − lead_s` and `t_event + duration`; when `beats.json` exists, snap keyframe times to
+the nearest beat; FOV from that camera's intrinsics (same formula as `scene3d.snapTo`).
+Frontend: fill the stub `frontend/src/director.js` — the hidden **🎬 Auto-path** button fetches
+`/api/director` and loads it via `camPath.fromJSON` (do not edit camPath.js).
+**Accept:** output round-trips through `camPath.fromJSON` (structure-locked test against the
+frozen shape); covers the top-N events; keyframe times land on beats when beats exist; live
+browser evidence of the path playing.
+
+### M12 — Cinematic export (lane E, frontend)
+Fill the stub `frontend/src/exportVideo.js`. The hidden **⏺ Export** button: seek to the first
+keyframe, play the path, capture `scene3d` canvas via `captureStream(30)` + the soundtrack via
+the `transport.captureAudioStream()` hook (foundation2 provides it — taps the WebAudio graph
+post-gain, spatial mix included), mux with `MediaRecorder` (vp9/opus webm), stop at the last
+keyframe, download `festival4d-cut.webm`. Restore all prior UI/transport state afterwards.
+**Accept:** exported file duration within ±5% of the path span, contains an audio track, plays
+in Chrome — browser evidence (file size, duration probe via ffprobe) in the status file.
+
+### M13 — Anchor manager & friend tags (lane F, frontend)
+Fill the stub `frontend/src/anchorPanel.js`: an always-available collapsible panel listing every
+anchor (color dot, label, jump-to, delete, rename/recolor via the new `PATCH /api/anchors/{id}`).
+Jump-to uses the foundation2 hook `scene3d.focusOn(x, y, z)`. Add the "friend tag" flow: a
+**＋ Tag** action that runs the existing M8 bbox-annotation flow *without* an event and names the
+resulting anchor (friends become labeled, colored anchors that x-ray onto every video for free).
+**Accept:** create → rename → recolor → jump → delete round-trip live in the browser; labels
+visible in the 3D scene and video overlays; anchors survive reload.
+
+### M14 — Photo mode (lane F, frontend)
+Fill the stub `frontend/src/photoMode.js`. **📷 / `P`**: pause, hide helpers + gizmos + anchor
+labels (foundation2 hook `scene3d.setHelpersVisible(false)`), render one frame at 3840-wide
+(preserving aspect) to an offscreen target, download PNG, restore everything.
+**Accept:** PNG ≥ 3840 px wide with no grid/axes/frusta/gizmos; works in free-roam and
+follow-cam; with a splat present it renders the splat (verify on the synthetic point cloud +
+note the splat path).
+
+### M15 — Moment FX (lane F, frontend)
+Fill the stub `frontend/src/fx.js`. When the playhead crosses a `pyro`/`confetti` event, fire a
+short particle burst at the event's resolved anchor (fall back to the stage centroid); on
+`bass_drop`, pulse the point-cloud/splat scale for ~a beat. Toggle via the hidden **✨** button;
+uses only `scene3d.addObject/removeObject`. Must fire when crossing markers in play *and* scrub.
+**Accept:** browser evidence of both effect types; frame rate stays ≥ 30 fps on the synthetic
+scene with FX enabled.
+
+### M16 — Server-side camera paths (lane G, backend + thin UI)
+Implement the foundation2-stubbed routes over the new `paths` table:
+`GET /api/paths` (list), `GET /api/paths/{id}`, `POST /api/paths {name, json}`,
+`DELETE /api/paths/{id}` — `json` is the frozen camPath JSON as text, validated on POST.
+Fill the stub `frontend/src/pathsStore.js`: save-current-path (name prompt), load dropdown,
+delete — wired to the hidden path-save controls.
+**Accept:** endpoint tests (roundtrip deep-equal, 404s, invalid-JSON 422) in
+`backend/tests/test_paths.py`; live save → reload page → load reproduces keyframes exactly.
+
+### M17 — Memory capsule: static shareable export (lane G, backend)
+`backend/festival4d/capsule.py`, CLI `python -m festival4d capsule [--out DIR]` (default
+`data/capsule/`). Build a **zero-backend** bundle:
+copy `frontend/dist` (error clearly if missing — tell the user to `npm run build`); copy media;
+bake every read-only API response to files at the *same relative paths* (`api/manifest`,
+`api/events`, `api/anchors`, `api/beats`, `api/videos/{id}/poses`, `api/pointcloud`, `api/splat`,
+`api/audio/{id}` — extracting audio if not yet cached); inject
+`<script>window.__F4D_API_BASE__=""</script>` into the copied `index.html`; set
+`"capsule": true` in the baked manifest (the frontend hides all write UI — foundation2 wires
+the flag); ship a stdlib **`serve.py`** in the bundle root that serves with HTTP Range support
+(video seeking needs 206s; plain `http.server` can't).
+**Accept:** `python -m festival4d capsule` on the synthetic project, then `python serve.py` in
+the bundle: app loads with **zero** requests to :8000, videos play *and seek*, 3D scene +
+timeline + spatial audio work, write UI absent; `backend/tests/test_capsule.py` covers bundle
+structure, baked-JSON validity, and serve.py Range (206) behavior.
+
+## Contracts (frozen at foundation2 merge)
+
+1. **beats.json** (and the `GET /api/beats` body):
+   `{"tempo_bpm": float|null, "beats_s": [float], "onsets": [{"t_global_s": float, "strength": float 0..1}], "generated_by": "festival4d.audio_features"}`
+   — all times are `t_global` seconds.
+2. **camPath JSON** (already shipped in `camPath.js`; now frozen):
+   `{"version": 1, "keyframes": [{"t_global": float, "pos": [x,y,z], "quat": [x,y,z,w], "fov": float}]}`
+   — **Three.js scene space**, quat order **[x,y,z,w]** (NOT COLMAP's [w,x,y,z]; convert via
+   `geometry.colmap_to_threejs` on the backend, never inline).
+3. **paths table:** `paths(id INTEGER PK, name TEXT NOT NULL, created_at TEXT, path_json TEXT NOT NULL)`
+   + the four routes in M16.
+4. **`PATCH /api/anchors/{id}`** `{label?, color?}` → updated anchor dict (404 on missing).
+5. **Runtime API base override:** `state.js` resolves `window.__F4D_API_BASE__` first, then
+   `VITE_API_BASE`, then the localhost default. **`manifest.capsule: true`** ⇒ read-only UI.
+6. **`transport.captureAudioStream()`** → `MediaStream` tapping the live WebAudio mix (and a
+   matching `releaseAudioStream()`); `scene3d.focusOn(x,y,z)` and `scene3d.setHelpersVisible(bool)`.
+7. **Frozen files during 5b:** everything frozen in phase 1 **plus**
+   `frontend/src/{transport,scene3d,state,main,camPath,annotate}.js` and `frontend/index.html`
+   (foundation2 pre-wires every button — hidden — and every module import; lanes fill their own
+   stub modules only). Believe a frozen file must change → `plan/CHANGE_REQUESTS.md`, work
+   around locally, integration2 adjudicates.
+
+## foundation2 scope (Phase 5a, one agent, branch `foundation2`)
+
+- DB: `paths` table + `db.py` helpers. API: `/api/beats`, `/api/director`, `/api/paths*`,
+  `PATCH /api/anchors/{id}` — implemented against synthetic data where cheap, stubbed with the
+  spec's graceful-degradation pattern where a lane owns the logic. CLI: `features`, `direct`,
+  `capsule` subcommands dispatching into stubs.
+- Backend stubs: `audio_features.py`, `director.py`, `capsule.py` (raise `NotImplementedError`;
+  API/CLI catch and degrade, exactly like phase 1).
+- Frontend: create empty stub modules `director.js`, `exportVideo.js`, `anchorPanel.js`,
+  `photoMode.js`, `fx.js`, `pathsStore.js`; import them from `main.js`; add ALL new buttons to
+  `index.html` (hidden until their module reports ready); add the transport/scene3d hooks and
+  the `__F4D_API_BASE__` + `manifest.capsule` wiring from the contracts above.
+- Tests for every new route shape (mirroring `test_api.py`'s exact-set style — note it locks
+  the manifest key set; add `capsule` consciously).
+- Update the ownership table below if anything moved. Suite ≥ 121 + your new tests. Merge fast.
+
+## integration2 scope (Phase 5c, one agent, branch `integration2`)
+
+- Merge order irrelevant; resolve CHANGE_REQUESTS; full synthetic pass of every milestone
+  (evidence per feature); raise the suite floor to the new count.
+- **Real-footage field test** (the actual milestone): 2–4 real overlapping clips through
+  `ingest → sync → reconstruct → events → features → direct → serve`, then capsule the result.
+  Log every failure as a `plan/ISSUES.md` entry (ISSUE-1 is the format) — fix nothing mid-test.
+- Tag `v0.2.0` when accepted by the coordinator.
+
+## File ownership (Phase 5b)
+
+| Owner | Files |
+|---|---|
+| Lane E | `backend/festival4d/audio_features.py`, `director.py`, `backend/tests/test_audio_features.py`, `test_director.py`, `frontend/src/director.js`, `exportVideo.js` |
+| Lane F | `frontend/src/anchorPanel.js`, `photoMode.js`, `fx.js` |
+| Lane G | `backend/festival4d/capsule.py`, `backend/tests/test_paths.py`, `test_capsule.py`, `frontend/src/pathsStore.js`, the paths-route bodies in the area foundation2 marks for lane G |
+| Every agent | its own `plan/status/<lane>.md` |
+| **Coordinator only** | `plan/DIRECTIVES.md` |
+| **Frozen during 5b** | contract item 7 above |
+
+## Phase-5 pitfalls
+
+1. **Quaternion order.** camPath is `[x,y,z,w]` (Three.js); COLMAP storage is `[w,x,y,z]`.
+   Every past pose bug in this repo was a convention slip — go through `geometry.colmap_to_threejs`
+   / `lib/pose.js`, never inline the math.
+2. **The pane you verify in throttles rAF** (backgrounded tab). Drive the loop with the
+   `window.__f4d` pump (`localStorage.setItem("f4dDebug","1")` in production builds); don't
+   file "playback frozen" issues against your own hidden tab. MediaRecorder export (M12) needs
+   a *focused* tab to capture real frames — say so in your evidence if you couldn't.
+3. **`test_api.py` locks response key-sets on purpose.** Adding a manifest key or route means
+   consciously updating the exact-set assertion in the same commit — a failing set-lock is a
+   feature, not an obstacle.
+4. **Range serving.** Any static path that serves video must answer 206 (spec M3 pitfall #3);
+   `python -m http.server` does not — that's why M17 ships `serve.py`.
+5. **Never block on a missing optional input.** No beats.json → director still works (no beat
+   snapping). No splat → photo mode shoots the point cloud. No audio track → transport already
+   falls back. Follow the degradation pattern everywhere; it's the house style.
+6. **Decoded audio is big** (~23 MB/min stereo). M12/M17 must not decode all tracks just
+   because they can — reuse what the transport already has.

plan/DIRECTIVES.md

@@ -4,6 +4,47 @@ Written **only** by the coordinator (the human's planning session). Agents: read
 
 ---
 
+## Round 4 — 2026-07-17 — Fresh-eyes upgrades landed; PHASE 5 begins (M10–M17)
+
+**Coordinator work landed directly on `main` (self-reviewed + live-verified):**
+- `95b7178` — sync solver caches one rFFT per signal across the pairwise GCC-PHAT stage
+  (1.8× on 6 cams × 10 min; equivalence locked by a new unequal-lengths test).
+- `c24af80` — **audio is now the master clock** (`GET /api/audio/{id}` AAC + WebAudio;
+  `t_global` from `AudioContext.currentTime`; all `<video>` muted picture-only; graceful
+  fallback) **+ spatial audio** (🎧 per-camera HRTF panners at live poses, listener = viewer
+  camera, gains 1/√N). Verified live: clock drives playback, mid-play source switching is
+  continuous, panners sit at the correct camera positions, toggle on/off mid-play is clean.
+- `db42c42` — chore: `python-multipart` locked (declared-but-unlocked drift), launch configs.
+- Suite is **121 passed**; frontend build clean. **The test floor is now 121.**
+
+**Contract amendments (coordinator-ratified, additive only):** new route `GET /api/audio/{id}`;
+`transport.js` is now the WebAudio clock owner (its header comment is the reference);
+the `window.__f4d` debug hook is available in production builds via `localStorage f4dDebug=1`.
+`test_api.py`'s manifest key-set lock is unchanged.
+
+**Directives — Phase 5 (spec: `plan/20-phase5.md`; lane briefs: `plan/lane-E/F/G-*.md`):**
+1. **Now: `foundation2` only** (one agent, branch `foundation2`, scope in 20-phase5.md).
+   Same rule as phase 1: contracts correctness beats speed — three lanes will hard-depend on
+   your route shapes, stubs, hooks, and UI wiring points. Maintain `plan/status/foundation2.md`.
+2. **After foundation2 merges — lanes in priority order** (all three concurrently if capacity
+   allows): **E** (director — longest, earliest contract validation), **G** (capsule — the
+   shareable payoff), **F** (social UX — smallest, pure frontend).
+3. **Worktree rule is MANDATORY this time:** every parallel agent creates its own `git worktree`
+   before its first edit (Round 1 standing lesson; the phase-2 shared-tree collision cost us a
+   stash cleanup). Single-agent phases (5a, 5c) may use the main working directory.
+4. **Merge policy unchanged:** merge to `main` per-milestone where the lane brief allows it;
+   never wait on another lane. Then `integration2` (branch `integration2`), which owns the
+   real-footage field test and the `v0.2.0` tag proposal — failures go to `plan/ISSUES.md`,
+   not into hasty fixes.
+5. Standing protocol (status files, evidence discipline, blocker rule, DIRECTIVES re-read after
+   every milestone) is unchanged from Round 0.
+
+**For the human:** (a) the Round 3 focused-browser sync check is still open — now also put on
+headphones, enable 🎧 3D audio, and fly around; (b) real overlapping clips for the field test
+remain the gating item for integration2; (c) optional: a trained splat for a real project makes
+photo mode (M14) and the capsule (M17) dramatically more impressive —
+`scripts/splat_via_modelbeast.sh`.
+
 ## Round 3 — 2026-07-16 — Integration review: v0.1.0 ACCEPTED; polish + field test next
 
 **Coordinator review of the integration work (independently verified):** `main` == `origin/main`

plan/README.md

@@ -2,6 +2,10 @@
 
 **Read [`../OPUS_BUILD_INSTRUCTIONS.md`](../OPUS_BUILD_INSTRUCTIONS.md) first — it is the canonical spec** (architecture, repo layout, data model, milestone details M0–M9, pitfalls). This directory only organizes *who builds what, in what order, without colliding*. Lane briefs reference spec milestones by number; they do not restate them.
 
+> **Current phase: 5** (M10–M17) — spec and organization in [`20-phase5.md`](20-phase5.md), lane
+> briefs `lane-E/F/G-*.md`. Phases 1–4 below are complete (v0.1.0 + polish); their protocol
+> sections (git, status, directives loop, evidence discipline) remain the standing rules.
+
 ## Phase map
 
 ```

plan/lane-E-director.md

@@ -0,0 +1,18 @@
+# Lane E — Director & export (branch `lane/e-director`)
+
+**Scope: phase-5 milestones M10 + M11 + M12** (beats/onsets, auto-director, cinematic export) — see [`20-phase5.md`](20-phase5.md).
+
+- Owned files: `backend/festival4d/audio_features.py`, `backend/festival4d/director.py`,
+  `backend/tests/test_audio_features.py`, `backend/tests/test_director.py`,
+  `frontend/src/director.js`, `frontend/src/exportVideo.js`. Nothing else.
+- Fill the stub bodies foundation2 left; signatures, routes, and CLI dispatch already exist.
+- Contracts you consume (frozen): `beats.json` shape, camPath JSON (quat `[x,y,z,w]`, Three.js
+  space — convert COLMAP poses via `geometry.colmap_to_threejs`, never inline),
+  `transport.captureAudioStream()`.
+- Build order M10 → M11 → M12: the director consumes beats; export consumes the director's path.
+  Each milestone is independently mergeable — don't hold M10/M11 hostage to M12.
+- Degradation: no beats → director skips beat-snapping; no poses → director returns a clear
+  error body, never a crash; export with no path → button stays disabled.
+
+**Done when:** M10–M12 acceptance passes (browser evidence for M11/M12 in your status file),
+`pytest` ≥ 121 + your tests, merged to `main` with synthetic end-to-end intact.

plan/lane-F-social.md

@@ -0,0 +1,17 @@
+# Lane F — Social & fun UX (branch `lane/f-social`)
+
+**Scope: phase-5 milestones M13 + M14 + M15** (anchor manager + friend tags, photo mode, moment FX) — see [`20-phase5.md`](20-phase5.md).
+
+- Owned files: `frontend/src/anchorPanel.js`, `frontend/src/photoMode.js`, `frontend/src/fx.js`.
+  Nothing else — buttons, imports, and hooks are pre-wired by foundation2.
+- Contracts you consume (frozen): `PATCH /api/anchors/{id}` + existing anchor GET/POST/DELETE,
+  `scene3d.focusOn(x,y,z)`, `scene3d.setHelpersVisible(bool)`, `scene3d.addObject/removeObject`,
+  the M8 annotation flow (drive it, don't modify `annotate.js`).
+- This is the "friends connect & remember" lane — labels, colors, and jump-to should feel like
+  tagging people, not editing database rows. Keep interactions one-click where possible.
+- All verification is in-browser: use the `__f4d` debug pump (phase-5 pitfall #2) and put
+  screenshots/measurements in your status file. No backend tests to write; the suite must
+  simply stay green.
+
+**Done when:** M13–M15 acceptance passes with browser evidence, frontend `npm run build` clean,
+merged to `main` with synthetic end-to-end intact.

plan/lane-G-capsule.md

@@ -0,0 +1,17 @@
+# Lane G — Persistence & capsule (branch `lane/g-capsule`)
+
+**Scope: phase-5 milestones M16 + M17** (server-side camera paths, static memory-capsule export) — see [`20-phase5.md`](20-phase5.md).
+
+- Owned files: `backend/festival4d/capsule.py`, `backend/tests/test_paths.py`,
+  `backend/tests/test_capsule.py`, `frontend/src/pathsStore.js`, plus the paths-route bodies in
+  the region foundation2 marks for lane G. Nothing else.
+- Contracts you consume (frozen): `paths` table + route shapes, camPath JSON (validate on POST),
+  `window.__F4D_API_BASE__` override, `manifest.capsule` read-only flag.
+- M17 discipline: the capsule bakes API responses to files at the **same relative paths** the
+  live API serves — the frontend must not know the difference. Ship `serve.py` (stdlib,
+  Range-capable) in the bundle root; acceptance explicitly includes video *seeking* under it.
+- The capsule is the shareable artifact of the whole project — treat "zero requests to :8000"
+  and "write UI absent" as hard acceptance, not nice-to-haves.
+
+**Done when:** M16–M17 acceptance passes (endpoint tests + a served-capsule browser pass with
+evidence), `pytest` ≥ 121 + your tests, merged to `main` with synthetic end-to-end intact.

uv.lock

@@ -441,6 +441,7 @@ dependencies = [
     { name = "openai" },
     { name = "opencv-python-headless" },
     { name = "pydantic" },
+    { name = "python-multipart" },
     { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
     { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
     { name = "soundfile" },
@@ -464,6 +465,7 @@ requires-dist = [
     { name = "opencv-python-headless", specifier = ">=4.9" },
     { name = "pydantic", specifier = ">=2.6" },
     { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" },
+    { name = "python-multipart", specifier = ">=0.0.9" },
     { name = "scipy", specifier = ">=1.11" },
     { name = "soundfile", specifier = ">=0.12" },
     { name = "sqlalchemy", specifier = ">=2.0" },
@@ -1276,6 +1278,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" },
 ]
 
+[[package]]
+name = "python-multipart"
+version = "0.0.32"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" },
+]
+
 [[package]]
 name = "pyyaml"
 version = "6.0.3"