From c24af8029e12b425efa609f302578dca7fabe2aa Mon Sep 17 00:00:00 2001
From: m3ultra <m3ultra@m3ultra.local>
Date: Fri, 17 Jul 2026 17:02:48 +1000
Subject: [PATCH] Audio is the master clock + spatial audio free-roam
 (fresh-eyes opts 1 & feature 1)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Master clock: the selected soundtrack (new GET /api/audio/{id} —
listening-quality stereo AAC, lazily ffmpeg-extracted and cached in
data/work/audio_hq/) plays through WebAudio, and t_global derives from
AudioContext.currentTime. Every <video> is muted picture-only, so hard
seeks and playbackRate corrections are inaudible (the unmuted video
used to warble on every nudge). Graceful fallback to the old
performance.now() clock + <video> audio when a track can't load, with
an in-place upgrade if the decode lands mid-play. Drift is compensated
in the track's playbackRate; _resync() now recomputes from the live
clock since the soundtrack keeps playing while rAF is throttled.

Spatial audio (🎧 3D audio, shown when poses exist): every enabled
camera's track plays through an HRTF PannerNode at its live
reconstructed pose, gains normalized 1/sqrt(N); the listener follows
the viewer camera each frame (free roam, follow-cam, and M9 paths).
Tracks that finish decoding mid-play join the mix in place; toggling,
seeking, rate and enable changes rebuild the graph at the current
t_global with no timeline jump.

The window.__f4d debug hook is now also available in production builds
via localStorage f4dDebug=1 (headless panes throttle rAF; the pump is
the only way to drive the loop there).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 README.md                    |  16 ++-
 backend/festival4d/api.py    |  28 ++++
 backend/festival4d/config.py |   1 +
 backend/festival4d/ingest.py |  25 ++++
 backend/tests/test_api.py    |  13 ++
 frontend/index.html          |   3 +
 frontend/src/main.js         |  13 +-
 frontend/src/scene3d.js      |  22 ++++
 frontend/src/state.js        |   3 +-
 frontend/src/transport.js    | 241 +++++++++++++++++++++++++++++++++--
 10 files changed, 346 insertions(+), 19 deletions(-)

diff --git a/README.md b/README.md
index cc7ae0a..07eae80 100644
--- a/README.md
+++ b/README.md
@@ -23,6 +23,9 @@ fully offline and local; the only optional cloud piece is moment classification.
   correct any misclassification.
 - **Click to locate things in 3D**: draw a box around an object in one or two videos and the
   app triangulates its 3D position, dropping an anchor that appears everywhere at once.
+- **Hear the scene in 3D** 🎧: toggle spatial audio and every camera becomes a positional
+  sound source at its reconstructed pose — fly toward a camera and you hear *that* spot in
+  the crowd get closer.
 
 ---
 
@@ -154,7 +157,8 @@ export GEMINI_API_KEY=...             # default provider (Gemini flash, native v
 | Play / pause | `Space` or the ⏵ button |
 | Nudge time ±1 s (±5 s) | `←` / `→` (hold `Shift`) |
 | Scrub | drag the timeline |
-| Pick the audio you hear | click a video (only one is unmuted at a time) |
+| Pick the audio you hear | click a video (one soundtrack at a time) |
+| Spatial audio | 🎧 **3D audio** (needs poses): every camera plays from its position in the scene; the mix follows the viewer camera as you fly |
 | Enable/disable a video | ◉ on the video |
 | Orbit the 3D scene | drag in the 3D pane |
 | Snap the 3D view to a camera | click its frustum, its ⛶ button, or press `1`–`9` |
@@ -194,9 +198,13 @@ data/raw/ ──ingest──▶ data/work/audio ──sync────▶ videos
   classifiers), `resolve` (annotation→3D), `api` (FastAPI, Range-capable video serving), `db`
   (SQLAlchemy/SQLite), `synthetic` (the test fixture).
 - **Frontend** — vanilla JS + Three.js. A single master clock (`transport.js`) derives each
-  video's local time from the shared timeline and continuously corrects playback; `lib/pose.js`
-  is the one COLMAP→Three.js pose conversion, mirrored from `geometry.py` and locked by a shared
-  test.
+  video's local time from the shared timeline and continuously corrects playback. When possible
+  the clock is **audio**: the selected soundtrack (`GET /api/audio/{id}`, extracted+cached
+  server-side) plays through WebAudio and `t_global` derives from `AudioContext.currentTime` —
+  so every video is muted picture-only and can be seeked/rate-trimmed with zero audible
+  artifacts (falls back to a `performance.now()` clock + `<video>` audio if the track can't
+  load). `lib/pose.js` is the one COLMAP→Three.js pose conversion, mirrored from `geometry.py`
+  and locked by a shared test.
 - **Timebase contract:** `t_video = (t_global − offset_ms/1000) · (1 + drift_ppm·1e−6)`; the
   reference video has offset 0.
 
diff --git a/backend/festival4d/api.py b/backend/festival4d/api.py
index 8d64ed6..8e90e79 100644
--- a/backend/festival4d/api.py
+++ b/backend/festival4d/api.py
@@ -194,6 +194,34 @@ def splat() -> FileResponse:
     )
 
 
+@app.get("/api/audio/{video_id}")
+def video_audio(video_id: int) -> FileResponse:
+    """Listening-quality audio track of a video (stereo AAC), for the WebAudio master clock.
+
+    Extracted lazily from the source video via ffmpeg on first request and cached in
+    ``data/work/audio_hq/`` (re-extracted if the source file is newer). The frontend
+    decodes this whole file with ``decodeAudioData`` and derives ``t_global`` from
+    ``AudioContext.currentTime`` — see ``frontend/src/transport.js``.
+    """
+    from festival4d import ingest
+    import subprocess
+
+    v = db.get_video(video_id)
+    if v is None:
+        raise HTTPException(status_code=404, detail=f"no video with id={video_id}")
+    src = config.RAW_DIR / v.filename
+    if not src.exists():
+        raise HTTPException(status_code=404, detail=f"source file missing: {v.filename}")
+    out = config.AUDIO_HQ_DIR / f"{video_id}.m4a"
+    if not out.exists() or out.stat().st_mtime < src.stat().st_mtime:
+        try:
+            ingest.extract_audio_hq(src, out)
+        except (subprocess.CalledProcessError, RuntimeError) as exc:
+            # No audio stream / no ffmpeg — the frontend falls back to <video> audio.
+            raise HTTPException(status_code=404, detail=f"no extractable audio: {exc}") from exc
+    return FileResponse(out, media_type="audio/mp4", filename=out.name)
+
+
 @app.get("/api/anchors")
 def get_anchors() -> list[dict]:
     return [_anchor_dict(a) for a in db.get_anchors()]
diff --git a/backend/festival4d/config.py b/backend/festival4d/config.py
index 492bd09..3615888 100644
--- a/backend/festival4d/config.py
+++ b/backend/festival4d/config.py
@@ -35,6 +35,7 @@ DATA_DIR = _data_dir()
 RAW_DIR = DATA_DIR / "raw"          # user drops source videos here (gitignored)
 WORK_DIR = DATA_DIR / "work"        # extracted wavs, frames, colmap workspace (gitignored)
 AUDIO_DIR = WORK_DIR / "audio"      # per-video mono 16 kHz wavs (ingest)
+AUDIO_HQ_DIR = WORK_DIR / "audio_hq"  # per-video listening-quality AAC (lazy, /api/audio/{id})
 FRAMES_DIR = WORK_DIR / "frames"    # sampled JPEGs for SfM
 COLMAP_DIR = WORK_DIR / "colmap"    # COLMAP workspace
 DB_PATH = DATA_DIR / "project.db"   # SQLite (gitignored)
diff --git a/backend/festival4d/ingest.py b/backend/festival4d/ingest.py
index e733941..75ecc2a 100644
--- a/backend/festival4d/ingest.py
+++ b/backend/festival4d/ingest.py
@@ -95,6 +95,31 @@ def extract_audio(path: Path, out_wav: Path, sample_rate: int = 16_000) -> Path:
     return out_wav
 
 
+def extract_audio_hq(path: Path, out_m4a: Path) -> Path:
+    """Extract a listening-quality stereo AAC track (48 kHz, 192 kbps) from ``path``.
+
+    Feeds the frontend's WebAudio master clock (``GET /api/audio/{id}``). AAC-in-m4a is
+    re-encoded (not stream-copied) so any source codec — including webm/Opus from browser
+    capture — decodes in every browser's ``decodeAudioData``. Written atomically (tmp +
+    rename) so a request racing the first extraction never reads a half-written file.
+    """
+    _require("ffmpeg")
+    out_m4a.parent.mkdir(parents=True, exist_ok=True)
+    tmp = out_m4a.with_suffix(".tmp.m4a")
+    subprocess.run(
+        ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error",
+         "-i", str(path),
+         "-vn",
+         "-ac", "2",
+         "-ar", "48000",
+         "-c:a", "aac", "-b:a", "192k",
+         str(tmp)],
+        check=True,
+    )
+    tmp.replace(out_m4a)
+    return out_m4a
+
+
 def run_ingest() -> list[dict]:
     """Ingest every video in ``config.RAW_DIR``: probe, register in DB, extract audio.
 
diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py
index 88a1a38..23d46a3 100644
--- a/backend/tests/test_api.py
+++ b/backend/tests/test_api.py
@@ -81,6 +81,19 @@ def test_pointcloud_is_ply(client):
     assert resp.content[:3] == b"ply"
 
 
+def test_audio_track_extracted_and_cached(client):
+    """GET /api/audio/{id}: lazily-extracted AAC for the WebAudio master clock."""
+    resp = client.get("/api/audio/1")
+    assert resp.status_code == 200
+    assert resp.headers["content-type"] == "audio/mp4"
+    assert len(resp.content) > 1000
+    # ftyp box near the start marks an MP4/M4A container.
+    assert b"ftyp" in resp.content[:64]
+    # Second request serves the cached file byte-identical (no re-extraction drift).
+    assert client.get("/api/audio/1").content == resp.content
+    assert client.get("/api/audio/999").status_code == 404
+
+
 def test_video_range_returns_206(client):
     """Spec pitfall #3: <video> seeking needs HTTP Range -> 206."""
     resp = client.get("/media/cam0.mp4", headers={"Range": "bytes=0-100"})
diff --git a/frontend/index.html b/frontend/index.html
index fcbb76e..0b12044 100644
--- a/frontend/index.html
+++ b/frontend/index.html
@@ -228,6 +228,9 @@
             <button class="btn" id="btn-path-export" title="Export path as JSON">⤓</button>
             <button class="btn" id="btn-path-import" title="Import path JSON">⤒</button>
             <button class="btn" id="btn-path-clear" title="Clear all keyframes">✕</button>
+            <span class="ctl-sep"></span>
+            <button class="btn" id="btn-spatial" style="display:none"
+                    title="Spatial audio: hear every camera from its position in the scene">🎧 3D audio</button>
             <input type="file" id="path-file" accept="application/json,.json" hidden />
           </div>
           <div id="scene-hint">drag to orbit · 1–9 snap to camera · K add keyframe · Esc detach</div>
diff --git a/frontend/src/main.js b/frontend/src/main.js
index 2595d8a..45cab0d 100644
--- a/frontend/src/main.js
+++ b/frontend/src/main.js
@@ -70,9 +70,10 @@ async function boot() {
     startLoop();
     loading.style.display = "none";
 
-    // Dev-only hook so the transport/scene can be driven in tests (e.g. a setTimeout pump when
-    // a headless pane throttles rAF). Never present in a production build.
-    if (import.meta.env?.DEV) {
+    // Debug hook so the transport/scene can be driven in tests (e.g. a setTimeout pump when
+    // a headless pane throttles rAF). Dev builds always; production only when opted in via
+    // localStorage.setItem("f4dDebug", "1").
+    if (import.meta.env?.DEV || localStorage.getItem("f4dDebug")) {
       window.__f4d = {
         state, transport, scene3d, timeline, cells: () => cells,
         poseAt, projectWorldToVideoPx,
@@ -190,6 +191,12 @@ function wireTransportControls() {
     }
     scene3d.freeRoam();
   });
+
+  // 🎧 spatial audio — only offered when there are camera poses to place emitters at.
+  const spatialBtn = el("btn-spatial");
+  if (state.hasPoses) spatialBtn.style.display = "";
+  spatialBtn.addEventListener("click", () => transport.setSpatial(!state.spatialAudio));
+  on("spatial", (enabled) => spatialBtn.classList.toggle("active", enabled));
 }
 
 function wireKeyboard() {
diff --git a/frontend/src/scene3d.js b/frontend/src/scene3d.js
index 172f2a3..542949a 100644
--- a/frontend/src/scene3d.js
+++ b/frontend/src/scene3d.js
@@ -10,6 +10,7 @@ import { colmapToThreejs } from "./lib/pose.js";
 import { poseAt } from "./lib/poseTrack.js";
 import { tVideoFromGlobal } from "./lib/timebase.js";
 import { state, emit } from "./state.js";
+import { transport } from "./transport.js";
 
 // Per-video colors (match the frustum, path line, and grid accents).
 export const VIDEO_COLORS = ["#ff5d73", "#4dd0ff", "#c9a2ff", "#ffcf5d", "#7dffb0", "#ff9d5d"];
@@ -25,6 +26,8 @@ const _m4 = new THREE.Matrix4();
 const _pos = new THREE.Vector3();
 const _quat = new THREE.Quaternion();
 const _scale = new THREE.Vector3();
+const _fwd = new THREE.Vector3();
+const _up = new THREE.Vector3();
 
 /** Build a camera-facing text label sprite (used for 3D anchor labels). */
 function makeLabelSprite(text) {
@@ -381,9 +384,28 @@ export class Scene3D {
       this.controls.update();
     }
 
+    this._updateAudio();
     this.renderer.render(this.scene, this.camera);
   }
 
+  // 🎧 Feed the transport's spatial audio: listener = viewer camera, emitters = camera rigs
+  // (rig.group already carries each camera's live pose from the loop above).
+  _updateAudio() {
+    if (!state.spatialAudio) return;
+    this.camera.getWorldPosition(_pos);
+    _fwd.set(0, 0, -1).applyQuaternion(this.camera.quaternion);
+    _up.set(0, 1, 0).applyQuaternion(this.camera.quaternion);
+    const positions = {};
+    for (const v of state.videos) {
+      const rig = this.rigs[v.id];
+      if (rig?.group.visible) positions[v.id] = rig.group.position;
+    }
+    transport.updateSpatial(
+      { px: _pos.x, py: _pos.y, pz: _pos.z, fx: _fwd.x, fy: _fwd.y, fz: _fwd.z, ux: _up.x, uy: _up.y, uz: _up.z },
+      positions
+    );
+  }
+
   _wirePicking() {
     const el = this.renderer.domElement;
     const ray = new THREE.Raycaster();
diff --git a/frontend/src/state.js b/frontend/src/state.js
index 748189f..220ae40 100644
--- a/frontend/src/state.js
+++ b/frontend/src/state.js
@@ -26,7 +26,8 @@ export const state = {
   tGlobal: 0,
   playing: false,
   rate: 1,
-  audioSourceId: null, // video id whose audio is unmuted (exactly one)
+  audioSourceId: null, // video id whose audio you hear (WebAudio soundtrack; fallback: unmuted <video>)
+  spatialAudio: false, // 🎧 every camera is a positional emitter at its 3D pose; listener = viewer cam
   enabled: {}, // id -> bool (per-video enable; disabled videos pause + dim + drop from sync)
   inRange: {}, // id -> bool (t_video within [0, duration] and enabled)
   syncErrorMs: {}, // id -> number | null (currentTime - target, ms; null when out of range)
diff --git a/frontend/src/transport.js b/frontend/src/transport.js
index 471174a..1400852 100644
--- a/frontend/src/transport.js
+++ b/frontend/src/transport.js
@@ -1,7 +1,14 @@
 // Master clock + per-video synchronization (spec M4 — the heart of the app).
 //
-// The master clock is derived from performance.now(); we NEVER trust a <video> element as the
-// clock (HTML5 video is not frame-accurate — pitfall #2). Correction is continuous:
+// AUDIO IS THE MASTER CLOCK when possible: the selected audio source's soundtrack (fetched from
+// /api/audio/{id}, decoded once) plays through WebAudio, and t_global is derived from
+// AudioContext.currentTime — sample-accurate, immune to rAF throttling, and NEVER corrected.
+// Every <video> element is muted picture-only and is seeked / rate-trimmed freely against that
+// clock with zero audible artifacts (previously the unmuted video's own corrections warbled).
+// If the track can't be fetched/decoded, we fall back per-session to the legacy behavior:
+// performance.now() clock + the selected video's element audio unmuted.
+// We NEVER trust a <video> element as the clock (HTML5 video is not frame-accurate —
+// pitfall #2). Correction is continuous:
 //   |err| > 150 ms          -> hard seek (fastSeek where available)
 //   20 ms < |err| <= 150 ms -> nudge playbackRate within master rate x[0.95, 1.05]
 //   |err| <= 20 ms          -> locked: playbackRate = master rate
@@ -12,9 +19,9 @@
 // better than sampling the frame-quantized currentTime in rAF). Browsers without rVFC fall back
 // to a currentTime measurement inside the rAF tick(). While PAUSED / seeking, videos are parked
 // on their target frame via currentTime. Videos whose target t_video falls outside [0, duration]
-// pause and are marked out-of-range (the grid dims them). Exactly one video's audio is unmuted.
+// pause and are marked out-of-range (the grid dims them).
 
-import { state, emit } from "./state.js";
+import { state, emit, on } from "./state.js";
 import { tVideoFromGlobal } from "./lib/timebase.js";
 
 const HARD_SEEK_S = 0.15; // > this error => hard seek
@@ -27,13 +34,29 @@ export class Transport {
     this._t0 = 0; // master-timeline anchor (seconds)
     this._wall0 = performance.now(); // wall-clock anchor (ms) matching _t0
     this._visWired = false;
+
+    // WebAudio master clock (see header). All lazy: nothing is created until needed.
+    this._actx = null; // AudioContext
+    this._ctx0 = 0; // AudioContext.currentTime anchor (s) matching _t0
+    this._audioActive = false; // audio clock is currently driving t_global
+    this._buffers = new Map(); // video id -> AudioBuffer (decoded) | null (unavailable)
+    this._decoding = new Set(); // video ids with a fetch/decode in flight
+    this._tracks = new Map(); // playing tracks: video id -> { src, gain, panner|null }
   }
 
   /** Register the video cells the transport drives and start their frame loops. */
   register(cells) {
     this.cells = cells;
     this._applyAudio();
+    this._loadTrack(state.audioSourceId); // start fetching the soundtrack immediately
     for (const cell of this.cells) this._startFrameLoop(cell);
+    if (!this._enabledWired) {
+      // In spatial mode a toggled video's emitter must join/leave the running mix.
+      on("enabled", () => {
+        if (state.spatialAudio) this._restartIfPlaying();
+      });
+      this._enabledWired = true;
+    }
     if (!this._visWired) {
       // A throttled/backgrounded tab stalls rAF while decoders free-run; re-lock on return.
       document.addEventListener("visibilitychange", () => {
@@ -46,7 +69,16 @@ export class Transport {
   /** Master-timeline seconds at a given wall-clock (ms) timestamp, clamped to [0, tGlobalMax]. */
   _clockAt(wallMs) {
     if (!state.playing) return this._t0;
-    const t = this._t0 + ((wallMs - this._wall0) / 1000) * state.rate;
+    let elapsed;
+    if (this._audioActive) {
+      // Audio clock: map the wall timestamp onto the AudioContext clock (the two clocks tick
+      // together; only their epochs differ, and this mapping is sub-ms), measure from anchor.
+      const ctxNow = this._actx.currentTime + (wallMs - performance.now()) / 1000;
+      elapsed = ctxNow - this._ctx0;
+    } else {
+      elapsed = (wallMs - this._wall0) / 1000;
+    }
+    const t = this._t0 + elapsed * state.rate;
     if (t <= 0) return 0;
     if (t >= state.tGlobalMax) return state.tGlobalMax;
     return t;
@@ -68,15 +100,17 @@ export class Transport {
     this._wall0 = performance.now();
     state.tGlobal = t;
     state.playing = true;
-    this._applyAudio(); // first play() is a user gesture -> browser lets us unmute
+    this._startAudio(t); // play() is a user gesture -> AudioContext may resume / videos unmute
     emit("play");
   }
 
   pause() {
     if (!state.playing) return;
-    state.tGlobal = this.computeTGlobal();
+    state.tGlobal = this.computeTGlobal(); // BEFORE stopping the audio clock's source node
     this._t0 = state.tGlobal;
     state.playing = false;
+    this._stopTracks();
+    this._audioActive = false;
     for (const { video } of this.cells) this._safePause(video);
     emit("pause");
   }
@@ -92,6 +126,7 @@ export class Transport {
     this._t0 = t;
     this._wall0 = performance.now();
     state.tGlobal = t;
+    if (state.playing) this._startAudio(t); // restart the soundtrack at the new position
     for (const cell of this.cells) {
       const target = this._targetFor(cell.meta, t);
       const inRange =
@@ -121,24 +156,205 @@ export class Transport {
     this._wall0 = performance.now();
     state.rate = r;
     for (const { video } of this.cells) video.playbackRate = r; // re-trimmed by correction
+    if (state.playing) this._startAudio(state.tGlobal); // restart the track at the new rate
     emit("rate", r);
   }
 
-  /** Choose which video's audio is unmuted (all others muted — avoids phasing chaos). */
+  /** Choose whose audio you hear (the WebAudio track when decoded, else that video unmuted). */
   setAudioSource(id) {
     state.audioSourceId = id;
-    this._applyAudio();
+    this._loadTrack(id);
+    this._restartIfPlaying();
     emit("audiosource", id);
   }
 
+  /** 🎧 Toggle spatial audio: every camera becomes a positional emitter at its 3D pose. */
+  setSpatial(enabled) {
+    state.spatialAudio = !!enabled;
+    if (state.spatialAudio) for (const { id } of this.cells) this._loadTrack(id);
+    this._restartIfPlaying();
+    emit("spatial", state.spatialAudio);
+  }
+
+  _restartIfPlaying() {
+    if (state.playing) this._startAudio(this.computeTGlobal());
+    else this._applyAudio();
+  }
+
   _applyAudio() {
+    // With a decoded soundtrack the WebAudio track is the only audible thing: every video is
+    // muted picture-only. Fallback (no track): the selected video's element audio, as before.
+    const trackReady = state.spatialAudio
+      ? [...this._buffers.values()].some((b) => b instanceof AudioBuffer)
+      : this._buffers.get(state.audioSourceId) instanceof AudioBuffer;
     for (const { id, video } of this.cells) {
-      const isSrc = id === state.audioSourceId;
+      const isSrc = !trackReady && id === state.audioSourceId;
       video.muted = !isSrc;
       if (isSrc) video.volume = 1;
     }
   }
 
+  // --- WebAudio master clock ----------------------------------------------------------------
+
+  _ensureCtx() {
+    if (!this._actx) this._actx = new (window.AudioContext || window.webkitAudioContext)();
+    return this._actx;
+  }
+
+  /** Fetch + decode a video's soundtrack once; upgrade the running clock when it lands. */
+  async _loadTrack(id) {
+    if (id == null || this._buffers.has(id) || this._decoding.has(id)) return;
+    if (typeof window === "undefined" || !(window.AudioContext || window.webkitAudioContext)) return;
+    this._decoding.add(id);
+    try {
+      const resp = await fetch(`${state.apiBase}/api/audio/${id}`);
+      if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
+      const buf = await this._ensureCtx().decodeAudioData(await resp.arrayBuffer());
+      // Decoded PCM is big (~23 MB/min stereo 48k): keep only a few tracks around
+      // (spatial mode needs one per camera, so the cap widens to the video count).
+      const cap = state.spatialAudio ? Math.max(this.cells.length, 3) : 3;
+      for (const key of this._buffers.keys()) {
+        if (this._buffers.size < cap) break;
+        if (key !== state.audioSourceId) this._buffers.delete(key);
+      }
+      this._buffers.set(id, buf);
+    } catch (err) {
+      console.warn(`[transport] soundtrack for video ${id} unavailable — using <video> audio`, err);
+      this._buffers.set(id, null);
+    } finally {
+      this._decoding.delete(id);
+    }
+    // A track landing mid-play joins in place: in spatial mode it enters the running mix; as
+    // the selected source it upgrades the fallback clock (both clocks agree on t_global).
+    const decoded = this._buffers.get(id) instanceof AudioBuffer;
+    if (
+      state.playing &&
+      decoded &&
+      (state.spatialAudio || (id === state.audioSourceId && !this._audioActive))
+    ) {
+      this._startAudio(this.computeTGlobal());
+    } else if (!state.playing) {
+      this._applyAudio();
+    }
+  }
+
+  /**
+   * (Re)start the soundtrack at t_global and re-anchor the master clock. Single-track mode
+   * plays the selected source; spatial mode plays every enabled camera's track through a
+   * PannerNode at its 3D pose. Falls back to the performance.now() clock (selected video
+   * unmuted) when no decoded track is available.
+   */
+  _startAudio(tGlobal) {
+    this._stopTracks();
+    this._t0 = tGlobal;
+    this._wall0 = performance.now();
+    const ready = (id) => this._buffers.get(id) instanceof AudioBuffer;
+    const ids = state.spatialAudio
+      ? this.cells.filter((c) => state.enabled[c.id] !== false && ready(c.id)).map((c) => c.id)
+      : ready(state.audioSourceId)
+        ? [state.audioSourceId]
+        : [];
+    if (ids.length === 0) {
+      this._audioActive = false;
+      this._applyAudio(); // fallback: unmute the selected video
+      return;
+    }
+    const ctx = this._ensureCtx();
+    if (ctx.state === "suspended") ctx.resume(); // play()/click gestures allow this
+    // N time-aligned copies of the same sound sum loud — normalize by 1/sqrt(N).
+    const gainValue = 1 / Math.sqrt(ids.length);
+    for (const id of ids) this._startTrack(ctx, id, tGlobal, state.spatialAudio, gainValue);
+    // Even with no live node (source out of range => silence), the audio clock drives time.
+    this._ctx0 = ctx.currentTime;
+    this._audioActive = true;
+    this._applyAudio();
+  }
+
+  _startTrack(ctx, id, tGlobal, spatial, gainValue) {
+    const buf = this._buffers.get(id);
+    const meta =
+      this.cells.find((c) => c.id === id)?.meta ?? state.videos.find((v) => v.id === id);
+    const drift = 1 + (meta?.drift_ppm || 0) * 1e-6;
+    const tv = tVideoFromGlobal(tGlobal, meta?.offset_ms || 0, meta?.drift_ppm || 0);
+    if (tv >= buf.duration) return; // this camera's audio is already over
+    const src = ctx.createBufferSource();
+    src.buffer = buf;
+    // The track's local time must advance at rate·(1+drift) per global second.
+    src.playbackRate.value = state.rate * drift;
+    const gain = ctx.createGain();
+    gain.gain.value = gainValue;
+    src.connect(gain);
+    let panner = null;
+    if (spatial) {
+      panner = ctx.createPanner();
+      panner.panningModel = "HRTF";
+      panner.distanceModel = "inverse";
+      panner.refDistance = 2; // scene units — cameras sit ~5–15 from the stage at COLMAP scale
+      panner.rolloffFactor = 0.6;
+      gain.connect(panner);
+      panner.connect(ctx.destination);
+    } else {
+      gain.connect(ctx.destination);
+    }
+    if (tv >= 0) src.start(0, tv);
+    else src.start(ctx.currentTime + -tv / state.rate, 0); // camera starts later than t_global
+    this._tracks.set(id, { src, gain, panner });
+  }
+
+  /**
+   * 🎧 Per-frame 3D audio update (called by scene3d): the listener follows the viewer camera
+   * and each playing track's panner sits at its camera's current pose. Takes plain numbers in
+   * scene (Three.js) space so the transport stays THREE-free.
+   */
+  updateSpatial(listener, positions) {
+    if (!state.spatialAudio || !this._audioActive || !this._actx) return;
+    const L = this._actx.listener;
+    if (L.positionX) {
+      L.positionX.value = listener.px;
+      L.positionY.value = listener.py;
+      L.positionZ.value = listener.pz;
+      L.forwardX.value = listener.fx;
+      L.forwardY.value = listener.fy;
+      L.forwardZ.value = listener.fz;
+      L.upX.value = listener.ux;
+      L.upY.value = listener.uy;
+      L.upZ.value = listener.uz;
+    } else {
+      // Safari: AudioListener still uses the legacy setters.
+      L.setPosition(listener.px, listener.py, listener.pz);
+      L.setOrientation(listener.fx, listener.fy, listener.fz, listener.ux, listener.uy, listener.uz);
+    }
+    for (const [id, { panner }] of this._tracks) {
+      const pos = positions[id];
+      if (!panner || !pos) continue;
+      if (panner.positionX) {
+        panner.positionX.value = pos.x;
+        panner.positionY.value = pos.y;
+        panner.positionZ.value = pos.z;
+      } else {
+        panner.setPosition(pos.x, pos.y, pos.z);
+      }
+    }
+  }
+
+  _stopTracks() {
+    for (const { src, gain, panner } of this._tracks.values()) {
+      try {
+        src.stop();
+      } catch {
+        /* never started or already ended */
+      }
+      try {
+        src.disconnect();
+        gain.disconnect();
+        panner?.disconnect();
+      } catch {
+        /* ignore */
+      }
+    }
+    this._tracks.clear();
+  }
+
   // Per-presented-frame correction via requestVideoFrameCallback (accurate). Self-sustaining:
   // each callback re-registers the next. No-ops while paused / out of range; rAF tick() handles
   // the fallback path for browsers without rVFC.
@@ -192,7 +408,10 @@ export class Transport {
 
   /** Re-lock every in-range video to the current master time (after a tab-visibility stall). */
   _resync() {
-    const t = state.tGlobal;
+    // Compute fresh: with the audio clock the soundtrack kept playing while rAF (and therefore
+    // state.tGlobal) was frozen, so the live clock may be well past the last stored value.
+    const t = this.computeTGlobal();
+    state.tGlobal = t;
     for (const cell of this.cells) {
       const target = this._targetFor(cell.meta, t);
       if (state.enabled[cell.id] !== false && target >= 0 && target <= cell.meta.duration_s) {