Audio is the master clock + spatial audio free-roam (fresh-eyes opts 1 & feature 1)
Master clock: the selected soundtrack (new GET /api/audio/{id} —
listening-quality stereo AAC, lazily ffmpeg-extracted and cached in
data/work/audio_hq/) plays through WebAudio, and t_global derives from
AudioContext.currentTime. Every <video> is muted picture-only, so hard
seeks and playbackRate corrections are inaudible (the unmuted video
used to warble on every nudge). Graceful fallback to the old
performance.now() clock + <video> audio when a track can't load, with
an in-place upgrade if the decode lands mid-play. Drift is compensated
in the track's playbackRate; _resync() now recomputes from the live
clock since the soundtrack keeps playing while rAF is throttled.
Spatial audio (🎧 3D audio, shown when poses exist): every enabled
camera's track plays through an HRTF PannerNode at its live
reconstructed pose, gains normalized 1/sqrt(N); the listener follows
the viewer camera each frame (free roam, follow-cam, and M9 paths).
Tracks that finish decoding mid-play join the mix in place; toggling,
seeking, rate and enable changes rebuild the graph at the current
t_global with no timeline jump.
The window.__f4d debug hook is now also available in production builds
via localStorage f4dDebug=1 (headless panes throttle rAF; the pump is
the only way to drive the loop there).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
m3ultra
parent
95b71788a3
commit
c24af8029e
16
README.md
16
README.md
@ -23,6 +23,9 @@ fully offline and local; the only optional cloud piece is moment classification.
|
||||
correct any misclassification.
|
||||
- **Click to locate things in 3D**: draw a box around an object in one or two videos and the
|
||||
app triangulates its 3D position, dropping an anchor that appears everywhere at once.
|
||||
- **Hear the scene in 3D** 🎧: toggle spatial audio and every camera becomes a positional
|
||||
sound source at its reconstructed pose — fly toward a camera and you hear *that* spot in
|
||||
the crowd get closer.
|
||||
|
||||
---
|
||||
|
||||
@ -154,7 +157,8 @@ export GEMINI_API_KEY=... # default provider (Gemini flash, native v
|
||||
| Play / pause | `Space` or the ⏵ button |
|
||||
| Nudge time ±1 s (±5 s) | `←` / `→` (hold `Shift`) |
|
||||
| Scrub | drag the timeline |
|
||||
| Pick the audio you hear | click a video (only one is unmuted at a time) |
|
||||
| Pick the audio you hear | click a video (one soundtrack at a time) |
|
||||
| Spatial audio | 🎧 **3D audio** (needs poses): every camera plays from its position in the scene; the mix follows the viewer camera as you fly |
|
||||
| Enable/disable a video | ◉ on the video |
|
||||
| Orbit the 3D scene | drag in the 3D pane |
|
||||
| Snap the 3D view to a camera | click its frustum, its ⛶ button, or press `1`–`9` |
|
||||
@ -194,9 +198,13 @@ data/raw/ ──ingest──▶ data/work/audio ──sync────▶ videos
|
||||
classifiers), `resolve` (annotation→3D), `api` (FastAPI, Range-capable video serving), `db`
|
||||
(SQLAlchemy/SQLite), `synthetic` (the test fixture).
|
||||
- **Frontend** — vanilla JS + Three.js. A single master clock (`transport.js`) derives each
|
||||
video's local time from the shared timeline and continuously corrects playback; `lib/pose.js`
|
||||
is the one COLMAP→Three.js pose conversion, mirrored from `geometry.py` and locked by a shared
|
||||
test.
|
||||
video's local time from the shared timeline and continuously corrects playback. When possible
|
||||
the clock is **audio**: the selected soundtrack (`GET /api/audio/{id}`, extracted+cached
|
||||
server-side) plays through WebAudio and `t_global` derives from `AudioContext.currentTime` —
|
||||
so every video is muted picture-only and can be seeked/rate-trimmed with zero audible
|
||||
artifacts (falls back to a `performance.now()` clock + `<video>` audio if the track can't
|
||||
load). `lib/pose.js` is the one COLMAP→Three.js pose conversion, mirrored from `geometry.py`
|
||||
and locked by a shared test.
|
||||
- **Timebase contract:** `t_video = (t_global − offset_ms/1000) · (1 + drift_ppm·1e−6)`; the
|
||||
reference video has offset 0.
|
||||
|
||||
|
||||
@ -194,6 +194,34 @@ def splat() -> FileResponse:
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/audio/{video_id}")
|
||||
def video_audio(video_id: int) -> FileResponse:
|
||||
"""Listening-quality audio track of a video (stereo AAC), for the WebAudio master clock.
|
||||
|
||||
Extracted lazily from the source video via ffmpeg on first request and cached in
|
||||
``data/work/audio_hq/`` (re-extracted if the source file is newer). The frontend
|
||||
decodes this whole file with ``decodeAudioData`` and derives ``t_global`` from
|
||||
``AudioContext.currentTime`` — see ``frontend/src/transport.js``.
|
||||
"""
|
||||
from festival4d import ingest
|
||||
import subprocess
|
||||
|
||||
v = db.get_video(video_id)
|
||||
if v is None:
|
||||
raise HTTPException(status_code=404, detail=f"no video with id={video_id}")
|
||||
src = config.RAW_DIR / v.filename
|
||||
if not src.exists():
|
||||
raise HTTPException(status_code=404, detail=f"source file missing: {v.filename}")
|
||||
out = config.AUDIO_HQ_DIR / f"{video_id}.m4a"
|
||||
if not out.exists() or out.stat().st_mtime < src.stat().st_mtime:
|
||||
try:
|
||||
ingest.extract_audio_hq(src, out)
|
||||
except (subprocess.CalledProcessError, RuntimeError) as exc:
|
||||
# No audio stream / no ffmpeg — the frontend falls back to <video> audio.
|
||||
raise HTTPException(status_code=404, detail=f"no extractable audio: {exc}") from exc
|
||||
return FileResponse(out, media_type="audio/mp4", filename=out.name)
|
||||
|
||||
|
||||
@app.get("/api/anchors")
|
||||
def get_anchors() -> list[dict]:
|
||||
return [_anchor_dict(a) for a in db.get_anchors()]
|
||||
|
||||
@ -35,6 +35,7 @@ DATA_DIR = _data_dir()
|
||||
RAW_DIR = DATA_DIR / "raw" # user drops source videos here (gitignored)
|
||||
WORK_DIR = DATA_DIR / "work" # extracted wavs, frames, colmap workspace (gitignored)
|
||||
AUDIO_DIR = WORK_DIR / "audio" # per-video mono 16 kHz wavs (ingest)
|
||||
AUDIO_HQ_DIR = WORK_DIR / "audio_hq" # per-video listening-quality AAC (lazy, /api/audio/{id})
|
||||
FRAMES_DIR = WORK_DIR / "frames" # sampled JPEGs for SfM
|
||||
COLMAP_DIR = WORK_DIR / "colmap" # COLMAP workspace
|
||||
DB_PATH = DATA_DIR / "project.db" # SQLite (gitignored)
|
||||
|
||||
@ -95,6 +95,31 @@ def extract_audio(path: Path, out_wav: Path, sample_rate: int = 16_000) -> Path:
|
||||
return out_wav
|
||||
|
||||
|
||||
def extract_audio_hq(path: Path, out_m4a: Path) -> Path:
|
||||
"""Extract a listening-quality stereo AAC track (48 kHz, 192 kbps) from ``path``.
|
||||
|
||||
Feeds the frontend's WebAudio master clock (``GET /api/audio/{id}``). AAC-in-m4a is
|
||||
re-encoded (not stream-copied) so any source codec — including webm/Opus from browser
|
||||
capture — decodes in every browser's ``decodeAudioData``. Written atomically (tmp +
|
||||
rename) so a request racing the first extraction never reads a half-written file.
|
||||
"""
|
||||
_require("ffmpeg")
|
||||
out_m4a.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = out_m4a.with_suffix(".tmp.m4a")
|
||||
subprocess.run(
|
||||
["ffmpeg", "-y", "-hide_banner", "-loglevel", "error",
|
||||
"-i", str(path),
|
||||
"-vn",
|
||||
"-ac", "2",
|
||||
"-ar", "48000",
|
||||
"-c:a", "aac", "-b:a", "192k",
|
||||
str(tmp)],
|
||||
check=True,
|
||||
)
|
||||
tmp.replace(out_m4a)
|
||||
return out_m4a
|
||||
|
||||
|
||||
def run_ingest() -> list[dict]:
|
||||
"""Ingest every video in ``config.RAW_DIR``: probe, register in DB, extract audio.
|
||||
|
||||
|
||||
@ -81,6 +81,19 @@ def test_pointcloud_is_ply(client):
|
||||
assert resp.content[:3] == b"ply"
|
||||
|
||||
|
||||
def test_audio_track_extracted_and_cached(client):
|
||||
"""GET /api/audio/{id}: lazily-extracted AAC for the WebAudio master clock."""
|
||||
resp = client.get("/api/audio/1")
|
||||
assert resp.status_code == 200
|
||||
assert resp.headers["content-type"] == "audio/mp4"
|
||||
assert len(resp.content) > 1000
|
||||
# ftyp box near the start marks an MP4/M4A container.
|
||||
assert b"ftyp" in resp.content[:64]
|
||||
# Second request serves the cached file byte-identical (no re-extraction drift).
|
||||
assert client.get("/api/audio/1").content == resp.content
|
||||
assert client.get("/api/audio/999").status_code == 404
|
||||
|
||||
|
||||
def test_video_range_returns_206(client):
|
||||
"""Spec pitfall #3: <video> seeking needs HTTP Range -> 206."""
|
||||
resp = client.get("/media/cam0.mp4", headers={"Range": "bytes=0-100"})
|
||||
|
||||
@ -228,6 +228,9 @@
|
||||
<button class="btn" id="btn-path-export" title="Export path as JSON">⤓</button>
|
||||
<button class="btn" id="btn-path-import" title="Import path JSON">⤒</button>
|
||||
<button class="btn" id="btn-path-clear" title="Clear all keyframes">✕</button>
|
||||
<span class="ctl-sep"></span>
|
||||
<button class="btn" id="btn-spatial" style="display:none"
|
||||
title="Spatial audio: hear every camera from its position in the scene">🎧 3D audio</button>
|
||||
<input type="file" id="path-file" accept="application/json,.json" hidden />
|
||||
</div>
|
||||
<div id="scene-hint">drag to orbit · 1–9 snap to camera · K add keyframe · Esc detach</div>
|
||||
|
||||
@ -70,9 +70,10 @@ async function boot() {
|
||||
startLoop();
|
||||
loading.style.display = "none";
|
||||
|
||||
// Dev-only hook so the transport/scene can be driven in tests (e.g. a setTimeout pump when
|
||||
// a headless pane throttles rAF). Never present in a production build.
|
||||
if (import.meta.env?.DEV) {
|
||||
// Debug hook so the transport/scene can be driven in tests (e.g. a setTimeout pump when
|
||||
// a headless pane throttles rAF). Dev builds always; production only when opted in via
|
||||
// localStorage.setItem("f4dDebug", "1").
|
||||
if (import.meta.env?.DEV || localStorage.getItem("f4dDebug")) {
|
||||
window.__f4d = {
|
||||
state, transport, scene3d, timeline, cells: () => cells,
|
||||
poseAt, projectWorldToVideoPx,
|
||||
@ -190,6 +191,12 @@ function wireTransportControls() {
|
||||
}
|
||||
scene3d.freeRoam();
|
||||
});
|
||||
|
||||
// 🎧 spatial audio — only offered when there are camera poses to place emitters at.
|
||||
const spatialBtn = el("btn-spatial");
|
||||
if (state.hasPoses) spatialBtn.style.display = "";
|
||||
spatialBtn.addEventListener("click", () => transport.setSpatial(!state.spatialAudio));
|
||||
on("spatial", (enabled) => spatialBtn.classList.toggle("active", enabled));
|
||||
}
|
||||
|
||||
function wireKeyboard() {
|
||||
|
||||
@ -10,6 +10,7 @@ import { colmapToThreejs } from "./lib/pose.js";
|
||||
import { poseAt } from "./lib/poseTrack.js";
|
||||
import { tVideoFromGlobal } from "./lib/timebase.js";
|
||||
import { state, emit } from "./state.js";
|
||||
import { transport } from "./transport.js";
|
||||
|
||||
// Per-video colors (match the frustum, path line, and grid accents).
|
||||
export const VIDEO_COLORS = ["#ff5d73", "#4dd0ff", "#c9a2ff", "#ffcf5d", "#7dffb0", "#ff9d5d"];
|
||||
@ -25,6 +26,8 @@ const _m4 = new THREE.Matrix4();
|
||||
const _pos = new THREE.Vector3();
|
||||
const _quat = new THREE.Quaternion();
|
||||
const _scale = new THREE.Vector3();
|
||||
const _fwd = new THREE.Vector3();
|
||||
const _up = new THREE.Vector3();
|
||||
|
||||
/** Build a camera-facing text label sprite (used for 3D anchor labels). */
|
||||
function makeLabelSprite(text) {
|
||||
@ -381,9 +384,28 @@ export class Scene3D {
|
||||
this.controls.update();
|
||||
}
|
||||
|
||||
this._updateAudio();
|
||||
this.renderer.render(this.scene, this.camera);
|
||||
}
|
||||
|
||||
// 🎧 Feed the transport's spatial audio: listener = viewer camera, emitters = camera rigs
|
||||
// (rig.group already carries each camera's live pose from the loop above).
|
||||
_updateAudio() {
|
||||
if (!state.spatialAudio) return;
|
||||
this.camera.getWorldPosition(_pos);
|
||||
_fwd.set(0, 0, -1).applyQuaternion(this.camera.quaternion);
|
||||
_up.set(0, 1, 0).applyQuaternion(this.camera.quaternion);
|
||||
const positions = {};
|
||||
for (const v of state.videos) {
|
||||
const rig = this.rigs[v.id];
|
||||
if (rig?.group.visible) positions[v.id] = rig.group.position;
|
||||
}
|
||||
transport.updateSpatial(
|
||||
{ px: _pos.x, py: _pos.y, pz: _pos.z, fx: _fwd.x, fy: _fwd.y, fz: _fwd.z, ux: _up.x, uy: _up.y, uz: _up.z },
|
||||
positions
|
||||
);
|
||||
}
|
||||
|
||||
_wirePicking() {
|
||||
const el = this.renderer.domElement;
|
||||
const ray = new THREE.Raycaster();
|
||||
|
||||
@ -26,7 +26,8 @@ export const state = {
|
||||
tGlobal: 0,
|
||||
playing: false,
|
||||
rate: 1,
|
||||
audioSourceId: null, // video id whose audio is unmuted (exactly one)
|
||||
audioSourceId: null, // video id whose audio you hear (WebAudio soundtrack; fallback: unmuted <video>)
|
||||
spatialAudio: false, // 🎧 every camera is a positional emitter at its 3D pose; listener = viewer cam
|
||||
enabled: {}, // id -> bool (per-video enable; disabled videos pause + dim + drop from sync)
|
||||
inRange: {}, // id -> bool (t_video within [0, duration] and enabled)
|
||||
syncErrorMs: {}, // id -> number | null (currentTime - target, ms; null when out of range)
|
||||
|
||||
@ -1,7 +1,14 @@
|
||||
// Master clock + per-video synchronization (spec M4 — the heart of the app).
|
||||
//
|
||||
// The master clock is derived from performance.now(); we NEVER trust a <video> element as the
|
||||
// clock (HTML5 video is not frame-accurate — pitfall #2). Correction is continuous:
|
||||
// AUDIO IS THE MASTER CLOCK when possible: the selected audio source's soundtrack (fetched from
|
||||
// /api/audio/{id}, decoded once) plays through WebAudio, and t_global is derived from
|
||||
// AudioContext.currentTime — sample-accurate, immune to rAF throttling, and NEVER corrected.
|
||||
// Every <video> element is muted picture-only and is seeked / rate-trimmed freely against that
|
||||
// clock with zero audible artifacts (previously the unmuted video's own corrections warbled).
|
||||
// If the track can't be fetched/decoded, we fall back per-session to the legacy behavior:
|
||||
// performance.now() clock + the selected video's element audio unmuted.
|
||||
// We NEVER trust a <video> element as the clock (HTML5 video is not frame-accurate —
|
||||
// pitfall #2). Correction is continuous:
|
||||
// |err| > 150 ms -> hard seek (fastSeek where available)
|
||||
// 20 ms < |err| <= 150 ms -> nudge playbackRate within master rate x[0.95, 1.05]
|
||||
// |err| <= 20 ms -> locked: playbackRate = master rate
|
||||
@ -12,9 +19,9 @@
|
||||
// better than sampling the frame-quantized currentTime in rAF). Browsers without rVFC fall back
|
||||
// to a currentTime measurement inside the rAF tick(). While PAUSED / seeking, videos are parked
|
||||
// on their target frame via currentTime. Videos whose target t_video falls outside [0, duration]
|
||||
// pause and are marked out-of-range (the grid dims them). Exactly one video's audio is unmuted.
|
||||
// pause and are marked out-of-range (the grid dims them).
|
||||
|
||||
import { state, emit } from "./state.js";
|
||||
import { state, emit, on } from "./state.js";
|
||||
import { tVideoFromGlobal } from "./lib/timebase.js";
|
||||
|
||||
const HARD_SEEK_S = 0.15; // > this error => hard seek
|
||||
@ -27,13 +34,29 @@ export class Transport {
|
||||
this._t0 = 0; // master-timeline anchor (seconds)
|
||||
this._wall0 = performance.now(); // wall-clock anchor (ms) matching _t0
|
||||
this._visWired = false;
|
||||
|
||||
// WebAudio master clock (see header). All lazy: nothing is created until needed.
|
||||
this._actx = null; // AudioContext
|
||||
this._ctx0 = 0; // AudioContext.currentTime anchor (s) matching _t0
|
||||
this._audioActive = false; // audio clock is currently driving t_global
|
||||
this._buffers = new Map(); // video id -> AudioBuffer (decoded) | null (unavailable)
|
||||
this._decoding = new Set(); // video ids with a fetch/decode in flight
|
||||
this._tracks = new Map(); // playing tracks: video id -> { src, gain, panner|null }
|
||||
}
|
||||
|
||||
/** Register the video cells the transport drives and start their frame loops. */
|
||||
register(cells) {
|
||||
this.cells = cells;
|
||||
this._applyAudio();
|
||||
this._loadTrack(state.audioSourceId); // start fetching the soundtrack immediately
|
||||
for (const cell of this.cells) this._startFrameLoop(cell);
|
||||
if (!this._enabledWired) {
|
||||
// In spatial mode a toggled video's emitter must join/leave the running mix.
|
||||
on("enabled", () => {
|
||||
if (state.spatialAudio) this._restartIfPlaying();
|
||||
});
|
||||
this._enabledWired = true;
|
||||
}
|
||||
if (!this._visWired) {
|
||||
// A throttled/backgrounded tab stalls rAF while decoders free-run; re-lock on return.
|
||||
document.addEventListener("visibilitychange", () => {
|
||||
@ -46,7 +69,16 @@ export class Transport {
|
||||
/** Master-timeline seconds at a given wall-clock (ms) timestamp, clamped to [0, tGlobalMax]. */
|
||||
_clockAt(wallMs) {
|
||||
if (!state.playing) return this._t0;
|
||||
const t = this._t0 + ((wallMs - this._wall0) / 1000) * state.rate;
|
||||
let elapsed;
|
||||
if (this._audioActive) {
|
||||
// Audio clock: map the wall timestamp onto the AudioContext clock (the two clocks tick
|
||||
// together; only their epochs differ, and this mapping is sub-ms), measure from anchor.
|
||||
const ctxNow = this._actx.currentTime + (wallMs - performance.now()) / 1000;
|
||||
elapsed = ctxNow - this._ctx0;
|
||||
} else {
|
||||
elapsed = (wallMs - this._wall0) / 1000;
|
||||
}
|
||||
const t = this._t0 + elapsed * state.rate;
|
||||
if (t <= 0) return 0;
|
||||
if (t >= state.tGlobalMax) return state.tGlobalMax;
|
||||
return t;
|
||||
@ -68,15 +100,17 @@ export class Transport {
|
||||
this._wall0 = performance.now();
|
||||
state.tGlobal = t;
|
||||
state.playing = true;
|
||||
this._applyAudio(); // first play() is a user gesture -> browser lets us unmute
|
||||
this._startAudio(t); // play() is a user gesture -> AudioContext may resume / videos unmute
|
||||
emit("play");
|
||||
}
|
||||
|
||||
pause() {
|
||||
if (!state.playing) return;
|
||||
state.tGlobal = this.computeTGlobal();
|
||||
state.tGlobal = this.computeTGlobal(); // BEFORE stopping the audio clock's source node
|
||||
this._t0 = state.tGlobal;
|
||||
state.playing = false;
|
||||
this._stopTracks();
|
||||
this._audioActive = false;
|
||||
for (const { video } of this.cells) this._safePause(video);
|
||||
emit("pause");
|
||||
}
|
||||
@ -92,6 +126,7 @@ export class Transport {
|
||||
this._t0 = t;
|
||||
this._wall0 = performance.now();
|
||||
state.tGlobal = t;
|
||||
if (state.playing) this._startAudio(t); // restart the soundtrack at the new position
|
||||
for (const cell of this.cells) {
|
||||
const target = this._targetFor(cell.meta, t);
|
||||
const inRange =
|
||||
@ -121,24 +156,205 @@ export class Transport {
|
||||
this._wall0 = performance.now();
|
||||
state.rate = r;
|
||||
for (const { video } of this.cells) video.playbackRate = r; // re-trimmed by correction
|
||||
if (state.playing) this._startAudio(state.tGlobal); // restart the track at the new rate
|
||||
emit("rate", r);
|
||||
}
|
||||
|
||||
/** Choose which video's audio is unmuted (all others muted — avoids phasing chaos). */
|
||||
/** Choose whose audio you hear (the WebAudio track when decoded, else that video unmuted). */
|
||||
setAudioSource(id) {
|
||||
state.audioSourceId = id;
|
||||
this._applyAudio();
|
||||
this._loadTrack(id);
|
||||
this._restartIfPlaying();
|
||||
emit("audiosource", id);
|
||||
}
|
||||
|
||||
/** 🎧 Toggle spatial audio: every camera becomes a positional emitter at its 3D pose. */
|
||||
setSpatial(enabled) {
|
||||
state.spatialAudio = !!enabled;
|
||||
if (state.spatialAudio) for (const { id } of this.cells) this._loadTrack(id);
|
||||
this._restartIfPlaying();
|
||||
emit("spatial", state.spatialAudio);
|
||||
}
|
||||
|
||||
_restartIfPlaying() {
|
||||
if (state.playing) this._startAudio(this.computeTGlobal());
|
||||
else this._applyAudio();
|
||||
}
|
||||
|
||||
_applyAudio() {
|
||||
// With a decoded soundtrack the WebAudio track is the only audible thing: every video is
|
||||
// muted picture-only. Fallback (no track): the selected video's element audio, as before.
|
||||
const trackReady = state.spatialAudio
|
||||
? [...this._buffers.values()].some((b) => b instanceof AudioBuffer)
|
||||
: this._buffers.get(state.audioSourceId) instanceof AudioBuffer;
|
||||
for (const { id, video } of this.cells) {
|
||||
const isSrc = id === state.audioSourceId;
|
||||
const isSrc = !trackReady && id === state.audioSourceId;
|
||||
video.muted = !isSrc;
|
||||
if (isSrc) video.volume = 1;
|
||||
}
|
||||
}
|
||||
|
||||
// --- WebAudio master clock ----------------------------------------------------------------
|
||||
|
||||
_ensureCtx() {
|
||||
if (!this._actx) this._actx = new (window.AudioContext || window.webkitAudioContext)();
|
||||
return this._actx;
|
||||
}
|
||||
|
||||
/** Fetch + decode a video's soundtrack once; upgrade the running clock when it lands. */
|
||||
async _loadTrack(id) {
|
||||
if (id == null || this._buffers.has(id) || this._decoding.has(id)) return;
|
||||
if (typeof window === "undefined" || !(window.AudioContext || window.webkitAudioContext)) return;
|
||||
this._decoding.add(id);
|
||||
try {
|
||||
const resp = await fetch(`${state.apiBase}/api/audio/${id}`);
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
||||
const buf = await this._ensureCtx().decodeAudioData(await resp.arrayBuffer());
|
||||
// Decoded PCM is big (~23 MB/min stereo 48k): keep only a few tracks around
|
||||
// (spatial mode needs one per camera, so the cap widens to the video count).
|
||||
const cap = state.spatialAudio ? Math.max(this.cells.length, 3) : 3;
|
||||
for (const key of this._buffers.keys()) {
|
||||
if (this._buffers.size < cap) break;
|
||||
if (key !== state.audioSourceId) this._buffers.delete(key);
|
||||
}
|
||||
this._buffers.set(id, buf);
|
||||
} catch (err) {
|
||||
console.warn(`[transport] soundtrack for video ${id} unavailable — using <video> audio`, err);
|
||||
this._buffers.set(id, null);
|
||||
} finally {
|
||||
this._decoding.delete(id);
|
||||
}
|
||||
// A track landing mid-play joins in place: in spatial mode it enters the running mix; as
|
||||
// the selected source it upgrades the fallback clock (both clocks agree on t_global).
|
||||
const decoded = this._buffers.get(id) instanceof AudioBuffer;
|
||||
if (
|
||||
state.playing &&
|
||||
decoded &&
|
||||
(state.spatialAudio || (id === state.audioSourceId && !this._audioActive))
|
||||
) {
|
||||
this._startAudio(this.computeTGlobal());
|
||||
} else if (!state.playing) {
|
||||
this._applyAudio();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* (Re)start the soundtrack at t_global and re-anchor the master clock. Single-track mode
|
||||
* plays the selected source; spatial mode plays every enabled camera's track through a
|
||||
* PannerNode at its 3D pose. Falls back to the performance.now() clock (selected video
|
||||
* unmuted) when no decoded track is available.
|
||||
*/
|
||||
_startAudio(tGlobal) {
|
||||
this._stopTracks();
|
||||
this._t0 = tGlobal;
|
||||
this._wall0 = performance.now();
|
||||
const ready = (id) => this._buffers.get(id) instanceof AudioBuffer;
|
||||
const ids = state.spatialAudio
|
||||
? this.cells.filter((c) => state.enabled[c.id] !== false && ready(c.id)).map((c) => c.id)
|
||||
: ready(state.audioSourceId)
|
||||
? [state.audioSourceId]
|
||||
: [];
|
||||
if (ids.length === 0) {
|
||||
this._audioActive = false;
|
||||
this._applyAudio(); // fallback: unmute the selected video
|
||||
return;
|
||||
}
|
||||
const ctx = this._ensureCtx();
|
||||
if (ctx.state === "suspended") ctx.resume(); // play()/click gestures allow this
|
||||
// N time-aligned copies of the same sound sum loud — normalize by 1/sqrt(N).
|
||||
const gainValue = 1 / Math.sqrt(ids.length);
|
||||
for (const id of ids) this._startTrack(ctx, id, tGlobal, state.spatialAudio, gainValue);
|
||||
// Even with no live node (source out of range => silence), the audio clock drives time.
|
||||
this._ctx0 = ctx.currentTime;
|
||||
this._audioActive = true;
|
||||
this._applyAudio();
|
||||
}
|
||||
|
||||
_startTrack(ctx, id, tGlobal, spatial, gainValue) {
|
||||
const buf = this._buffers.get(id);
|
||||
const meta =
|
||||
this.cells.find((c) => c.id === id)?.meta ?? state.videos.find((v) => v.id === id);
|
||||
const drift = 1 + (meta?.drift_ppm || 0) * 1e-6;
|
||||
const tv = tVideoFromGlobal(tGlobal, meta?.offset_ms || 0, meta?.drift_ppm || 0);
|
||||
if (tv >= buf.duration) return; // this camera's audio is already over
|
||||
const src = ctx.createBufferSource();
|
||||
src.buffer = buf;
|
||||
// The track's local time must advance at rate·(1+drift) per global second.
|
||||
src.playbackRate.value = state.rate * drift;
|
||||
const gain = ctx.createGain();
|
||||
gain.gain.value = gainValue;
|
||||
src.connect(gain);
|
||||
let panner = null;
|
||||
if (spatial) {
|
||||
panner = ctx.createPanner();
|
||||
panner.panningModel = "HRTF";
|
||||
panner.distanceModel = "inverse";
|
||||
panner.refDistance = 2; // scene units — cameras sit ~5–15 from the stage at COLMAP scale
|
||||
panner.rolloffFactor = 0.6;
|
||||
gain.connect(panner);
|
||||
panner.connect(ctx.destination);
|
||||
} else {
|
||||
gain.connect(ctx.destination);
|
||||
}
|
||||
if (tv >= 0) src.start(0, tv);
|
||||
else src.start(ctx.currentTime + -tv / state.rate, 0); // camera starts later than t_global
|
||||
this._tracks.set(id, { src, gain, panner });
|
||||
}
|
||||
|
||||
/**
|
||||
* 🎧 Per-frame 3D audio update (called by scene3d): the listener follows the viewer camera
|
||||
* and each playing track's panner sits at its camera's current pose. Takes plain numbers in
|
||||
* scene (Three.js) space so the transport stays THREE-free.
|
||||
*/
|
||||
updateSpatial(listener, positions) {
|
||||
if (!state.spatialAudio || !this._audioActive || !this._actx) return;
|
||||
const L = this._actx.listener;
|
||||
if (L.positionX) {
|
||||
L.positionX.value = listener.px;
|
||||
L.positionY.value = listener.py;
|
||||
L.positionZ.value = listener.pz;
|
||||
L.forwardX.value = listener.fx;
|
||||
L.forwardY.value = listener.fy;
|
||||
L.forwardZ.value = listener.fz;
|
||||
L.upX.value = listener.ux;
|
||||
L.upY.value = listener.uy;
|
||||
L.upZ.value = listener.uz;
|
||||
} else {
|
||||
// Safari: AudioListener still uses the legacy setters.
|
||||
L.setPosition(listener.px, listener.py, listener.pz);
|
||||
L.setOrientation(listener.fx, listener.fy, listener.fz, listener.ux, listener.uy, listener.uz);
|
||||
}
|
||||
for (const [id, { panner }] of this._tracks) {
|
||||
const pos = positions[id];
|
||||
if (!panner || !pos) continue;
|
||||
if (panner.positionX) {
|
||||
panner.positionX.value = pos.x;
|
||||
panner.positionY.value = pos.y;
|
||||
panner.positionZ.value = pos.z;
|
||||
} else {
|
||||
panner.setPosition(pos.x, pos.y, pos.z);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_stopTracks() {
|
||||
for (const { src, gain, panner } of this._tracks.values()) {
|
||||
try {
|
||||
src.stop();
|
||||
} catch {
|
||||
/* never started or already ended */
|
||||
}
|
||||
try {
|
||||
src.disconnect();
|
||||
gain.disconnect();
|
||||
panner?.disconnect();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
this._tracks.clear();
|
||||
}
|
||||
|
||||
// Per-presented-frame correction via requestVideoFrameCallback (accurate). Self-sustaining:
|
||||
// each callback re-registers the next. No-ops while paused / out of range; rAF tick() handles
|
||||
// the fallback path for browsers without rVFC.
|
||||
@ -192,7 +408,10 @@ export class Transport {
|
||||
|
||||
/** Re-lock every in-range video to the current master time (after a tab-visibility stall). */
|
||||
_resync() {
|
||||
const t = state.tGlobal;
|
||||
// Compute fresh: with the audio clock the soundtrack kept playing while rAF (and therefore
|
||||
// state.tGlobal) was frozen, so the live clock may be well past the last stored value.
|
||||
const t = this.computeTGlobal();
|
||||
state.tGlobal = t;
|
||||
for (const cell of this.cells) {
|
||||
const target = this._targetFor(cell.meta, t);
|
||||
if (state.enabled[cell.id] !== false && target >= 0 && target <= cell.meta.duration_s) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user