// timeline.js — SCENEGOD master clock, tracks, keyframe evaluation (Lane B). // Talks to Stage ONLY through the PLAN §4.2 API. NO DOM, NO three.js import, // so it runs headless under `node` for timeline_test.mjs. All the vector / // quaternion math it needs is inlined below (a few lines each). // ---- tiny math (self-contained; euler XYZ radians) -------------------------- const lerp = (a, b, u) => a + (b - a) * u; const lerpArr = (a, b, u) => a.map((v, i) => lerp(v, b[i], u)); function eulerToQuat([x, y, z]) { // XYZ order, matches three const c1 = Math.cos(x / 2), s1 = Math.sin(x / 2); const c2 = Math.cos(y / 2), s2 = Math.sin(y / 2); const c3 = Math.cos(z / 2), s3 = Math.sin(z / 2); return [ s1 * c2 * c3 + c1 * s2 * s3, c1 * s2 * c3 - s1 * c2 * s3, c1 * c2 * s3 + s1 * s2 * c3, c1 * c2 * c3 - s1 * s2 * s3, ]; } function quatToEuler([x, y, z, w]) { // XYZ order const m11 = 1 - 2 * (y * y + z * z), m12 = 2 * (x * y - z * w), m13 = 2 * (x * z + y * w); const m22 = 1 - 2 * (x * x + z * z), m23 = 2 * (y * z - x * w); const m33 = 1 - 2 * (x * x + y * y); const ey = Math.asin(Math.max(-1, Math.min(1, m13))); let ex, ez; if (Math.abs(m13) < 0.9999999) { ex = Math.atan2(-m23, m33); ez = Math.atan2(-m12, m11); } else { ex = Math.atan2(m22 !== undefined ? 2 * (y * z + x * w) : 0, m22); ez = 0; } return [ex, ey, ez]; } function slerp(a, b, u) { // quaternion slerp, sign-safe let [ax, ay, az, aw] = a, [bx, by, bz, bw] = b; let cos = ax * bx + ay * by + az * bz + aw * bw; if (cos < 0) { bx = -bx; by = -by; bz = -bz; bw = -bw; cos = -cos; } if (cos > 0.9995) { // nearly parallel → nlerp const q = [lerp(ax, bx, u), lerp(ay, by, u), lerp(az, bz, u), lerp(aw, bw, u)]; const n = Math.hypot(...q) || 1; return q.map((v) => v / n); } const t = Math.acos(cos), s = Math.sin(t); const wa = Math.sin((1 - u) * t) / s, wb = Math.sin(u * t) / s; return [ax * wa + bx * wb, ay * wa + by * wb, az * wa + bz * wb, aw * wa + bw * wb]; } function ease(u, kind) { switch (kind) { case 'step': return 0; // hold start value across segment case 'in': return u * u * u; case 'out': return 1 - Math.pow(1 - u, 3); case 'inout': return u < 0.5 ? 4 * u * u * u : 1 - Math.pow(-2 * u + 2, 3) / 2; case 'linear': default: return u; } } // color helpers for param track (sRGB lerp is fine for v1 per B2) const isColor = (v) => typeof v === 'string' && v[0] === '#'; function hexToRgb(h) { const n = parseInt(h.slice(1), 16); return [(n >> 16) & 255, (n >> 8) & 255, n & 255]; } function rgbToHex([r, g, b]) { return '#' + [r, g, b].map((c) => Math.round(Math.max(0, Math.min(255, c))).toString(16).padStart(2, '0')).join(''); } // find bracketing keys in a t-sorted array. ponytail: linear scan — key arrays // are tiny (handfuls); swap for binary search only if a lane ever profiles hot. function bracket(keys, t) { if (keys.length === 0) return null; if (t <= keys[0].t) return [keys[0], keys[0], 0]; const last = keys[keys.length - 1]; if (t >= last.t) return [last, last, 0]; for (let i = 0; i < keys.length - 1; i++) { if (t >= keys[i].t && t < keys[i + 1].t) { const span = keys[i + 1].t - keys[i].t; return [keys[i], keys[i + 1], span > 0 ? (t - keys[i].t) / span : 0]; } } return [last, last, 0]; } // ---- Timeline --------------------------------------------------------------- export class Timeline { constructor(stage) { this.stage = stage; this.scene = { version: 1, name: 'untitled', fps: 30, duration: 10, entities: [], cameraCuts: [], audio: [] }; this.time = 0; this.playing = false; this._tickCbs = []; this._loadCbs = []; this._raf = null; this._last = 0; this._qcache = new WeakMap(); // key obj -> cached quat (keeps keys pure for round-trip) this._clipActions = new Map(); // block obj -> {action, clip} this._activeCam = undefined; // last camera pushed (avoid re-spamming setActiveCamera) this.undoStack = []; if (typeof window !== 'undefined') { window.addEventListener('scenegod:capturekey', (e) => { const id = e.detail && e.detail.id; if (id) this.addKey(id, 'transform', { t: this.time, ...this.stage.entityTransform(id), ease: 'inout' }); }); // Lane A dock drops a clip on a character → add a block at the playhead. // REVIEW-FIX: this is fire-and-forget from an event handler, so a rejected // promise vanished — a non-mixamo rig (CC_Base_*) failed the retarget and // the drop just silently did nothing. Surface every failure. window.addEventListener('scenegod:clipdrop', (e) => { if (e.detail && e.detail.id) this.addClipDrop(e.detail).catch((err) => this._fail('clip', err)); }); // M5 DIRECTOR MODE: Lane A presets dispatch ops → normal keys/cuts at // the playhead (contract: lanes/A §M5.3 / PLAN §5 M5). window.addEventListener('scenegod:direct', (e) => { if (e.detail && e.detail.op) { try { this.applyDirect(e.detail); } catch (err) { this._fail('direct', err); } } }); } // SYNC2 seam: mirror live stage adds/removes into scene.entities so // dock-added entities can be keyframed without a scene load. if (stage && stage.onChange) stage.onChange((en) => this._mirror(en)); } // One place for "the user asked for something and it didn't happen": console + a // toast if the UI provides one (tlui installs window.sgToast). Never swallow. _fail(what, err) { console.error(`[scenegod:${what}]`, err); const msg = (err && err.message) || String(err); if (typeof window !== 'undefined' && typeof window.sgToast === 'function') window.sgToast(`${what}: ${msg}`); return null; } _mirror(en) { if (!en || !en.id) return; const list = this.scene.entities; const i = list.findIndex((x) => x.id === en.id); if (!this.stage.getEntity(en.id)) { // gone from stage → drop entity + its tracks if (i >= 0) list.splice(i, 1); return; } if (i >= 0) return; // known — timeline already owns its tracks list.push({ id: en.id, kind: en.kind, label: en.label, source: en.source ? structuredClone(en.source) : { type: 'upload' }, params: en.params ? structuredClone(en.params) : {}, transform: this.stage.entityTransform(en.id), tracks: { transform: [], params: [], clips: [] }, }); } get fps() { return this.scene.fps; } get duration() { return this.scene.duration; } get entities() { return this.scene.entities; } get cameraCuts() { return this.scene.cameraCuts; } // ---- load / save (lossless round-trip) ---- load(sceneJson) { this.scene = structuredClone(sceneJson); if (this.scene.fps == null) this.scene.fps = 30; if (this.scene.duration == null) this.scene.duration = 10; this.scene.entities = this.scene.entities || []; this.scene.cameraCuts = this.scene.cameraCuts || []; this.scene.audio = this.scene.audio || []; for (const e of this.scene.entities) this._sortTracks(e); this.scene.cameraCuts.sort((a, b) => a.t - b.t); this._clipActions.clear(); this._activeCam = undefined; this.seek(0); for (const cb of this._loadCbs) cb(this.scene); // UI: rebuild rows + refresh scene bar (SYNC1 #3) return this; } onLoad(cb) { this._loadCbs.push(cb); } toJSON() { return structuredClone(this.scene); } // Apply a whole scene JSON: stage first (builds the entity objects), then the // tracks, then clip preload. Load and "new from template" share this so the // two paths can never drift apart. `template` (Lane C's presentation metadata: // title/description/thumb/order) is dropped here — it must never ride into a // saved scene through the lossless round-trip. async applyScene(json) { const scene = structuredClone(json); delete scene.template; if (this.stage.applyState) await this.stage.applyState(scene); this.load(scene); // fires onLoad → UI rebuilds rows + scene bar await this.preload(); return scene; } // "Is there anything here the user would hate to lose?" — the confirm-before- // clobber predicate for New-from-template. Any entity at all counts (a dock- // added, not-yet-keyframed character is still work), as do cuts and audio. hasContent() { const s = this.scene; if ((s.entities || []).length) return true; if ((s.cameraCuts || []).length) return true; if ((s.audio || []).length) return true; return false; } // Programmatic duration change (dur field or code). Keys past the new end are // kept (lossless) but unreachable; clamp the playhead and refresh the UI — // plain `scene.duration = x` skips both. Reuses the onLoad hook to redraw + // resync the scene bar. setDuration(x) { this.scene.duration = Math.max(0.1, +x || 0.1); if (this.time > this.scene.duration) this.time = this.scene.duration; for (const cb of this._loadCbs) cb(this.scene); this.seek(this.time); return this.scene.duration; } // Lane A clip-drop: fetch the clip (for its duration), drop a block at the // playhead, add a default crossfade where it abuts a neighbour, preload. async addClipDrop({ id, path, clipIndex = 0 }) { const clip = await this.stage.prepareClip(id, path, clipIndex); const dur = (clip && clip.duration) || 1; this.addClipBlock(id, { path, clipIndex, start: this.time, in: 0, out: dur, loop: 1, fade: 0 }); this._setAbutFades(id); await this.preload(); this.seek(this.time); // re-evaluate + redraw } _setAbutFades(id) { // 0.25s fade on any block that abuts its successor const blocks = (this._tracks(id).clips) || []; for (let i = 0; i < blocks.length - 1; i++) { const b = blocks[i], n = blocks[i + 1]; const end = b.start + ((b.out - b.in) || 0) * (b.loop || 1); if (Math.abs(end - n.start) < 1e-3 && !b.fade) b.fade = 0.25; } } _sortTracks(e) { const tr = e.tracks; if (!tr) return; if (tr.transform) tr.transform.sort((a, b) => a.t - b.t); if (tr.params) tr.params.sort((a, b) => a.t - b.t); if (tr.morphs) tr.morphs.sort((a, b) => a.t - b.t); if (tr.clips) tr.clips.sort((a, b) => a.start - b.start); } _quat(key) { let q = this._qcache.get(key); if (!q) { q = eulerToQuat(key.rot || [0, 0, 0]); this._qcache.set(key, q); } return q; } // ---- clock ---- onTick(cb) { this._tickCbs.push(cb); } play() { if (this.playing || typeof requestAnimationFrame === 'undefined') return; this.playing = true; this._last = performance.now(); const loop = (now) => { if (!this.playing) return; const dt = (now - this._last) / 1000; this._last = now; let t = this.time + dt; if (t >= this.duration) { t = this.duration; this.playing = false; } this.seek(t); if (this.playing) this._raf = requestAnimationFrame(loop); }; this._raf = requestAnimationFrame(loop); } pause() { this.playing = false; if (this._raf) cancelAnimationFrame(this._raf); this._raf = null; if (this.stage.entityVideo) this.evaluate(this.time); // M6: videos pause with the clock } seek(t) { this.time = Math.max(0, Math.min(this.duration, t)); this.evaluate(this.time); for (const cb of this._tickCbs) cb(this.time); } step(frame) { this.time = Math.max(0, Math.min(this.duration, frame / this.fps)); this.evaluate(this.time); return this.time; } // ---- clip preload (async fetch/retarget once; evaluate stays sync) ---- async preload() { for (const e of this.entities) { const clips = e.tracks && e.tracks.clips; if (!clips) continue; const mixer = this.stage.entityMixer(e.id); for (const b of clips) { if (this._clipActions.has(b)) continue; const clip = await this.stage.prepareClip(e.id, b.path, b.clipIndex || 0); const action = mixer.clipAction(clip); action.play(); action.paused = true; action.enabled = false; action.weight = 0; this._clipActions.set(b, { action, clip }); } } } // ---- evaluation ---- evaluate(t) { for (const e of this.entities) { if (e.kind === 'camera' || e.tracks) this._evalTransform(e, t); this._evalParams(e, t); if (e.tracks && e.tracks.morphs) this._evalMorphs(e, t); if (e.tracks && e.tracks.clips) this._evalClips(e, t); if ((e.kind === 'backdrop' || e.kind === 'screen') && this.stage.entityVideo) this._evalVideo(e, t); } this._evalCameraCuts(t); } // M6 FLOW VIDEO PLATES: video backdrops/screens follow the master clock. // Cheap scrub-follow — seek only when drift >50ms; play/pause with the // clock. NEVER awaits (step() stays sync — the deterministic render path // awaits stage.syncVideos(t) per frame instead, Lane A/C's seam). _evalVideo(e, t) { const v = this.stage.entityVideo(e.id); if (!v) return; // nullable: image backdrop / not loaded const dur = v.duration; if (Number.isFinite(dur) && dur > 0) { const want = Math.max(0, t - ((e.params && e.params.videoStart) || 0)) % dur; if (Math.abs(v.currentTime - want) > 0.05) v.currentTime = want; } if (this.playing && v.paused) { const p = v.play(); if (p && p.catch) p.catch(() => {}); } else if (!this.playing && !v.paused) v.pause(); } _evalMorphs(e, t) { // viseme/blendshape weights → stage.setMorph (M4-B) const keys = e.tracks && e.tracks.morphs; if (!keys || !keys.length || !this.stage.setMorph) return; const byKey = new Map(); for (const k of keys) { if (!byKey.has(k.key)) byKey.set(k.key, []); byKey.get(k.key).push(k); } for (const [name, ks] of byKey) { const [k0, k1, u] = bracket(ks, t); const w = k0 === k1 ? k0.value : lerp(k0.value, k1.value, ease(u, k0.ease)); this.stage.setMorph(e.id, name, w); } } _evalTransform(e, t) { const keys = e.tracks && e.tracks.transform; if (!keys || keys.length === 0) return; // 0 keys → leave rest pose alone const br = bracket(keys, t); const [k0, k1, u] = br; if (k0 === k1) { this.stage.setTransform(e.id, { pos: k0.pos, rot: k0.rot, scale: k0.scale }); return; } const eu = ease(u, k0.ease); const pos = lerpArr(k0.pos, k1.pos, eu); const scale = typeof k0.scale === 'number' ? lerp(k0.scale, k1.scale, eu) : lerpArr(k0.scale, k1.scale, eu); const rot = k0.ease === 'step' ? k0.rot.slice() : quatToEuler(slerp(this._quat(k0), this._quat(k1), eu)); this.stage.setTransform(e.id, { pos, rot, scale }); } _evalParams(e, t) { const keys = e.tracks && e.tracks.params; if (!keys || keys.length === 0) return; const byKey = new Map(); for (const k of keys) { if (!byKey.has(k.key)) byKey.set(k.key, []); byKey.get(k.key).push(k); } for (const [name, ks] of byKey) { const br = bracket(ks, t); const [k0, k1, u] = br; let val; if (k0 === k1) val = k0.value; else { const eu = ease(u, k0.ease); val = isColor(k0.value) ? rgbToHex(lerpArr(hexToRgb(k0.value), hexToRgb(k1.value), eu)) : lerp(k0.value, k1.value, eu); } this.stage.setParam(e.id, name, val); } } _evalClips(e, t) { const blocks = e.tracks.clips; if (!blocks || !blocks.length) return; const mixer = this.stage.entityMixer(e.id); if (!mixer) return; // SYNC1: real Stage returns null for non-characters (stub always had one) for (let i = 0; i < blocks.length; i++) { const b = blocks[i]; const rec = this._clipActions.get(b); if (!rec) continue; // not preloaded yet → silent const span = (b.out - b.in) || 0.0001; const blockEnd = b.start + span * (b.loop || 1); // pre-roll: a predecessor's fade pulls THIS block's playback start earlier // by pred.fade so it advances during the crossfade (continuous local time). const pred = blocks[i - 1]; const preroll = (pred && pred.fade > 0) ? pred.fade : 0; const inStart = b.start - preroll; let w = 0, local = b.in; if (t >= inStart && t < blockEnd) { local = b.in + ((t - inStart) % span); // continuous across the boundary w = 1; if (t < b.start) w = (t - inStart) / preroll; // fade-IN 0→1 over the pred's fade window const succ = blocks[i + 1]; // fade-OUT 1→0 over own fade window if (succ && b.fade > 0 && t >= blockEnd - b.fade) w = Math.min(w, (blockEnd - t) / b.fade); } rec.action.time = local; rec.action.weight = Math.max(0, w); rec.action.enabled = w > 0; } mixer.update(0); // set-time style — exact when scrubbing } _evalCameraCuts(t) { const cuts = this.cameraCuts; let cam = null; for (const c of cuts) { if (c.t <= t) cam = c.camera; else break; } if (cam !== this._activeCam) { this._activeCam = cam; this.stage.setActiveCamera(cam); } } // ---- keyframe / block mutators (keep sorted, push inverse for undo) ---- _tracks(id) { const e = this.scene.entities.find((x) => x.id === id); if (!e) throw new Error(`no entity ${id}`); e.tracks = e.tracks || {}; return e.tracks; } addKey(id, track, key) { const tr = this._tracks(id); const arr = (tr[track] = tr[track] || []); // a key replaces any existing at the same t; named tracks (params/morphs) // are keyed by (t, key name) since many names share the lane. const named = track === 'params' || track === 'morphs'; const same = (k) => k.t === key.t && (!named || k.key === key.key); const prevIdx = arr.findIndex(same); const prev = prevIdx >= 0 ? arr[prevIdx] : null; if (prevIdx >= 0) arr.splice(prevIdx, 1); arr.push(key); arr.sort((a, b) => a.t - b.t); this._qcache.delete(key); this.undoStack.push({ undo: () => { const i = arr.indexOf(key); if (i >= 0) arr.splice(i, 1); if (prev) { arr.push(prev); arr.sort((a, b) => a.t - b.t); } } }); return key; } moveKey(id, track, key, newT) { const oldT = key.t; key.t = newT; this._tracks(id)[track].sort((a, b) => a.t - b.t); this.undoStack.push({ undo: () => { key.t = oldT; this._tracks(id)[track].sort((a, b) => a.t - b.t); } }); } deleteKey(id, track, key) { const arr = this._tracks(id)[track] || []; const i = arr.indexOf(key); if (i < 0) return; arr.splice(i, 1); this.undoStack.push({ undo: () => { arr.push(key); arr.sort((a, b) => a.t - b.t); } }); } addClipBlock(id, block) { const tr = this._tracks(id); const arr = (tr.clips = tr.clips || []); arr.push(block); arr.sort((a, b) => a.start - b.start); this._clipActions.delete(block); this.undoStack.push({ undo: () => { const i = arr.indexOf(block); if (i >= 0) arr.splice(i, 1); } }); return block; } moveClipBlock(id, block, newStart) { const old = block.start; block.start = newStart; this._tracks(id).clips.sort((a, b) => a.start - b.start); this.undoStack.push({ undo: () => { block.start = old; this._tracks(id).clips.sort((a, b) => a.start - b.start); } }); } trimClipBlock(id, block, { in: inV, out: outV } = {}) { const oldIn = block.in, oldOut = block.out; if (inV != null) block.in = inV; if (outV != null) block.out = outV; this.undoStack.push({ undo: () => { block.in = oldIn; block.out = oldOut; } }); } addCut(t, cameraId) { const cut = { t, camera: cameraId }; this.scene.cameraCuts.push(cut); this.scene.cameraCuts.sort((a, b) => a.t - b.t); this._activeCam = undefined; // force re-eval of active cam this.undoStack.push({ undo: () => { const i = this.scene.cameraCuts.indexOf(cut); if (i >= 0) this.scene.cameraCuts.splice(i, 1); } }); return cut; } removeCut(cut) { const i = this.scene.cameraCuts.indexOf(cut); if (i < 0) return; this.scene.cameraCuts.splice(i, 1); this._activeCam = undefined; this.undoStack.push({ undo: () => { this.scene.cameraCuts.splice(i, 0, cut); this._activeCam = undefined; } }); } undo() { const op = this.undoStack.pop(); if (op) op.undo(); this.evaluate(this.time); } // Run fn (which may push many undo entries) and collapse everything it // pushed into ONE undo entry — a multi-key preset op reverts in one Ctrl+Z. group(fn) { const n = this.undoStack.length; const r = fn(); if (this.undoStack.length > n + 1) { const ops = this.undoStack.splice(n); this.undoStack.push({ undo: () => { for (let i = ops.length - 1; i >= 0; i--) ops[i].undo(); } }); } return r; } // ---- M5 DIRECTOR MODE: scenegod:direct ops → keys/cuts at the playhead. // Accepts BOTH Lane A's live presets.js payloads (values spread at top level: // shot {camId, pos, rot, fov, …, entityIds:[camId, subjectId]} // light {sun:{color,intensity}, ambient:{…}, bg, sunId, ambientId} // mark {id, pos?, rot?, entityIds:[id]} ) // and the PLAN §5 contract shapes ({transform:{pos,rot}}, {params:{k:v}}, // walkTo). One op = ONE undo entry (group()). applyDirect(d) { const t = this.time; const key = (id, k) => this.addKey(id, 'params', { t, key: k[0], value: k[1], ease: 'inout' }); this.group(() => { if (d.op === 'shot') { const camId = d.camId ?? (d.entityIds && d.entityIds[0]); const cur = this.stage.entityTransform(camId); const tr = d.transform || d; // Lane A spreads pos/rot at top level this.addKey(camId, 'transform', { t, pos: tr.pos || cur.pos, rot: tr.rot || cur.rot, scale: tr.scale ?? cur.scale ?? 1, ease: 'inout', }); if (d.fov != null) key(camId, ['fov', d.fov]); this.addCut(t, camId); } else if (d.op === 'light') { if (d.sunId != null && d.sun) { // Lane A applyLight shape key(d.sunId, ['color', d.sun.color]); key(d.sunId, ['intensity', d.sun.intensity]); // sun boom pos/aim moved per preset — capture it so presets scrub this.addKey(d.sunId, 'transform', { t, ...this.stage.entityTransform(d.sunId), ease: 'inout' }); if (d.ambientId != null && d.ambient) { key(d.ambientId, ['color', d.ambient.color]); key(d.ambientId, ['intensity', d.ambient.intensity]); if (d.bg != null) key(d.ambientId, ['bg', d.bg]); } } else { // generic: entityId(s) + params map const ids = d.entityIds || (d.entityId != null ? [d.entityId] : []); for (const id of ids) for (const kv of Object.entries(d.params || {})) key(id, kv); } } else if (d.op === 'mark') { const ids = d.entityIds || (d.entityId != null ? [d.entityId] : (d.id != null ? [d.id] : [])); for (const id of ids) { const cur = this.stage.entityTransform(id); const tr = d.transform || d; // Lane A spreads pos/rot at top level const target = { pos: tr.pos || cur.pos, rot: tr.rot || cur.rot, scale: tr.scale ?? cur.scale ?? 1 }; if (d.walkTo) { // walk: from prevPos (or current) → mark over 2s const from = d.prevPos ? { ...cur, pos: d.prevPos } : cur; this.addKey(id, 'transform', { t, ...from, ease: 'inout' }); this.addKey(id, 'transform', { t: t + 2, ...target, ease: 'inout' }); } else { this.addKey(id, 'transform', { t, ...target, ease: 'inout' }); } } } }); this.seek(t); // re-evaluate: cut/keys take effect now } // ---- audio track (M4-B). Clips are {path, start, gain}; duration is a // client-side draw concern (decode + cache in tlui). Server muxes from // scene.audio at render, so step()/evaluate never touch audio. ---- addAudio(clip) { this.scene.audio = this.scene.audio || []; this.scene.audio.push(clip); this.undoStack.push({ undo: () => { const i = this.scene.audio.indexOf(clip); if (i >= 0) this.scene.audio.splice(i, 1); } }); return clip; } moveAudio(clip, newStart) { const old = clip.start; clip.start = Math.max(0, newStart); this.undoStack.push({ undo: () => { clip.start = old; } }); } setAudioGain(clip, gain) { const old = clip.gain; clip.gain = gain; this.undoStack.push({ undo: () => { clip.gain = old; } }); } removeAudio(clip) { const i = (this.scene.audio || []).indexOf(clip); if (i < 0) return; this.scene.audio.splice(i, 1); this.undoStack.push({ undo: () => { this.scene.audio.splice(i, 0, clip); } }); } // Trim within the source file, like clip blocks: {in, out} seconds. Absent = // untrimmed (schema stays {path,start,gain} for untouched clips). Server-side // mux honoring in/out is a Lane C REQUEST (logged); draft Web Audio playback // honors it client-side already. trimAudio(clip, { in: inV, out: outV } = {}) { const oldIn = clip.in, oldOut = clip.out; if (inV != null) clip.in = Math.max(0, inV); if (outV != null) clip.out = Math.max((clip.in || 0) + 0.05, outV); this.undoStack.push({ undo: () => { if (oldIn === undefined) delete clip.in; else clip.in = oldIn; if (oldOut === undefined) delete clip.out; else clip.out = oldOut; } }); } // Head trim: dragging an audio block's LEFT edge eats into the source rather // than moving the block — `in` grows by exactly what `start` does, so what you // still hear stays put on the timeline. The pair is ONE undo entry. trimAudioIn(clip, newStart, srcDur = null) { const inT = clip.in || 0; const outT = clip.out ?? srcDur; const lo = -Math.min(inT, clip.start); // can't rewind past the file head or t=0 const hi = outT != null ? Math.max(0, outT - inT - 0.05) : Infinity; // always leave a sliver const d = Math.max(lo, Math.min(hi, newStart - clip.start)); if (!d) return; this.group(() => { this.trimAudio(clip, { in: inT + d }); this.moveAudio(clip, clip.start + d); }); } // ---- M9-B1: BEAT GRID (Lane C's `GET beats?path=`) ------------------------- // Stored ON THE SCENE (`scene.beatGrid`) so it survives save/load — the grid is // the musical spine of an edit, not a UI preference. It is a single top-level // object, NOT per-audio-clip: one track sets the tempo of a music video, and // "which grid is the timeline snapping to" must have exactly one answer. // Legal by inspection: server.py:235 `validate_scene` only looks at version / // entities / cameraCuts, so an extra top-level key passes, and load()'s // structuredClone carries it through the lossless round-trip for free. // // {path, bpm, confidence, meter, period, offset, beats[], downbeats[]} // // `beats`/`downbeats` stay in FILE seconds exactly as Lane C returns them; // `offset` is the timeline second the file's t=0 sits at. The effective offset // is re-derived from the audio clip with the same path whenever one is still // on the timeline, so dragging the audio block drags the grid with it. setBeatGrid(grid) { if (!grid) { delete this.scene.beatGrid; return null; } this.scene.beatGrid = grid; return grid; } clearBeatGrid() { delete this.scene.beatGrid; } // All four delegate to the pure module helpers at the bottom of this file — // the FILM panel aligns ANOTHER scene's shot list to ITS grid, with no // Timeline instance in hand, and must get identical arithmetic. beatOffset() { return beatOffsetOf(this.scene); } _beatPeriod(kind) { return beatPeriodOf(this.scene.beatGrid, kind); } // Grid positions in TIMELINE seconds, extended at the constant tempo across the // whole scene so the drawn lines and snapToGrid() agree everywhere — not only // where the audio file happens to have samples. gridTimes(kind = 'beat') { return gridTimesOf(this.scene.beatGrid, kind, this.beatOffset(), this.duration); } // Nearest grid line to t, in timeline seconds. null = no grid (caller falls // back to its fixed increment). Outside the analysed range the constant tempo // is extrapolated, same as gridTimes. snapToGrid(t, kind = 'beat') { return snapToGridWith(this.scene.beatGrid, t, kind, this.beatOffset()); } // ---- M10-B1: AUTO-CUT — a beat grid + 2 cameras → cuts on the downbeats ---- // Deliberately dumb and predictable: every Nth downbeat in range gets a cut, // cameras cycle in SCENE order (2 cameras = strict A/B, which is what a DJ // edit looks like), and the result is ordinary cameraCuts the user can then // hand-edit. No randomness, no energy analysis, no "smart" variation. // Cameras in scene order, plus any live-on-stage camera the model hasn't // mirrored yet (same merge tlui uses to build its rows). cameraIds() { const ids = this.scene.entities.filter((e) => e.kind === 'camera').map((e) => e.id); if (this.stage && this.stage.entities) { for (const se of this.stage.entities()) if (se.kind === 'camera' && !ids.includes(se.id)) ids.push(se.id); } return ids; } // Pure: what autoCut WOULD do. The popover shows `cuts.length` before the user // commits, and Apply runs the same plan — so the count on screen is the count // you get. `{ok:false, error}` for every refusal (no grid / <2 cameras / empty // range); never a silent no-op. autoCutPlan(opts = {}) { const g = this.scene.beatGrid; if (!g) return { ok: false, error: 'no beat grid — press ♪ on an audio row to detect one first' }; const cadence = Math.max(1, Math.round(+opts.cadence || 2)); const all = this.cameraIds(); const cams = opts.cameras && opts.cameras.length ? all.filter((id) => opts.cameras.includes(id)) : all; if (cams.length < 2) { return { ok: false, error: `auto-cut needs at least 2 cameras — ${cams.length === 1 ? 'only 1 is' : 'none are'} selected in this scene` }; } const from = Math.max(0, opts.from == null ? 0 : +opts.from); const to = Math.min(this.duration, opts.to == null ? this.duration : +opts.to); const bars = this.gridTimes('bar').filter((t) => t >= from - 1e-9 && t <= to + 1e-9); if (!bars.length) return { ok: false, error: `no downbeats between ${from.toFixed(2)}s and ${to.toFixed(2)}s` }; const times = bars.filter((_, i) => i % cadence === 0); // Don't open with the camera that is already live going into the range — // otherwise "from the playhead" can land the same camera twice in a row. const eps = 0.5 / (this.fps || 30); const before = this.cameraCuts.filter((c) => c.t < times[0] - eps); const prior = before.length ? before[before.length - 1] : null; const phase = prior && prior.camera === cams[0] ? 1 : 0; const cuts = times.map((t, i) => ({ t, camera: cams[(i + phase) % cams.length] })); // An existing cut ON a placed position is REPLACED (half a frame either way, // so a hand-placed cut near the downbeat is absorbed rather than duplicated // into a flash frame). Cuts elsewhere in the range are left alone. const replaced = this.cameraCuts.filter((c) => times.some((t) => Math.abs(c.t - t) <= eps)); // Hand-placed cuts elsewhere in the range are NOT touched — deleting someone's // edit is over-reach — with ONE exception (SYNC13, orchestrator): a kept cut // that selects the camera already live at that instant is a literal no-op. It // cuts from cam2 to cam2, nothing happens on screen, and it still rides into // the saved scene looking like an edit. Those are dropped; the rest are // reported as `kept` so the popover can own the interleave out loud. // The "already live" camera is the EFFECTIVE one after the auto cuts are // merged in, so walk the merged list in time order rather than comparing // against the previous array element. const survivors = this.cameraCuts.filter((c) => !replaced.includes(c)); const merged = [...survivors.map((c) => ({ c, auto: false })), ...cuts.map((c) => ({ c, auto: true }))] .sort((a, b) => a.c.t - b.c.t); const kept = [], dropped = []; let eff = null; for (const m of merged) { if (!m.auto && m.c.t >= times[0] && m.c.t <= times[times.length - 1]) { if (m.c.camera === eff) { dropped.push(m.c); continue; } // cuts from X to X = nothing kept.push(m.c); } eff = m.c.camera; } return { ok: true, cuts, replaced, kept, dropped, cameras: cams, cadence, barPeriod: this._beatPeriod('bar'), confidence: g.confidence ?? 0, bpm: g.bpm, from, to, }; } // Apply the plan as ONE undo entry. Refuses loudly (throws) rather than // half-applying — the caller already has autoCutPlan() to check first. autoCut(opts = {}) { const plan = this.autoCutPlan(opts); if (!plan.ok) throw new Error(plan.error); this.group(() => { for (const c of plan.replaced) this.removeCut(c); for (const c of plan.dropped) this.removeCut(c); // no-op cuts (cam X → cam X) for (const c of plan.cuts) this.addCut(c.t, c.camera); }); this.seek(this.time); // the new cut under the playhead goes live return plan; } // Rhubarb lip-sync JSON → morph keys. { mouthCues: [{start, end, value}] }. // ponytail: per-cue ramp (shape 1 at cue start → 0 at cue end); hold/crossfade // tuning deferred until it looks wrong on a real blendshape character. importRhubarb(id, rhubarbJson, t0 = 0) { const cues = (rhubarbJson && rhubarbJson.mouthCues) || []; this.group(() => { // one undo reverts the whole import for (const c of cues) { if (!c.value) continue; this.addKey(id, 'morphs', { t: t0 + c.start, key: c.value, value: 1, ease: 'linear' }); this.addKey(id, 'morphs', { t: t0 + c.end, key: c.value, value: 0, ease: 'linear' }); } }); return (this._tracks(id).morphs || []).length; } } // ---- pure beat-grid helpers ------------------------------------------------ // The Timeline methods above are thin wrappers over these. They exist as free // functions because the FILM panel (M10-B2) aligns the shots of scenes that are // NOT loaded — it holds a grid and a shot list, not a Timeline — and it must get // bit-identical arithmetic to what the user saw while cutting. // Where the grid's file t=0 sits on that scene's timeline: the audio clip that // carries the grid wins (drag the audio, drag the grid), else the stored offset. export function beatOffsetOf(scene) { const g = scene && scene.beatGrid; if (!g) return 0; const clip = ((scene && scene.audio) || []).find((c) => c.path === g.path); return clip ? clip.start - (clip.in || 0) : (g.offset || 0); } export function beatPeriodOf(grid, kind = 'beat') { if (!grid) return 0; const per = grid.period || (grid.bpm ? 60 / grid.bpm : 0); return kind === 'bar' ? per * (grid.meter || 4) : per; } export function gridTimesOf(grid, kind = 'beat', off = 0, span = 0) { if (!grid) return []; const src = kind === 'bar' ? (grid.downbeats || []) : (grid.beats || []); if (!src.length) return []; const per = beatPeriodOf(grid, kind); const inFile = src.map((b) => b + off); if (!(per > 0.05)) return inFile.filter((t) => t >= 0); // nonsense tempo → no extrapolation const before = []; for (let x = inFile[0] - per; x >= 0; x -= per) before.push(x); const after = []; for (let x = inFile[inFile.length - 1] + per; x <= span; x += per) after.push(x); return [...before.reverse(), ...inFile.filter((t) => t >= 0), ...after]; } export function snapToGridWith(grid, t, kind = 'beat', off = 0) { if (!grid) return null; const src = kind === 'bar' ? grid.downbeats : grid.beats; if (!src || !src.length) return null; const per = beatPeriodOf(grid, kind); const x = t - off; const last = src[src.length - 1]; let r; if (per > 0 && x <= src[0]) r = src[0] + Math.round((x - src[0]) / per) * per; else if (per > 0 && x >= last) r = last + Math.round((x - last) / per) * per; else { let lo = 0, hi = src.length - 1; while (hi - lo > 1) { const m = (lo + hi) >> 1; if (src[m] <= x) lo = m; else hi = m; } r = (x - src[lo] <= src[hi] - x) ? src[lo] : src[hi]; } r += off; if (r < 0 && per > 0) r += per; // never snap off the front of the timeline return Math.max(0, r); } // M10-B2: snap a FILM shot list's in/out to downbeats so the shot boundaries // land musically. Order is never touched. A shot that would collapse (in >= out // after snapping, e.g. a shot shorter than a bar) is left EXACTLY as it was and // reported — silently mangling someone's edit is worse than not helping. // `durations[i]` (a scene's length, 0/absent = unknown) keeps `out` inside its // scene: an out past the end walks back whole bars until it fits. export function alignShotsToGrid(shots, grid, off = 0, durations = []) { const bar = beatPeriodOf(grid, 'bar'); const r6 = (x) => Math.round(x * 1e6) / 1e6; // kill float noise in the panel's number fields const out = []; const skipped = []; (shots || []).forEach((sh, i) => { const a0 = +sh.in || 0, b0 = +sh.out || 0; let a = snapToGridWith(grid, a0, 'bar', off); let b = snapToGridWith(grid, b0, 'bar', off); const lim = +durations[i] || 0; if (a == null || b == null) { out.push({ ...sh }); skipped.push({ index: i, scene: sh.scene, why: 'no downbeats' }); return; } // Walk back a bar at a time and RE-SNAP each step: `b - bar` is a bar-period // away from a grid line, not on one (Lane C rounds beats to 4dp, the period // to 6dp), and "lands on a downbeat" has to mean the real one. for (let guard = 0; lim && bar > 0 && b > lim + 1e-9 && guard < 64; guard++) b = snapToGridWith(grid, b - bar, 'bar', off); if (!(b > a + 1e-9)) { out.push({ ...sh }); skipped.push({ index: i, scene: sh.scene, why: 'would collapse' }); return; } out.push({ ...sh, in: r6(a), out: r6(b) }); }); return { shots: out, skipped }; } // Lane C's /beats response → the scene-stored grid (M9-B1). Pure, so the shape // contract is testable headless against a real captured payload; the fetch + the // per-path cache live in tlui, like every other server call in this lane. // `offset` = the timeline second where the file's t=0 sits (audio clip // `start - (in||0)`, per Lane C's log: "beat + audioClip.start - (audioClip.in||0)"). export function beatGridFrom(res, offset = 0) { return { path: res.path, bpm: res.bpm, confidence: res.confidence ?? 0, meter: res.meter || 4, period: res.period || (res.bpm ? 60 / res.bpm : 0), offset, beats: res.beats || [], downbeats: res.downbeats || [], }; } export default Timeline;