ba29d0c6d6
- timeline.js: morphs track kind evaluated via stage.setMorph (lerp per viseme name), audio mutators addAudio/moveAudio/setAudioGain/removeAudio (undo-able), importRhubarb(id, json, t0) mapping mouthCues → morph keys (per-cue ramp), load() ensures scene.audio[], addKey keyed by (t,name) for morphs too. - stagestub.js: setMorph (records) + visemeTargets seam. - timeline_test.mjs: morph lerp, rhubarb keys, audio mutators + undo round-trip. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
434 lines
18 KiB
JavaScript
434 lines
18 KiB
JavaScript
// timeline.js — SCENEGOD master clock, tracks, keyframe evaluation (Lane B).
|
|
// Talks to Stage ONLY through the PLAN §4.2 API. NO DOM, NO three.js import,
|
|
// so it runs headless under `node` for timeline_test.mjs. All the vector /
|
|
// quaternion math it needs is inlined below (a few lines each).
|
|
|
|
// ---- tiny math (self-contained; euler XYZ radians) --------------------------
|
|
const lerp = (a, b, u) => a + (b - a) * u;
|
|
const lerpArr = (a, b, u) => a.map((v, i) => lerp(v, b[i], u));
|
|
|
|
function eulerToQuat([x, y, z]) { // XYZ order, matches three
|
|
const c1 = Math.cos(x / 2), s1 = Math.sin(x / 2);
|
|
const c2 = Math.cos(y / 2), s2 = Math.sin(y / 2);
|
|
const c3 = Math.cos(z / 2), s3 = Math.sin(z / 2);
|
|
return [
|
|
s1 * c2 * c3 + c1 * s2 * s3,
|
|
c1 * s2 * c3 - s1 * c2 * s3,
|
|
c1 * c2 * s3 + s1 * s2 * c3,
|
|
c1 * c2 * c3 - s1 * s2 * s3,
|
|
];
|
|
}
|
|
function quatToEuler([x, y, z, w]) { // XYZ order
|
|
const m11 = 1 - 2 * (y * y + z * z), m12 = 2 * (x * y - z * w), m13 = 2 * (x * z + y * w);
|
|
const m22 = 1 - 2 * (x * x + z * z), m23 = 2 * (y * z - x * w);
|
|
const m33 = 1 - 2 * (x * x + y * y);
|
|
const ey = Math.asin(Math.max(-1, Math.min(1, m13)));
|
|
let ex, ez;
|
|
if (Math.abs(m13) < 0.9999999) { ex = Math.atan2(-m23, m33); ez = Math.atan2(-m12, m11); }
|
|
else { ex = Math.atan2(m22 !== undefined ? 2 * (y * z + x * w) : 0, m22); ez = 0; }
|
|
return [ex, ey, ez];
|
|
}
|
|
function slerp(a, b, u) { // quaternion slerp, sign-safe
|
|
let [ax, ay, az, aw] = a, [bx, by, bz, bw] = b;
|
|
let cos = ax * bx + ay * by + az * bz + aw * bw;
|
|
if (cos < 0) { bx = -bx; by = -by; bz = -bz; bw = -bw; cos = -cos; }
|
|
if (cos > 0.9995) { // nearly parallel → nlerp
|
|
const q = [lerp(ax, bx, u), lerp(ay, by, u), lerp(az, bz, u), lerp(aw, bw, u)];
|
|
const n = Math.hypot(...q) || 1;
|
|
return q.map((v) => v / n);
|
|
}
|
|
const t = Math.acos(cos), s = Math.sin(t);
|
|
const wa = Math.sin((1 - u) * t) / s, wb = Math.sin(u * t) / s;
|
|
return [ax * wa + bx * wb, ay * wa + by * wb, az * wa + bz * wb, aw * wa + bw * wb];
|
|
}
|
|
|
|
function ease(u, kind) {
|
|
switch (kind) {
|
|
case 'step': return 0; // hold start value across segment
|
|
case 'in': return u * u * u;
|
|
case 'out': return 1 - Math.pow(1 - u, 3);
|
|
case 'inout': return u < 0.5 ? 4 * u * u * u : 1 - Math.pow(-2 * u + 2, 3) / 2;
|
|
case 'linear': default: return u;
|
|
}
|
|
}
|
|
|
|
// color helpers for param track (sRGB lerp is fine for v1 per B2)
|
|
const isColor = (v) => typeof v === 'string' && v[0] === '#';
|
|
function hexToRgb(h) { const n = parseInt(h.slice(1), 16); return [(n >> 16) & 255, (n >> 8) & 255, n & 255]; }
|
|
function rgbToHex([r, g, b]) { return '#' + [r, g, b].map((c) => Math.round(Math.max(0, Math.min(255, c))).toString(16).padStart(2, '0')).join(''); }
|
|
|
|
// find bracketing keys in a t-sorted array. ponytail: linear scan — key arrays
|
|
// are tiny (handfuls); swap for binary search only if a lane ever profiles hot.
|
|
function bracket(keys, t) {
|
|
if (keys.length === 0) return null;
|
|
if (t <= keys[0].t) return [keys[0], keys[0], 0];
|
|
const last = keys[keys.length - 1];
|
|
if (t >= last.t) return [last, last, 0];
|
|
for (let i = 0; i < keys.length - 1; i++) {
|
|
if (t >= keys[i].t && t < keys[i + 1].t) {
|
|
const span = keys[i + 1].t - keys[i].t;
|
|
return [keys[i], keys[i + 1], span > 0 ? (t - keys[i].t) / span : 0];
|
|
}
|
|
}
|
|
return [last, last, 0];
|
|
}
|
|
|
|
// ---- Timeline ---------------------------------------------------------------
|
|
export class Timeline {
|
|
constructor(stage) {
|
|
this.stage = stage;
|
|
this.scene = { version: 1, name: 'untitled', fps: 30, duration: 10, entities: [], cameraCuts: [], audio: [] };
|
|
this.time = 0;
|
|
this.playing = false;
|
|
this._tickCbs = [];
|
|
this._loadCbs = [];
|
|
this._raf = null;
|
|
this._last = 0;
|
|
this._qcache = new WeakMap(); // key obj -> cached quat (keeps keys pure for round-trip)
|
|
this._clipActions = new Map(); // block obj -> {action, clip}
|
|
this._activeCam = undefined; // last camera pushed (avoid re-spamming setActiveCamera)
|
|
this.undoStack = [];
|
|
|
|
if (typeof window !== 'undefined') {
|
|
window.addEventListener('scenegod:capturekey', (e) => {
|
|
const id = e.detail && e.detail.id;
|
|
if (id) this.addKey(id, 'transform', { t: this.time, ...this.stage.entityTransform(id), ease: 'inout' });
|
|
});
|
|
// Lane A dock drops a clip on a character → add a block at the playhead.
|
|
window.addEventListener('scenegod:clipdrop', (e) => {
|
|
if (e.detail && e.detail.id) this.addClipDrop(e.detail);
|
|
});
|
|
}
|
|
// SYNC2 seam: mirror live stage adds/removes into scene.entities so
|
|
// dock-added entities can be keyframed without a scene load.
|
|
if (stage && stage.onChange) stage.onChange((en) => this._mirror(en));
|
|
}
|
|
|
|
_mirror(en) {
|
|
if (!en || !en.id) return;
|
|
const list = this.scene.entities;
|
|
const i = list.findIndex((x) => x.id === en.id);
|
|
if (!this.stage.getEntity(en.id)) { // gone from stage → drop entity + its tracks
|
|
if (i >= 0) list.splice(i, 1);
|
|
return;
|
|
}
|
|
if (i >= 0) return; // known — timeline already owns its tracks
|
|
list.push({
|
|
id: en.id, kind: en.kind, label: en.label,
|
|
source: en.source ? structuredClone(en.source) : { type: 'upload' },
|
|
params: en.params ? structuredClone(en.params) : {},
|
|
transform: this.stage.entityTransform(en.id),
|
|
tracks: { transform: [], params: [], clips: [] },
|
|
});
|
|
}
|
|
|
|
get fps() { return this.scene.fps; }
|
|
get duration() { return this.scene.duration; }
|
|
get entities() { return this.scene.entities; }
|
|
get cameraCuts() { return this.scene.cameraCuts; }
|
|
|
|
// ---- load / save (lossless round-trip) ----
|
|
load(sceneJson) {
|
|
this.scene = structuredClone(sceneJson);
|
|
if (this.scene.fps == null) this.scene.fps = 30;
|
|
if (this.scene.duration == null) this.scene.duration = 10;
|
|
this.scene.entities = this.scene.entities || [];
|
|
this.scene.cameraCuts = this.scene.cameraCuts || [];
|
|
this.scene.audio = this.scene.audio || [];
|
|
for (const e of this.scene.entities) this._sortTracks(e);
|
|
this.scene.cameraCuts.sort((a, b) => a.t - b.t);
|
|
this._clipActions.clear();
|
|
this._activeCam = undefined;
|
|
this.seek(0);
|
|
for (const cb of this._loadCbs) cb(this.scene); // UI: rebuild rows + refresh scene bar (SYNC1 #3)
|
|
return this;
|
|
}
|
|
onLoad(cb) { this._loadCbs.push(cb); }
|
|
toJSON() { return structuredClone(this.scene); }
|
|
|
|
// Programmatic duration change (dur field or code). Keys past the new end are
|
|
// kept (lossless) but unreachable; clamp the playhead and refresh the UI —
|
|
// plain `scene.duration = x` skips both. Reuses the onLoad hook to redraw +
|
|
// resync the scene bar.
|
|
setDuration(x) {
|
|
this.scene.duration = Math.max(0.1, +x || 0.1);
|
|
if (this.time > this.scene.duration) this.time = this.scene.duration;
|
|
for (const cb of this._loadCbs) cb(this.scene);
|
|
this.seek(this.time);
|
|
return this.scene.duration;
|
|
}
|
|
|
|
// Lane A clip-drop: fetch the clip (for its duration), drop a block at the
|
|
// playhead, add a default crossfade where it abuts a neighbour, preload.
|
|
async addClipDrop({ id, path, clipIndex = 0 }) {
|
|
const clip = await this.stage.prepareClip(id, path, clipIndex);
|
|
const dur = (clip && clip.duration) || 1;
|
|
this.addClipBlock(id, { path, clipIndex, start: this.time, in: 0, out: dur, loop: 1, fade: 0 });
|
|
this._setAbutFades(id);
|
|
await this.preload();
|
|
this.seek(this.time); // re-evaluate + redraw
|
|
}
|
|
_setAbutFades(id) { // 0.25s fade on any block that abuts its successor
|
|
const blocks = (this._tracks(id).clips) || [];
|
|
for (let i = 0; i < blocks.length - 1; i++) {
|
|
const b = blocks[i], n = blocks[i + 1];
|
|
const end = b.start + ((b.out - b.in) || 0) * (b.loop || 1);
|
|
if (Math.abs(end - n.start) < 1e-3 && !b.fade) b.fade = 0.25;
|
|
}
|
|
}
|
|
|
|
_sortTracks(e) {
|
|
const tr = e.tracks;
|
|
if (!tr) return;
|
|
if (tr.transform) tr.transform.sort((a, b) => a.t - b.t);
|
|
if (tr.params) tr.params.sort((a, b) => a.t - b.t);
|
|
if (tr.morphs) tr.morphs.sort((a, b) => a.t - b.t);
|
|
if (tr.clips) tr.clips.sort((a, b) => a.start - b.start);
|
|
}
|
|
_quat(key) {
|
|
let q = this._qcache.get(key);
|
|
if (!q) { q = eulerToQuat(key.rot || [0, 0, 0]); this._qcache.set(key, q); }
|
|
return q;
|
|
}
|
|
|
|
// ---- clock ----
|
|
onTick(cb) { this._tickCbs.push(cb); }
|
|
play() {
|
|
if (this.playing || typeof requestAnimationFrame === 'undefined') return;
|
|
this.playing = true;
|
|
this._last = performance.now();
|
|
const loop = (now) => {
|
|
if (!this.playing) return;
|
|
const dt = (now - this._last) / 1000;
|
|
this._last = now;
|
|
let t = this.time + dt;
|
|
if (t >= this.duration) { t = this.duration; this.playing = false; }
|
|
this.seek(t);
|
|
if (this.playing) this._raf = requestAnimationFrame(loop);
|
|
};
|
|
this._raf = requestAnimationFrame(loop);
|
|
}
|
|
pause() { this.playing = false; if (this._raf) cancelAnimationFrame(this._raf); this._raf = null; }
|
|
seek(t) {
|
|
this.time = Math.max(0, Math.min(this.duration, t));
|
|
this.evaluate(this.time);
|
|
for (const cb of this._tickCbs) cb(this.time);
|
|
}
|
|
step(frame) { this.time = Math.max(0, Math.min(this.duration, frame / this.fps)); this.evaluate(this.time); return this.time; }
|
|
|
|
// ---- clip preload (async fetch/retarget once; evaluate stays sync) ----
|
|
async preload() {
|
|
for (const e of this.entities) {
|
|
const clips = e.tracks && e.tracks.clips;
|
|
if (!clips) continue;
|
|
const mixer = this.stage.entityMixer(e.id);
|
|
for (const b of clips) {
|
|
if (this._clipActions.has(b)) continue;
|
|
const clip = await this.stage.prepareClip(e.id, b.path, b.clipIndex || 0);
|
|
const action = mixer.clipAction(clip);
|
|
action.play(); action.paused = true; action.enabled = false; action.weight = 0;
|
|
this._clipActions.set(b, { action, clip });
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- evaluation ----
|
|
evaluate(t) {
|
|
for (const e of this.entities) {
|
|
if (e.kind === 'camera' || e.tracks) this._evalTransform(e, t);
|
|
this._evalParams(e, t);
|
|
if (e.tracks && e.tracks.morphs) this._evalMorphs(e, t);
|
|
if (e.tracks && e.tracks.clips) this._evalClips(e, t);
|
|
}
|
|
this._evalCameraCuts(t);
|
|
}
|
|
|
|
_evalMorphs(e, t) { // viseme/blendshape weights → stage.setMorph (M4-B)
|
|
const keys = e.tracks && e.tracks.morphs;
|
|
if (!keys || !keys.length || !this.stage.setMorph) return;
|
|
const byKey = new Map();
|
|
for (const k of keys) { if (!byKey.has(k.key)) byKey.set(k.key, []); byKey.get(k.key).push(k); }
|
|
for (const [name, ks] of byKey) {
|
|
const [k0, k1, u] = bracket(ks, t);
|
|
const w = k0 === k1 ? k0.value : lerp(k0.value, k1.value, ease(u, k0.ease));
|
|
this.stage.setMorph(e.id, name, w);
|
|
}
|
|
}
|
|
|
|
_evalTransform(e, t) {
|
|
const keys = e.tracks && e.tracks.transform;
|
|
if (!keys || keys.length === 0) return; // 0 keys → leave rest pose alone
|
|
const br = bracket(keys, t);
|
|
const [k0, k1, u] = br;
|
|
if (k0 === k1) { this.stage.setTransform(e.id, { pos: k0.pos, rot: k0.rot, scale: k0.scale }); return; }
|
|
const eu = ease(u, k0.ease);
|
|
const pos = lerpArr(k0.pos, k1.pos, eu);
|
|
const scale = typeof k0.scale === 'number'
|
|
? lerp(k0.scale, k1.scale, eu)
|
|
: lerpArr(k0.scale, k1.scale, eu);
|
|
const rot = k0.ease === 'step' ? k0.rot.slice() : quatToEuler(slerp(this._quat(k0), this._quat(k1), eu));
|
|
this.stage.setTransform(e.id, { pos, rot, scale });
|
|
}
|
|
|
|
_evalParams(e, t) {
|
|
const keys = e.tracks && e.tracks.params;
|
|
if (!keys || keys.length === 0) return;
|
|
const byKey = new Map();
|
|
for (const k of keys) { if (!byKey.has(k.key)) byKey.set(k.key, []); byKey.get(k.key).push(k); }
|
|
for (const [name, ks] of byKey) {
|
|
const br = bracket(ks, t);
|
|
const [k0, k1, u] = br;
|
|
let val;
|
|
if (k0 === k1) val = k0.value;
|
|
else {
|
|
const eu = ease(u, k0.ease);
|
|
val = isColor(k0.value)
|
|
? rgbToHex(lerpArr(hexToRgb(k0.value), hexToRgb(k1.value), eu))
|
|
: lerp(k0.value, k1.value, eu);
|
|
}
|
|
this.stage.setParam(e.id, name, val);
|
|
}
|
|
}
|
|
|
|
_evalClips(e, t) {
|
|
const blocks = e.tracks.clips;
|
|
if (!blocks || !blocks.length) return;
|
|
const mixer = this.stage.entityMixer(e.id);
|
|
if (!mixer) return; // SYNC1: real Stage returns null for non-characters (stub always had one)
|
|
for (let i = 0; i < blocks.length; i++) {
|
|
const b = blocks[i];
|
|
const rec = this._clipActions.get(b);
|
|
if (!rec) continue; // not preloaded yet → silent
|
|
const span = (b.out - b.in) || 0.0001;
|
|
const blockEnd = b.start + span * (b.loop || 1);
|
|
// pre-roll: a predecessor's fade pulls THIS block's playback start earlier
|
|
// by pred.fade so it advances during the crossfade (continuous local time).
|
|
const pred = blocks[i - 1];
|
|
const preroll = (pred && pred.fade > 0) ? pred.fade : 0;
|
|
const inStart = b.start - preroll;
|
|
let w = 0, local = b.in;
|
|
if (t >= inStart && t < blockEnd) {
|
|
local = b.in + ((t - inStart) % span); // continuous across the boundary
|
|
w = 1;
|
|
if (t < b.start) w = (t - inStart) / preroll; // fade-IN 0→1 over the pred's fade window
|
|
const succ = blocks[i + 1]; // fade-OUT 1→0 over own fade window
|
|
if (succ && b.fade > 0 && t >= blockEnd - b.fade) w = Math.min(w, (blockEnd - t) / b.fade);
|
|
}
|
|
rec.action.time = local;
|
|
rec.action.weight = Math.max(0, w);
|
|
rec.action.enabled = w > 0;
|
|
}
|
|
mixer.update(0); // set-time style — exact when scrubbing
|
|
}
|
|
|
|
_evalCameraCuts(t) {
|
|
const cuts = this.cameraCuts;
|
|
let cam = null;
|
|
for (const c of cuts) { if (c.t <= t) cam = c.camera; else break; }
|
|
if (cam !== this._activeCam) { this._activeCam = cam; this.stage.setActiveCamera(cam); }
|
|
}
|
|
|
|
// ---- keyframe / block mutators (keep sorted, push inverse for undo) ----
|
|
_tracks(id) {
|
|
const e = this.scene.entities.find((x) => x.id === id);
|
|
if (!e) throw new Error(`no entity ${id}`);
|
|
e.tracks = e.tracks || {};
|
|
return e.tracks;
|
|
}
|
|
addKey(id, track, key) {
|
|
const tr = this._tracks(id);
|
|
const arr = (tr[track] = tr[track] || []);
|
|
// a key replaces any existing at the same t; named tracks (params/morphs)
|
|
// are keyed by (t, key name) since many names share the lane.
|
|
const named = track === 'params' || track === 'morphs';
|
|
const same = (k) => k.t === key.t && (!named || k.key === key.key);
|
|
const prevIdx = arr.findIndex(same);
|
|
const prev = prevIdx >= 0 ? arr[prevIdx] : null;
|
|
if (prevIdx >= 0) arr.splice(prevIdx, 1);
|
|
arr.push(key);
|
|
arr.sort((a, b) => a.t - b.t);
|
|
this._qcache.delete(key);
|
|
this.undoStack.push({ undo: () => { const i = arr.indexOf(key); if (i >= 0) arr.splice(i, 1); if (prev) { arr.push(prev); arr.sort((a, b) => a.t - b.t); } } });
|
|
return key;
|
|
}
|
|
moveKey(id, track, key, newT) {
|
|
const oldT = key.t; key.t = newT;
|
|
this._tracks(id)[track].sort((a, b) => a.t - b.t);
|
|
this.undoStack.push({ undo: () => { key.t = oldT; this._tracks(id)[track].sort((a, b) => a.t - b.t); } });
|
|
}
|
|
deleteKey(id, track, key) {
|
|
const arr = this._tracks(id)[track] || [];
|
|
const i = arr.indexOf(key);
|
|
if (i < 0) return;
|
|
arr.splice(i, 1);
|
|
this.undoStack.push({ undo: () => { arr.push(key); arr.sort((a, b) => a.t - b.t); } });
|
|
}
|
|
addClipBlock(id, block) {
|
|
const tr = this._tracks(id);
|
|
const arr = (tr.clips = tr.clips || []);
|
|
arr.push(block); arr.sort((a, b) => a.start - b.start);
|
|
this._clipActions.delete(block);
|
|
this.undoStack.push({ undo: () => { const i = arr.indexOf(block); if (i >= 0) arr.splice(i, 1); } });
|
|
return block;
|
|
}
|
|
moveClipBlock(id, block, newStart) {
|
|
const old = block.start; block.start = newStart;
|
|
this._tracks(id).clips.sort((a, b) => a.start - b.start);
|
|
this.undoStack.push({ undo: () => { block.start = old; this._tracks(id).clips.sort((a, b) => a.start - b.start); } });
|
|
}
|
|
trimClipBlock(id, block, { in: inV, out: outV } = {}) {
|
|
const oldIn = block.in, oldOut = block.out;
|
|
if (inV != null) block.in = inV;
|
|
if (outV != null) block.out = outV;
|
|
this.undoStack.push({ undo: () => { block.in = oldIn; block.out = oldOut; } });
|
|
}
|
|
addCut(t, cameraId) {
|
|
const cut = { t, camera: cameraId };
|
|
this.scene.cameraCuts.push(cut);
|
|
this.scene.cameraCuts.sort((a, b) => a.t - b.t);
|
|
this._activeCam = undefined; // force re-eval of active cam
|
|
this.undoStack.push({ undo: () => { const i = this.scene.cameraCuts.indexOf(cut); if (i >= 0) this.scene.cameraCuts.splice(i, 1); } });
|
|
return cut;
|
|
}
|
|
undo() { const op = this.undoStack.pop(); if (op) op.undo(); this.evaluate(this.time); }
|
|
|
|
// ---- audio track (M4-B). Clips are {path, start, gain}; duration is a
|
|
// client-side draw concern (decode + cache in tlui). Server muxes from
|
|
// scene.audio at render, so step()/evaluate never touch audio. ----
|
|
addAudio(clip) {
|
|
this.scene.audio = this.scene.audio || [];
|
|
this.scene.audio.push(clip);
|
|
this.undoStack.push({ undo: () => { const i = this.scene.audio.indexOf(clip); if (i >= 0) this.scene.audio.splice(i, 1); } });
|
|
return clip;
|
|
}
|
|
moveAudio(clip, newStart) {
|
|
const old = clip.start; clip.start = Math.max(0, newStart);
|
|
this.undoStack.push({ undo: () => { clip.start = old; } });
|
|
}
|
|
setAudioGain(clip, gain) {
|
|
const old = clip.gain; clip.gain = gain;
|
|
this.undoStack.push({ undo: () => { clip.gain = old; } });
|
|
}
|
|
removeAudio(clip) {
|
|
const i = (this.scene.audio || []).indexOf(clip);
|
|
if (i < 0) return;
|
|
this.scene.audio.splice(i, 1);
|
|
this.undoStack.push({ undo: () => { this.scene.audio.splice(i, 0, clip); } });
|
|
}
|
|
|
|
// Rhubarb lip-sync JSON → morph keys. { mouthCues: [{start, end, value}] }.
|
|
// ponytail: per-cue ramp (shape 1 at cue start → 0 at cue end); hold/crossfade
|
|
// tuning deferred until it looks wrong on a real blendshape character.
|
|
importRhubarb(id, rhubarbJson, t0 = 0) {
|
|
const cues = (rhubarbJson && rhubarbJson.mouthCues) || [];
|
|
for (const c of cues) {
|
|
if (!c.value) continue;
|
|
this.addKey(id, 'morphs', { t: t0 + c.start, key: c.value, value: 1, ease: 'linear' });
|
|
this.addKey(id, 'morphs', { t: t0 + c.end, key: c.value, value: 0, ease: 'linear' });
|
|
}
|
|
return (this._tracks(id).morphs || []).length;
|
|
}
|
|
}
|
|
|
|
export default Timeline;
|