From ba29d0c6d6a0339778ffe7911130232c109016ae Mon Sep 17 00:00:00 2001 From: type-two Date: Sat, 18 Jul 2026 21:13:35 +1000 Subject: [PATCH] [laneB] M4-B model: audio mutators, morphs track eval, importRhubarb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - timeline.js: morphs track kind evaluated via stage.setMorph (lerp per viseme name), audio mutators addAudio/moveAudio/setAudioGain/removeAudio (undo-able), importRhubarb(id, json, t0) mapping mouthCues → morph keys (per-cue ramp), load() ensures scene.audio[], addKey keyed by (t,name) for morphs too. - stagestub.js: setMorph (records) + visemeTargets seam. - timeline_test.mjs: morph lerp, rhubarb keys, audio mutators + undo round-trip. Co-Authored-By: Claude Opus 4.8 --- scenegod/web/stagestub.js | 4 +++ scenegod/web/timeline.js | 58 ++++++++++++++++++++++++++++++++-- scenegod/web/timeline_test.mjs | 31 ++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/scenegod/web/stagestub.js b/scenegod/web/stagestub.js index 865d913..95966b8 100644 --- a/scenegod/web/stagestub.js +++ b/scenegod/web/stagestub.js @@ -97,6 +97,10 @@ export class StageStub { setActiveCamera(id) { this._active = id; this.applied.push({ type: 'camera', id }); console.debug('stub.setActiveCamera', id); } renderActiveCamera(_canvas) { /* noop in stub */ } + // M4-B viseme seam (real Stage: Lane A). Entities may carry a `visemes` array. + setMorph(id, name, weight) { this.applied.push({ type: 'morph', id, name, weight }); console.debug('stub.setMorph', id, name, weight); } + visemeTargets(id) { const e = this._entities.get(id); return (e && e.visemes) || []; } + captureState() { return this.entities().map(clone); } async applyState(scene) { this._entities.clear(); diff --git a/scenegod/web/timeline.js b/scenegod/web/timeline.js index 2a0d20c..64fe39b 100644 --- a/scenegod/web/timeline.js +++ b/scenegod/web/timeline.js @@ -134,6 +134,7 @@ export class Timeline { if (this.scene.duration == null) this.scene.duration = 10; this.scene.entities = this.scene.entities || []; this.scene.cameraCuts = this.scene.cameraCuts || []; + this.scene.audio = this.scene.audio || []; for (const e of this.scene.entities) this._sortTracks(e); this.scene.cameraCuts.sort((a, b) => a.t - b.t); this._clipActions.clear(); @@ -181,6 +182,7 @@ export class Timeline { if (!tr) return; if (tr.transform) tr.transform.sort((a, b) => a.t - b.t); if (tr.params) tr.params.sort((a, b) => a.t - b.t); + if (tr.morphs) tr.morphs.sort((a, b) => a.t - b.t); if (tr.clips) tr.clips.sort((a, b) => a.start - b.start); } _quat(key) { @@ -235,11 +237,24 @@ export class Timeline { for (const e of this.entities) { if (e.kind === 'camera' || e.tracks) this._evalTransform(e, t); this._evalParams(e, t); + if (e.tracks && e.tracks.morphs) this._evalMorphs(e, t); if (e.tracks && e.tracks.clips) this._evalClips(e, t); } this._evalCameraCuts(t); } + _evalMorphs(e, t) { // viseme/blendshape weights → stage.setMorph (M4-B) + const keys = e.tracks && e.tracks.morphs; + if (!keys || !keys.length || !this.stage.setMorph) return; + const byKey = new Map(); + for (const k of keys) { if (!byKey.has(k.key)) byKey.set(k.key, []); byKey.get(k.key).push(k); } + for (const [name, ks] of byKey) { + const [k0, k1, u] = bracket(ks, t); + const w = k0 === k1 ? k0.value : lerp(k0.value, k1.value, ease(u, k0.ease)); + this.stage.setMorph(e.id, name, w); + } + } + _evalTransform(e, t) { const keys = e.tracks && e.tracks.transform; if (!keys || keys.length === 0) return; // 0 keys → leave rest pose alone @@ -323,8 +338,10 @@ export class Timeline { addKey(id, track, key) { const tr = this._tracks(id); const arr = (tr[track] = tr[track] || []); - // for transform/param a "key" replaces any existing at same t (+key name) - const same = (k) => k.t === key.t && (track !== 'params' || k.key === key.key); + // a key replaces any existing at the same t; named tracks (params/morphs) + // are keyed by (t, key name) since many names share the lane. + const named = track === 'params' || track === 'morphs'; + const same = (k) => k.t === key.t && (!named || k.key === key.key); const prevIdx = arr.findIndex(same); const prev = prevIdx >= 0 ? arr[prevIdx] : null; if (prevIdx >= 0) arr.splice(prevIdx, 1); @@ -374,6 +391,43 @@ export class Timeline { return cut; } undo() { const op = this.undoStack.pop(); if (op) op.undo(); this.evaluate(this.time); } + + // ---- audio track (M4-B). Clips are {path, start, gain}; duration is a + // client-side draw concern (decode + cache in tlui). Server muxes from + // scene.audio at render, so step()/evaluate never touch audio. ---- + addAudio(clip) { + this.scene.audio = this.scene.audio || []; + this.scene.audio.push(clip); + this.undoStack.push({ undo: () => { const i = this.scene.audio.indexOf(clip); if (i >= 0) this.scene.audio.splice(i, 1); } }); + return clip; + } + moveAudio(clip, newStart) { + const old = clip.start; clip.start = Math.max(0, newStart); + this.undoStack.push({ undo: () => { clip.start = old; } }); + } + setAudioGain(clip, gain) { + const old = clip.gain; clip.gain = gain; + this.undoStack.push({ undo: () => { clip.gain = old; } }); + } + removeAudio(clip) { + const i = (this.scene.audio || []).indexOf(clip); + if (i < 0) return; + this.scene.audio.splice(i, 1); + this.undoStack.push({ undo: () => { this.scene.audio.splice(i, 0, clip); } }); + } + + // Rhubarb lip-sync JSON → morph keys. { mouthCues: [{start, end, value}] }. + // ponytail: per-cue ramp (shape 1 at cue start → 0 at cue end); hold/crossfade + // tuning deferred until it looks wrong on a real blendshape character. + importRhubarb(id, rhubarbJson, t0 = 0) { + const cues = (rhubarbJson && rhubarbJson.mouthCues) || []; + for (const c of cues) { + if (!c.value) continue; + this.addKey(id, 'morphs', { t: t0 + c.start, key: c.value, value: 1, ease: 'linear' }); + this.addKey(id, 'morphs', { t: t0 + c.end, key: c.value, value: 0, ease: 'linear' }); + } + return (this._tracks(id).morphs || []).length; + } } export default Timeline; diff --git a/scenegod/web/timeline_test.mjs b/scenegod/web/timeline_test.mjs index b00ba79..0c31f63 100644 --- a/scenegod/web/timeline_test.mjs +++ b/scenegod/web/timeline_test.mjs @@ -163,4 +163,35 @@ assert.equal(tlDur.time, 5, 'setDuration clamps the playhead into range'); assert.equal(tlDur.scene.entities[0].tracks.transform.length, 1, 'keys past the new end are kept (lossless)'); assert.ok(durNotified >= 1, 'setDuration notifies the UI via onLoad'); +// ---- M4-B: morphs track lerps → stage.setMorph ---- +const morphStage = new StageStub(); +await morphStage.addEntity({ id: 'face', kind: 'character', visemes: ['A', 'B', 'X'] }); +const tlMo = new Timeline(morphStage); +tlMo.load({ version: 1, fps: 30, duration: 4, cameraCuts: [], entities: [ + { id: 'face', kind: 'character', tracks: { morphs: [ + { t: 0, key: 'A', value: 0, ease: 'linear' }, { t: 2, key: 'A', value: 1, ease: 'linear' }, + ] } }, +] }); +morphStage.applied.length = 0; +tlMo.step(30); // t=1 → A at 0.5 +const mp = lastIn(morphStage.applied, (a) => a.type === 'morph' && a.name === 'A'); +assert.ok(mp && near(mp.weight, 0.5, 1e-6), `morph A weight ~0.5, got ${mp && mp.weight}`); + +// ---- M4-B: importRhubarb → morph keys (1 at cue start, 0 at cue end) ---- +const rj = { mouthCues: [{ start: 0.0, end: 0.3, value: 'A' }, { start: 0.3, end: 0.6, value: 'B' }] }; +tlMo.importRhubarb('face', rj, 1.0); // offset by 1s +const mk = tlMo.scene.entities[0].tracks.morphs.filter((k) => k.key === 'A' || k.key === 'B').map((k) => k.key + '@' + k.t + '=' + k.value); +assert.ok(mk.includes('B@1.3=1') && mk.includes('B@1.6=0'), `rhubarb B keys wrong: ${mk.join(',')}`); + +// ---- M4-B: audio mutators + undo ---- +const tlA = new Timeline(new StageStub()); +tlA.load({ version: 1, fps: 30, duration: 10, entities: [], cameraCuts: [] }); +const aclip = tlA.addAudio({ path: 'audio/vo1.wav', start: 2, gain: 1 }); +assert.equal(tlA.scene.audio.length, 1, 'addAudio appends'); +tlA.moveAudio(aclip, 3.5); assert.equal(aclip.start, 3.5, 'moveAudio sets start'); +tlA.setAudioGain(aclip, 0.4); assert.equal(aclip.gain, 0.4, 'setAudioGain sets gain'); +assert.deepEqual(tlA.toJSON().audio, [{ path: 'audio/vo1.wav', start: 3.5, gain: 0.4 }], 'audio survives round-trip'); +tlA.removeAudio(aclip); assert.equal(tlA.scene.audio.length, 0, 'removeAudio drops it'); +tlA.undo(); assert.equal(tlA.scene.audio.length, 1, 'undo restores removed audio'); + console.log('OK — timeline_test.mjs: all assertions passed');