[laneB] M4-B model: audio mutators, morphs track eval, importRhubarb

- timeline.js: morphs track kind evaluated via stage.setMorph (lerp per viseme
  name), audio mutators addAudio/moveAudio/setAudioGain/removeAudio (undo-able),
  importRhubarb(id, json, t0) mapping mouthCues → morph keys (per-cue ramp),
  load() ensures scene.audio[], addKey keyed by (t,name) for morphs too.
- stagestub.js: setMorph (records) + visemeTargets seam.
- timeline_test.mjs: morph lerp, rhubarb keys, audio mutators + undo round-trip.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
type-two 2026-07-18 21:13:35 +10:00
parent 355ad2fe33
commit ba29d0c6d6
3 changed files with 91 additions and 2 deletions

View File

@ -97,6 +97,10 @@ export class StageStub {
setActiveCamera(id) { this._active = id; this.applied.push({ type: 'camera', id }); console.debug('stub.setActiveCamera', id); } setActiveCamera(id) { this._active = id; this.applied.push({ type: 'camera', id }); console.debug('stub.setActiveCamera', id); }
renderActiveCamera(_canvas) { /* noop in stub */ } renderActiveCamera(_canvas) { /* noop in stub */ }
// M4-B viseme seam (real Stage: Lane A). Entities may carry a `visemes` array.
setMorph(id, name, weight) { this.applied.push({ type: 'morph', id, name, weight }); console.debug('stub.setMorph', id, name, weight); }
visemeTargets(id) { const e = this._entities.get(id); return (e && e.visemes) || []; }
captureState() { return this.entities().map(clone); } captureState() { return this.entities().map(clone); }
async applyState(scene) { async applyState(scene) {
this._entities.clear(); this._entities.clear();

View File

@ -134,6 +134,7 @@ export class Timeline {
if (this.scene.duration == null) this.scene.duration = 10; if (this.scene.duration == null) this.scene.duration = 10;
this.scene.entities = this.scene.entities || []; this.scene.entities = this.scene.entities || [];
this.scene.cameraCuts = this.scene.cameraCuts || []; this.scene.cameraCuts = this.scene.cameraCuts || [];
this.scene.audio = this.scene.audio || [];
for (const e of this.scene.entities) this._sortTracks(e); for (const e of this.scene.entities) this._sortTracks(e);
this.scene.cameraCuts.sort((a, b) => a.t - b.t); this.scene.cameraCuts.sort((a, b) => a.t - b.t);
this._clipActions.clear(); this._clipActions.clear();
@ -181,6 +182,7 @@ export class Timeline {
if (!tr) return; if (!tr) return;
if (tr.transform) tr.transform.sort((a, b) => a.t - b.t); if (tr.transform) tr.transform.sort((a, b) => a.t - b.t);
if (tr.params) tr.params.sort((a, b) => a.t - b.t); if (tr.params) tr.params.sort((a, b) => a.t - b.t);
if (tr.morphs) tr.morphs.sort((a, b) => a.t - b.t);
if (tr.clips) tr.clips.sort((a, b) => a.start - b.start); if (tr.clips) tr.clips.sort((a, b) => a.start - b.start);
} }
_quat(key) { _quat(key) {
@ -235,11 +237,24 @@ export class Timeline {
for (const e of this.entities) { for (const e of this.entities) {
if (e.kind === 'camera' || e.tracks) this._evalTransform(e, t); if (e.kind === 'camera' || e.tracks) this._evalTransform(e, t);
this._evalParams(e, t); this._evalParams(e, t);
if (e.tracks && e.tracks.morphs) this._evalMorphs(e, t);
if (e.tracks && e.tracks.clips) this._evalClips(e, t); if (e.tracks && e.tracks.clips) this._evalClips(e, t);
} }
this._evalCameraCuts(t); this._evalCameraCuts(t);
} }
_evalMorphs(e, t) { // viseme/blendshape weights → stage.setMorph (M4-B)
const keys = e.tracks && e.tracks.morphs;
if (!keys || !keys.length || !this.stage.setMorph) return;
const byKey = new Map();
for (const k of keys) { if (!byKey.has(k.key)) byKey.set(k.key, []); byKey.get(k.key).push(k); }
for (const [name, ks] of byKey) {
const [k0, k1, u] = bracket(ks, t);
const w = k0 === k1 ? k0.value : lerp(k0.value, k1.value, ease(u, k0.ease));
this.stage.setMorph(e.id, name, w);
}
}
_evalTransform(e, t) { _evalTransform(e, t) {
const keys = e.tracks && e.tracks.transform; const keys = e.tracks && e.tracks.transform;
if (!keys || keys.length === 0) return; // 0 keys → leave rest pose alone if (!keys || keys.length === 0) return; // 0 keys → leave rest pose alone
@ -323,8 +338,10 @@ export class Timeline {
addKey(id, track, key) { addKey(id, track, key) {
const tr = this._tracks(id); const tr = this._tracks(id);
const arr = (tr[track] = tr[track] || []); const arr = (tr[track] = tr[track] || []);
// for transform/param a "key" replaces any existing at same t (+key name) // a key replaces any existing at the same t; named tracks (params/morphs)
const same = (k) => k.t === key.t && (track !== 'params' || k.key === key.key); // are keyed by (t, key name) since many names share the lane.
const named = track === 'params' || track === 'morphs';
const same = (k) => k.t === key.t && (!named || k.key === key.key);
const prevIdx = arr.findIndex(same); const prevIdx = arr.findIndex(same);
const prev = prevIdx >= 0 ? arr[prevIdx] : null; const prev = prevIdx >= 0 ? arr[prevIdx] : null;
if (prevIdx >= 0) arr.splice(prevIdx, 1); if (prevIdx >= 0) arr.splice(prevIdx, 1);
@ -374,6 +391,43 @@ export class Timeline {
return cut; return cut;
} }
undo() { const op = this.undoStack.pop(); if (op) op.undo(); this.evaluate(this.time); } undo() { const op = this.undoStack.pop(); if (op) op.undo(); this.evaluate(this.time); }
// ---- audio track (M4-B). Clips are {path, start, gain}; duration is a
// client-side draw concern (decode + cache in tlui). Server muxes from
// scene.audio at render, so step()/evaluate never touch audio. ----
addAudio(clip) {
this.scene.audio = this.scene.audio || [];
this.scene.audio.push(clip);
this.undoStack.push({ undo: () => { const i = this.scene.audio.indexOf(clip); if (i >= 0) this.scene.audio.splice(i, 1); } });
return clip;
}
moveAudio(clip, newStart) {
const old = clip.start; clip.start = Math.max(0, newStart);
this.undoStack.push({ undo: () => { clip.start = old; } });
}
setAudioGain(clip, gain) {
const old = clip.gain; clip.gain = gain;
this.undoStack.push({ undo: () => { clip.gain = old; } });
}
removeAudio(clip) {
const i = (this.scene.audio || []).indexOf(clip);
if (i < 0) return;
this.scene.audio.splice(i, 1);
this.undoStack.push({ undo: () => { this.scene.audio.splice(i, 0, clip); } });
}
// Rhubarb lip-sync JSON → morph keys. { mouthCues: [{start, end, value}] }.
// ponytail: per-cue ramp (shape 1 at cue start → 0 at cue end); hold/crossfade
// tuning deferred until it looks wrong on a real blendshape character.
importRhubarb(id, rhubarbJson, t0 = 0) {
const cues = (rhubarbJson && rhubarbJson.mouthCues) || [];
for (const c of cues) {
if (!c.value) continue;
this.addKey(id, 'morphs', { t: t0 + c.start, key: c.value, value: 1, ease: 'linear' });
this.addKey(id, 'morphs', { t: t0 + c.end, key: c.value, value: 0, ease: 'linear' });
}
return (this._tracks(id).morphs || []).length;
}
} }
export default Timeline; export default Timeline;

View File

@ -163,4 +163,35 @@ assert.equal(tlDur.time, 5, 'setDuration clamps the playhead into range');
assert.equal(tlDur.scene.entities[0].tracks.transform.length, 1, 'keys past the new end are kept (lossless)'); assert.equal(tlDur.scene.entities[0].tracks.transform.length, 1, 'keys past the new end are kept (lossless)');
assert.ok(durNotified >= 1, 'setDuration notifies the UI via onLoad'); assert.ok(durNotified >= 1, 'setDuration notifies the UI via onLoad');
// ---- M4-B: morphs track lerps → stage.setMorph ----
const morphStage = new StageStub();
await morphStage.addEntity({ id: 'face', kind: 'character', visemes: ['A', 'B', 'X'] });
const tlMo = new Timeline(morphStage);
tlMo.load({ version: 1, fps: 30, duration: 4, cameraCuts: [], entities: [
{ id: 'face', kind: 'character', tracks: { morphs: [
{ t: 0, key: 'A', value: 0, ease: 'linear' }, { t: 2, key: 'A', value: 1, ease: 'linear' },
] } },
] });
morphStage.applied.length = 0;
tlMo.step(30); // t=1 → A at 0.5
const mp = lastIn(morphStage.applied, (a) => a.type === 'morph' && a.name === 'A');
assert.ok(mp && near(mp.weight, 0.5, 1e-6), `morph A weight ~0.5, got ${mp && mp.weight}`);
// ---- M4-B: importRhubarb → morph keys (1 at cue start, 0 at cue end) ----
const rj = { mouthCues: [{ start: 0.0, end: 0.3, value: 'A' }, { start: 0.3, end: 0.6, value: 'B' }] };
tlMo.importRhubarb('face', rj, 1.0); // offset by 1s
const mk = tlMo.scene.entities[0].tracks.morphs.filter((k) => k.key === 'A' || k.key === 'B').map((k) => k.key + '@' + k.t + '=' + k.value);
assert.ok(mk.includes('B@1.3=1') && mk.includes('B@1.6=0'), `rhubarb B keys wrong: ${mk.join(',')}`);
// ---- M4-B: audio mutators + undo ----
const tlA = new Timeline(new StageStub());
tlA.load({ version: 1, fps: 30, duration: 10, entities: [], cameraCuts: [] });
const aclip = tlA.addAudio({ path: 'audio/vo1.wav', start: 2, gain: 1 });
assert.equal(tlA.scene.audio.length, 1, 'addAudio appends');
tlA.moveAudio(aclip, 3.5); assert.equal(aclip.start, 3.5, 'moveAudio sets start');
tlA.setAudioGain(aclip, 0.4); assert.equal(aclip.gain, 0.4, 'setAudioGain sets gain');
assert.deepEqual(tlA.toJSON().audio, [{ path: 'audio/vo1.wav', start: 3.5, gain: 0.4 }], 'audio survives round-trip');
tlA.removeAudio(aclip); assert.equal(tlA.scene.audio.length, 0, 'removeAudio drops it');
tlA.undo(); assert.equal(tlA.scene.audio.length, 1, 'undo restores removed audio');
console.log('OK — timeline_test.mjs: all assertions passed'); console.log('OK — timeline_test.mjs: all assertions passed');