[laneB] M4-B model: audio mutators, morphs track eval, importRhubarb
- timeline.js: morphs track kind evaluated via stage.setMorph (lerp per viseme name), audio mutators addAudio/moveAudio/setAudioGain/removeAudio (undo-able), importRhubarb(id, json, t0) mapping mouthCues → morph keys (per-cue ramp), load() ensures scene.audio[], addKey keyed by (t,name) for morphs too. - stagestub.js: setMorph (records) + visemeTargets seam. - timeline_test.mjs: morph lerp, rhubarb keys, audio mutators + undo round-trip. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
355ad2fe33
commit
ba29d0c6d6
@ -97,6 +97,10 @@ export class StageStub {
|
||||
setActiveCamera(id) { this._active = id; this.applied.push({ type: 'camera', id }); console.debug('stub.setActiveCamera', id); }
|
||||
renderActiveCamera(_canvas) { /* noop in stub */ }
|
||||
|
||||
// M4-B viseme seam (real Stage: Lane A). Entities may carry a `visemes` array.
|
||||
setMorph(id, name, weight) { this.applied.push({ type: 'morph', id, name, weight }); console.debug('stub.setMorph', id, name, weight); }
|
||||
visemeTargets(id) { const e = this._entities.get(id); return (e && e.visemes) || []; }
|
||||
|
||||
captureState() { return this.entities().map(clone); }
|
||||
async applyState(scene) {
|
||||
this._entities.clear();
|
||||
|
||||
@ -134,6 +134,7 @@ export class Timeline {
|
||||
if (this.scene.duration == null) this.scene.duration = 10;
|
||||
this.scene.entities = this.scene.entities || [];
|
||||
this.scene.cameraCuts = this.scene.cameraCuts || [];
|
||||
this.scene.audio = this.scene.audio || [];
|
||||
for (const e of this.scene.entities) this._sortTracks(e);
|
||||
this.scene.cameraCuts.sort((a, b) => a.t - b.t);
|
||||
this._clipActions.clear();
|
||||
@ -181,6 +182,7 @@ export class Timeline {
|
||||
if (!tr) return;
|
||||
if (tr.transform) tr.transform.sort((a, b) => a.t - b.t);
|
||||
if (tr.params) tr.params.sort((a, b) => a.t - b.t);
|
||||
if (tr.morphs) tr.morphs.sort((a, b) => a.t - b.t);
|
||||
if (tr.clips) tr.clips.sort((a, b) => a.start - b.start);
|
||||
}
|
||||
_quat(key) {
|
||||
@ -235,11 +237,24 @@ export class Timeline {
|
||||
for (const e of this.entities) {
|
||||
if (e.kind === 'camera' || e.tracks) this._evalTransform(e, t);
|
||||
this._evalParams(e, t);
|
||||
if (e.tracks && e.tracks.morphs) this._evalMorphs(e, t);
|
||||
if (e.tracks && e.tracks.clips) this._evalClips(e, t);
|
||||
}
|
||||
this._evalCameraCuts(t);
|
||||
}
|
||||
|
||||
_evalMorphs(e, t) { // viseme/blendshape weights → stage.setMorph (M4-B)
|
||||
const keys = e.tracks && e.tracks.morphs;
|
||||
if (!keys || !keys.length || !this.stage.setMorph) return;
|
||||
const byKey = new Map();
|
||||
for (const k of keys) { if (!byKey.has(k.key)) byKey.set(k.key, []); byKey.get(k.key).push(k); }
|
||||
for (const [name, ks] of byKey) {
|
||||
const [k0, k1, u] = bracket(ks, t);
|
||||
const w = k0 === k1 ? k0.value : lerp(k0.value, k1.value, ease(u, k0.ease));
|
||||
this.stage.setMorph(e.id, name, w);
|
||||
}
|
||||
}
|
||||
|
||||
_evalTransform(e, t) {
|
||||
const keys = e.tracks && e.tracks.transform;
|
||||
if (!keys || keys.length === 0) return; // 0 keys → leave rest pose alone
|
||||
@ -323,8 +338,10 @@ export class Timeline {
|
||||
addKey(id, track, key) {
|
||||
const tr = this._tracks(id);
|
||||
const arr = (tr[track] = tr[track] || []);
|
||||
// for transform/param a "key" replaces any existing at same t (+key name)
|
||||
const same = (k) => k.t === key.t && (track !== 'params' || k.key === key.key);
|
||||
// a key replaces any existing at the same t; named tracks (params/morphs)
|
||||
// are keyed by (t, key name) since many names share the lane.
|
||||
const named = track === 'params' || track === 'morphs';
|
||||
const same = (k) => k.t === key.t && (!named || k.key === key.key);
|
||||
const prevIdx = arr.findIndex(same);
|
||||
const prev = prevIdx >= 0 ? arr[prevIdx] : null;
|
||||
if (prevIdx >= 0) arr.splice(prevIdx, 1);
|
||||
@ -374,6 +391,43 @@ export class Timeline {
|
||||
return cut;
|
||||
}
|
||||
undo() { const op = this.undoStack.pop(); if (op) op.undo(); this.evaluate(this.time); }
|
||||
|
||||
// ---- audio track (M4-B). Clips are {path, start, gain}; duration is a
|
||||
// client-side draw concern (decode + cache in tlui). Server muxes from
|
||||
// scene.audio at render, so step()/evaluate never touch audio. ----
|
||||
addAudio(clip) {
|
||||
this.scene.audio = this.scene.audio || [];
|
||||
this.scene.audio.push(clip);
|
||||
this.undoStack.push({ undo: () => { const i = this.scene.audio.indexOf(clip); if (i >= 0) this.scene.audio.splice(i, 1); } });
|
||||
return clip;
|
||||
}
|
||||
moveAudio(clip, newStart) {
|
||||
const old = clip.start; clip.start = Math.max(0, newStart);
|
||||
this.undoStack.push({ undo: () => { clip.start = old; } });
|
||||
}
|
||||
setAudioGain(clip, gain) {
|
||||
const old = clip.gain; clip.gain = gain;
|
||||
this.undoStack.push({ undo: () => { clip.gain = old; } });
|
||||
}
|
||||
removeAudio(clip) {
|
||||
const i = (this.scene.audio || []).indexOf(clip);
|
||||
if (i < 0) return;
|
||||
this.scene.audio.splice(i, 1);
|
||||
this.undoStack.push({ undo: () => { this.scene.audio.splice(i, 0, clip); } });
|
||||
}
|
||||
|
||||
// Rhubarb lip-sync JSON → morph keys. { mouthCues: [{start, end, value}] }.
|
||||
// ponytail: per-cue ramp (shape 1 at cue start → 0 at cue end); hold/crossfade
|
||||
// tuning deferred until it looks wrong on a real blendshape character.
|
||||
importRhubarb(id, rhubarbJson, t0 = 0) {
|
||||
const cues = (rhubarbJson && rhubarbJson.mouthCues) || [];
|
||||
for (const c of cues) {
|
||||
if (!c.value) continue;
|
||||
this.addKey(id, 'morphs', { t: t0 + c.start, key: c.value, value: 1, ease: 'linear' });
|
||||
this.addKey(id, 'morphs', { t: t0 + c.end, key: c.value, value: 0, ease: 'linear' });
|
||||
}
|
||||
return (this._tracks(id).morphs || []).length;
|
||||
}
|
||||
}
|
||||
|
||||
export default Timeline;
|
||||
|
||||
@ -163,4 +163,35 @@ assert.equal(tlDur.time, 5, 'setDuration clamps the playhead into range');
|
||||
assert.equal(tlDur.scene.entities[0].tracks.transform.length, 1, 'keys past the new end are kept (lossless)');
|
||||
assert.ok(durNotified >= 1, 'setDuration notifies the UI via onLoad');
|
||||
|
||||
// ---- M4-B: morphs track lerps → stage.setMorph ----
|
||||
const morphStage = new StageStub();
|
||||
await morphStage.addEntity({ id: 'face', kind: 'character', visemes: ['A', 'B', 'X'] });
|
||||
const tlMo = new Timeline(morphStage);
|
||||
tlMo.load({ version: 1, fps: 30, duration: 4, cameraCuts: [], entities: [
|
||||
{ id: 'face', kind: 'character', tracks: { morphs: [
|
||||
{ t: 0, key: 'A', value: 0, ease: 'linear' }, { t: 2, key: 'A', value: 1, ease: 'linear' },
|
||||
] } },
|
||||
] });
|
||||
morphStage.applied.length = 0;
|
||||
tlMo.step(30); // t=1 → A at 0.5
|
||||
const mp = lastIn(morphStage.applied, (a) => a.type === 'morph' && a.name === 'A');
|
||||
assert.ok(mp && near(mp.weight, 0.5, 1e-6), `morph A weight ~0.5, got ${mp && mp.weight}`);
|
||||
|
||||
// ---- M4-B: importRhubarb → morph keys (1 at cue start, 0 at cue end) ----
|
||||
const rj = { mouthCues: [{ start: 0.0, end: 0.3, value: 'A' }, { start: 0.3, end: 0.6, value: 'B' }] };
|
||||
tlMo.importRhubarb('face', rj, 1.0); // offset by 1s
|
||||
const mk = tlMo.scene.entities[0].tracks.morphs.filter((k) => k.key === 'A' || k.key === 'B').map((k) => k.key + '@' + k.t + '=' + k.value);
|
||||
assert.ok(mk.includes('B@1.3=1') && mk.includes('B@1.6=0'), `rhubarb B keys wrong: ${mk.join(',')}`);
|
||||
|
||||
// ---- M4-B: audio mutators + undo ----
|
||||
const tlA = new Timeline(new StageStub());
|
||||
tlA.load({ version: 1, fps: 30, duration: 10, entities: [], cameraCuts: [] });
|
||||
const aclip = tlA.addAudio({ path: 'audio/vo1.wav', start: 2, gain: 1 });
|
||||
assert.equal(tlA.scene.audio.length, 1, 'addAudio appends');
|
||||
tlA.moveAudio(aclip, 3.5); assert.equal(aclip.start, 3.5, 'moveAudio sets start');
|
||||
tlA.setAudioGain(aclip, 0.4); assert.equal(aclip.gain, 0.4, 'setAudioGain sets gain');
|
||||
assert.deepEqual(tlA.toJSON().audio, [{ path: 'audio/vo1.wav', start: 3.5, gain: 0.4 }], 'audio survives round-trip');
|
||||
tlA.removeAudio(aclip); assert.equal(tlA.scene.audio.length, 0, 'removeAudio drops it');
|
||||
tlA.undo(); assert.equal(tlA.scene.audio.length, 1, 'undo restores removed audio');
|
||||
|
||||
console.log('OK — timeline_test.mjs: all assertions passed');
|
||||
|
||||
Loading…
Reference in New Issue
Block a user