[laneA] M4-A visemes + applyState source:none fix
- stage: detectVisemes() maps CC/ARKit/plain morph names -> canonical
A/E/I/O/U/MBP/FV/L at character load; visemeTargets(id) + setMorph(id,name,w)
for the timeline to drive lip sync; visemeSelfCheck() asserts the matcher
- dock: inspector 🗣 badge (greyed when a rig has no mouth morphs)
- stage: camera/light default source:{type:'none'} (PLAN §4.1); applyState
skips only upload mesh/backdrop bytes -> rig survives save/load (SYNC4)
Verified live at :8020: selfcheck ok=[A,FV,L,MBP,U]; save/load keeps cam+light.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
type-two
parent
ba29d0c6d6
commit
df4e06f047
@ -147,3 +147,41 @@ BLOCKED/REQUESTS: none.
|
||||
|
||||
NEXT: hold at M3-polish boundary. Available for SYNC-3 findings (finalRender drives
|
||||
`pause`/`resume`/`renderActiveCamera` — kept stable per the hard contract).
|
||||
|
||||
## 2026-07-18 session 4 (SYNC4 fix + M4-A visemes)
|
||||
DONE:
|
||||
- **applyState/source clean fix.** Absorbed the orchestrator's SYNC4 hotfix and did the schema-correct
|
||||
version: `addEntity` defaults camera/light `source` to `{type:'none'}` (PLAN §4.1); `captureState`
|
||||
emits it; `applyState` skips only entities that need bytes it lacks (`type:'upload'` AND
|
||||
kind∈{character,prop,backdrop}), so `'none'` always rebuilds and legacy `'upload'` cams still
|
||||
rebuild. Verified live: save→load of {character, camera, light} → srcTypes
|
||||
`[character:assets, camera:none, light:none]`, all 3 come back (rig no longer lost).
|
||||
- **M4-A viseme lane** (my deliverable = detection + setMorph hook + badge; Rhubarb→keys is Lane B):
|
||||
- `detectVisemes(root)` (stage.js): scans every skinned mesh's `morphTargetDictionary`, loosely
|
||||
maps names → canonical visemes A/E/I/O/U/MBP/FV/L (covers CC `V_Open/V_Explosive/V_Dental_Lip/
|
||||
V_Tongue_*`, ARKit `jawOpen/mouthPucker/mouthFunnel/mouthClose`, plain vowels). Built once at
|
||||
character load; stored on the entity as `{morphs, visemes}` (head/teeth/tongue meshes sharing a
|
||||
morph name are grouped so one call drives them all).
|
||||
- `stage.visemeTargets(id)` → canonical names present; `stage.setMorph(id, name, weight)` drives a
|
||||
canonical viseme OR a raw morph name across all mapped meshes.
|
||||
- inspector 🗣 badge (dock.js): lists detected visemes, greyed `🗣 no visemes` when the rig has none.
|
||||
- `visemeSelfCheck()` exported: synthetic dict of CC+ARKit+blink names → asserts A/U/MBP/FV/L
|
||||
detected, E/I/O NOT (no blink/brow false positives), and setMorph('A') drives the mapped indices
|
||||
only. Verified in-browser: `ok=true, found=[A,FV,L,MBP,U]`.
|
||||
|
||||
VERIFICATION (:8020, zero console errors): viseme self-check ok; man/lady show greyed 🗣 badge
|
||||
(neither rig carries mouth morphs — lady.glb is a CC body export with 0 morph targets, confirmed by
|
||||
reading the GLB); save/load round-trip keeps cameras+lights.
|
||||
|
||||
DECISIONS:
|
||||
- Viseme matcher is intentionally loose (orchestrator asked "match loosely") and detection-only; the
|
||||
authoritative Rhubarb-letter→canonical mapping + keyframing is Lane B's. `setMorph` also accepts
|
||||
raw morph names so B can drive anything the matcher missed.
|
||||
|
||||
BLOCKED/REQUESTS:
|
||||
- @orchestrator: no viseme-bearing test character in `assets/` (man.fbx and lady.glb both have zero
|
||||
mouth morphs), so the LIT badge + a visible mouth move are proven only by the synthetic self-check,
|
||||
not on a real asset. If you can drop a CC/ARKit head with visemes into `assets/characters/test/`,
|
||||
I'll confirm live and Lane B can validate Rhubarb end-to-end.
|
||||
|
||||
NEXT: hold at M4-A boundary. Available for a live viseme asset check and any SYNC findings.
|
||||
|
||||
@ -163,6 +163,14 @@ export class Dock {
|
||||
el.appendChild(this._vec3('rot', t.rot, v => this.stage.setTransform(en.id, { rot:v })));
|
||||
el.appendChild(this._num('scale', t.scale, v => this.stage.setTransform(en.id, { scale:v })));
|
||||
|
||||
if(en.kind === 'character'){
|
||||
const vis = this.stage.visemeTargets ? this.stage.visemeTargets(en.id) : [];
|
||||
const badge = document.createElement('div');
|
||||
badge.className = 'viseme' + (vis.length ? '' : ' off');
|
||||
badge.textContent = vis.length ? `🗣 visemes: ${vis.join(' ')}` : '🗣 no visemes';
|
||||
badge.title = vis.length ? 'lip-sync targets detected' : 'no mouth morph targets on this rig';
|
||||
el.appendChild(badge);
|
||||
}
|
||||
if(en.kind === 'camera'){
|
||||
el.appendChild(this._num('fov', en.params.fov ?? 45, v => this.stage.setParam(en.id, 'fov', v)));
|
||||
const act = document.createElement('button'); act.textContent = '◉ set active camera'; act.className = 'wide';
|
||||
|
||||
@ -8,6 +8,46 @@ import { parseAny, captureRest, disposeRoot, stats, bakeRetarget } from './room3
|
||||
|
||||
const ASSET_URL = p => '/assets/file?path=' + encodeURIComponent(p);
|
||||
|
||||
// Loose viseme matcher: map a character's morph-target names (CC "V_Open", ARKit "jawOpen", plain
|
||||
// "AA"…) to a canonical set the timeline drives. Detection only — Rhubarb→keyframes is Lane B's.
|
||||
const VISEMES = {
|
||||
A: [/\bah\b/, /(^|_)aa?($|_)/, /jaw[_ ]?open/, /(^|[_-])open($|[_-])/, /mouth_?open/],
|
||||
E: [/\bee\b/, /(^|_)e[hr]?($|_)/, /wide/, /smile/],
|
||||
I: [/(^|_)i[hy]?($|_)/, /affric/],
|
||||
O: [/\boh\b/, /(^|_)o($|_)/, /tight[_-]?o/, /round/],
|
||||
U: [/\boo\b/, /(^|_)u[w]?($|_)/, /pucker/, /funnel/, /w[_-]?oo/, /(^|_)tight($|_)/],
|
||||
MBP: [/mbp/, /b[_-]?m[_-]?p/, /explos/, /mouth_?close/, /press/, /lip[_-]?open/, /(^|_)pp($|_)/],
|
||||
FV: [/(^|_)f[_-]?v($|_)/, /dental[_-]?lip/, /(^|_)ff($|_)/],
|
||||
L: [/t[_-]?l[_-]?d[_-]?n/, /tongue/, /(^|_)l($|_)/, /(^|_)dd($|_)/, /(^|_)nn($|_)/],
|
||||
};
|
||||
function detectVisemes(root){
|
||||
const morphs = new Map(); // rawName(lower) -> [{mesh, idx}] (head, teeth, tongue often share a name)
|
||||
root.traverse(o => { if(o.isMesh && o.morphTargetDictionary)
|
||||
for(const [name, idx] of Object.entries(o.morphTargetDictionary)){
|
||||
const k = name.toLowerCase(); (morphs.get(k) || morphs.set(k, []).get(k)).push({mesh:o, idx}); } });
|
||||
const visemes = {};
|
||||
for(const [canon, pats] of Object.entries(VISEMES)){
|
||||
const hits = [...morphs.keys()].filter(k => pats.some(p => p.test(k)));
|
||||
if(hits.length) visemes[canon] = hits;
|
||||
}
|
||||
return { morphs, visemes };
|
||||
}
|
||||
// runnable check for the matcher (assert-based). Call visemeSelfCheck() from the console.
|
||||
export function visemeSelfCheck(){
|
||||
const dict = {'V_Open':0,'V_Explosive':1,'jawOpen':2,'mouthPucker':3,'mouthFunnel':4,
|
||||
'mouthClose':5,'V_Dental_Lip':6,'V_Tongue_up':7,'EyeBlink_L':8,'browDown_L':9};
|
||||
const mesh = { isMesh:true, morphTargetDictionary:dict, morphTargetInfluences:new Array(10).fill(0) };
|
||||
const { morphs, visemes } = detectVisemes({ traverse(cb){ cb(mesh); } });
|
||||
const has = v => v in visemes;
|
||||
const detectOk = ['A','U','MBP','FV','L'].every(has) // open/jaw, pucker/funnel, explos/close, dental, tongue
|
||||
&& !has('E') && !has('I') && !has('O'); // blink/brow must NOT register as visemes
|
||||
// canonical 'A' drives every mesh morph mapped to it (jawOpen idx2 + V_Open idx0)
|
||||
for(const r of visemes.A) for(const {idx} of morphs.get(r)) mesh.morphTargetInfluences[idx] = 0.7;
|
||||
const driveOk = mesh.morphTargetInfluences[0] === 0.7 && mesh.morphTargetInfluences[2] === 0.7
|
||||
&& mesh.morphTargetInfluences[8] === 0; // blink untouched
|
||||
return { ok: detectOk && driveOk, found: Object.keys(visemes).sort().join(','), visemes };
|
||||
}
|
||||
|
||||
export class Stage {
|
||||
constructor(viewEl){
|
||||
this.view = viewEl;
|
||||
@ -100,16 +140,18 @@ export class Stage {
|
||||
if(this._idc <= (parseInt(String(id).slice(1)) || 0)) this._idc = parseInt(String(id).slice(1)) || this._idc;
|
||||
const wrapper = new THREE.Group();
|
||||
wrapper.userData.entityId = id;
|
||||
const noAsset = desc.kind === 'camera' || desc.kind === 'light'; // rig entities have no file
|
||||
const en = { id, kind: desc.kind, label: desc.label || desc.kind,
|
||||
source: desc.source || null, params: {...(desc.params||{})},
|
||||
wrapper, root:null, mixer:null, rest:null, cam:null, light:null };
|
||||
source: desc.source || (noAsset ? {type:'none'} : null), params: {...(desc.params||{})},
|
||||
wrapper, root:null, mixer:null, rest:null, cam:null, light:null, morphs:null, visemes:null };
|
||||
|
||||
if(desc.kind === 'character' || desc.kind === 'prop'){
|
||||
const { root } = await this._loadAsset(desc);
|
||||
this._normalize(root, desc.kind === 'character');
|
||||
root.traverse(o=>{ if(o.isMesh){ o.castShadow = true; o.receiveShadow = true; } if(o.isSkinnedMesh) o.frustumCulled=false; });
|
||||
wrapper.add(root); en.root = root;
|
||||
if(desc.kind === 'character'){ en.rest = captureRest(root); en.mixer = new THREE.AnimationMixer(root); }
|
||||
if(desc.kind === 'character'){ en.rest = captureRest(root); en.mixer = new THREE.AnimationMixer(root);
|
||||
const v = detectVisemes(root); en.morphs = v.morphs; en.visemes = v.visemes; }
|
||||
} else if(desc.kind === 'backdrop'){
|
||||
en.root = await this._buildBackdrop(en); wrapper.add(en.root);
|
||||
} else if(desc.kind === 'camera'){
|
||||
@ -312,6 +354,16 @@ export class Stage {
|
||||
}
|
||||
entityMixer(id){ return this._entities.get(id)?.mixer || null; }
|
||||
|
||||
// ---- visemes / morph targets (lip sync) ----
|
||||
visemeTargets(id){ return Object.keys(this._entities.get(id)?.visemes || {}); }
|
||||
// drive a canonical viseme (A/E/I/O/U/MBP/FV/L) or a raw morph name; sets every mapped mesh morph
|
||||
setMorph(id, name, weight){
|
||||
const en = this._entities.get(id); if(!en || !en.morphs) return;
|
||||
const raws = (en.visemes && en.visemes[name]) || (en.morphs.has(name.toLowerCase()) ? [name.toLowerCase()] : []);
|
||||
for(const r of raws) for(const {mesh, idx} of en.morphs.get(r))
|
||||
if(mesh.morphTargetInfluences) mesh.morphTargetInfluences[idx] = weight;
|
||||
}
|
||||
|
||||
// M1 temp helper: play a prepared clip on a character (Timeline replaces this in M2).
|
||||
playClip(id, clip, {loop=true} = {}){
|
||||
const en = this._entities.get(id); if(!en || !en.mixer) return;
|
||||
@ -377,9 +429,10 @@ export class Stage {
|
||||
async applyState(sceneJson){
|
||||
for(const en of this.entities()) this.removeEntity(en.id);
|
||||
for(const d of (sceneJson.entities || [])){
|
||||
// SYNC4: cameras/lights carry source:'upload' but need no bytes — never skip them,
|
||||
// or scene loads silently lose the whole directing rig (cams + sun).
|
||||
if(d.source && d.source.type === 'upload' && d.kind !== 'camera' && d.kind !== 'light'){
|
||||
// skip only what needs asset BYTES we don't have: an 'upload' mesh or backdrop image.
|
||||
// camera/light use source:{type:'none'} (legacy scenes: 'upload') and always rebuild.
|
||||
const needsBytes = d.kind === 'character' || d.kind === 'prop' || d.kind === 'backdrop';
|
||||
if(d.source && d.source.type === 'upload' && needsBytes){
|
||||
console.warn('skipping upload entity (no bytes):', d.id); continue; }
|
||||
try { await this.addEntity(d); } catch(err){ console.error('addEntity failed', d.id, err); }
|
||||
}
|
||||
|
||||
@ -59,6 +59,9 @@ header .tag{font-family:var(--mono);font-size:10px;color:var(--teal);letter-spac
|
||||
button.wide{width:100%;margin:8px 0 0;background:var(--panel);border:1px solid var(--line2);color:var(--ink);
|
||||
border-radius:7px;padding:8px;font-family:var(--mono);font-size:11px;cursor:pointer}
|
||||
button.wide:hover{border-color:var(--teal)}
|
||||
.viseme{font-family:var(--mono);font-size:10.5px;color:var(--teal);background:var(--bg2);
|
||||
border:1px solid var(--line2);border-radius:6px;padding:6px 8px;margin:8px 0 0;word-break:break-word}
|
||||
.viseme.off{color:var(--mut);opacity:.6}
|
||||
|
||||
/* drop + toast */
|
||||
body.hot::after{content:'drop to load';position:fixed;inset:0;display:flex;align-items:center;justify-content:center;
|
||||
|
||||
@ -29,8 +29,13 @@ const CSS = `
|
||||
#timeline .tl-names .row.head{margin-top:${RULER_H}px}
|
||||
#timeline .tl-names .row .k{color:#565f6b;margin-right:6px;font-size:10px}
|
||||
#timeline .tl-names .row.sub{color:#8b949e;padding-left:20px;font-size:11px}
|
||||
#timeline .tl-names .row input.gain{width:38px;margin-left:auto;background:#0d1117;border:1px solid #2b323c;color:#c9d1d9;font:10px ui-monospace;border-radius:3px;padding:1px 2px}
|
||||
#timeline .tl-lanes{flex:1;position:relative;overflow:hidden}
|
||||
#timeline canvas{display:block;width:100%;height:100%}
|
||||
.tl-menu{position:fixed;z-index:9999;background:#12161c;border:1px solid #2b323c;border-radius:4px;max-height:240px;overflow:auto;font:12px ui-monospace;color:#c9d1d9;min-width:160px}
|
||||
.tl-menu .item{padding:4px 10px;cursor:pointer;white-space:nowrap}
|
||||
.tl-menu .item:hover{background:#2b323c}
|
||||
.tl-menu .empty{padding:6px 10px;color:#565f6b}
|
||||
`;
|
||||
|
||||
export class TimelineUI {
|
||||
@ -43,9 +48,16 @@ export class TimelineUI {
|
||||
this._selKeys = []; // box-selected keys: [{id, track, key}]
|
||||
this._snapOn = true;
|
||||
this._snapStep = 'frame'; // 'frame' | seconds string
|
||||
this._audioBuf = new Map(); // path -> decoded AudioBuffer
|
||||
this._audioSrcs = []; // live BufferSourceNodes during playback
|
||||
this._wasPlaying = false;
|
||||
this._injectCSS();
|
||||
this._build();
|
||||
this.tl.onTick(() => this.draw());
|
||||
this.tl.onTick(() => { // natural end-of-play stops audio (timeline auto-pauses)
|
||||
this.draw();
|
||||
if (this._wasPlaying && !this.tl.playing) this._audioStop();
|
||||
this._wasPlaying = this.tl.playing;
|
||||
});
|
||||
this.tl.onLoad(() => { this._syncRows(); this._refreshBar(); }); // SYNC1 #3a/#3b
|
||||
if (this.stage.onChange) this.stage.onChange(() => this._syncRows());
|
||||
this._syncRows();
|
||||
@ -78,6 +90,7 @@ export class TimelineUI {
|
||||
<select class="snapstep" title="snap increment">
|
||||
<option value="frame">frame</option><option value="0.1">0.1s</option>
|
||||
<option value="0.25">0.25s</option><option value="1">1s</option></select>
|
||||
<button data-act="audio" title="add audio clip at playhead">🎵</button>
|
||||
<span class="spring"></span>
|
||||
<button data-act="save">Save</button>
|
||||
<button data-act="load">Load</button>`;
|
||||
@ -91,6 +104,8 @@ export class TimelineUI {
|
||||
this.$play.onclick = () => this._toggle();
|
||||
bar.querySelector('[data-act="save"]').onclick = () => this.save();
|
||||
bar.querySelector('[data-act="load"]').onclick = () => this.load(this.$name.value);
|
||||
this.$audioBtn = bar.querySelector('[data-act="audio"]');
|
||||
this.$audioBtn.onclick = () => this._pickAudio();
|
||||
this.$dur.onchange = () => this.tl.setDuration(this.$dur.value); // clamps + refreshes bar
|
||||
this.$name.onchange = () => { this.tl.scene.name = this.$name.value; };
|
||||
this.$snap.onchange = () => { this._snapOn = this.$snap.checked; };
|
||||
@ -127,8 +142,15 @@ export class TimelineUI {
|
||||
if (e.kind === 'character') rows.push({ type: 'clip', id: e.id, label: '↳ clips' });
|
||||
const paramable = (e.tracks?.params?.length > 0) || e.kind === 'camera' || e.kind === 'light';
|
||||
if (paramable) rows.push({ type: 'params', id: e.id, label: '↳ params' });
|
||||
// viseme lane only when the character actually has morph targets (M4-B)
|
||||
const hasMorphs = (e.tracks?.morphs?.length > 0) ||
|
||||
(e.kind === 'character' && this.stage.visemeTargets && (this.stage.visemeTargets(e.id) || []).length > 0);
|
||||
if (hasMorphs) rows.push({ type: 'morphs', id: e.id, label: '↳ visemes' });
|
||||
}
|
||||
rows.push({ type: 'cameras', label: 'Cameras' });
|
||||
for (const clip of (this.tl.scene.audio || [])) {
|
||||
rows.push({ type: 'audio', clip, label: '🎵 ' + (clip.path || '').split('/').pop() });
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
@ -137,10 +159,17 @@ export class TimelineUI {
|
||||
this.$names.innerHTML = '';
|
||||
this.rows.forEach((r, i) => {
|
||||
const d = document.createElement('div');
|
||||
const sub = r.type === 'clip' || r.type === 'params';
|
||||
const sub = r.type === 'clip' || r.type === 'params' || r.type === 'morphs' || r.type === 'audio';
|
||||
d.className = 'row' + (i === 0 ? ' head' : '') + (sub ? ' sub' : '');
|
||||
if (r.type === 'transform') d.innerHTML = `<span class="k">${(r.kind || '?')[0].toUpperCase()}</span>${r.label}`;
|
||||
else d.textContent = r.label;
|
||||
else if (r.type === 'audio') {
|
||||
d.append(r.label);
|
||||
const g = document.createElement('input');
|
||||
g.type = 'number'; g.className = 'gain'; g.step = '0.1'; g.min = '0'; g.title = 'gain';
|
||||
g.value = r.clip.gain ?? 1;
|
||||
g.onchange = () => this.tl.setAudioGain(r.clip, Math.max(0, +g.value || 0));
|
||||
d.appendChild(g);
|
||||
} else d.textContent = r.label;
|
||||
this.$names.appendChild(d);
|
||||
});
|
||||
// size lanes to content so no row (esp. Cameras) is clipped as rows grow.
|
||||
@ -193,8 +222,10 @@ export class TimelineUI {
|
||||
c.strokeStyle = '#1b2027'; c.beginPath(); c.moveTo(0, y + ROW_H); c.lineTo(w, y + ROW_H); c.stroke();
|
||||
if (r.type === 'transform') this._drawKeys(c, r, y);
|
||||
else if (r.type === 'params') this._drawParams(c, r, y);
|
||||
else if (r.type === 'morphs') this._drawMorphs(c, r, y);
|
||||
else if (r.type === 'clip') this._drawClips(c, r, y);
|
||||
else if (r.type === 'cameras') this._drawCuts(c, y);
|
||||
else if (r.type === 'audio') this._drawAudio(c, r, y);
|
||||
});
|
||||
|
||||
// box-select overlay
|
||||
@ -272,6 +303,28 @@ export class TimelineUI {
|
||||
this._hits.push({ kind: 'cut', cut, x, y: cy });
|
||||
}
|
||||
}
|
||||
_drawMorphs(c, r, y) {
|
||||
const cy = y + ROW_H / 2;
|
||||
const keys = (this._te(r.id)?.tracks?.morphs) || [];
|
||||
for (const k of keys) {
|
||||
const x = this._t2x(k.t);
|
||||
this._diamond(c, x, cy, 4, this._paramColor(k.key), this._selected(k));
|
||||
this._hits.push({ kind: 'key', id: r.id, track: 'morphs', key: k, x, y: cy });
|
||||
}
|
||||
}
|
||||
_drawAudio(c, r, y) {
|
||||
const clip = r.clip;
|
||||
const dur = this._audioDur(clip.path);
|
||||
const x0 = this._t2x(clip.start);
|
||||
const x1 = dur != null ? this._t2x(clip.start + dur) : x0 + 60; // fallback width until decoded
|
||||
const bw = Math.max(3, x1 - x0);
|
||||
c.fillStyle = '#2e7d5b'; c.fillRect(x0, y + 5, bw, ROW_H - 10);
|
||||
c.strokeStyle = '#57c99a'; c.strokeRect(x0 + 0.5, y + 5.5, bw - 1, ROW_H - 11);
|
||||
c.fillStyle = '#dff5ea'; c.font = '10px ui-monospace';
|
||||
c.save(); c.beginPath(); c.rect(x0, y, bw, ROW_H); c.clip();
|
||||
c.fillText((clip.path || '').split('/').pop() + (dur == null ? ' …' : ''), x0 + 4, y + ROW_H / 2 + 3); c.restore();
|
||||
this._hits.push({ kind: 'audioblock', clip, x0, x1, y });
|
||||
}
|
||||
|
||||
// ---- interaction ----
|
||||
_local(e) { const r = this.canvas.getBoundingClientRect(); return { x: e.clientX - r.left, y: e.clientY - r.top }; }
|
||||
@ -285,9 +338,10 @@ export class TimelineUI {
|
||||
if (h.kind === 'key' || h.kind === 'cut') { if (Math.abs(h.x - x) < 7 && Math.abs(h.y - y) < 9) return h; }
|
||||
else if (h.kind === 'edge') { if (Math.abs(h.x - x) < 5 && y > h.y && y < h.y + ROW_H) return h; }
|
||||
}
|
||||
for (const h of this._hits) if (h.kind === 'block' && x >= h.x0 && x <= h.x1 && y > h.y && y < h.y + ROW_H) return h;
|
||||
for (const h of this._hits) if ((h.kind === 'block' || h.kind === 'audioblock') && x >= h.x0 && x <= h.x1 && y > h.y && y < h.y + ROW_H) return h;
|
||||
return null;
|
||||
}
|
||||
_seek(t) { this.tl.seek(t); if (this.tl.playing) this._audioStart(); } // user seek: reschedule audio
|
||||
_keysInRect(x0, y0, x1, y1) {
|
||||
const ax = Math.min(x0, x1), bx = Math.max(x0, x1), ay = Math.min(y0, y1), by = Math.max(y0, y1);
|
||||
const out = [];
|
||||
@ -297,7 +351,7 @@ export class TimelineUI {
|
||||
|
||||
_down(e) {
|
||||
const { x, y } = this._local(e);
|
||||
if (y < RULER_H) { this._drag = { kind: 'seek' }; this.tl.seek(this._x2t(x)); return; }
|
||||
if (y < RULER_H) { this._drag = { kind: 'seek' }; this._seek(this._x2t(x)); return; }
|
||||
const h = this._hitAt(x, y);
|
||||
if (!h) { this._drag = { kind: 'box', x0: x, y0: y, x, y, moved: false }; return; } // empty → box/seek
|
||||
if (h.kind === 'key') {
|
||||
@ -306,17 +360,19 @@ export class TimelineUI {
|
||||
} else { this._selKeys = []; this._drag = { kind: 'key', h }; }
|
||||
} else if (h.kind === 'edge') this._drag = { kind: 'trim', h };
|
||||
else if (h.kind === 'block') this._drag = { kind: 'block', h, grabT: this._x2t(x) - h.block.start };
|
||||
else if (h.kind === 'audioblock') this._drag = { kind: 'audio', h, grabT: this._x2t(x) - h.clip.start };
|
||||
else if (h.kind === 'cut') this._drag = { kind: 'cut', h };
|
||||
}
|
||||
_move(e) {
|
||||
if (!this._drag) return;
|
||||
const { x, y } = this._local(e);
|
||||
const t = this._x2t(x), d = this._drag;
|
||||
if (d.kind === 'seek') this.tl.seek(t);
|
||||
if (d.kind === 'seek') this._seek(t);
|
||||
else if (d.kind === 'key') { this.tl.moveKey(d.h.id, d.h.track, d.h.key, this._snap(t, e)); this.draw(); }
|
||||
else if (d.kind === 'multikey') { for (const s of d.orig) this.tl.moveKey(s.id, s.track, s.key, this._snap(s.t0 + (t - d.grabT), e)); this.draw(); }
|
||||
else if (d.kind === 'cut') { d.h.cut.t = this._snap(t, e); this.tl.cameraCuts.sort((a, b) => a.t - b.t); this.tl._activeCam = undefined; this.draw(); }
|
||||
else if (d.kind === 'block') { this.tl.moveClipBlock(d.h.id, d.h.block, Math.max(0, this._snap(t - d.grabT, e))); this.draw(); }
|
||||
else if (d.kind === 'audio') { this.tl.moveAudio(d.h.clip, this._snap(t - d.grabT, e)); this.draw(); }
|
||||
else if (d.kind === 'trim') { const b = d.h.block; this.tl.trimClipBlock(d.h.id, b, { out: Math.max(b.in + 0.05, b.in + (t - b.start)) }); this.draw(); }
|
||||
else if (d.kind === 'box') { d.x = x; d.y = y; d.moved = Math.abs(x - d.x0) > 4 || Math.abs(y - d.y0) > 4; this.draw(); }
|
||||
}
|
||||
@ -324,7 +380,7 @@ export class TimelineUI {
|
||||
const d = this._drag; this._drag = null;
|
||||
if (!d) return;
|
||||
if (d.kind === 'box') {
|
||||
if (!d.moved) this.tl.seek(this._x2t(d.x0)); // click on empty = seek
|
||||
if (!d.moved) this._seek(this._x2t(d.x0)); // click on empty = seek
|
||||
else { this._selKeys = this._keysInRect(d.x0, d.y0, d.x, d.y); this.draw(); }
|
||||
}
|
||||
}
|
||||
@ -358,12 +414,13 @@ export class TimelineUI {
|
||||
const h = this._hitAt(x, y);
|
||||
if (h && h.kind === 'key') { this.tl.deleteKey(h.id, h.track, h.key); this._selKeys = this._selKeys.filter((s) => s.key !== h.key); this.draw(); }
|
||||
else if (h && h.kind === 'cut') { const i = this.tl.cameraCuts.indexOf(h.cut); if (i >= 0) this.tl.cameraCuts.splice(i, 1); this.tl._activeCam = undefined; this.draw(); }
|
||||
else if (h && h.kind === 'audioblock') { this.tl.removeAudio(h.clip); this._syncRows(); }
|
||||
}
|
||||
_key(e) {
|
||||
if (e.target && /INPUT|TEXTAREA/.test(e.target.tagName)) return;
|
||||
if (e.code === 'Space') { e.preventDefault(); this._toggle(); }
|
||||
else if (e.key === 'Home') this.tl.seek(0);
|
||||
else if (e.key === 'End') this.tl.seek(this.tl.duration);
|
||||
else if (e.key === 'Home') this._seek(0);
|
||||
else if (e.key === 'End') this._seek(this.tl.duration);
|
||||
else if ((e.ctrlKey || e.metaKey) && e.key === 'z') { e.preventDefault(); this.tl.undo(); this.draw(); }
|
||||
else if ((e.key === 'Delete' || e.key === 'Backspace') && this._selKeys.length) {
|
||||
e.preventDefault();
|
||||
@ -371,7 +428,75 @@ export class TimelineUI {
|
||||
this._selKeys = []; this.draw();
|
||||
}
|
||||
}
|
||||
_toggle() { this.tl.playing ? this.tl.pause() : this.tl.play(); this.draw(); }
|
||||
_toggle() {
|
||||
if (this.tl.playing) { this.tl.pause(); this._audioStop(); }
|
||||
else { this.tl.play(); this._audioStart(); }
|
||||
this._wasPlaying = this.tl.playing;
|
||||
this.draw();
|
||||
}
|
||||
|
||||
// ---- Web Audio (draft playback synced to the master clock). step()/render
|
||||
// NEVER touch this — the server muxes from scene.audio deterministically. ----
|
||||
_ensureCtx() {
|
||||
if (!this._audioCtx) { const C = window.AudioContext || window.webkitAudioContext; if (C) this._audioCtx = new C(); }
|
||||
return this._audioCtx;
|
||||
}
|
||||
_audioDur(path) { const b = this._audioBuf.get(path); return b ? b.duration : null; }
|
||||
async _decode(path) {
|
||||
if (this._audioBuf.has(path)) return this._audioBuf.get(path);
|
||||
const ctx = this._ensureCtx(); if (!ctx) return null;
|
||||
try {
|
||||
const res = await fetch('/assets/file?path=' + encodeURIComponent(path));
|
||||
if (!res.ok) throw new Error(res.status);
|
||||
const buf = await ctx.decodeAudioData(await res.arrayBuffer());
|
||||
this._audioBuf.set(path, buf);
|
||||
this.draw(); // width now known
|
||||
return buf;
|
||||
} catch { return null; } // keep the block at fallback width
|
||||
}
|
||||
_decodeAll() { for (const clip of (this.tl.scene.audio || [])) this._decode(clip.path); }
|
||||
_audioStart() {
|
||||
const ctx = this._ensureCtx(); if (!ctx) return;
|
||||
if (ctx.state === 'suspended') ctx.resume();
|
||||
this._audioStop();
|
||||
const now = ctx.currentTime, t = this.tl.time;
|
||||
for (const clip of (this.tl.scene.audio || [])) {
|
||||
const buf = this._audioBuf.get(clip.path); if (!buf) continue;
|
||||
const offset = t - clip.start; // seconds into the clip at current playhead
|
||||
if (offset >= buf.duration) continue; // already finished
|
||||
const src = ctx.createBufferSource(); src.buffer = buf;
|
||||
const g = ctx.createGain(); g.gain.value = clip.gain ?? 1;
|
||||
src.connect(g).connect(ctx.destination);
|
||||
const when = now + Math.max(0, clip.start - t);
|
||||
src.start(when, Math.max(0, offset));
|
||||
this._audioSrcs.push(src);
|
||||
}
|
||||
}
|
||||
_audioStop() { for (const s of this._audioSrcs) { try { s.stop(); } catch { /* already stopped */ } } this._audioSrcs = []; }
|
||||
|
||||
async _pickAudio() {
|
||||
let list = [];
|
||||
try { const r = await fetch('/assets/tree'); if (r.ok) list = (await r.json()).audio || []; } catch { /* offline */ }
|
||||
const menu = document.createElement('div'); menu.className = 'tl-menu';
|
||||
if (!list.length) menu.innerHTML = '<div class="empty">no audio in /assets/tree</div>';
|
||||
for (const it of list) {
|
||||
const path = it.path || it;
|
||||
const item = document.createElement('div'); item.className = 'item';
|
||||
item.textContent = (it.name || path).split('/').pop();
|
||||
item.onclick = async () => {
|
||||
this.tl.addAudio({ path, start: this.tl.time, gain: 1 });
|
||||
menu.remove();
|
||||
await this._decode(path);
|
||||
this._syncRows();
|
||||
};
|
||||
menu.appendChild(item);
|
||||
}
|
||||
document.body.appendChild(menu);
|
||||
const rect = this.$audioBtn.getBoundingClientRect();
|
||||
menu.style.left = rect.left + 'px'; menu.style.top = (rect.bottom + 2) + 'px';
|
||||
const close = (ev) => { if (!menu.contains(ev.target)) { menu.remove(); document.removeEventListener('mousedown', close); } };
|
||||
setTimeout(() => document.addEventListener('mousedown', close), 0);
|
||||
}
|
||||
|
||||
// ---- persistence (Lane C endpoints; localStorage fallback) ----
|
||||
async save() {
|
||||
@ -396,6 +521,7 @@ export class TimelineUI {
|
||||
if (this.stage.applyState) await this.stage.applyState(json);
|
||||
this.tl.load(json); // fires onLoad → _syncRows + _refreshBar
|
||||
await this.tl.preload();
|
||||
this._decodeAll(); // decode audio for lane widths + playback
|
||||
this.draw();
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user