diff --git a/logs/laneA.md b/logs/laneA.md index 990630b..c17d7f4 100644 --- a/logs/laneA.md +++ b/logs/laneA.md @@ -147,3 +147,41 @@ BLOCKED/REQUESTS: none. NEXT: hold at M3-polish boundary. Available for SYNC-3 findings (finalRender drives `pause`/`resume`/`renderActiveCamera` — kept stable per the hard contract). + +## 2026-07-18 session 4 (SYNC4 fix + M4-A visemes) +DONE: +- **applyState/source clean fix.** Absorbed the orchestrator's SYNC4 hotfix and did the schema-correct + version: `addEntity` defaults camera/light `source` to `{type:'none'}` (PLAN §4.1); `captureState` + emits it; `applyState` skips only entities that need bytes it lacks (`type:'upload'` AND + kind∈{character,prop,backdrop}), so `'none'` always rebuilds and legacy `'upload'` cams still + rebuild. Verified live: save→load of {character, camera, light} → srcTypes + `[character:assets, camera:none, light:none]`, all 3 come back (rig no longer lost). +- **M4-A viseme lane** (my deliverable = detection + setMorph hook + badge; Rhubarb→keys is Lane B): + - `detectVisemes(root)` (stage.js): scans every skinned mesh's `morphTargetDictionary`, loosely + maps names → canonical visemes A/E/I/O/U/MBP/FV/L (covers CC `V_Open/V_Explosive/V_Dental_Lip/ + V_Tongue_*`, ARKit `jawOpen/mouthPucker/mouthFunnel/mouthClose`, plain vowels). Built once at + character load; stored on the entity as `{morphs, visemes}` (head/teeth/tongue meshes sharing a + morph name are grouped so one call drives them all). + - `stage.visemeTargets(id)` → canonical names present; `stage.setMorph(id, name, weight)` drives a + canonical viseme OR a raw morph name across all mapped meshes. + - inspector 🗣 badge (dock.js): lists detected visemes, greyed `🗣 no visemes` when the rig has none. + - `visemeSelfCheck()` exported: synthetic dict of CC+ARKit+blink names → asserts A/U/MBP/FV/L + detected, E/I/O NOT (no blink/brow false positives), and setMorph('A') drives the mapped indices + only. Verified in-browser: `ok=true, found=[A,FV,L,MBP,U]`. + +VERIFICATION (:8020, zero console errors): viseme self-check ok; man/lady show greyed 🗣 badge +(neither rig carries mouth morphs — lady.glb is a CC body export with 0 morph targets, confirmed by +reading the GLB); save/load round-trip keeps cameras+lights. + +DECISIONS: +- Viseme matcher is intentionally loose (orchestrator asked "match loosely") and detection-only; the + authoritative Rhubarb-letter→canonical mapping + keyframing is Lane B's. `setMorph` also accepts + raw morph names so B can drive anything the matcher missed. + +BLOCKED/REQUESTS: +- @orchestrator: no viseme-bearing test character in `assets/` (man.fbx and lady.glb both have zero + mouth morphs), so the LIT badge + a visible mouth move are proven only by the synthetic self-check, + not on a real asset. If you can drop a CC/ARKit head with visemes into `assets/characters/test/`, + I'll confirm live and Lane B can validate Rhubarb end-to-end. + +NEXT: hold at M4-A boundary. Available for a live viseme asset check and any SYNC findings. diff --git a/scenegod/web/dock.js b/scenegod/web/dock.js index c6891fe..9928dc0 100644 --- a/scenegod/web/dock.js +++ b/scenegod/web/dock.js @@ -163,6 +163,14 @@ export class Dock { el.appendChild(this._vec3('rot', t.rot, v => this.stage.setTransform(en.id, { rot:v }))); el.appendChild(this._num('scale', t.scale, v => this.stage.setTransform(en.id, { scale:v }))); + if(en.kind === 'character'){ + const vis = this.stage.visemeTargets ? this.stage.visemeTargets(en.id) : []; + const badge = document.createElement('div'); + badge.className = 'viseme' + (vis.length ? '' : ' off'); + badge.textContent = vis.length ? `🗣 visemes: ${vis.join(' ')}` : '🗣 no visemes'; + badge.title = vis.length ? 'lip-sync targets detected' : 'no mouth morph targets on this rig'; + el.appendChild(badge); + } if(en.kind === 'camera'){ el.appendChild(this._num('fov', en.params.fov ?? 45, v => this.stage.setParam(en.id, 'fov', v))); const act = document.createElement('button'); act.textContent = '◉ set active camera'; act.className = 'wide'; diff --git a/scenegod/web/stage.js b/scenegod/web/stage.js index dfb9980..de7bf82 100644 --- a/scenegod/web/stage.js +++ b/scenegod/web/stage.js @@ -8,6 +8,46 @@ import { parseAny, captureRest, disposeRoot, stats, bakeRetarget } from './room3 const ASSET_URL = p => '/assets/file?path=' + encodeURIComponent(p); +// Loose viseme matcher: map a character's morph-target names (CC "V_Open", ARKit "jawOpen", plain +// "AA"…) to a canonical set the timeline drives. Detection only — Rhubarb→keyframes is Lane B's. +const VISEMES = { + A: [/\bah\b/, /(^|_)aa?($|_)/, /jaw[_ ]?open/, /(^|[_-])open($|[_-])/, /mouth_?open/], + E: [/\bee\b/, /(^|_)e[hr]?($|_)/, /wide/, /smile/], + I: [/(^|_)i[hy]?($|_)/, /affric/], + O: [/\boh\b/, /(^|_)o($|_)/, /tight[_-]?o/, /round/], + U: [/\boo\b/, /(^|_)u[w]?($|_)/, /pucker/, /funnel/, /w[_-]?oo/, /(^|_)tight($|_)/], + MBP: [/mbp/, /b[_-]?m[_-]?p/, /explos/, /mouth_?close/, /press/, /lip[_-]?open/, /(^|_)pp($|_)/], + FV: [/(^|_)f[_-]?v($|_)/, /dental[_-]?lip/, /(^|_)ff($|_)/], + L: [/t[_-]?l[_-]?d[_-]?n/, /tongue/, /(^|_)l($|_)/, /(^|_)dd($|_)/, /(^|_)nn($|_)/], +}; +function detectVisemes(root){ + const morphs = new Map(); // rawName(lower) -> [{mesh, idx}] (head, teeth, tongue often share a name) + root.traverse(o => { if(o.isMesh && o.morphTargetDictionary) + for(const [name, idx] of Object.entries(o.morphTargetDictionary)){ + const k = name.toLowerCase(); (morphs.get(k) || morphs.set(k, []).get(k)).push({mesh:o, idx}); } }); + const visemes = {}; + for(const [canon, pats] of Object.entries(VISEMES)){ + const hits = [...morphs.keys()].filter(k => pats.some(p => p.test(k))); + if(hits.length) visemes[canon] = hits; + } + return { morphs, visemes }; +} +// runnable check for the matcher (assert-based). Call visemeSelfCheck() from the console. +export function visemeSelfCheck(){ + const dict = {'V_Open':0,'V_Explosive':1,'jawOpen':2,'mouthPucker':3,'mouthFunnel':4, + 'mouthClose':5,'V_Dental_Lip':6,'V_Tongue_up':7,'EyeBlink_L':8,'browDown_L':9}; + const mesh = { isMesh:true, morphTargetDictionary:dict, morphTargetInfluences:new Array(10).fill(0) }; + const { morphs, visemes } = detectVisemes({ traverse(cb){ cb(mesh); } }); + const has = v => v in visemes; + const detectOk = ['A','U','MBP','FV','L'].every(has) // open/jaw, pucker/funnel, explos/close, dental, tongue + && !has('E') && !has('I') && !has('O'); // blink/brow must NOT register as visemes + // canonical 'A' drives every mesh morph mapped to it (jawOpen idx2 + V_Open idx0) + for(const r of visemes.A) for(const {idx} of morphs.get(r)) mesh.morphTargetInfluences[idx] = 0.7; + const driveOk = mesh.morphTargetInfluences[0] === 0.7 && mesh.morphTargetInfluences[2] === 0.7 + && mesh.morphTargetInfluences[8] === 0; // blink untouched + return { ok: detectOk && driveOk, found: Object.keys(visemes).sort().join(','), visemes }; +} + export class Stage { constructor(viewEl){ this.view = viewEl; @@ -100,16 +140,18 @@ export class Stage { if(this._idc <= (parseInt(String(id).slice(1)) || 0)) this._idc = parseInt(String(id).slice(1)) || this._idc; const wrapper = new THREE.Group(); wrapper.userData.entityId = id; + const noAsset = desc.kind === 'camera' || desc.kind === 'light'; // rig entities have no file const en = { id, kind: desc.kind, label: desc.label || desc.kind, - source: desc.source || null, params: {...(desc.params||{})}, - wrapper, root:null, mixer:null, rest:null, cam:null, light:null }; + source: desc.source || (noAsset ? {type:'none'} : null), params: {...(desc.params||{})}, + wrapper, root:null, mixer:null, rest:null, cam:null, light:null, morphs:null, visemes:null }; if(desc.kind === 'character' || desc.kind === 'prop'){ const { root } = await this._loadAsset(desc); this._normalize(root, desc.kind === 'character'); root.traverse(o=>{ if(o.isMesh){ o.castShadow = true; o.receiveShadow = true; } if(o.isSkinnedMesh) o.frustumCulled=false; }); wrapper.add(root); en.root = root; - if(desc.kind === 'character'){ en.rest = captureRest(root); en.mixer = new THREE.AnimationMixer(root); } + if(desc.kind === 'character'){ en.rest = captureRest(root); en.mixer = new THREE.AnimationMixer(root); + const v = detectVisemes(root); en.morphs = v.morphs; en.visemes = v.visemes; } } else if(desc.kind === 'backdrop'){ en.root = await this._buildBackdrop(en); wrapper.add(en.root); } else if(desc.kind === 'camera'){ @@ -312,6 +354,16 @@ export class Stage { } entityMixer(id){ return this._entities.get(id)?.mixer || null; } + // ---- visemes / morph targets (lip sync) ---- + visemeTargets(id){ return Object.keys(this._entities.get(id)?.visemes || {}); } + // drive a canonical viseme (A/E/I/O/U/MBP/FV/L) or a raw morph name; sets every mapped mesh morph + setMorph(id, name, weight){ + const en = this._entities.get(id); if(!en || !en.morphs) return; + const raws = (en.visemes && en.visemes[name]) || (en.morphs.has(name.toLowerCase()) ? [name.toLowerCase()] : []); + for(const r of raws) for(const {mesh, idx} of en.morphs.get(r)) + if(mesh.morphTargetInfluences) mesh.morphTargetInfluences[idx] = weight; + } + // M1 temp helper: play a prepared clip on a character (Timeline replaces this in M2). playClip(id, clip, {loop=true} = {}){ const en = this._entities.get(id); if(!en || !en.mixer) return; @@ -377,9 +429,10 @@ export class Stage { async applyState(sceneJson){ for(const en of this.entities()) this.removeEntity(en.id); for(const d of (sceneJson.entities || [])){ - // SYNC4: cameras/lights carry source:'upload' but need no bytes — never skip them, - // or scene loads silently lose the whole directing rig (cams + sun). - if(d.source && d.source.type === 'upload' && d.kind !== 'camera' && d.kind !== 'light'){ + // skip only what needs asset BYTES we don't have: an 'upload' mesh or backdrop image. + // camera/light use source:{type:'none'} (legacy scenes: 'upload') and always rebuild. + const needsBytes = d.kind === 'character' || d.kind === 'prop' || d.kind === 'backdrop'; + if(d.source && d.source.type === 'upload' && needsBytes){ console.warn('skipping upload entity (no bytes):', d.id); continue; } try { await this.addEntity(d); } catch(err){ console.error('addEntity failed', d.id, err); } } diff --git a/scenegod/web/style.css b/scenegod/web/style.css index a7b3d1f..992dd48 100644 --- a/scenegod/web/style.css +++ b/scenegod/web/style.css @@ -59,6 +59,9 @@ header .tag{font-family:var(--mono);font-size:10px;color:var(--teal);letter-spac button.wide{width:100%;margin:8px 0 0;background:var(--panel);border:1px solid var(--line2);color:var(--ink); border-radius:7px;padding:8px;font-family:var(--mono);font-size:11px;cursor:pointer} button.wide:hover{border-color:var(--teal)} +.viseme{font-family:var(--mono);font-size:10.5px;color:var(--teal);background:var(--bg2); + border:1px solid var(--line2);border-radius:6px;padding:6px 8px;margin:8px 0 0;word-break:break-word} +.viseme.off{color:var(--mut);opacity:.6} /* drop + toast */ body.hot::after{content:'drop to load';position:fixed;inset:0;display:flex;align-items:center;justify-content:center; diff --git a/scenegod/web/tlui.js b/scenegod/web/tlui.js index 3e7c329..124030d 100644 --- a/scenegod/web/tlui.js +++ b/scenegod/web/tlui.js @@ -29,8 +29,13 @@ const CSS = ` #timeline .tl-names .row.head{margin-top:${RULER_H}px} #timeline .tl-names .row .k{color:#565f6b;margin-right:6px;font-size:10px} #timeline .tl-names .row.sub{color:#8b949e;padding-left:20px;font-size:11px} +#timeline .tl-names .row input.gain{width:38px;margin-left:auto;background:#0d1117;border:1px solid #2b323c;color:#c9d1d9;font:10px ui-monospace;border-radius:3px;padding:1px 2px} #timeline .tl-lanes{flex:1;position:relative;overflow:hidden} #timeline canvas{display:block;width:100%;height:100%} +.tl-menu{position:fixed;z-index:9999;background:#12161c;border:1px solid #2b323c;border-radius:4px;max-height:240px;overflow:auto;font:12px ui-monospace;color:#c9d1d9;min-width:160px} +.tl-menu .item{padding:4px 10px;cursor:pointer;white-space:nowrap} +.tl-menu .item:hover{background:#2b323c} +.tl-menu .empty{padding:6px 10px;color:#565f6b} `; export class TimelineUI { @@ -43,9 +48,16 @@ export class TimelineUI { this._selKeys = []; // box-selected keys: [{id, track, key}] this._snapOn = true; this._snapStep = 'frame'; // 'frame' | seconds string + this._audioBuf = new Map(); // path -> decoded AudioBuffer + this._audioSrcs = []; // live BufferSourceNodes during playback + this._wasPlaying = false; this._injectCSS(); this._build(); - this.tl.onTick(() => this.draw()); + this.tl.onTick(() => { // natural end-of-play stops audio (timeline auto-pauses) + this.draw(); + if (this._wasPlaying && !this.tl.playing) this._audioStop(); + this._wasPlaying = this.tl.playing; + }); this.tl.onLoad(() => { this._syncRows(); this._refreshBar(); }); // SYNC1 #3a/#3b if (this.stage.onChange) this.stage.onChange(() => this._syncRows()); this._syncRows(); @@ -78,6 +90,7 @@ export class TimelineUI { + `; @@ -91,6 +104,8 @@ export class TimelineUI { this.$play.onclick = () => this._toggle(); bar.querySelector('[data-act="save"]').onclick = () => this.save(); bar.querySelector('[data-act="load"]').onclick = () => this.load(this.$name.value); + this.$audioBtn = bar.querySelector('[data-act="audio"]'); + this.$audioBtn.onclick = () => this._pickAudio(); this.$dur.onchange = () => this.tl.setDuration(this.$dur.value); // clamps + refreshes bar this.$name.onchange = () => { this.tl.scene.name = this.$name.value; }; this.$snap.onchange = () => { this._snapOn = this.$snap.checked; }; @@ -127,8 +142,15 @@ export class TimelineUI { if (e.kind === 'character') rows.push({ type: 'clip', id: e.id, label: '↳ clips' }); const paramable = (e.tracks?.params?.length > 0) || e.kind === 'camera' || e.kind === 'light'; if (paramable) rows.push({ type: 'params', id: e.id, label: '↳ params' }); + // viseme lane only when the character actually has morph targets (M4-B) + const hasMorphs = (e.tracks?.morphs?.length > 0) || + (e.kind === 'character' && this.stage.visemeTargets && (this.stage.visemeTargets(e.id) || []).length > 0); + if (hasMorphs) rows.push({ type: 'morphs', id: e.id, label: '↳ visemes' }); } rows.push({ type: 'cameras', label: 'Cameras' }); + for (const clip of (this.tl.scene.audio || [])) { + rows.push({ type: 'audio', clip, label: '🎵 ' + (clip.path || '').split('/').pop() }); + } return rows; } @@ -137,10 +159,17 @@ export class TimelineUI { this.$names.innerHTML = ''; this.rows.forEach((r, i) => { const d = document.createElement('div'); - const sub = r.type === 'clip' || r.type === 'params'; + const sub = r.type === 'clip' || r.type === 'params' || r.type === 'morphs' || r.type === 'audio'; d.className = 'row' + (i === 0 ? ' head' : '') + (sub ? ' sub' : ''); if (r.type === 'transform') d.innerHTML = `${(r.kind || '?')[0].toUpperCase()}${r.label}`; - else d.textContent = r.label; + else if (r.type === 'audio') { + d.append(r.label); + const g = document.createElement('input'); + g.type = 'number'; g.className = 'gain'; g.step = '0.1'; g.min = '0'; g.title = 'gain'; + g.value = r.clip.gain ?? 1; + g.onchange = () => this.tl.setAudioGain(r.clip, Math.max(0, +g.value || 0)); + d.appendChild(g); + } else d.textContent = r.label; this.$names.appendChild(d); }); // size lanes to content so no row (esp. Cameras) is clipped as rows grow. @@ -193,8 +222,10 @@ export class TimelineUI { c.strokeStyle = '#1b2027'; c.beginPath(); c.moveTo(0, y + ROW_H); c.lineTo(w, y + ROW_H); c.stroke(); if (r.type === 'transform') this._drawKeys(c, r, y); else if (r.type === 'params') this._drawParams(c, r, y); + else if (r.type === 'morphs') this._drawMorphs(c, r, y); else if (r.type === 'clip') this._drawClips(c, r, y); else if (r.type === 'cameras') this._drawCuts(c, y); + else if (r.type === 'audio') this._drawAudio(c, r, y); }); // box-select overlay @@ -272,6 +303,28 @@ export class TimelineUI { this._hits.push({ kind: 'cut', cut, x, y: cy }); } } + _drawMorphs(c, r, y) { + const cy = y + ROW_H / 2; + const keys = (this._te(r.id)?.tracks?.morphs) || []; + for (const k of keys) { + const x = this._t2x(k.t); + this._diamond(c, x, cy, 4, this._paramColor(k.key), this._selected(k)); + this._hits.push({ kind: 'key', id: r.id, track: 'morphs', key: k, x, y: cy }); + } + } + _drawAudio(c, r, y) { + const clip = r.clip; + const dur = this._audioDur(clip.path); + const x0 = this._t2x(clip.start); + const x1 = dur != null ? this._t2x(clip.start + dur) : x0 + 60; // fallback width until decoded + const bw = Math.max(3, x1 - x0); + c.fillStyle = '#2e7d5b'; c.fillRect(x0, y + 5, bw, ROW_H - 10); + c.strokeStyle = '#57c99a'; c.strokeRect(x0 + 0.5, y + 5.5, bw - 1, ROW_H - 11); + c.fillStyle = '#dff5ea'; c.font = '10px ui-monospace'; + c.save(); c.beginPath(); c.rect(x0, y, bw, ROW_H); c.clip(); + c.fillText((clip.path || '').split('/').pop() + (dur == null ? ' …' : ''), x0 + 4, y + ROW_H / 2 + 3); c.restore(); + this._hits.push({ kind: 'audioblock', clip, x0, x1, y }); + } // ---- interaction ---- _local(e) { const r = this.canvas.getBoundingClientRect(); return { x: e.clientX - r.left, y: e.clientY - r.top }; } @@ -285,9 +338,10 @@ export class TimelineUI { if (h.kind === 'key' || h.kind === 'cut') { if (Math.abs(h.x - x) < 7 && Math.abs(h.y - y) < 9) return h; } else if (h.kind === 'edge') { if (Math.abs(h.x - x) < 5 && y > h.y && y < h.y + ROW_H) return h; } } - for (const h of this._hits) if (h.kind === 'block' && x >= h.x0 && x <= h.x1 && y > h.y && y < h.y + ROW_H) return h; + for (const h of this._hits) if ((h.kind === 'block' || h.kind === 'audioblock') && x >= h.x0 && x <= h.x1 && y > h.y && y < h.y + ROW_H) return h; return null; } + _seek(t) { this.tl.seek(t); if (this.tl.playing) this._audioStart(); } // user seek: reschedule audio _keysInRect(x0, y0, x1, y1) { const ax = Math.min(x0, x1), bx = Math.max(x0, x1), ay = Math.min(y0, y1), by = Math.max(y0, y1); const out = []; @@ -297,7 +351,7 @@ export class TimelineUI { _down(e) { const { x, y } = this._local(e); - if (y < RULER_H) { this._drag = { kind: 'seek' }; this.tl.seek(this._x2t(x)); return; } + if (y < RULER_H) { this._drag = { kind: 'seek' }; this._seek(this._x2t(x)); return; } const h = this._hitAt(x, y); if (!h) { this._drag = { kind: 'box', x0: x, y0: y, x, y, moved: false }; return; } // empty → box/seek if (h.kind === 'key') { @@ -306,17 +360,19 @@ export class TimelineUI { } else { this._selKeys = []; this._drag = { kind: 'key', h }; } } else if (h.kind === 'edge') this._drag = { kind: 'trim', h }; else if (h.kind === 'block') this._drag = { kind: 'block', h, grabT: this._x2t(x) - h.block.start }; + else if (h.kind === 'audioblock') this._drag = { kind: 'audio', h, grabT: this._x2t(x) - h.clip.start }; else if (h.kind === 'cut') this._drag = { kind: 'cut', h }; } _move(e) { if (!this._drag) return; const { x, y } = this._local(e); const t = this._x2t(x), d = this._drag; - if (d.kind === 'seek') this.tl.seek(t); + if (d.kind === 'seek') this._seek(t); else if (d.kind === 'key') { this.tl.moveKey(d.h.id, d.h.track, d.h.key, this._snap(t, e)); this.draw(); } else if (d.kind === 'multikey') { for (const s of d.orig) this.tl.moveKey(s.id, s.track, s.key, this._snap(s.t0 + (t - d.grabT), e)); this.draw(); } else if (d.kind === 'cut') { d.h.cut.t = this._snap(t, e); this.tl.cameraCuts.sort((a, b) => a.t - b.t); this.tl._activeCam = undefined; this.draw(); } else if (d.kind === 'block') { this.tl.moveClipBlock(d.h.id, d.h.block, Math.max(0, this._snap(t - d.grabT, e))); this.draw(); } + else if (d.kind === 'audio') { this.tl.moveAudio(d.h.clip, this._snap(t - d.grabT, e)); this.draw(); } else if (d.kind === 'trim') { const b = d.h.block; this.tl.trimClipBlock(d.h.id, b, { out: Math.max(b.in + 0.05, b.in + (t - b.start)) }); this.draw(); } else if (d.kind === 'box') { d.x = x; d.y = y; d.moved = Math.abs(x - d.x0) > 4 || Math.abs(y - d.y0) > 4; this.draw(); } } @@ -324,7 +380,7 @@ export class TimelineUI { const d = this._drag; this._drag = null; if (!d) return; if (d.kind === 'box') { - if (!d.moved) this.tl.seek(this._x2t(d.x0)); // click on empty = seek + if (!d.moved) this._seek(this._x2t(d.x0)); // click on empty = seek else { this._selKeys = this._keysInRect(d.x0, d.y0, d.x, d.y); this.draw(); } } } @@ -358,12 +414,13 @@ export class TimelineUI { const h = this._hitAt(x, y); if (h && h.kind === 'key') { this.tl.deleteKey(h.id, h.track, h.key); this._selKeys = this._selKeys.filter((s) => s.key !== h.key); this.draw(); } else if (h && h.kind === 'cut') { const i = this.tl.cameraCuts.indexOf(h.cut); if (i >= 0) this.tl.cameraCuts.splice(i, 1); this.tl._activeCam = undefined; this.draw(); } + else if (h && h.kind === 'audioblock') { this.tl.removeAudio(h.clip); this._syncRows(); } } _key(e) { if (e.target && /INPUT|TEXTAREA/.test(e.target.tagName)) return; if (e.code === 'Space') { e.preventDefault(); this._toggle(); } - else if (e.key === 'Home') this.tl.seek(0); - else if (e.key === 'End') this.tl.seek(this.tl.duration); + else if (e.key === 'Home') this._seek(0); + else if (e.key === 'End') this._seek(this.tl.duration); else if ((e.ctrlKey || e.metaKey) && e.key === 'z') { e.preventDefault(); this.tl.undo(); this.draw(); } else if ((e.key === 'Delete' || e.key === 'Backspace') && this._selKeys.length) { e.preventDefault(); @@ -371,7 +428,75 @@ export class TimelineUI { this._selKeys = []; this.draw(); } } - _toggle() { this.tl.playing ? this.tl.pause() : this.tl.play(); this.draw(); } + _toggle() { + if (this.tl.playing) { this.tl.pause(); this._audioStop(); } + else { this.tl.play(); this._audioStart(); } + this._wasPlaying = this.tl.playing; + this.draw(); + } + + // ---- Web Audio (draft playback synced to the master clock). step()/render + // NEVER touch this — the server muxes from scene.audio deterministically. ---- + _ensureCtx() { + if (!this._audioCtx) { const C = window.AudioContext || window.webkitAudioContext; if (C) this._audioCtx = new C(); } + return this._audioCtx; + } + _audioDur(path) { const b = this._audioBuf.get(path); return b ? b.duration : null; } + async _decode(path) { + if (this._audioBuf.has(path)) return this._audioBuf.get(path); + const ctx = this._ensureCtx(); if (!ctx) return null; + try { + const res = await fetch('/assets/file?path=' + encodeURIComponent(path)); + if (!res.ok) throw new Error(res.status); + const buf = await ctx.decodeAudioData(await res.arrayBuffer()); + this._audioBuf.set(path, buf); + this.draw(); // width now known + return buf; + } catch { return null; } // keep the block at fallback width + } + _decodeAll() { for (const clip of (this.tl.scene.audio || [])) this._decode(clip.path); } + _audioStart() { + const ctx = this._ensureCtx(); if (!ctx) return; + if (ctx.state === 'suspended') ctx.resume(); + this._audioStop(); + const now = ctx.currentTime, t = this.tl.time; + for (const clip of (this.tl.scene.audio || [])) { + const buf = this._audioBuf.get(clip.path); if (!buf) continue; + const offset = t - clip.start; // seconds into the clip at current playhead + if (offset >= buf.duration) continue; // already finished + const src = ctx.createBufferSource(); src.buffer = buf; + const g = ctx.createGain(); g.gain.value = clip.gain ?? 1; + src.connect(g).connect(ctx.destination); + const when = now + Math.max(0, clip.start - t); + src.start(when, Math.max(0, offset)); + this._audioSrcs.push(src); + } + } + _audioStop() { for (const s of this._audioSrcs) { try { s.stop(); } catch { /* already stopped */ } } this._audioSrcs = []; } + + async _pickAudio() { + let list = []; + try { const r = await fetch('/assets/tree'); if (r.ok) list = (await r.json()).audio || []; } catch { /* offline */ } + const menu = document.createElement('div'); menu.className = 'tl-menu'; + if (!list.length) menu.innerHTML = '
no audio in /assets/tree
'; + for (const it of list) { + const path = it.path || it; + const item = document.createElement('div'); item.className = 'item'; + item.textContent = (it.name || path).split('/').pop(); + item.onclick = async () => { + this.tl.addAudio({ path, start: this.tl.time, gain: 1 }); + menu.remove(); + await this._decode(path); + this._syncRows(); + }; + menu.appendChild(item); + } + document.body.appendChild(menu); + const rect = this.$audioBtn.getBoundingClientRect(); + menu.style.left = rect.left + 'px'; menu.style.top = (rect.bottom + 2) + 'px'; + const close = (ev) => { if (!menu.contains(ev.target)) { menu.remove(); document.removeEventListener('mousedown', close); } }; + setTimeout(() => document.addEventListener('mousedown', close), 0); + } // ---- persistence (Lane C endpoints; localStorage fallback) ---- async save() { @@ -396,6 +521,7 @@ export class TimelineUI { if (this.stage.applyState) await this.stage.applyState(json); this.tl.load(json); // fires onLoad → _syncRows + _refreshBar await this.tl.preload(); + this._decodeAll(); // decode audio for lane widths + playback this.draw(); } }