// stage.js — the 3D stage: renderer, IBL scene, entities on wrappers, gizmo, retarget cache. // Timeline (Lane B) talks to the stage ONLY through this API (PLAN §4.2). import * as THREE from 'three'; import { OrbitControls } from 'three/addons/controls/OrbitControls.js'; import { TransformControls } from 'three/addons/controls/TransformControls.js'; import { RoomEnvironment } from 'three/addons/environments/RoomEnvironment.js'; import { parseAny, captureRest, disposeRoot, stats, bakeRetarget } from './room3d.js'; const ASSET_URL = p => '/assets/file?path=' + encodeURIComponent(p); // Loose viseme matcher: map a character's morph-target names (CC "V_Open", ARKit "jawOpen", plain // "AA"…) to a canonical set the timeline drives. Detection only — Rhubarb→keyframes is Lane B's. const VISEMES = { A: [/\bah\b/, /(^|_)aa?($|_)/, /jaw[_ ]?open/, /(^|[_-])open($|[_-])/, /mouth_?open/], E: [/\bee\b/, /(^|_)e[hr]?($|_)/, /wide/, /smile/], I: [/(^|_)i[hy]?($|_)/, /affric/], O: [/\boh\b/, /(^|_)o($|_)/, /tight[_-]?o/, /round/], U: [/\boo\b/, /(^|_)u[w]?($|_)/, /pucker/, /funnel/, /w[_-]?oo/, /(^|_)tight($|_)/], MBP: [/mbp/, /b[_-]?m[_-]?p/, /explos/, /mouth_?close/, /press/, /lip[_-]?open/, /(^|_)pp($|_)/], FV: [/(^|_)f[_-]?v($|_)/, /dental[_-]?lip/, /(^|_)ff($|_)/], L: [/t[_-]?l[_-]?d[_-]?n/, /tongue/, /(^|_)l($|_)/, /(^|_)dd($|_)/, /(^|_)nn($|_)/], }; function detectVisemes(root){ const morphs = new Map(); // rawName(lower) -> [{mesh, idx}] (head, teeth, tongue often share a name) root.traverse(o => { if(o.isMesh && o.morphTargetDictionary) for(const [name, idx] of Object.entries(o.morphTargetDictionary)){ const k = name.toLowerCase(); (morphs.get(k) || morphs.set(k, []).get(k)).push({mesh:o, idx}); } }); const visemes = {}; // SYNC5: ARKit side suffixes (_L/_R) made /(^|_)l($|_)/ swallow eyeblink_l & co — // never match non-mouth regions, and match against the side-stripped name. const NONMOUTH = /eye|blink|brow|squint|look|nose|cheek/; for(const [canon, pats] of Object.entries(VISEMES)){ const hits = [...morphs.keys()].filter(k => !NONMOUTH.test(k) && pats.some(p => p.test(k.replace(/(.)[_-][lr]$/, '$1')))); if(hits.length) visemes[canon] = hits; } return { morphs, visemes }; } // runnable check for the matcher (assert-based). Call visemeSelfCheck() from the console. export function visemeSelfCheck(){ const dict = {'V_Open':0,'V_Explosive':1,'jawOpen':2,'mouthPucker':3,'mouthFunnel':4, 'mouthClose':5,'V_Dental_Lip':6,'V_Tongue_up':7,'EyeBlink_L':8,'browDown_L':9,'mouthSmile_L':10}; const mesh = { isMesh:true, morphTargetDictionary:dict, morphTargetInfluences:new Array(11).fill(0) }; const { morphs, visemes } = detectVisemes({ traverse(cb){ cb(mesh); } }); const has = v => v in visemes; const detectOk = ['A','U','MBP','FV','L'].every(has) // open/jaw, pucker/funnel, explos/close, dental, tongue && !has('I') && !has('O') // SYNC5 regression guard: side-suffixed ARKit names must land on the RIGHT viseme — // L is tongue only (no blink/brow/smile bleed), E gets the side-stripped smile. && visemes.L.length === 1 && visemes.L[0] === 'v_tongue_up' && has('E') && visemes.E.includes('mouthsmile_l'); // canonical 'A' drives every mesh morph mapped to it (jawOpen idx2 + V_Open idx0) for(const r of visemes.A) for(const {idx} of morphs.get(r)) mesh.morphTargetInfluences[idx] = 0.7; const driveOk = mesh.morphTargetInfluences[0] === 0.7 && mesh.morphTargetInfluences[2] === 0.7 && mesh.morphTargetInfluences[8] === 0; // blink untouched return { ok: detectOk && driveOk, found: Object.keys(visemes).sort().join(','), visemes }; } export class Stage { constructor(viewEl){ this.view = viewEl; this._entities = new Map(); this._selCbs = []; this._changeCbs = []; this._selected = null; this._activeCamId = null; this._mixerAuto = true; // Stage advances mixers each frame until Timeline takes over this._idc = 0; this._clipSrc = new Map(); // path -> {root, rest, anims} (parse+rest cached) this._baked = new Map(); // id|path|index -> AnimationClip const r = this.renderer = new THREE.WebGLRenderer({antialias:true, preserveDrawingBuffer:true}); r.setPixelRatio(devicePixelRatio); r.shadowMap.enabled = true; view.appendChild(r.domElement); this._paused = false; // M3 final render freezes the director loop, then drives renderActiveCamera // PiP overlay — the active camera's view (bottom-right). CSS sizes it (~24% w); the buffer // stays small. Click to swap which view is main vs inset. const pip = this._pip = document.createElement('canvas'); pip.width = 320; pip.height = 180; pip.className = 'pip'; pip.style.display = 'none'; pip.title = 'click to swap with the main view'; this._pipSwap = false; pip.addEventListener('click', () => { this._pipSwap = !this._pipSwap; }); view.appendChild(pip); const scene = this.scene = new THREE.Scene(); const cam = this._dirCam = new THREE.PerspectiveCamera(45, 1, 0.01, 3000); cam.position.set(0, 1.6, 5); const ctr = this.controls = new OrbitControls(cam, r.domElement); ctr.enableDamping = true; ctr.target.set(0, 1.0, 0); const pmrem = new THREE.PMREMGenerator(r); scene.environment = pmrem.fromScene(new RoomEnvironment(), 0.04).texture; pmrem.dispose(); // default lights — replaced/dimmed once the scene carries light entities this._defHemi = new THREE.HemisphereLight(0xffffff, 0x555a66, 2.2); scene.add(this._defHemi); this._defKey = new THREE.DirectionalLight(0xffffff, 1.8); this._defKey.position.set(3, 5, 2); scene.add(this._defKey); const floor = new THREE.Mesh(new THREE.CircleGeometry(30, 64).rotateX(-Math.PI/2), new THREE.MeshStandardMaterial({color:0x141a20, roughness:.97})); floor.receiveShadow = true; scene.add(floor); scene.add(new THREE.GridHelper(60, 60, 0x2a323b, 0x1c232b)); // gizmo const tc = this._tc = new TransformControls(cam, r.domElement); tc.addEventListener('dragging-changed', e => { ctr.enabled = !e.value; }); tc.addEventListener('mouseUp', () => { if(this._selected) this._fireChange(this.getEntity(this._selected)); }); scene.add(tc); addEventListener('keydown', e => { if(!this._selected) return; if(e.key==='w') tc.setMode('translate'); else if(e.key==='e') tc.setMode('rotate'); else if(e.key==='r') tc.setMode('scale'); }); r.domElement.addEventListener('click', e => this._pick(e)); this.clock = new THREE.Clock(); this._resize(); addEventListener('resize', () => this._resize()); const loop = () => { requestAnimationFrame(loop); const dt = this.clock.getDelta(); if(this._paused) return; // M3: caller drives rendering deterministically ctr.update(); if(this._mixerAuto) for(const en of this._entities.values()) en.mixer && en.mixer.update(dt); // inset first (it scribbles onto the main canvas), then the main view last so it wins if(this._activeCamId && pip.style.display !== 'none') this._render(this._pipSwap ? this._dirCam : this._activeCam(), pip); this._render((this._pipSwap && this._activeCamId) ? this._activeCam() : this._dirCam, null); }; loop(); } _resize(){ const w=this.view.clientWidth, h=this.view.clientHeight; this.renderer.setSize(w, h, false); this._dirCam.aspect = w/h; this._dirCam.updateProjectionMatrix(); // size the inset here — CSS aspect-ratio/% is unreliable on a const pw = Math.max(180, Math.min(340, w*0.24)); this._pip.style.width = Math.round(pw)+'px'; this._pip.style.height = Math.round(pw*9/16)+'px'; } // ---- callbacks ---- onSelect(cb){ this._selCbs.push(cb); } onChange(cb){ this._changeCbs.push(cb); } _fireChange(en){ for(const cb of this._changeCbs) cb(en); } // ---- entity lifecycle ---- entities(){ return [...this._entities.values()]; } getEntity(id){ return this._entities.get(id); } async addEntity(desc){ const id = desc.id || ('e' + (++this._idc)); if(this._idc <= (parseInt(String(id).slice(1)) || 0)) this._idc = parseInt(String(id).slice(1)) || this._idc; const wrapper = new THREE.Group(); wrapper.userData.entityId = id; const noAsset = desc.kind === 'camera' || desc.kind === 'light'; // rig entities have no file const en = { id, kind: desc.kind, label: desc.label || desc.kind, source: desc.source || (noAsset ? {type:'none'} : null), params: {...(desc.params||{})}, wrapper, root:null, mixer:null, rest:null, cam:null, light:null, morphs:null, visemes:null }; if(desc.kind === 'character' || desc.kind === 'prop'){ const { root } = await this._loadAsset(desc); this._normalize(root, desc.kind === 'character'); root.traverse(o=>{ if(o.isMesh){ o.castShadow = true; o.receiveShadow = true; } if(o.isSkinnedMesh) o.frustumCulled=false; }); wrapper.add(root); en.root = root; if(desc.kind === 'character'){ en.rest = captureRest(root); en.mixer = new THREE.AnimationMixer(root); const v = detectVisemes(root); en.morphs = v.morphs; en.visemes = v.visemes; } } else if(desc.kind === 'backdrop'){ en.root = await this._buildBackdrop(en); wrapper.add(en.root); } else if(desc.kind === 'camera'){ const fov = en.params.fov || 45; en.cam = new THREE.PerspectiveCamera(fov, 16/9, 0.05, 3000); wrapper.add(en.cam); const proxy = new THREE.Mesh(new THREE.ConeGeometry(0.18, 0.35, 4).rotateX(-Math.PI/2), new THREE.MeshBasicMaterial({color:0x43c0cf, wireframe:true})); proxy.position.z = 0.18; wrapper.add(proxy); } else if(desc.kind === 'light'){ this._buildLight(en); } else throw new Error('unknown kind: ' + desc.kind); this.scene.add(wrapper); this._entities.set(id, en); if(desc.transform) this.setTransform(id, desc.transform); this._fireChange(en); // tlui/dock build a row per entity off onChange (incl. applyState loads) return en; } removeEntity(id){ const en = this._entities.get(id); if(!en) return; if(this._selected === id) this.select(null); if(en.root) disposeRoot(en.root); if(en.light) en.light.parent && en.light.parent.remove(en.light); this.scene.remove(en.wrapper); this._entities.delete(id); for(const k of [...this._baked.keys()]) if(k.startsWith(id+'|')) this._baked.delete(k); if(en.kind === 'light') this._refreshDefaults(); this._fireChange(en); // trigger a row resync so the deleted entity drops out } async _loadAsset(desc){ let buf, name; if(desc.source && desc.source.type === 'upload'){ if(!desc._buf) throw new Error('upload entity has no buffer'); buf = desc._buf; name = desc.source.path; } else { const path = desc.source.path; const res = await fetch(ASSET_URL(path)); if(!res.ok) throw new Error('asset fetch ' + res.status + ': ' + path); buf = await res.arrayBuffer(); name = path; } return parseAny(buf, name); } _normalize(root, isChar){ root.updateMatrixWorld(true); const box = new THREE.Box3().setFromObject(root); if(box.isEmpty() || !isFinite(box.min.x)) return; if(isChar){ const size = box.getSize(new THREE.Vector3()); const s = 1.7 / (Math.max(size.x, size.y, size.z) || 1); root.scale.setScalar(s); root.updateMatrixWorld(true); } const b = new THREE.Box3().setFromObject(root); root.position.x -= (b.min.x + b.max.x)/2; root.position.z -= (b.min.z + b.max.z)/2; root.position.y -= b.min.y; // feet on the floor } async _buildBackdrop(en){ const p = en.params; const url = en.source ? (en.source.type === 'upload' ? en.source._url : ASSET_URL(p.image || en.source.path)) : ASSET_URL(p.image); const tex = await new THREE.TextureLoader().loadAsync(url); tex.colorSpace = THREE.SRGBColorSpace; const aspect = (tex.image.width / tex.image.height) || 1; const w = p.width || 10, h = w / aspect; const mat = new THREE.MeshStandardMaterial({map:tex, roughness:1, metalness:0}); const grp = new THREE.Group(); if(p.mode === 'dome'){ const dome = new THREE.Mesh(new THREE.SphereGeometry(40, 40, 24), new THREE.MeshBasicMaterial({map:tex, side:THREE.BackSide})); grp.add(dome); } else { const wall = new THREE.Mesh(new THREE.PlaneGeometry(w, h), mat); wall.position.y = h/2; wall.receiveShadow = true; grp.add(wall); if(p.mode === 'corner'){ // cheap "L": duplicate lower third laid flat as ground const gh = h/3; const gm = new THREE.Mesh(new THREE.PlaneGeometry(w, gh), mat.clone()); gm.rotation.x = -Math.PI/2; gm.position.z = gh/2; gm.receiveShadow = true; grp.add(gm); wall.position.z = -gh/2; } } return grp; } _buildLight(en){ const p = en.params; if(p.type === 'ambient'){ en.light = new THREE.HemisphereLight(new THREE.Color(p.color||'#ffffff'), 0x444450, p.intensity ?? 1.0); en.wrapper.add(en.light); } else { const dir = en.light = new THREE.DirectionalLight(new THREE.Color(p.color||'#ffffff'), p.intensity ?? 1.5); dir.castShadow = p.castShadow ?? true; dir.shadow.mapSize.set(1024,1024); dir.shadow.camera.far = 60; dir.position.set(0,0,0); const tgt = new THREE.Object3D(); tgt.position.set(0,0,-1); en.wrapper.add(tgt); dir.target = tgt; en.wrapper.add(dir); const proxy = new THREE.Mesh(new THREE.SphereGeometry(0.15, 12, 8), new THREE.MeshBasicMaterial({color:0xffe08a})); en.wrapper.add(proxy); } this._refreshDefaults(); } // dim built-in lights once scene lights exist _refreshDefaults(){ const hasKey = this.entities().some(e => e.kind==='light' && e.params.type!=='ambient'); const hasAmb = this.entities().some(e => e.kind==='light' && e.params.type==='ambient'); this._defKey.visible = !hasKey; this._defHemi.intensity = hasAmb ? 0 : 2.2; } // ---- selection + gizmo ---- select(id){ this._selected = id; if(id == null){ this._tc.detach(); } else { const en = this._entities.get(id); if(en) this._tc.attach(en.wrapper); } for(const cb of this._selCbs) cb(id ? this._entities.get(id) : null); } _pick(e){ if(this._tc.dragging) return; const rect = this.renderer.domElement.getBoundingClientRect(); const ndc = new THREE.Vector2( ((e.clientX - rect.left)/rect.width)*2 - 1, -((e.clientY - rect.top)/rect.height)*2 + 1); const rc = new THREE.Raycaster(); rc.setFromCamera(ndc, this._dirCam); const wraps = this.entities().map(en => en.wrapper); const hits = rc.intersectObjects(wraps, true); for(const h of hits){ let o = h.object; while(o && o.userData.entityId == null) o = o.parent; if(o){ this.select(o.userData.entityId); return; } } this.select(null); } // ---- transforms / params (Timeline drives these every tick) ---- entityTransform(id){ const w = this._entities.get(id)?.wrapper; if(!w) return null; return { pos:[w.position.x, w.position.y, w.position.z], rot:[w.rotation.x, w.rotation.y, w.rotation.z], scale: w.scale.x }; } setTransform(id, {pos, rot, scale}){ const w = this._entities.get(id)?.wrapper; if(!w) return; if(pos) w.position.set(pos[0], pos[1], pos[2]); if(rot) w.rotation.set(rot[0], rot[1], rot[2]); if(scale != null) w.scale.setScalar(scale); } setParam(id, key, value){ const en = this._entities.get(id); if(!en) return; en.params[key] = value; if(en.cam && key === 'fov'){ en.cam.fov = value; en.cam.updateProjectionMatrix(); } if(en.light){ if(key === 'intensity') en.light.intensity = value; if(key === 'color') en.light.color.set(value); if(key === 'castShadow' && en.light.isDirectionalLight) en.light.castShadow = !!value; } if(en.kind === 'backdrop' && (key === 'mode' || key === 'image' || key === 'width')) this._rebuildBackdrop(en); // geometry depends on these — rebuild the mesh } async _rebuildBackdrop(en){ const old = en.root; en.root = await this._buildBackdrop(en); en.wrapper.add(en.root); if(old){ en.wrapper.remove(old); disposeRoot(old); } } // ---- clips (bake is expensive — cache per path parse and per (entity,path,index) bake) ---- async prepareClip(id, path, clipIndex=0){ const en = this._entities.get(id); if(!en || en.kind !== 'character') throw new Error('prepareClip: not a character: ' + id); const bkey = `${id}|${path}|${clipIndex}`; if(this._baked.has(bkey)) return this._baked.get(bkey); let src = this._clipSrc.get(path); if(!src){ const res = await fetch(ASSET_URL(path)); if(!res.ok) throw new Error('clip fetch ' + res.status + ': ' + path); const { root, anims } = await parseAny(await res.arrayBuffer(), path); src = { root, anims, rest: captureRest(root) }; this._clipSrc.set(path, src); } const clip = src.anims[clipIndex]; if(!clip) throw new Error('no clip index ' + clipIndex + ' in ' + path); const baked = bakeRetarget(clip, src.root, src.rest, en.root, en.rest); this._baked.set(bkey, baked); return baked; } // same bake, from in-memory bytes (dropped clip file) instead of a server path async prepareClipUpload(id, buf, name, clipIndex=0){ const en = this._entities.get(id); if(!en || en.kind !== 'character') throw new Error('prepareClipUpload: not a character: ' + id); const { root, anims } = await parseAny(buf, name); const clip = anims[clipIndex]; if(!clip) throw new Error('no animation tracks in ' + name); return bakeRetarget(clip, root, captureRest(root), en.root, en.rest); } entityMixer(id){ return this._entities.get(id)?.mixer || null; } // ---- visemes / morph targets (lip sync) ---- visemeTargets(id){ return Object.keys(this._entities.get(id)?.visemes || {}); } // drive a canonical viseme (A/E/I/O/U/MBP/FV/L) or a raw morph name; sets every mapped mesh morph setMorph(id, name, weight){ const en = this._entities.get(id); if(!en || !en.morphs) return; const raws = (en.visemes && en.visemes[name]) || (en.morphs.has(name.toLowerCase()) ? [name.toLowerCase()] : []); for(const r of raws) for(const {mesh, idx} of en.morphs.get(r)) if(mesh.morphTargetInfluences) mesh.morphTargetInfluences[idx] = weight; } // M1 temp helper: play a prepared clip on a character (Timeline replaces this in M2). playClip(id, clip, {loop=true} = {}){ const en = this._entities.get(id); if(!en || !en.mixer) return; en.mixer.stopAllAction(); const a = en.mixer.clipAction(clip); a.setLoop(loop ? THREE.LoopRepeat : THREE.LoopOnce, Infinity); a.clampWhenFinished = !loop; a.reset().play(); } setMixerAuto(on){ this._mixerAuto = on; } // Timeline calls setMixerAuto(false) at integration // ---- cameras ---- setActiveCamera(id){ this._activeCamId = id; this._pip.style.display = (id && this._entities.get(id)?.cam) ? 'block' : 'none'; } _activeCam(){ const en = this._activeCamId && this._entities.get(this._activeCamId); return en && en.cam ? en.cam : this._dirCam; } // render `cam` to the main WebGL canvas at the target's aspect; if `canvas` given, blit there and // restore the cam's aspect (so using a cam for the inset never distorts it in the main view). _render(cam, canvas){ const w = canvas ? canvas.width : this.renderer.domElement.width; const h = canvas ? canvas.height : this.renderer.domElement.height; const prev = cam.aspect; if(cam.isPerspectiveCamera){ cam.aspect = w/h; cam.updateProjectionMatrix(); } this.renderer.render(this.scene, cam); if(canvas){ canvas.getContext('2d').drawImage(this.renderer.domElement, 0, 0, canvas.width, canvas.height); if(cam.isPerspectiveCamera && cam.aspect !== prev){ cam.aspect = prev; cam.updateProjectionMatrix(); } } } // PiP: pass the small overlay canvas. Final render (M3): pass null after pause() — the active cam // lands on the main WebGL canvas at its current size for readPixels/toBlob. Hard contract, keep it. renderActiveCamera(canvas){ this._render(this._activeCam(), canvas); } pause(){ this._paused = true; } resume(){ this._paused = false; this.clock.getDelta(); } // swallow the paused gap so mixers don't jump // ---- pointer→world helpers (drag-from-dock placement) ---- screenToFloor(clientX, clientY){ const rect = this.renderer.domElement.getBoundingClientRect(); const ndc = new THREE.Vector2(((clientX-rect.left)/rect.width)*2-1, -((clientY-rect.top)/rect.height)*2+1); const rc = new THREE.Raycaster(); rc.setFromCamera(ndc, this._dirCam); const hit = new THREE.Vector3(); return rc.ray.intersectPlane(new THREE.Plane(new THREE.Vector3(0,1,0), 0), hit) ? [hit.x, 0, hit.z] : [0,0,0]; } entityAt(clientX, clientY, kind){ const rect = this.renderer.domElement.getBoundingClientRect(); const ndc = new THREE.Vector2(((clientX-rect.left)/rect.width)*2-1, -((clientY-rect.top)/rect.height)*2+1); const rc = new THREE.Raycaster(); rc.setFromCamera(ndc, this._dirCam); const hits = rc.intersectObjects(this.entities().map(e=>e.wrapper), true); for(const h of hits){ let o=h.object; while(o && o.userData.entityId==null) o=o.parent; if(o){ const en=this._entities.get(o.userData.entityId); if(en && (!kind || en.kind===kind)) return en.id; } } return null; } // ---- scene JSON round-trip (entities only; Timeline owns tracks/duration/cuts) ---- captureState(){ return this.entities().map(en => ({ id: en.id, kind: en.kind, label: en.label, source: en.source, params: {...en.params}, transform: this.entityTransform(en.id), upload: en.source && en.source.type === 'upload' || undefined, })); } async applyState(sceneJson){ for(const en of this.entities()) this.removeEntity(en.id); for(const d of (sceneJson.entities || [])){ // skip only what needs asset BYTES we don't have: an 'upload' mesh or backdrop image. // camera/light use source:{type:'none'} (legacy scenes: 'upload') and always rebuild. const needsBytes = d.kind === 'character' || d.kind === 'prop' || d.kind === 'backdrop'; if(d.source && d.source.type === 'upload' && needsBytes){ console.warn('skipping upload entity (no bytes):', d.id); continue; } try { await this.addEntity(d); } catch(err){ console.error('addEntity failed', d.id, err); } } } }