20af1b20c2
Lane A: grammar.js (shot/angle/focal/light/mark data), presets.js (bbox shot solver, light/mark appliers, DIRECT bar UI, NL box), stage video plates (VideoTexture plane/corner/dome, screen kind, syncVideos), dock video cards. Lane B: scenegod:direct → grouped-undo keys/cuts, 🗣 lip-sync from audio via /rhubarb, Cmd+wheel zoom / wheel scroll, audio trim, video clock-follow via nullable entityVideo seam. Lane C: POST /director (LLM proxy, server-side op validation vs vocab+scene, mock-tested), render.js awaits stage.syncVideos. Sync6 (orchestrator): VIDEO_EXTS in server tree/media types (video primary, img poster as thumb), stage.entityVideo accessor, syncVideos forces videoTex.needsUpdate (rVFC never fires in hidden tabs → black plates). Verified: 9 server groups + timeline + presets suites green; browser SYNC: one-click MCU framing lands camera+fov keys+cut (single undo), golden_hour relight keys, plate scrub-follow frame-exact, finalRender mp4 shows the Flow golden-hour street plate behind an animated character. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
566 lines
29 KiB
JavaScript
566 lines
29 KiB
JavaScript
// stage.js — the 3D stage: renderer, IBL scene, entities on wrappers, gizmo, retarget cache.
|
|
// Timeline (Lane B) talks to the stage ONLY through this API (PLAN §4.2).
|
|
import * as THREE from 'three';
|
|
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
|
|
import { TransformControls } from 'three/addons/controls/TransformControls.js';
|
|
import { RoomEnvironment } from 'three/addons/environments/RoomEnvironment.js';
|
|
import { parseAny, captureRest, disposeRoot, stats, bakeRetarget } from './room3d.js';
|
|
|
|
const ASSET_URL = p => '/assets/file?path=' + encodeURIComponent(p);
|
|
|
|
// Loose viseme matcher: map a character's morph-target names (CC "V_Open", ARKit "jawOpen", plain
|
|
// "AA"…) to a canonical set the timeline drives. Detection only — Rhubarb→keyframes is Lane B's.
|
|
const VISEMES = {
|
|
A: [/\bah\b/, /(^|_)aa?($|_)/, /jaw[_ ]?open/, /(^|[_-])open($|[_-])/, /mouth_?open/],
|
|
E: [/\bee\b/, /(^|_)e[hr]?($|_)/, /wide/, /smile/],
|
|
I: [/(^|_)i[hy]?($|_)/, /affric/],
|
|
O: [/\boh\b/, /(^|_)o($|_)/, /tight[_-]?o/, /round/],
|
|
U: [/\boo\b/, /(^|_)u[w]?($|_)/, /pucker/, /funnel/, /w[_-]?oo/, /(^|_)tight($|_)/],
|
|
MBP: [/mbp/, /b[_-]?m[_-]?p/, /explos/, /mouth_?close/, /press/, /lip[_-]?open/, /(^|_)pp($|_)/],
|
|
FV: [/(^|_)f[_-]?v($|_)/, /dental[_-]?lip/, /(^|_)ff($|_)/],
|
|
L: [/t[_-]?l[_-]?d[_-]?n/, /tongue/, /(^|_)l($|_)/, /(^|_)dd($|_)/, /(^|_)nn($|_)/],
|
|
};
|
|
function detectVisemes(root){
|
|
const morphs = new Map(); // rawName(lower) -> [{mesh, idx}] (head, teeth, tongue often share a name)
|
|
root.traverse(o => { if(o.isMesh && o.morphTargetDictionary)
|
|
for(const [name, idx] of Object.entries(o.morphTargetDictionary)){
|
|
const k = name.toLowerCase(); (morphs.get(k) || morphs.set(k, []).get(k)).push({mesh:o, idx}); } });
|
|
const visemes = {};
|
|
// SYNC5: ARKit side suffixes (_L/_R) made /(^|_)l($|_)/ swallow eyeblink_l & co —
|
|
// never match non-mouth regions, and match against the side-stripped name.
|
|
const NONMOUTH = /eye|blink|brow|squint|look|nose|cheek/;
|
|
for(const [canon, pats] of Object.entries(VISEMES)){
|
|
const hits = [...morphs.keys()].filter(k => !NONMOUTH.test(k)
|
|
&& pats.some(p => p.test(k.replace(/(.)[_-][lr]$/, '$1'))));
|
|
if(hits.length) visemes[canon] = hits;
|
|
}
|
|
return { morphs, visemes };
|
|
}
|
|
// runnable check for the matcher (assert-based). Call visemeSelfCheck() from the console.
|
|
export function visemeSelfCheck(){
|
|
const dict = {'V_Open':0,'V_Explosive':1,'jawOpen':2,'mouthPucker':3,'mouthFunnel':4,
|
|
'mouthClose':5,'V_Dental_Lip':6,'V_Tongue_up':7,'EyeBlink_L':8,'browDown_L':9,'mouthSmile_L':10};
|
|
const mesh = { isMesh:true, morphTargetDictionary:dict, morphTargetInfluences:new Array(11).fill(0) };
|
|
const { morphs, visemes } = detectVisemes({ traverse(cb){ cb(mesh); } });
|
|
const has = v => v in visemes;
|
|
const detectOk = ['A','U','MBP','FV','L'].every(has) // open/jaw, pucker/funnel, explos/close, dental, tongue
|
|
&& !has('I') && !has('O')
|
|
// SYNC5 regression guard: side-suffixed ARKit names must land on the RIGHT viseme —
|
|
// L is tongue only (no blink/brow/smile bleed), E gets the side-stripped smile.
|
|
&& visemes.L.length === 1 && visemes.L[0] === 'v_tongue_up'
|
|
&& has('E') && visemes.E.includes('mouthsmile_l');
|
|
// canonical 'A' drives every mesh morph mapped to it (jawOpen idx2 + V_Open idx0)
|
|
for(const r of visemes.A) for(const {idx} of morphs.get(r)) mesh.morphTargetInfluences[idx] = 0.7;
|
|
const driveOk = mesh.morphTargetInfluences[0] === 0.7 && mesh.morphTargetInfluences[2] === 0.7
|
|
&& mesh.morphTargetInfluences[8] === 0; // blink untouched
|
|
return { ok: detectOk && driveOk, found: Object.keys(visemes).sort().join(','), visemes };
|
|
}
|
|
|
|
export class Stage {
|
|
constructor(viewEl){
|
|
this.view = viewEl;
|
|
this._entities = new Map();
|
|
this._selCbs = []; this._changeCbs = [];
|
|
this._selected = null;
|
|
this._activeCamId = null;
|
|
this._mixerAuto = true; // Stage advances mixers each frame until Timeline takes over
|
|
this._idc = 0;
|
|
this._clipSrc = new Map(); // path -> {root, rest, anims} (parse+rest cached)
|
|
this._baked = new Map(); // id|path|index -> AnimationClip
|
|
|
|
const r = this.renderer = new THREE.WebGLRenderer({antialias:true, preserveDrawingBuffer:true});
|
|
r.setPixelRatio(devicePixelRatio);
|
|
r.shadowMap.enabled = true;
|
|
view.appendChild(r.domElement);
|
|
this._paused = false; // M3 final render freezes the director loop, then drives renderActiveCamera
|
|
|
|
// PiP overlay — the active camera's view (bottom-right). CSS sizes it (~24% w); the buffer
|
|
// stays small. Click to swap which view is main vs inset.
|
|
const pip = this._pip = document.createElement('canvas');
|
|
pip.width = 320; pip.height = 180; pip.className = 'pip'; pip.style.display = 'none';
|
|
pip.title = 'click to swap with the main view';
|
|
this._pipSwap = false;
|
|
pip.addEventListener('click', () => { this._pipSwap = !this._pipSwap; });
|
|
view.appendChild(pip);
|
|
|
|
const scene = this.scene = new THREE.Scene();
|
|
const cam = this._dirCam = new THREE.PerspectiveCamera(45, 1, 0.01, 3000);
|
|
cam.position.set(0, 1.6, 5);
|
|
const ctr = this.controls = new OrbitControls(cam, r.domElement);
|
|
ctr.enableDamping = true; ctr.target.set(0, 1.0, 0);
|
|
|
|
const pmrem = new THREE.PMREMGenerator(r);
|
|
scene.environment = pmrem.fromScene(new RoomEnvironment(), 0.04).texture; pmrem.dispose();
|
|
// default lights — replaced/dimmed once the scene carries light entities
|
|
this._defHemi = new THREE.HemisphereLight(0xffffff, 0x555a66, 2.2); scene.add(this._defHemi);
|
|
this._defKey = new THREE.DirectionalLight(0xffffff, 1.8); this._defKey.position.set(3, 5, 2); scene.add(this._defKey);
|
|
|
|
const floor = new THREE.Mesh(new THREE.CircleGeometry(30, 64).rotateX(-Math.PI/2),
|
|
new THREE.MeshStandardMaterial({color:0x141a20, roughness:.97}));
|
|
floor.receiveShadow = true; scene.add(floor);
|
|
scene.add(new THREE.GridHelper(60, 60, 0x2a323b, 0x1c232b));
|
|
|
|
// gizmo
|
|
const tc = this._tc = new TransformControls(cam, r.domElement);
|
|
tc.addEventListener('dragging-changed', e => { ctr.enabled = !e.value; });
|
|
tc.addEventListener('mouseUp', () => { if(this._selected) this._fireChange(this.getEntity(this._selected)); });
|
|
scene.add(tc);
|
|
addEventListener('keydown', e => {
|
|
if(!this._selected) return;
|
|
if(e.key==='w') tc.setMode('translate');
|
|
else if(e.key==='e') tc.setMode('rotate');
|
|
else if(e.key==='r') tc.setMode('scale');
|
|
});
|
|
|
|
r.domElement.addEventListener('click', e => this._pick(e));
|
|
this.clock = new THREE.Clock();
|
|
this._resize(); addEventListener('resize', () => this._resize());
|
|
const loop = () => { requestAnimationFrame(loop);
|
|
const dt = this.clock.getDelta();
|
|
if(this._paused) return; // M3: caller drives rendering deterministically
|
|
ctr.update();
|
|
if(this._mixerAuto) for(const en of this._entities.values()) en.mixer && en.mixer.update(dt);
|
|
// inset first (it scribbles onto the main canvas), then the main view last so it wins
|
|
if(this._activeCamId && pip.style.display !== 'none')
|
|
this._render(this._pipSwap ? this._dirCam : this._activeCam(), pip);
|
|
this._render((this._pipSwap && this._activeCamId) ? this._activeCam() : this._dirCam, null);
|
|
};
|
|
loop();
|
|
}
|
|
|
|
_resize(){ const w=this.view.clientWidth, h=this.view.clientHeight;
|
|
this.renderer.setSize(w, h, false); this._dirCam.aspect = w/h; this._dirCam.updateProjectionMatrix();
|
|
// size the inset here — CSS aspect-ratio/% is unreliable on a <canvas>
|
|
const pw = Math.max(180, Math.min(340, w*0.24));
|
|
this._pip.style.width = Math.round(pw)+'px'; this._pip.style.height = Math.round(pw*9/16)+'px'; }
|
|
|
|
// ---- callbacks ----
|
|
onSelect(cb){ this._selCbs.push(cb); }
|
|
onChange(cb){ this._changeCbs.push(cb); }
|
|
_fireChange(en){ for(const cb of this._changeCbs) cb(en); }
|
|
|
|
// ---- entity lifecycle ----
|
|
entities(){ return [...this._entities.values()]; }
|
|
getEntity(id){ return this._entities.get(id); }
|
|
|
|
async addEntity(desc){
|
|
const id = desc.id || ('e' + (++this._idc));
|
|
if(this._idc <= (parseInt(String(id).slice(1)) || 0)) this._idc = parseInt(String(id).slice(1)) || this._idc;
|
|
const wrapper = new THREE.Group();
|
|
wrapper.userData.entityId = id;
|
|
const noAsset = desc.kind === 'camera' || desc.kind === 'light'; // rig entities have no file
|
|
const en = { id, kind: desc.kind, label: desc.label || desc.kind,
|
|
source: desc.source || (noAsset ? {type:'none'} : null), params: {...(desc.params||{})},
|
|
wrapper, root:null, mixer:null, rest:null, cam:null, light:null, morphs:null, visemes:null, video:null };
|
|
|
|
if(desc.kind === 'character' || desc.kind === 'prop'){
|
|
const { root } = await this._loadAsset(desc);
|
|
this._normalize(root, desc.kind === 'character');
|
|
root.traverse(o=>{ if(o.isMesh){ o.castShadow = true; o.receiveShadow = true; } if(o.isSkinnedMesh) o.frustumCulled=false; });
|
|
wrapper.add(root); en.root = root;
|
|
if(desc.kind === 'character'){ en.rest = captureRest(root); en.mixer = new THREE.AnimationMixer(root);
|
|
const v = detectVisemes(root); en.morphs = v.morphs; en.visemes = v.visemes; }
|
|
} else if(desc.kind === 'backdrop'){
|
|
en.root = await this._buildBackdrop(en); wrapper.add(en.root);
|
|
} else if(desc.kind === 'screen'){
|
|
en.root = await this._buildScreen(en); wrapper.add(en.root);
|
|
} else if(desc.kind === 'camera'){
|
|
const fov = en.params.fov || 45;
|
|
en.cam = new THREE.PerspectiveCamera(fov, 16/9, 0.05, 3000);
|
|
wrapper.add(en.cam);
|
|
const proxy = new THREE.Mesh(new THREE.ConeGeometry(0.18, 0.35, 4).rotateX(-Math.PI/2),
|
|
new THREE.MeshBasicMaterial({color:0x43c0cf, wireframe:true}));
|
|
proxy.position.z = 0.18; wrapper.add(proxy);
|
|
} else if(desc.kind === 'light'){
|
|
this._buildLight(en);
|
|
} else throw new Error('unknown kind: ' + desc.kind);
|
|
|
|
this.scene.add(wrapper);
|
|
this._entities.set(id, en);
|
|
if(desc.transform) this.setTransform(id, desc.transform);
|
|
this._fireChange(en); // tlui/dock build a row per entity off onChange (incl. applyState loads)
|
|
return en;
|
|
}
|
|
|
|
removeEntity(id){
|
|
const en = this._entities.get(id); if(!en) return;
|
|
if(this._selected === id) this.select(null);
|
|
this._dropVideo(en);
|
|
if(en.root) disposeRoot(en.root);
|
|
if(en.light) en.light.parent && en.light.parent.remove(en.light);
|
|
this.scene.remove(en.wrapper);
|
|
this._entities.delete(id);
|
|
for(const k of [...this._baked.keys()]) if(k.startsWith(id+'|')) this._baked.delete(k);
|
|
if(en.kind === 'light') this._refreshDefaults();
|
|
this._fireChange(en); // trigger a row resync so the deleted entity drops out
|
|
}
|
|
|
|
async _loadAsset(desc){
|
|
let buf, name;
|
|
if(desc.source && desc.source.type === 'upload'){
|
|
if(!desc._buf) throw new Error('upload entity has no buffer');
|
|
buf = desc._buf; name = desc.source.path;
|
|
} else {
|
|
const path = desc.source.path;
|
|
const res = await fetch(ASSET_URL(path));
|
|
if(!res.ok) throw new Error('asset fetch ' + res.status + ': ' + path);
|
|
buf = await res.arrayBuffer(); name = path;
|
|
}
|
|
return parseAny(buf, name);
|
|
}
|
|
|
|
_normalize(root, isChar){
|
|
root.updateMatrixWorld(true);
|
|
const box = new THREE.Box3().setFromObject(root);
|
|
if(box.isEmpty() || !isFinite(box.min.x)) return;
|
|
if(isChar){
|
|
const size = box.getSize(new THREE.Vector3());
|
|
const s = 1.7 / (Math.max(size.x, size.y, size.z) || 1);
|
|
root.scale.setScalar(s); root.updateMatrixWorld(true);
|
|
}
|
|
const b = new THREE.Box3().setFromObject(root);
|
|
root.position.x -= (b.min.x + b.max.x)/2;
|
|
root.position.z -= (b.min.z + b.max.z)/2;
|
|
root.position.y -= b.min.y; // feet on the floor
|
|
}
|
|
|
|
// Backdrop plate — image OR video (M6). params.mode: plane|corner|dome|video ('video' = a
|
|
// plane; the other modes also accept .mp4 sources). Video plates show their poster jpg (the
|
|
// server's same-stem sidecar) until first play/seek, then swap to the live VideoTexture.
|
|
async _buildBackdrop(en){
|
|
const p = en.params;
|
|
const src = p.image || (en.source && en.source.path) || '';
|
|
const upload = en.source && en.source.type === 'upload';
|
|
const url = upload ? en.source._url : ASSET_URL(src);
|
|
const isVideo = p.mode === 'video' || /\.(mp4|webm|mov)$/i.test(src);
|
|
let tex, aspect, vidTex = null, poster = null;
|
|
if(isVideo){
|
|
const { video, texture } = await this._makeVideo(url, src);
|
|
en.video = video; en.videoTex = vidTex = texture;
|
|
aspect = (video.videoWidth / video.videoHeight) || 16/9;
|
|
poster = upload ? null : await this._posterTex(src);
|
|
tex = poster || vidTex;
|
|
} else {
|
|
tex = await new THREE.TextureLoader().loadAsync(url);
|
|
tex.colorSpace = THREE.SRGBColorSpace;
|
|
aspect = (tex.image.width / tex.image.height) || 1;
|
|
}
|
|
const w = p.width || 10, h = w / aspect;
|
|
const grp = new THREE.Group(), mats = [];
|
|
if(p.mode === 'dome'){
|
|
const dm = new THREE.MeshBasicMaterial({map:tex, side:THREE.BackSide}); mats.push(dm);
|
|
grp.add(new THREE.Mesh(new THREE.SphereGeometry(40, 40, 24), dm));
|
|
} else {
|
|
const mat = new THREE.MeshStandardMaterial({map:tex, roughness:1, metalness:0}); mats.push(mat);
|
|
const wall = new THREE.Mesh(new THREE.PlaneGeometry(w, h), mat);
|
|
wall.position.y = h/2; wall.receiveShadow = true; grp.add(wall);
|
|
if(p.mode === 'corner'){ // cheap "L": duplicate lower third laid flat as ground
|
|
const gh = h/3, gm = mat.clone(); mats.push(gm);
|
|
const g = new THREE.Mesh(new THREE.PlaneGeometry(w, gh), gm);
|
|
g.rotation.x = -Math.PI/2; g.position.z = gh/2; g.receiveShadow = true; grp.add(g);
|
|
wall.position.z = -gh/2;
|
|
}
|
|
}
|
|
if(vidTex && poster) this._swapOnPlay(en.video, vidTex, mats);
|
|
if(en.video && this._mixerAuto) en.video.play().catch(()=>{}); // ambient preview until a timeline owns the clock
|
|
return grp;
|
|
}
|
|
|
|
// In-set display (M6 `screen` kind): a video plane with an emissive boost so TVs/club
|
|
// screens/jumbotrons read "on" under any lighting preset. params {video, width, emissive}.
|
|
// ponytail: CRT curve skipped — flat plane until a real TV asset asks for it.
|
|
async _buildScreen(en){
|
|
const p = en.params;
|
|
const src = p.video || (en.source && en.source.path) || '';
|
|
const upload = en.source && en.source.type === 'upload';
|
|
const { video, texture } = await this._makeVideo(upload ? en.source._url : ASSET_URL(src), src);
|
|
en.video = video; en.videoTex = texture;
|
|
const aspect = (video.videoWidth / video.videoHeight) || 16/9;
|
|
const poster = upload ? null : await this._posterTex(src);
|
|
const tex = poster || texture;
|
|
const w = p.width || 3, h = w / aspect;
|
|
const mat = new THREE.MeshStandardMaterial({ map:tex, emissive:0xffffff, emissiveMap:tex,
|
|
emissiveIntensity: p.emissive ?? 0.6, roughness:0.4, metalness:0 });
|
|
if(poster) this._swapOnPlay(video, texture, [mat]);
|
|
if(this._mixerAuto) video.play().catch(()=>{});
|
|
const mesh = new THREE.Mesh(new THREE.PlaneGeometry(w, h), mat);
|
|
mesh.position.y = h/2;
|
|
const grp = new THREE.Group(); grp.add(mesh);
|
|
return grp;
|
|
}
|
|
|
|
// <video> element + VideoTexture, resolved once metadata (dimensions/duration) is in
|
|
_makeVideo(url, srcPath){
|
|
return new Promise((resolve, reject) => {
|
|
const v = document.createElement('video');
|
|
v.muted = true; v.loop = true; v.playsInline = true; v.preload = 'auto'; v.crossOrigin = 'anonymous';
|
|
v.src = url;
|
|
const texture = new THREE.VideoTexture(v);
|
|
texture.colorSpace = THREE.SRGBColorSpace;
|
|
v.addEventListener('loadedmetadata', () => resolve({ video: v, texture }), { once:true });
|
|
v.addEventListener('error', () => reject(new Error('video load failed: ' + srcPath)), { once:true });
|
|
});
|
|
}
|
|
_posterTex(src){ // same-stem jpg via the server's sidecar route; null when absent
|
|
return new THREE.TextureLoader().loadAsync('/assets/thumb?path=' + encodeURIComponent(src))
|
|
.then(t => { t.colorSpace = THREE.SRGBColorSpace; return t; })
|
|
.catch(() => null);
|
|
}
|
|
_swapOnPlay(video, texture, mats){ // poster stays up until the video actually shows a frame
|
|
const swap = () => {
|
|
video.removeEventListener('playing', swap); video.removeEventListener('seeked', swap);
|
|
for(const m of mats){ m.map = texture; if(m.emissiveMap) m.emissiveMap = texture; m.needsUpdate = true; }
|
|
};
|
|
video.addEventListener('playing', swap); video.addEventListener('seeked', swap);
|
|
}
|
|
_dropVideo(en){
|
|
if(!en.video) return;
|
|
en.video.pause(); en.video.removeAttribute('src'); en.video.load(); en.video = null;
|
|
}
|
|
|
|
_buildLight(en){
|
|
const p = en.params;
|
|
if(p.type === 'ambient'){
|
|
en.light = new THREE.HemisphereLight(new THREE.Color(p.color||'#ffffff'), 0x444450, p.intensity ?? 1.0);
|
|
en.wrapper.add(en.light);
|
|
if(p.bg) this.scene.background = new THREE.Color(p.bg); // saved lighting preset restores its backdrop color
|
|
} else {
|
|
const dir = en.light = new THREE.DirectionalLight(new THREE.Color(p.color||'#ffffff'), p.intensity ?? 1.5);
|
|
dir.castShadow = p.castShadow ?? true;
|
|
dir.shadow.mapSize.set(1024,1024); dir.shadow.camera.far = 60;
|
|
dir.position.set(0,0,0);
|
|
const tgt = new THREE.Object3D(); tgt.position.set(0,0,-1); en.wrapper.add(tgt); dir.target = tgt;
|
|
en.wrapper.add(dir);
|
|
const proxy = new THREE.Mesh(new THREE.SphereGeometry(0.15, 12, 8),
|
|
new THREE.MeshBasicMaterial({color:0xffe08a}));
|
|
en.wrapper.add(proxy);
|
|
}
|
|
this._refreshDefaults();
|
|
}
|
|
|
|
// dim built-in lights once scene lights exist
|
|
_refreshDefaults(){
|
|
const hasKey = this.entities().some(e => e.kind==='light' && e.params.type!=='ambient');
|
|
const hasAmb = this.entities().some(e => e.kind==='light' && e.params.type==='ambient');
|
|
this._defKey.visible = !hasKey;
|
|
this._defHemi.intensity = hasAmb ? 0 : 2.2;
|
|
}
|
|
|
|
// ---- selection + gizmo ----
|
|
select(id){
|
|
this._selected = id;
|
|
if(id == null){ this._tc.detach(); }
|
|
else { const en = this._entities.get(id); if(en) this._tc.attach(en.wrapper); }
|
|
for(const cb of this._selCbs) cb(id ? this._entities.get(id) : null);
|
|
}
|
|
_pick(e){
|
|
if(this._tc.dragging) return;
|
|
const rect = this.renderer.domElement.getBoundingClientRect();
|
|
const ndc = new THREE.Vector2(
|
|
((e.clientX - rect.left)/rect.width)*2 - 1,
|
|
-((e.clientY - rect.top)/rect.height)*2 + 1);
|
|
const rc = new THREE.Raycaster(); rc.setFromCamera(ndc, this._dirCam);
|
|
const wraps = this.entities().map(en => en.wrapper);
|
|
const hits = rc.intersectObjects(wraps, true);
|
|
for(const h of hits){
|
|
let o = h.object; while(o && o.userData.entityId == null) o = o.parent;
|
|
if(o){ this.select(o.userData.entityId); return; }
|
|
}
|
|
this.select(null);
|
|
}
|
|
|
|
// ---- transforms / params (Timeline drives these every tick) ----
|
|
entityTransform(id){
|
|
const w = this._entities.get(id)?.wrapper; if(!w) return null;
|
|
return { pos:[w.position.x, w.position.y, w.position.z],
|
|
rot:[w.rotation.x, w.rotation.y, w.rotation.z],
|
|
scale: w.scale.x };
|
|
}
|
|
setTransform(id, {pos, rot, scale}){
|
|
const w = this._entities.get(id)?.wrapper; if(!w) return;
|
|
if(pos) w.position.set(pos[0], pos[1], pos[2]);
|
|
if(rot) w.rotation.set(rot[0], rot[1], rot[2]);
|
|
if(scale != null) w.scale.setScalar(scale);
|
|
}
|
|
setParam(id, key, value){
|
|
const en = this._entities.get(id); if(!en) return;
|
|
en.params[key] = value;
|
|
if(en.cam && key === 'fov'){ en.cam.fov = value; en.cam.updateProjectionMatrix(); }
|
|
if(en.light){
|
|
if(key === 'intensity') en.light.intensity = value;
|
|
if(key === 'color') en.light.color.set(value);
|
|
if(key === 'castShadow' && en.light.isDirectionalLight) en.light.castShadow = !!value;
|
|
if(key === 'bg') this.scene.background = new THREE.Color(value); // bg rides the ambient entity (keyable, persists)
|
|
}
|
|
const rebuild = () => this._rebuild(en).catch(e => console.warn('media rebuild failed:', en.id, e));
|
|
if(en.kind === 'backdrop' && (key === 'mode' || key === 'image' || key === 'width'))
|
|
rebuild(); // geometry depends on these — rebuild the mesh
|
|
if(en.kind === 'screen'){
|
|
if(key === 'emissive' && en.root)
|
|
en.root.traverse(o => { if(o.isMesh) o.material.emissiveIntensity = value; });
|
|
if(key === 'video' || key === 'width') rebuild();
|
|
}
|
|
// videoStart is data for the timeline/syncVideos — nothing to rebuild
|
|
}
|
|
async _rebuild(en){ // backdrop/screen media rebuild (mode/image/video/width changed)
|
|
const old = en.root;
|
|
this._dropVideo(en);
|
|
en.root = en.kind === 'screen' ? await this._buildScreen(en) : await this._buildBackdrop(en);
|
|
en.wrapper.add(en.root);
|
|
if(old){ en.wrapper.remove(old); disposeRoot(old); }
|
|
}
|
|
|
|
// ---- clips (bake is expensive — cache per path parse and per (entity,path,index) bake) ----
|
|
async prepareClip(id, path, clipIndex=0){
|
|
const en = this._entities.get(id);
|
|
if(!en || en.kind !== 'character') throw new Error('prepareClip: not a character: ' + id);
|
|
const bkey = `${id}|${path}|${clipIndex}`;
|
|
if(this._baked.has(bkey)) return this._baked.get(bkey);
|
|
let src = this._clipSrc.get(path);
|
|
if(!src){
|
|
const res = await fetch(ASSET_URL(path));
|
|
if(!res.ok) throw new Error('clip fetch ' + res.status + ': ' + path);
|
|
const { root, anims } = await parseAny(await res.arrayBuffer(), path);
|
|
src = { root, anims, rest: captureRest(root) };
|
|
this._clipSrc.set(path, src);
|
|
}
|
|
const clip = src.anims[clipIndex];
|
|
if(!clip) throw new Error('no clip index ' + clipIndex + ' in ' + path);
|
|
const baked = bakeRetarget(clip, src.root, src.rest, en.root, en.rest);
|
|
this._baked.set(bkey, baked);
|
|
return baked;
|
|
}
|
|
// same bake, from in-memory bytes (dropped clip file) instead of a server path
|
|
async prepareClipUpload(id, buf, name, clipIndex=0){
|
|
const en = this._entities.get(id);
|
|
if(!en || en.kind !== 'character') throw new Error('prepareClipUpload: not a character: ' + id);
|
|
const { root, anims } = await parseAny(buf, name);
|
|
const clip = anims[clipIndex];
|
|
if(!clip) throw new Error('no animation tracks in ' + name);
|
|
return bakeRetarget(clip, root, captureRest(root), en.root, en.rest);
|
|
}
|
|
entityMixer(id){ return this._entities.get(id)?.mixer || null; }
|
|
|
|
// ---- visemes / morph targets (lip sync) ----
|
|
visemeTargets(id){ return Object.keys(this._entities.get(id)?.visemes || {}); }
|
|
// drive a canonical viseme (A/E/I/O/U/MBP/FV/L) or a raw morph name; sets every mapped mesh morph
|
|
setMorph(id, name, weight){
|
|
const en = this._entities.get(id); if(!en || !en.morphs) return;
|
|
const raws = (en.visemes && en.visemes[name]) || (en.morphs.has(name.toLowerCase()) ? [name.toLowerCase()] : []);
|
|
for(const r of raws) for(const {mesh, idx} of en.morphs.get(r))
|
|
if(mesh.morphTargetInfluences) mesh.morphTargetInfluences[idx] = weight;
|
|
}
|
|
|
|
// M1 temp helper: play a prepared clip on a character (Timeline replaces this in M2).
|
|
playClip(id, clip, {loop=true} = {}){
|
|
const en = this._entities.get(id); if(!en || !en.mixer) return;
|
|
en.mixer.stopAllAction();
|
|
const a = en.mixer.clipAction(clip);
|
|
a.setLoop(loop ? THREE.LoopRepeat : THREE.LoopOnce, Infinity);
|
|
a.clampWhenFinished = !loop; a.reset().play();
|
|
}
|
|
|
|
setMixerAuto(on){ this._mixerAuto = on; } // Timeline calls setMixerAuto(false) at integration
|
|
|
|
// ---- cameras ----
|
|
setActiveCamera(id){
|
|
this._activeCamId = id;
|
|
this._pip.style.display = (id && this._entities.get(id)?.cam) ? 'block' : 'none';
|
|
}
|
|
_activeCam(){ const en = this._activeCamId && this._entities.get(this._activeCamId);
|
|
return en && en.cam ? en.cam : this._dirCam; }
|
|
// render `cam` to the main WebGL canvas at the target's aspect; if `canvas` given, blit there and
|
|
// restore the cam's aspect (so using a cam for the inset never distorts it in the main view).
|
|
_render(cam, canvas){
|
|
const w = canvas ? canvas.width : this.renderer.domElement.width;
|
|
const h = canvas ? canvas.height : this.renderer.domElement.height;
|
|
const prev = cam.aspect;
|
|
if(cam.isPerspectiveCamera){ cam.aspect = w/h; cam.updateProjectionMatrix(); }
|
|
this.renderer.render(this.scene, cam);
|
|
if(canvas){ canvas.getContext('2d').drawImage(this.renderer.domElement, 0, 0, canvas.width, canvas.height);
|
|
if(cam.isPerspectiveCamera && cam.aspect !== prev){ cam.aspect = prev; cam.updateProjectionMatrix(); } }
|
|
}
|
|
// PiP: pass the small overlay canvas. Final render (M3): pass null after pause() — the active cam
|
|
// lands on the main WebGL canvas at its current size for readPixels/toBlob. Hard contract, keep it.
|
|
renderActiveCamera(canvas){ this._render(this._activeCam(), canvas); }
|
|
pause(){ this._paused = true; }
|
|
resume(){ this._paused = false; this.clock.getDelta(); } // swallow the paused gap so mixers don't jump
|
|
|
|
// ---- director-mode helpers (M5 presets.js solvers are three-free — plain arrays) ----
|
|
entityBBox(id){
|
|
const en = this._entities.get(id); if(!en) return null;
|
|
en.wrapper.updateMatrixWorld(true);
|
|
const box = new THREE.Box3().setFromObject(en.wrapper);
|
|
return (box.isEmpty() || !isFinite(box.min.x)) ? null : { min: box.min.toArray(), max: box.max.toArray() };
|
|
}
|
|
cameraWorldPos(){ // active camera if set, else the director orbit cam
|
|
const p = new THREE.Vector3(); this._activeCam().getWorldPosition(p); return p.toArray();
|
|
}
|
|
|
|
// ---- video plates (M6): master-clock sync + deterministic render seeks ----
|
|
videos(){ return this.entities().filter(e => e.video)
|
|
.map(e => ({ id: e.id, video: e.video, start: e.params.videoStart || 0 })); }
|
|
entityVideo(id){ return this._entities.get(id)?.video || null; } // SYNC6: Lane B's per-entity seam
|
|
setVideoPlaying(on){ for(const { video } of this.videos()) on ? video.play().catch(()=>{}) : video.pause(); }
|
|
// Seek every stage video to timeline time t and resolve once all fire 'seeked' —
|
|
// render.js awaits this per frame so plates are frame-exact, never mid-decode. Hard contract.
|
|
async syncVideos(t){
|
|
const jobs = [];
|
|
// SYNC6: r160 VideoTexture auto-updates via requestVideoFrameCallback, which never
|
|
// fires in a hidden/background tab — force the GPU upload or renders show black plates.
|
|
for(const en of this._entities.values()) if(en.videoTex) en.videoTex.needsUpdate = true;
|
|
for(const { video, start } of this.videos()){
|
|
if(video.readyState < 1 || !isFinite(video.duration) || video.duration <= 0) continue;
|
|
const vt = (((t - start) % video.duration) + video.duration) % video.duration;
|
|
if(!video.paused) video.pause();
|
|
if(Math.abs(video.currentTime - vt) < 0.004) continue; // already on this frame
|
|
jobs.push(new Promise(res => {
|
|
const done = () => { video.removeEventListener('seeked', done); res(); };
|
|
video.addEventListener('seeked', done);
|
|
setTimeout(done, 1500); // ponytail: never hang a render on a stuck decoder
|
|
video.currentTime = vt;
|
|
}));
|
|
}
|
|
await Promise.all(jobs);
|
|
}
|
|
|
|
// ---- pointer→world helpers (drag-from-dock placement) ----
|
|
screenToFloor(clientX, clientY){
|
|
const rect = this.renderer.domElement.getBoundingClientRect();
|
|
const ndc = new THREE.Vector2(((clientX-rect.left)/rect.width)*2-1, -((clientY-rect.top)/rect.height)*2+1);
|
|
const rc = new THREE.Raycaster(); rc.setFromCamera(ndc, this._dirCam);
|
|
const hit = new THREE.Vector3();
|
|
return rc.ray.intersectPlane(new THREE.Plane(new THREE.Vector3(0,1,0), 0), hit) ? [hit.x, 0, hit.z] : [0,0,0];
|
|
}
|
|
entityAt(clientX, clientY, kind){
|
|
const rect = this.renderer.domElement.getBoundingClientRect();
|
|
const ndc = new THREE.Vector2(((clientX-rect.left)/rect.width)*2-1, -((clientY-rect.top)/rect.height)*2+1);
|
|
const rc = new THREE.Raycaster(); rc.setFromCamera(ndc, this._dirCam);
|
|
const hits = rc.intersectObjects(this.entities().map(e=>e.wrapper), true);
|
|
for(const h of hits){ let o=h.object; while(o && o.userData.entityId==null) o=o.parent;
|
|
if(o){ const en=this._entities.get(o.userData.entityId); if(en && (!kind || en.kind===kind)) return en.id; } }
|
|
return null;
|
|
}
|
|
|
|
// ---- scene JSON round-trip (entities only; Timeline owns tracks/duration/cuts) ----
|
|
captureState(){
|
|
return this.entities().map(en => ({
|
|
id: en.id, kind: en.kind, label: en.label,
|
|
source: en.source, params: {...en.params},
|
|
transform: this.entityTransform(en.id),
|
|
upload: en.source && en.source.type === 'upload' || undefined,
|
|
}));
|
|
}
|
|
async applyState(sceneJson){
|
|
for(const en of this.entities()) this.removeEntity(en.id);
|
|
for(const d of (sceneJson.entities || [])){
|
|
// skip only what needs asset BYTES we don't have: an 'upload' mesh, backdrop or screen.
|
|
// camera/light use source:{type:'none'} (legacy scenes: 'upload') and always rebuild.
|
|
const needsBytes = d.kind === 'character' || d.kind === 'prop' || d.kind === 'backdrop' || d.kind === 'screen';
|
|
if(d.source && d.source.type === 'upload' && needsBytes){
|
|
console.warn('skipping upload entity (no bytes):', d.id); continue; }
|
|
try { await this.addEntity(d); } catch(err){ console.error('addEntity failed', d.id, err); }
|
|
}
|
|
}
|
|
}
|