f20a8d5d4e
MOVES grammar + solveMove(): push in/pull out (one rung along the shot ladder,
along the aim ray), truck L/R (lateral, parallel, no re-aim), crane up/down
(vertical then re-aim, floor-clamped), orbit L/R (constant distance, +/-25deg),
zoom in/out (focal only - pos and rot returned identical). Amounts are
frame-relative so a move reads the same on a CU as a WS. Emits a new 'move' op
on scenegod:direct with explicit from/to states, pre-clamped duration and ease;
the START is applied to the live camera immediately.
'+ angle' creates a NEW camera from the current shot settings instead of
reframing the active one - the friction hit while building a 4-camera scene by
hand. 🎥 frame now prefers the live camera so it adjusts what you just made
rather than silently editing camera 1. fitAnchor exposed in the backdrop
inspector (it was inert from the UI).
Verified live: push_in 1.975->1.588m fov fixed; zoom_in travel 0.000 with
pos/rot identical, fov 37.85->26.99; orbit_L constant distance; truck_R
parallel. 4 suites green.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
703 lines
38 KiB
JavaScript
703 lines
38 KiB
JavaScript
// stage.js — the 3D stage: renderer, IBL scene, entities on wrappers, gizmo, retarget cache.
|
|
// Timeline (Lane B) talks to the stage ONLY through this API (PLAN §4.2).
|
|
import * as THREE from 'three';
|
|
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
|
|
import { TransformControls } from 'three/addons/controls/TransformControls.js';
|
|
import { RoomEnvironment } from 'three/addons/environments/RoomEnvironment.js';
|
|
import { parseAny, captureRest, disposeRoot, stats, bakeRetarget } from './room3d.js';
|
|
|
|
const ASSET_URL = p => 'assets/file?path=' + encodeURIComponent(p);
|
|
|
|
// Loose viseme matcher: map a character's morph-target names (CC "V_Open", ARKit "jawOpen", plain
|
|
// "AA"…) to a canonical set the timeline drives. Detection only — Rhubarb→keyframes is Lane B's.
|
|
const VISEMES = {
|
|
A: [/\bah\b/, /(^|_)aa?($|_)/, /jaw[_ ]?open/, /(^|[_-])open($|[_-])/, /mouth_?open/],
|
|
E: [/\bee\b/, /(^|_)e[hr]?($|_)/, /wide/, /smile/],
|
|
I: [/(^|_)i[hy]?($|_)/, /affric/],
|
|
O: [/\boh\b/, /(^|_)o($|_)/, /tight[_-]?o/, /round/],
|
|
U: [/\boo\b/, /(^|_)u[w]?($|_)/, /pucker/, /funnel/, /w[_-]?oo/, /(^|_)tight($|_)/],
|
|
MBP: [/mbp/, /b[_-]?m[_-]?p/, /explos/, /mouth_?close/, /press/, /lip[_-]?open/, /(^|_)pp($|_)/],
|
|
FV: [/(^|_)f[_-]?v($|_)/, /dental[_-]?lip/, /(^|_)ff($|_)/],
|
|
L: [/t[_-]?l[_-]?d[_-]?n/, /tongue/, /(^|_)l($|_)/, /(^|_)dd($|_)/, /(^|_)nn($|_)/],
|
|
};
|
|
function detectVisemes(root){
|
|
const morphs = new Map(); // rawName(lower) -> [{mesh, idx}] (head, teeth, tongue often share a name)
|
|
root.traverse(o => { if(o.isMesh && o.morphTargetDictionary)
|
|
for(const [name, idx] of Object.entries(o.morphTargetDictionary)){
|
|
const k = name.toLowerCase(); (morphs.get(k) || morphs.set(k, []).get(k)).push({mesh:o, idx}); } });
|
|
const visemes = {};
|
|
// SYNC5: ARKit side suffixes (_L/_R) made /(^|_)l($|_)/ swallow eyeblink_l & co —
|
|
// never match non-mouth regions, and match against the side-stripped name.
|
|
const NONMOUTH = /eye|blink|brow|squint|look|nose|cheek/;
|
|
for(const [canon, pats] of Object.entries(VISEMES)){
|
|
const hits = [...morphs.keys()].filter(k => !NONMOUTH.test(k)
|
|
&& pats.some(p => p.test(k.replace(/(.)[_-][lr]$/, '$1'))));
|
|
if(hits.length) visemes[canon] = hits;
|
|
}
|
|
return { morphs, visemes };
|
|
}
|
|
// runnable check for the matcher (assert-based). Call visemeSelfCheck() from the console.
|
|
export function visemeSelfCheck(){
|
|
const dict = {'V_Open':0,'V_Explosive':1,'jawOpen':2,'mouthPucker':3,'mouthFunnel':4,
|
|
'mouthClose':5,'V_Dental_Lip':6,'V_Tongue_up':7,'EyeBlink_L':8,'browDown_L':9,'mouthSmile_L':10};
|
|
const mesh = { isMesh:true, morphTargetDictionary:dict, morphTargetInfluences:new Array(11).fill(0) };
|
|
const { morphs, visemes } = detectVisemes({ traverse(cb){ cb(mesh); } });
|
|
const has = v => v in visemes;
|
|
const detectOk = ['A','U','MBP','FV','L'].every(has) // open/jaw, pucker/funnel, explos/close, dental, tongue
|
|
&& !has('I') && !has('O')
|
|
// SYNC5 regression guard: side-suffixed ARKit names must land on the RIGHT viseme —
|
|
// L is tongue only (no blink/brow/smile bleed), E gets the side-stripped smile.
|
|
&& visemes.L.length === 1 && visemes.L[0] === 'v_tongue_up'
|
|
&& has('E') && visemes.E.includes('mouthsmile_l');
|
|
// canonical 'A' drives every mesh morph mapped to it (jawOpen idx2 + V_Open idx0)
|
|
for(const r of visemes.A) for(const {idx} of morphs.get(r)) mesh.morphTargetInfluences[idx] = 0.7;
|
|
const driveOk = mesh.morphTargetInfluences[0] === 0.7 && mesh.morphTargetInfluences[2] === 0.7
|
|
&& mesh.morphTargetInfluences[8] === 0; // blink untouched
|
|
return { ok: detectOk && driveOk, found: Object.keys(visemes).sort().join(','), visemes };
|
|
}
|
|
|
|
// M7 backdrop auto-fit, kept pure so node can check the algebra (stage_test.mjs): the frustum is
|
|
// `fh` tall at `dist`, and the plate scales to COVER it (either axis, whichever is tighter) + bleed.
|
|
export const BLEED = 1.06;
|
|
export function fitScale(fovDeg, aspect, dist, planeW, planeH){
|
|
const fh = 2 * Math.tan(fovDeg * Math.PI / 360) * dist, fw = fh * aspect;
|
|
return { fh, fw, s: Math.max(fw / planeW, fh / planeH) * BLEED };
|
|
}
|
|
|
|
export class Stage {
|
|
constructor(viewEl){
|
|
this.view = viewEl;
|
|
this._entities = new Map();
|
|
this._selCbs = []; this._changeCbs = [];
|
|
this._selected = null;
|
|
this._activeCamId = null;
|
|
this._mixerAuto = true; // Stage advances mixers each frame until Timeline takes over
|
|
this._idc = 0;
|
|
this._clipSrc = new Map(); // path -> {root, rest, anims} (parse+rest cached)
|
|
this._baked = new Map(); // id|path|index -> AnimationClip
|
|
|
|
const r = this.renderer = new THREE.WebGLRenderer({antialias:true, preserveDrawingBuffer:true});
|
|
r.setPixelRatio(devicePixelRatio);
|
|
r.shadowMap.enabled = true;
|
|
view.appendChild(r.domElement);
|
|
this._paused = false; // M3 final render freezes the director loop, then drives renderActiveCamera
|
|
|
|
// PiP overlay — the active camera's view (bottom-right). CSS sizes it (~24% w); the buffer
|
|
// stays small. Click to swap which view is main vs inset.
|
|
const pip = this._pip = document.createElement('canvas');
|
|
pip.width = 320; pip.height = 180; pip.className = 'pip'; pip.style.display = 'none';
|
|
pip.title = 'click to swap with the main view';
|
|
this._pipSwap = false;
|
|
pip.addEventListener('click', () => { this._pipSwap = !this._pipSwap; });
|
|
view.appendChild(pip);
|
|
|
|
const scene = this.scene = new THREE.Scene();
|
|
const cam = this._dirCam = new THREE.PerspectiveCamera(45, 1, 0.01, 3000);
|
|
cam.position.set(0, 1.6, 5);
|
|
const ctr = this.controls = new OrbitControls(cam, r.domElement);
|
|
ctr.enableDamping = true; ctr.target.set(0, 1.0, 0);
|
|
|
|
const pmrem = new THREE.PMREMGenerator(r);
|
|
scene.environment = pmrem.fromScene(new RoomEnvironment(), 0.04).texture; pmrem.dispose();
|
|
// default lights — replaced/dimmed once the scene carries light entities
|
|
this._defHemi = new THREE.HemisphereLight(0xffffff, 0x555a66, 2.2); scene.add(this._defHemi);
|
|
this._defKey = new THREE.DirectionalLight(0xffffff, 1.8); this._defKey.position.set(3, 5, 2); scene.add(this._defKey);
|
|
|
|
const floor = this._floor = new THREE.Mesh(new THREE.CircleGeometry(30, 64).rotateX(-Math.PI/2),
|
|
new THREE.MeshStandardMaterial({color:0x141a20, roughness:.97}));
|
|
floor.receiveShadow = true; scene.add(floor);
|
|
this._floorBase = new THREE.Color(0x141a20);
|
|
const grid = this._grid = new THREE.GridHelper(60, 60, 0x2a323b, 0x1c232b);
|
|
grid.material.transparent = true; scene.add(grid);
|
|
|
|
// gizmo
|
|
const tc = this._tc = new TransformControls(cam, r.domElement);
|
|
tc.addEventListener('dragging-changed', e => { ctr.enabled = !e.value; });
|
|
tc.addEventListener('mouseUp', () => { if(this._selected) this._fireChange(this.getEntity(this._selected)); });
|
|
scene.add(tc);
|
|
addEventListener('keydown', e => {
|
|
if(!this._selected) return;
|
|
if(e.key==='w') tc.setMode('translate');
|
|
else if(e.key==='e') tc.setMode('rotate');
|
|
else if(e.key==='r') tc.setMode('scale');
|
|
});
|
|
|
|
r.domElement.addEventListener('click', e => this._pick(e));
|
|
this.clock = new THREE.Clock();
|
|
this._resize(); addEventListener('resize', () => this._resize());
|
|
const loop = () => { requestAnimationFrame(loop);
|
|
const dt = this.clock.getDelta();
|
|
if(this._paused) return; // M3: caller drives rendering deterministically
|
|
ctr.update();
|
|
if(this._mixerAuto) for(const en of this._entities.values()) en.mixer && en.mixer.update(dt);
|
|
// inset first (it scribbles onto the main canvas), then the main view last so it wins
|
|
if(this._activeCamId && pip.style.display !== 'none')
|
|
this._render(this._pipSwap ? this._dirCam : this._activeCam(), pip);
|
|
this._render((this._pipSwap && this._activeCamId) ? this._activeCam() : this._dirCam, null);
|
|
};
|
|
loop();
|
|
}
|
|
|
|
_resize(){ const w=this.view.clientWidth, h=this.view.clientHeight;
|
|
this.renderer.setSize(w, h, false); this._dirCam.aspect = w/h; this._dirCam.updateProjectionMatrix();
|
|
// size the inset here — CSS aspect-ratio/% is unreliable on a <canvas>
|
|
const pw = Math.max(180, Math.min(340, w*0.24));
|
|
this._pip.style.width = Math.round(pw)+'px'; this._pip.style.height = Math.round(pw*9/16)+'px';
|
|
this.fitBackdrops(); // frame aspect changed → auto-fit plates no longer fill it
|
|
}
|
|
|
|
// ---- callbacks ----
|
|
onSelect(cb){ this._selCbs.push(cb); }
|
|
onChange(cb){ this._changeCbs.push(cb); }
|
|
_fireChange(en){ for(const cb of this._changeCbs) cb(en); }
|
|
|
|
// ---- entity lifecycle ----
|
|
entities(){ return [...this._entities.values()]; }
|
|
getEntity(id){ return this._entities.get(id); }
|
|
|
|
async addEntity(desc){
|
|
const id = desc.id || ('e' + (++this._idc));
|
|
if(this._idc <= (parseInt(String(id).slice(1)) || 0)) this._idc = parseInt(String(id).slice(1)) || this._idc;
|
|
const wrapper = new THREE.Group();
|
|
wrapper.userData.entityId = id;
|
|
const noAsset = desc.kind === 'camera' || desc.kind === 'light'; // rig entities have no file
|
|
const en = { id, kind: desc.kind, label: desc.label || desc.kind,
|
|
source: desc.source || (noAsset ? {type:'none'} : null), params: {...(desc.params||{})},
|
|
wrapper, root:null, mixer:null, rest:null, cam:null, light:null, morphs:null, visemes:null,
|
|
video:null, ground:null };
|
|
|
|
if(desc.kind === 'character' || desc.kind === 'prop'){
|
|
const { root } = await this._loadAsset(desc);
|
|
this._normalize(root, desc.kind === 'character');
|
|
root.traverse(o=>{ if(o.isMesh){ o.castShadow = true; o.receiveShadow = true; } if(o.isSkinnedMesh) o.frustumCulled=false; });
|
|
wrapper.add(root); en.root = root;
|
|
if(desc.kind === 'character'){ en.rest = captureRest(root); en.mixer = new THREE.AnimationMixer(root);
|
|
const v = detectVisemes(root); en.morphs = v.morphs; en.visemes = v.visemes; }
|
|
} else if(desc.kind === 'backdrop'){
|
|
en.root = await this._buildBackdrop(en); wrapper.add(en.root);
|
|
} else if(desc.kind === 'screen'){
|
|
en.root = await this._buildScreen(en); wrapper.add(en.root);
|
|
} else if(desc.kind === 'camera'){
|
|
const fov = en.params.fov || 45;
|
|
en.cam = new THREE.PerspectiveCamera(fov, 16/9, 0.05, 3000);
|
|
wrapper.add(en.cam);
|
|
const proxy = new THREE.Mesh(new THREE.ConeGeometry(0.18, 0.35, 4).rotateX(-Math.PI/2),
|
|
new THREE.MeshBasicMaterial({color:0x43c0cf, wireframe:true}));
|
|
proxy.position.z = 0.18; proxy.userData.chrome = true; // SYNC7: editor-only, never in a render
|
|
wrapper.add(proxy); en.proxy = proxy;
|
|
} else if(desc.kind === 'light'){
|
|
this._buildLight(en);
|
|
} else throw new Error('unknown kind: ' + desc.kind);
|
|
|
|
this.scene.add(wrapper);
|
|
this._entities.set(id, en);
|
|
if(desc.transform) this.setTransform(id, desc.transform);
|
|
if(en.kind === 'backdrop'){ this._fitBackdrop(en); this._sampleGround(en); } // M7: world tie-in
|
|
// _buildLight ran BEFORE the entity was registered, so its _refreshDefaults couldn't see it —
|
|
// without this the built-in hemi stayed at 2.2 and washed every lighting preset out.
|
|
if(en.kind === 'light'){ this._refreshDefaults(); this._applyWorld(); }
|
|
this._fireChange(en); // tlui/dock build a row per entity off onChange (incl. applyState loads)
|
|
return en;
|
|
}
|
|
|
|
removeEntity(id){
|
|
const en = this._entities.get(id); if(!en) return;
|
|
if(this._selected === id) this.select(null);
|
|
this._dropVideo(en);
|
|
if(en.root) disposeRoot(en.root);
|
|
if(en.light) en.light.parent && en.light.parent.remove(en.light);
|
|
this.scene.remove(en.wrapper);
|
|
this._entities.delete(id);
|
|
for(const k of [...this._baked.keys()]) if(k.startsWith(id+'|')) this._baked.delete(k);
|
|
if(en.kind === 'light') this._refreshDefaults();
|
|
if(en.kind === 'backdrop' || en.kind === 'light') this._applyWorld(); // floor/grid/fog follow the plates
|
|
this._fireChange(en); // trigger a row resync so the deleted entity drops out
|
|
}
|
|
|
|
async _loadAsset(desc){
|
|
let buf, name;
|
|
if(desc.source && desc.source.type === 'upload'){
|
|
if(!desc._buf) throw new Error('upload entity has no buffer');
|
|
buf = desc._buf; name = desc.source.path;
|
|
} else {
|
|
const path = desc.source.path;
|
|
const res = await fetch(ASSET_URL(path));
|
|
if(!res.ok) throw new Error('asset fetch ' + res.status + ': ' + path);
|
|
buf = await res.arrayBuffer(); name = path;
|
|
}
|
|
return parseAny(buf, name);
|
|
}
|
|
|
|
_normalize(root, isChar){
|
|
root.updateMatrixWorld(true);
|
|
const box = new THREE.Box3().setFromObject(root);
|
|
if(box.isEmpty() || !isFinite(box.min.x)) return;
|
|
if(isChar){
|
|
const size = box.getSize(new THREE.Vector3());
|
|
const s = 1.7 / (Math.max(size.x, size.y, size.z) || 1);
|
|
root.scale.setScalar(s); root.updateMatrixWorld(true);
|
|
}
|
|
const b = new THREE.Box3().setFromObject(root);
|
|
root.position.x -= (b.min.x + b.max.x)/2;
|
|
root.position.z -= (b.min.z + b.max.z)/2;
|
|
root.position.y -= b.min.y; // feet on the floor
|
|
}
|
|
|
|
// Backdrop plate — image OR video (M6). params.mode: plane|corner|dome|video ('video' = a
|
|
// plane; the other modes also accept .mp4 sources). Video plates show their poster jpg (the
|
|
// server's same-stem sidecar) until first play/seek, then swap to the live VideoTexture.
|
|
async _buildBackdrop(en){
|
|
const p = en.params;
|
|
const src = p.image || (en.source && en.source.path) || '';
|
|
const upload = en.source && en.source.type === 'upload';
|
|
const url = upload ? en.source._url : ASSET_URL(src);
|
|
const isVideo = p.mode === 'video' || /\.(mp4|webm|mov)$/i.test(src);
|
|
let tex, aspect, vidTex = null, poster = null;
|
|
if(isVideo){
|
|
const { video, texture } = await this._makeVideo(url, src);
|
|
en.video = video; en.videoTex = vidTex = texture;
|
|
aspect = (video.videoWidth / video.videoHeight) || 16/9;
|
|
poster = upload ? null : await this._posterTex(src);
|
|
tex = poster || vidTex;
|
|
} else {
|
|
tex = await new THREE.TextureLoader().loadAsync(url);
|
|
tex.colorSpace = THREE.SRGBColorSpace;
|
|
aspect = (tex.image.width / tex.image.height) || 1;
|
|
}
|
|
const w = p.width || 10, h = w / aspect;
|
|
const grp = new THREE.Group(), mats = [];
|
|
// fog:false — the plate IS the distance; atmosphere is for what stands in front of it
|
|
if(p.mode === 'dome'){
|
|
const dm = new THREE.MeshBasicMaterial({map:tex, side:THREE.BackSide, fog:false}); mats.push(dm);
|
|
grp.add(new THREE.Mesh(new THREE.SphereGeometry(40, 40, 24), dm));
|
|
} else {
|
|
const mat = new THREE.MeshStandardMaterial({map:tex, roughness:1, metalness:0, fog:false}); mats.push(mat);
|
|
const wall = new THREE.Mesh(new THREE.PlaneGeometry(w, h), mat);
|
|
wall.position.y = h/2; wall.receiveShadow = true; grp.add(wall);
|
|
if(p.mode === 'corner'){ // cheap "L": duplicate lower third laid flat as ground
|
|
const gh = h/3, gm = mat.clone(); mats.push(gm);
|
|
const g = new THREE.Mesh(new THREE.PlaneGeometry(w, gh), gm);
|
|
g.rotation.x = -Math.PI/2; g.position.z = gh/2; g.receiveShadow = true; grp.add(g);
|
|
wall.position.z = -gh/2;
|
|
}
|
|
}
|
|
if(vidTex && poster) this._swapOnPlay(en.video, vidTex, mats);
|
|
if(en.video && this._mixerAuto) en.video.play().catch(()=>{}); // ambient preview until a timeline owns the clock
|
|
return grp;
|
|
}
|
|
|
|
// In-set display (M6 `screen` kind): a video plane with an emissive boost so TVs/club
|
|
// screens/jumbotrons read "on" under any lighting preset. params {video, width, emissive}.
|
|
// ponytail: CRT curve skipped — flat plane until a real TV asset asks for it.
|
|
async _buildScreen(en){
|
|
const p = en.params;
|
|
const src = p.video || (en.source && en.source.path) || '';
|
|
const upload = en.source && en.source.type === 'upload';
|
|
const { video, texture } = await this._makeVideo(upload ? en.source._url : ASSET_URL(src), src);
|
|
en.video = video; en.videoTex = texture;
|
|
const aspect = (video.videoWidth / video.videoHeight) || 16/9;
|
|
const poster = upload ? null : await this._posterTex(src);
|
|
const tex = poster || texture;
|
|
const w = p.width || 3, h = w / aspect;
|
|
const mat = new THREE.MeshStandardMaterial({ map:tex, emissive:0xffffff, emissiveMap:tex,
|
|
emissiveIntensity: p.emissive ?? 0.6, roughness:0.4, metalness:0 });
|
|
if(poster) this._swapOnPlay(video, texture, [mat]);
|
|
if(this._mixerAuto) video.play().catch(()=>{});
|
|
const mesh = new THREE.Mesh(new THREE.PlaneGeometry(w, h), mat);
|
|
mesh.position.y = h/2;
|
|
const grp = new THREE.Group(); grp.add(mesh);
|
|
return grp;
|
|
}
|
|
|
|
// <video> element + VideoTexture, resolved once metadata (dimensions/duration) is in
|
|
_makeVideo(url, srcPath){
|
|
return new Promise((resolve, reject) => {
|
|
const v = document.createElement('video');
|
|
v.muted = true; v.loop = true; v.playsInline = true; v.preload = 'auto'; v.crossOrigin = 'anonymous';
|
|
v.src = url;
|
|
const texture = new THREE.VideoTexture(v);
|
|
texture.colorSpace = THREE.SRGBColorSpace;
|
|
v.addEventListener('loadedmetadata', () => resolve({ video: v, texture }), { once:true });
|
|
v.addEventListener('error', () => reject(new Error('video load failed: ' + srcPath)), { once:true });
|
|
});
|
|
}
|
|
_posterTex(src){ // same-stem jpg via the server's sidecar route; null when absent
|
|
return new THREE.TextureLoader().loadAsync('assets/thumb?path=' + encodeURIComponent(src))
|
|
.then(t => { t.colorSpace = THREE.SRGBColorSpace; return t; })
|
|
.catch(() => null);
|
|
}
|
|
_swapOnPlay(video, texture, mats){ // poster stays up until the video actually shows a frame
|
|
const swap = () => {
|
|
video.removeEventListener('playing', swap); video.removeEventListener('seeked', swap);
|
|
for(const m of mats){ m.map = texture; if(m.emissiveMap) m.emissiveMap = texture; m.needsUpdate = true; }
|
|
};
|
|
video.addEventListener('playing', swap); video.addEventListener('seeked', swap);
|
|
}
|
|
_dropVideo(en){
|
|
if(!en.video) return;
|
|
en.video.pause(); en.video.removeAttribute('src'); en.video.load(); en.video = null;
|
|
}
|
|
|
|
// ---- world-integrated backdrops (M7) ---------------------------------------------------------
|
|
// A plate used to read as a billboard in the void: a gap between its bottom edge and the grid, a
|
|
// floor that belongs to a different scene, no air between camera and horizon. Three cheap fixes,
|
|
// all recomputed on load / param change / camera change — NEVER per frame.
|
|
// params.fit:'frustum' size+place the plate so it fills the active camera's frustum (+bleed)
|
|
// params.groundTint tint the stage floor to the plate's bottom rows (default on)
|
|
// params `fog` on the ambient light keyable FogExp2 density, coloured from the plate
|
|
|
|
_activeAspect(){ const c = this._activeCam();
|
|
return c.aspect || (this.view.clientWidth / this.view.clientHeight) || 16/9; }
|
|
|
|
fitBackdrops(){ for(const en of this._entities.values())
|
|
if(en.kind === 'backdrop') this._fitBackdrop(en); }
|
|
|
|
// Scale/park the plate on the active camera's axis so it covers the frame at params.fitDist.
|
|
// ponytail: fit OWNS the backdrop's wrapper transform — keyframe a plate only with fit:'free'.
|
|
_fitBackdrop(en){
|
|
const p = en.params, root = en.root;
|
|
if(!root || p.fit !== 'frustum') return;
|
|
if(p.mode === 'dome' || p.mode === 'corner') return; // dome already wraps; corner needs its floor flat
|
|
const mesh = root.children[0], g = mesh && mesh.geometry && mesh.geometry.parameters;
|
|
if(!g || !g.width) return;
|
|
const cam = this._activeCam(); cam.updateMatrixWorld(true);
|
|
const d = p.fitDist || 14; // far enough to read as background
|
|
const { s } = fitScale(cam.fov, this._activeAspect(), d, g.width, g.height);
|
|
root.scale.setScalar(s);
|
|
// SYNC7: which row of the plate sits at eye level. 1.0 parked the plate's TOP EDGE on the
|
|
// camera axis (void above centre-frame); 0.5 put the image's middle — road tarmac, for a
|
|
// street plate — at eye height. params.fitAnchor is the fraction of image height at the
|
|
// lens axis: 0.75 lands a typical plate's horizon near eye level and still fills upward.
|
|
const anchor = p.fitAnchor == null ? 0.75 : p.fitAnchor;
|
|
root.position.set(0, (0.5 - anchor) * g.height * s, 0);
|
|
const camPos = cam.getWorldPosition(new THREE.Vector3());
|
|
const fwd = new THREE.Vector3(0, 0, -1).applyQuaternion(cam.getWorldQuaternion(new THREE.Quaternion()));
|
|
en.wrapper.position.copy(camPos).addScaledVector(fwd, d);
|
|
en.wrapper.lookAt(camPos); // plane normal is +Z
|
|
en.wrapper.scale.setScalar(1);
|
|
}
|
|
_unfitBackdrop(en){ if(en.root){ en.root.scale.setScalar(1); en.root.position.set(0,0,0); } }
|
|
|
|
// average colour of the plate's bottom rows — the ground the character should be standing on.
|
|
// Works for images and for video (poster or current frame); same-origin, so the canvas is clean.
|
|
_plateGround(en){
|
|
const mesh = en.root && en.root.children[0], tex = mesh && mesh.material && mesh.material.map;
|
|
const img = tex && tex.image;
|
|
if(!img || !(img.width || img.videoWidth)) return null;
|
|
if(img.videoWidth && img.readyState < 2) return null; // no decoded frame yet → sample black
|
|
try {
|
|
const c = document.createElement('canvas'); c.width = c.height = 24;
|
|
const g = c.getContext('2d', { willReadFrequently: true });
|
|
g.drawImage(img, 0, 0, 24, 24);
|
|
const d = g.getImageData(0, 17, 24, 7).data; // bottom ~quarter
|
|
let r = 0, gr = 0, b = 0; const n = d.length / 4;
|
|
for(let i = 0; i < d.length; i += 4){ r += d[i]; gr += d[i+1]; b += d[i+2]; }
|
|
return new THREE.Color().setRGB(r/n/255, gr/n/255, b/n/255, THREE.SRGBColorSpace);
|
|
} catch(e){ return null; } // tainted canvas / no frame yet
|
|
}
|
|
_sampleGround(en){
|
|
const col = this._plateGround(en);
|
|
if(col) en.ground = col;
|
|
else if(en.video) // no decoded frame yet — take the first one that lands
|
|
en.video.addEventListener('loadeddata', () => this._sampleGround(en), { once:true });
|
|
this._applyWorld(); // grid/fog follow the plate even when sampling came back empty
|
|
}
|
|
|
|
// floor tint + grid fade + fog, derived from whatever plates are on stage
|
|
_applyWorld(){
|
|
const plates = this.entities().filter(e => e.kind === 'backdrop');
|
|
const tint = plates.filter(e => e.ground && (e.params.groundTint ?? true)).pop();
|
|
const col = tint ? tint.ground : null;
|
|
this._floor.material.color.copy(col ? col.clone().multiplyScalar(0.85) : this._floorBase);
|
|
this._grid.material.opacity = plates.length ? 0.12 : 1; // fade the studio grid out under a plate
|
|
const amb = this.entities().find(e => e.kind === 'light' && e.params.type === 'ambient');
|
|
const dens = amb && amb.params.fog != null ? amb.params.fog : (col ? 0.02 : 0); // subtle default
|
|
const had = !!this.scene.fog;
|
|
if(col && dens > 0){ // Lane B can key `fog` per tick — mutate, don't reallocate
|
|
if(had){ this.scene.fog.color.copy(col); this.scene.fog.density = dens; }
|
|
else this.scene.fog = new THREE.FogExp2(col.getHex(), dens);
|
|
} else this.scene.fog = null;
|
|
// toggling fog on/off changes the shader program — materials must recompile (once, not per frame)
|
|
if(had !== !!this.scene.fog) this.scene.traverse(o => { const m = o.material;
|
|
if(m) (Array.isArray(m) ? m : [m]).forEach(x => x.needsUpdate = true); });
|
|
}
|
|
|
|
_buildLight(en){
|
|
const p = en.params;
|
|
if(p.type === 'ambient'){
|
|
en.light = new THREE.HemisphereLight(new THREE.Color(p.color||'#ffffff'), 0x444450, p.intensity ?? 1.0);
|
|
en.wrapper.add(en.light);
|
|
if(p.bg) this.scene.background = new THREE.Color(p.bg); // saved lighting preset restores its backdrop color
|
|
} else {
|
|
const dir = en.light = new THREE.DirectionalLight(new THREE.Color(p.color||'#ffffff'), p.intensity ?? 1.5);
|
|
dir.castShadow = p.castShadow ?? true;
|
|
dir.shadow.mapSize.set(1024,1024); dir.shadow.camera.far = 60;
|
|
dir.position.set(0,0,0);
|
|
const tgt = new THREE.Object3D(); tgt.position.set(0,0,-1); en.wrapper.add(tgt); dir.target = tgt;
|
|
en.wrapper.add(dir);
|
|
const proxy = new THREE.Mesh(new THREE.SphereGeometry(0.15, 12, 8),
|
|
new THREE.MeshBasicMaterial({color:0xffe08a}));
|
|
en.wrapper.add(proxy);
|
|
}
|
|
this._refreshDefaults();
|
|
}
|
|
|
|
// dim built-in lights once scene lights exist
|
|
_refreshDefaults(){
|
|
const hasKey = this.entities().some(e => e.kind==='light' && e.params.type!=='ambient');
|
|
const hasAmb = this.entities().some(e => e.kind==='light' && e.params.type==='ambient');
|
|
this._defKey.visible = !hasKey;
|
|
this._defHemi.intensity = hasAmb ? 0 : 2.2;
|
|
}
|
|
|
|
// ---- selection + gizmo ----
|
|
select(id){
|
|
this._selected = id;
|
|
if(id == null){ this._tc.detach(); }
|
|
else { const en = this._entities.get(id); if(en) this._tc.attach(en.wrapper); }
|
|
for(const cb of this._selCbs) cb(id ? this._entities.get(id) : null);
|
|
}
|
|
_pick(e){
|
|
if(this._tc.dragging) return;
|
|
const rect = this.renderer.domElement.getBoundingClientRect();
|
|
const ndc = new THREE.Vector2(
|
|
((e.clientX - rect.left)/rect.width)*2 - 1,
|
|
-((e.clientY - rect.top)/rect.height)*2 + 1);
|
|
const rc = new THREE.Raycaster(); rc.setFromCamera(ndc, this._dirCam);
|
|
const wraps = this.entities().map(en => en.wrapper);
|
|
const hits = rc.intersectObjects(wraps, true);
|
|
for(const h of hits){
|
|
let o = h.object; while(o && o.userData.entityId == null) o = o.parent;
|
|
if(o){ this.select(o.userData.entityId); return; }
|
|
}
|
|
this.select(null);
|
|
}
|
|
|
|
// ---- transforms / params (Timeline drives these every tick) ----
|
|
entityTransform(id){
|
|
const w = this._entities.get(id)?.wrapper; if(!w) return null;
|
|
return { pos:[w.position.x, w.position.y, w.position.z],
|
|
rot:[w.rotation.x, w.rotation.y, w.rotation.z],
|
|
scale: w.scale.x };
|
|
}
|
|
setTransform(id, {pos, rot, scale}){
|
|
const en = this._entities.get(id); const w = en?.wrapper; if(!w) return;
|
|
// SYNC7: a frustum-fitted plate is placed BY THE LENS, not by the user. The timeline
|
|
// re-applies every entity's rest transform each tick, which dragged the plate back to
|
|
// its authored pos and left a black bar at frame top. Camera owns it; ignore the rest.
|
|
if(en.kind === 'backdrop' && en.params && en.params.fit === 'frustum'){ this._fitBackdrop(en); return; }
|
|
if(pos) w.position.set(pos[0], pos[1], pos[2]);
|
|
if(rot) w.rotation.set(rot[0], rot[1], rot[2]);
|
|
if(scale != null) w.scale.setScalar(scale);
|
|
}
|
|
setParam(id, key, value){
|
|
const en = this._entities.get(id); if(!en) return;
|
|
en.params[key] = value;
|
|
if(en.cam && key === 'fov'){ en.cam.fov = value; en.cam.updateProjectionMatrix();
|
|
if(en.id === this._activeCamId) this.fitBackdrops(); } // the frame changed shape
|
|
if(en.light){
|
|
if(key === 'intensity') en.light.intensity = value;
|
|
if(key === 'color') en.light.color.set(value);
|
|
if(key === 'castShadow' && en.light.isDirectionalLight) en.light.castShadow = !!value;
|
|
if(key === 'bg') this.scene.background = new THREE.Color(value); // bg rides the ambient entity (keyable, persists)
|
|
if(key === 'fog') this._applyWorld(); // keyable atmosphere, same pattern as bg
|
|
}
|
|
if(en.kind === 'backdrop'){
|
|
if(key === 'fit') value === 'frustum' ? this._fitBackdrop(en) : this._unfitBackdrop(en);
|
|
if(key === 'fitDist' || key === 'fitAnchor') this._fitBackdrop(en);
|
|
if(key === 'groundTint') this._applyWorld();
|
|
}
|
|
const rebuild = () => this._rebuild(en).catch(e => console.warn('media rebuild failed:', en.id, e));
|
|
if(en.kind === 'backdrop' && (key === 'mode' || key === 'image' || key === 'width'))
|
|
rebuild(); // geometry depends on these — rebuild the mesh
|
|
if(en.kind === 'screen'){
|
|
if(key === 'emissive' && en.root)
|
|
en.root.traverse(o => { if(o.isMesh) o.material.emissiveIntensity = value; });
|
|
if(key === 'video' || key === 'width') rebuild();
|
|
}
|
|
// videoStart is data for the timeline/syncVideos — nothing to rebuild
|
|
}
|
|
async _rebuild(en){ // backdrop/screen media rebuild (mode/image/video/width changed)
|
|
const old = en.root;
|
|
this._dropVideo(en);
|
|
en.root = en.kind === 'screen' ? await this._buildScreen(en) : await this._buildBackdrop(en);
|
|
en.wrapper.add(en.root);
|
|
if(old){ en.wrapper.remove(old); disposeRoot(old); }
|
|
if(en.kind === 'backdrop'){ this._fitBackdrop(en); en.ground = null; this._sampleGround(en); }
|
|
}
|
|
|
|
// ---- clips (bake is expensive — cache per path parse and per (entity,path,index) bake) ----
|
|
async prepareClip(id, path, clipIndex=0){
|
|
const en = this._entities.get(id);
|
|
if(!en || en.kind !== 'character') throw new Error('prepareClip: not a character: ' + id);
|
|
const bkey = `${id}|${path}|${clipIndex}`;
|
|
if(this._baked.has(bkey)) return this._baked.get(bkey);
|
|
let src = this._clipSrc.get(path);
|
|
if(!src){
|
|
const res = await fetch(ASSET_URL(path));
|
|
if(!res.ok) throw new Error('clip fetch ' + res.status + ': ' + path);
|
|
const { root, anims } = await parseAny(await res.arrayBuffer(), path);
|
|
src = { root, anims, rest: captureRest(root) };
|
|
this._clipSrc.set(path, src);
|
|
}
|
|
const clip = src.anims[clipIndex];
|
|
if(!clip) throw new Error('no clip index ' + clipIndex + ' in ' + path);
|
|
const baked = bakeRetarget(clip, src.root, src.rest, en.root, en.rest);
|
|
this._baked.set(bkey, baked);
|
|
return baked;
|
|
}
|
|
// same bake, from in-memory bytes (dropped clip file) instead of a server path
|
|
async prepareClipUpload(id, buf, name, clipIndex=0){
|
|
const en = this._entities.get(id);
|
|
if(!en || en.kind !== 'character') throw new Error('prepareClipUpload: not a character: ' + id);
|
|
const { root, anims } = await parseAny(buf, name);
|
|
const clip = anims[clipIndex];
|
|
if(!clip) throw new Error('no animation tracks in ' + name);
|
|
return bakeRetarget(clip, root, captureRest(root), en.root, en.rest);
|
|
}
|
|
entityMixer(id){ return this._entities.get(id)?.mixer || null; }
|
|
|
|
// ---- visemes / morph targets (lip sync) ----
|
|
visemeTargets(id){ return Object.keys(this._entities.get(id)?.visemes || {}); }
|
|
// drive a canonical viseme (A/E/I/O/U/MBP/FV/L) or a raw morph name; sets every mapped mesh morph
|
|
setMorph(id, name, weight){
|
|
const en = this._entities.get(id); if(!en || !en.morphs) return;
|
|
const raws = (en.visemes && en.visemes[name]) || (en.morphs.has(name.toLowerCase()) ? [name.toLowerCase()] : []);
|
|
for(const r of raws) for(const {mesh, idx} of en.morphs.get(r))
|
|
if(mesh.morphTargetInfluences) mesh.morphTargetInfluences[idx] = weight;
|
|
}
|
|
|
|
// M1 temp helper: play a prepared clip on a character (Timeline replaces this in M2).
|
|
playClip(id, clip, {loop=true} = {}){
|
|
const en = this._entities.get(id); if(!en || !en.mixer) return;
|
|
en.mixer.stopAllAction();
|
|
const a = en.mixer.clipAction(clip);
|
|
a.setLoop(loop ? THREE.LoopRepeat : THREE.LoopOnce, Infinity);
|
|
a.clampWhenFinished = !loop; a.reset().play();
|
|
}
|
|
|
|
setMixerAuto(on){ this._mixerAuto = on; } // Timeline calls setMixerAuto(false) at integration
|
|
|
|
// ---- cameras ----
|
|
setActiveCamera(id){
|
|
this._activeCamId = id;
|
|
this._pip.style.display = (id && this._entities.get(id)?.cam) ? 'block' : 'none';
|
|
this.fitBackdrops(); // auto-fit plates belong to whichever camera is shooting
|
|
}
|
|
activeCamera(){ return this._activeCamId || null; } // entity id, or null = director orbit cam
|
|
_activeCam(){ const en = this._activeCamId && this._entities.get(this._activeCamId);
|
|
return en && en.cam ? en.cam : this._dirCam; }
|
|
// render `cam` to the main WebGL canvas at the target's aspect; if `canvas` given, blit there and
|
|
// restore the cam's aspect (so using a cam for the inset never distorts it in the main view).
|
|
// SYNC7: gizmo + helpers were being BAKED INTO THE FINAL MP4 (a green translate
|
|
// arrow across every frame). Anything that's editor chrome hides while paused —
|
|
// pause() is exactly "the caller is rendering output now".
|
|
_chrome(visible){
|
|
const set = o => { if(o) o.visible = visible; };
|
|
set(this._tc); set(this._grid);
|
|
this.scene.traverse(o => { if(o.userData.chrome || o.isTransformControls || o.type === 'GridHelper'
|
|
|| o.isCameraHelper || o.isDirectionalLightHelper || o.isBox3Helper || o.isSkeletonHelper) o.visible = visible; });
|
|
for(const en of this._entities.values()) if(en.helper) en.helper.visible = visible;
|
|
}
|
|
_render(cam, canvas){
|
|
const w = canvas ? canvas.width : this.renderer.domElement.width;
|
|
const h = canvas ? canvas.height : this.renderer.domElement.height;
|
|
const prev = cam.aspect;
|
|
if(cam.isPerspectiveCamera){ cam.aspect = w/h; cam.updateProjectionMatrix(); }
|
|
// SYNC7: plates are fitted to the LIVE viewport aspect, but output renders at their own
|
|
// (960x540 etc). A narrower output frame is taller than what the plate was sized for →
|
|
// black bars top and bottom of every rendered film. Refit to the aspect we're about to
|
|
// draw at. Pure algebra (no texture sampling), so per-frame is fine.
|
|
if(cam.isPerspectiveCamera && Math.abs((prev || 0) - cam.aspect) > 1e-4) this.fitBackdrops();
|
|
if(this._paused) this._chrome(false);
|
|
this.renderer.render(this.scene, cam);
|
|
if(this._paused) this._chrome(true);
|
|
if(canvas){ canvas.getContext('2d').drawImage(this.renderer.domElement, 0, 0, canvas.width, canvas.height);
|
|
if(cam.isPerspectiveCamera && cam.aspect !== prev){ cam.aspect = prev; cam.updateProjectionMatrix(); } }
|
|
}
|
|
// PiP: pass the small overlay canvas. Final render (M3): pass null after pause() — the active cam
|
|
// lands on the main WebGL canvas at its current size for readPixels/toBlob. Hard contract, keep it.
|
|
renderActiveCamera(canvas){ this._render(this._activeCam(), canvas); }
|
|
pause(){ this._paused = true; }
|
|
resume(){ this._paused = false; this.clock.getDelta(); } // swallow the paused gap so mixers don't jump
|
|
|
|
// ---- director-mode helpers (M5 presets.js solvers are three-free — plain arrays) ----
|
|
entityBBox(id){
|
|
const en = this._entities.get(id); if(!en) return null;
|
|
en.wrapper.updateMatrixWorld(true);
|
|
const box = new THREE.Box3().setFromObject(en.wrapper);
|
|
return (box.isEmpty() || !isFinite(box.min.x)) ? null : { min: box.min.toArray(), max: box.max.toArray() };
|
|
}
|
|
cameraWorldPos(){ // active camera if set, else the director orbit cam
|
|
const p = new THREE.Vector3(); this._activeCam().getWorldPosition(p); return p.toArray();
|
|
}
|
|
viewAspect(){ return this._activeAspect(); } // M11: a truck is framed in the REAL frame, not 16:9
|
|
|
|
// ---- video plates (M6): master-clock sync + deterministic render seeks ----
|
|
videos(){ return this.entities().filter(e => e.video)
|
|
.map(e => ({ id: e.id, video: e.video, start: e.params.videoStart || 0 })); }
|
|
entityVideo(id){ return this._entities.get(id)?.video || null; } // SYNC6: Lane B's per-entity seam
|
|
setVideoPlaying(on){ for(const { video } of this.videos()) on ? video.play().catch(()=>{}) : video.pause(); }
|
|
// Seek every stage video to timeline time t and resolve once all fire 'seeked' —
|
|
// render.js awaits this per frame so plates are frame-exact, never mid-decode. Hard contract.
|
|
async syncVideos(t){
|
|
const jobs = [];
|
|
// SYNC6: r160 VideoTexture auto-updates via requestVideoFrameCallback, which never
|
|
// fires in a hidden/background tab — force the GPU upload or renders show black plates.
|
|
for(const en of this._entities.values()) if(en.videoTex) en.videoTex.needsUpdate = true;
|
|
for(const { video, start } of this.videos()){
|
|
if(video.readyState < 1 || !isFinite(video.duration) || video.duration <= 0) continue;
|
|
const vt = (((t - start) % video.duration) + video.duration) % video.duration;
|
|
if(!video.paused) video.pause();
|
|
if(Math.abs(video.currentTime - vt) < 0.004) continue; // already on this frame
|
|
jobs.push(new Promise(res => {
|
|
const done = () => { video.removeEventListener('seeked', done); res(); };
|
|
video.addEventListener('seeked', done);
|
|
setTimeout(done, 1500); // ponytail: never hang a render on a stuck decoder
|
|
video.currentTime = vt;
|
|
}));
|
|
}
|
|
await Promise.all(jobs);
|
|
}
|
|
|
|
// ---- pointer→world helpers (drag-from-dock placement) ----
|
|
screenToFloor(clientX, clientY){
|
|
const rect = this.renderer.domElement.getBoundingClientRect();
|
|
const ndc = new THREE.Vector2(((clientX-rect.left)/rect.width)*2-1, -((clientY-rect.top)/rect.height)*2+1);
|
|
const rc = new THREE.Raycaster(); rc.setFromCamera(ndc, this._dirCam);
|
|
const hit = new THREE.Vector3();
|
|
return rc.ray.intersectPlane(new THREE.Plane(new THREE.Vector3(0,1,0), 0), hit) ? [hit.x, 0, hit.z] : [0,0,0];
|
|
}
|
|
entityAt(clientX, clientY, kind){
|
|
const rect = this.renderer.domElement.getBoundingClientRect();
|
|
const ndc = new THREE.Vector2(((clientX-rect.left)/rect.width)*2-1, -((clientY-rect.top)/rect.height)*2+1);
|
|
const rc = new THREE.Raycaster(); rc.setFromCamera(ndc, this._dirCam);
|
|
const hits = rc.intersectObjects(this.entities().map(e=>e.wrapper), true);
|
|
for(const h of hits){ let o=h.object; while(o && o.userData.entityId==null) o=o.parent;
|
|
if(o){ const en=this._entities.get(o.userData.entityId); if(en && (!kind || en.kind===kind)) return en.id; } }
|
|
return null;
|
|
}
|
|
|
|
// ---- scene JSON round-trip (entities only; Timeline owns tracks/duration/cuts) ----
|
|
captureState(){
|
|
return this.entities().map(en => ({
|
|
id: en.id, kind: en.kind, label: en.label,
|
|
source: en.source, params: {...en.params},
|
|
transform: this.entityTransform(en.id),
|
|
upload: en.source && en.source.type === 'upload' || undefined,
|
|
}));
|
|
}
|
|
async applyState(sceneJson){
|
|
for(const en of this.entities()) this.removeEntity(en.id);
|
|
for(const d of (sceneJson.entities || [])){
|
|
// skip only what needs asset BYTES we don't have: an 'upload' mesh, backdrop or screen.
|
|
// camera/light use source:{type:'none'} (legacy scenes: 'upload') and always rebuild.
|
|
const needsBytes = d.kind === 'character' || d.kind === 'prop' || d.kind === 'backdrop' || d.kind === 'screen';
|
|
if(d.source && d.source.type === 'upload' && needsBytes){
|
|
console.warn('skipping upload entity (no bytes):', d.id); continue; }
|
|
try { await this.addEntity(d); } catch(err){ console.error('addEntity failed', d.id, err); }
|
|
}
|
|
this.fitBackdrops(); // cameras usually load after their plates
|
|
}
|
|
}
|