// Three.js scene (spec M5): point cloud (/api/pointcloud via PLYLoader), each video's camera // path as a line + current-pose frustum wireframe (colored per video), OrbitControls free roam, // and snap-to-camera. ALL COLMAP->Three.js conversion goes through the FROZEN lib/pose.js // helper (colmapToThreejs) — the diag(1,-1,-1) math is never reimplemented here. import * as THREE from "three"; import { OrbitControls } from "three/addons/controls/OrbitControls.js"; import { PLYLoader } from "three/addons/loaders/PLYLoader.js"; import { colmapToThreejs } from "./lib/pose.js"; import { poseAt } from "./lib/poseTrack.js"; import { tVideoFromGlobal } from "./lib/timebase.js"; import { state, emit } from "./state.js"; // Per-video colors (match the frustum, path line, and grid accents). export const VIDEO_COLORS = ["#ff5d73", "#4dd0ff", "#c9a2ff", "#ffcf5d", "#7dffb0", "#ff9d5d"]; export function videoColor(index) { return VIDEO_COLORS[index % VIDEO_COLORS.length]; } const FRUSTUM_DEPTH = 1.2; const DEFAULT_FOV = 50; const STAGE_TARGET = new THREE.Vector3(0, 0.5, 0); const _m4 = new THREE.Matrix4(); const _pos = new THREE.Vector3(); const _quat = new THREE.Quaternion(); const _scale = new THREE.Vector3(); export class Scene3D { constructor() { this.videoIndex = {}; // id -> index (for stable colors) this.rigs = {}; // id -> { frustum, marker, pick, path } this._tween = null; this._raf = null; } init(canvas) { this.canvas = canvas; const parent = canvas.parentElement; const w = parent.clientWidth || 1; const h = parent.clientHeight || 1; this.renderer = new THREE.WebGLRenderer({ canvas, antialias: true, alpha: false }); this.renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2)); this.renderer.setSize(w, h, false); this.renderer.setClearColor(0x0a0c11, 1); this.scene = new THREE.Scene(); this.camera = new THREE.PerspectiveCamera(DEFAULT_FOV, w / h, 0.05, 3000); this.camera.position.set(12, 9, 15); this.camera.lookAt(STAGE_TARGET); this.controls = new OrbitControls(this.camera, this.renderer.domElement); this.controls.enableDamping = true; this.controls.dampingFactor = 0.08; this.controls.target.copy(STAGE_TARGET); this.controls.update(); // Lights (points ignore lighting, but frusta/markers use MeshBasic so this is mostly cosmetic). this.scene.add(new THREE.AmbientLight(0xffffff, 0.9)); const grid = new THREE.GridHelper(40, 40, 0x22384a, 0x161c26); grid.position.y = 0; this.scene.add(grid); const axes = new THREE.AxesHelper(1.5); this.scene.add(axes); this._buildRigs(); this._loadPointCloud(); this._wirePicking(); this._wireResize(parent); } _buildRigs() { state.videos.forEach((v, i) => { this.videoIndex[v.id] = i; const color = new THREE.Color(videoColor(i)); const group = new THREE.Group(); // Frustum wireframe (built in Three.js camera-local space: looks down -z, +y up). const poses = state.poses[v.id] || []; const intr = poses[0]?.intrinsics; const W = v.width; const H = v.height; const hx = intr ? (FRUSTUM_DEPTH * W) / (2 * intr.fx) : FRUSTUM_DEPTH * 0.6; const hy = intr ? (FRUSTUM_DEPTH * H) / (2 * intr.fy) : FRUSTUM_DEPTH * 0.34; const d = FRUSTUM_DEPTH; const c = [ [-hx, -hy, -d], [hx, -hy, -d], [hx, hy, -d], [-hx, hy, -d], ]; const apex = [0, 0, 0]; const segs = [ apex, c[0], apex, c[1], apex, c[2], apex, c[3], c[0], c[1], c[1], c[2], c[2], c[3], c[3], c[0], ]; const fg = new THREE.BufferGeometry(); fg.setAttribute("position", new THREE.Float32BufferAttribute(segs.flat(), 3)); const frustum = new THREE.LineSegments( fg, new THREE.LineBasicMaterial({ color }) ); frustum.matrixAutoUpdate = true; group.add(frustum); // Camera-center marker (visible) + an invisible larger sphere for click picking. const marker = new THREE.Mesh( new THREE.SphereGeometry(0.12, 12, 12), new THREE.MeshBasicMaterial({ color }) ); group.add(marker); this.scene.add(group); const pick = new THREE.Mesh( new THREE.SphereGeometry(0.55, 8, 8), new THREE.MeshBasicMaterial({ visible: false }) ); pick.userData.videoId = v.id; group.add(pick); // Camera path line (one vertex per stored pose center). let path = null; if (poses.length > 1) { const pts = []; for (const p of poses) { const { position } = colmapToThreejs(p.q, p.t); pts.push(position[0], position[1], position[2]); } const pg = new THREE.BufferGeometry(); pg.setAttribute("position", new THREE.Float32BufferAttribute(pts, 3)); path = new THREE.Line( pg, new THREE.LineBasicMaterial({ color, transparent: true, opacity: 0.5 }) ); this.scene.add(path); } this.rigs[v.id] = { group, frustum, marker, pick, path }; }); } _loadPointCloud() { const loader = new PLYLoader(); loader.load( state.apiBase + "/api/pointcloud", (geometry) => { geometry.computeBoundingBox(); const hasColor = !!geometry.getAttribute("color"); const material = new THREE.PointsMaterial({ size: 0.05, sizeAttenuation: true, vertexColors: hasColor, color: hasColor ? 0xffffff : 0x88ccff, }); this.points = new THREE.Points(geometry, material); this.scene.add(this.points); emit("pointcloud-loaded", geometry.getAttribute("position")?.count ?? 0); }, undefined, (err) => { console.warn("[scene3d] point cloud load failed", err); } ); } // Live pose (COLMAP convention) for a video at the current master time, or null. _livePose(videoId) { const poses = state.poses[videoId]; if (!poses || poses.length === 0) return null; const meta = state.videos.find((v) => v.id === videoId); const tv = tVideoFromGlobal(state.tGlobal, meta.offset_ms || 0, meta.drift_ppm || 0); return poseAt(poses, tv); } /** Snap the viewer camera to a video's pose and follow it while playing. */ snapTo(videoId) { const pose = this._livePose(videoId); if (!pose) return; state.followCameraId = videoId; this.controls.enabled = false; const targetFov = (2 * Math.atan(state.videos.find((v) => v.id === videoId).height / (2 * pose.intrinsics.fy)) * 180) / Math.PI; this._tween = { start: performance.now(), dur: 600, fromPos: this.camera.position.clone(), fromQuat: this.camera.quaternion.clone(), fromFov: this.camera.fov, toFov: targetFov, }; emit("follow", videoId); } /** Detach to free roam (OrbitControls). */ freeRoam() { if (state.followCameraId == null) return; state.followCameraId = null; this._tween = null; this.camera.fov = DEFAULT_FOV; this.camera.updateProjectionMatrix(); // Re-anchor OrbitControls to the stage, orbiting from the current position. this.controls.target.copy(STAGE_TARGET); this.controls.enabled = true; this.controls.update(); emit("follow", null); } _applyPoseToCamera(pose, targetFov) { const { matrixWorld } = colmapToThreejs(pose.q, pose.t); _m4.fromArray(matrixWorld); _m4.decompose(_pos, _quat, _scale); const t = this._tween; if (t) { const k = Math.min(1, (performance.now() - t.start) / t.dur); const e = k * k * (3 - 2 * k); // smoothstep this.camera.position.lerpVectors(t.fromPos, _pos, e); this.camera.quaternion.copy(t.fromQuat).slerp(_quat, e); this.camera.fov = t.fromFov + (t.toFov - t.fromFov) * e; this.camera.updateProjectionMatrix(); if (k >= 1) this._tween = null; } else { this.camera.position.copy(_pos); this.camera.quaternion.copy(_quat); if (Math.abs(this.camera.fov - targetFov) > 1e-3) { this.camera.fov = targetFov; this.camera.updateProjectionMatrix(); } } } /** Per-frame update: move frusta to current poses, drive follow-cam, render. */ update() { for (const v of state.videos) { const rig = this.rigs[v.id]; if (!rig) continue; const pose = this._livePose(v.id); const on = pose && state.enabled[v.id] !== false; rig.group.visible = !!on; if (rig.path) rig.path.visible = state.enabled[v.id] !== false; if (!on) continue; const { matrixWorld } = colmapToThreejs(pose.q, pose.t); _m4.fromArray(matrixWorld); _m4.decompose(rig.group.position, rig.group.quaternion, rig.group.scale); } if (state.followCameraId != null) { const pose = this._livePose(state.followCameraId); if (pose) { const meta = state.videos.find((v) => v.id === state.followCameraId); const targetFov = (2 * Math.atan(meta.height / (2 * pose.intrinsics.fy)) * 180) / Math.PI; this._applyPoseToCamera(pose, targetFov); } } else { this.controls.update(); } this.renderer.render(this.scene, this.camera); } _wirePicking() { const el = this.renderer.domElement; const ray = new THREE.Raycaster(); const ndc = new THREE.Vector2(); let downX = 0; let downY = 0; el.addEventListener("pointerdown", (e) => { downX = e.clientX; downY = e.clientY; }); el.addEventListener("pointerup", (e) => { if (Math.hypot(e.clientX - downX, e.clientY - downY) > 5) return; // was a drag, not a click const rect = el.getBoundingClientRect(); ndc.x = ((e.clientX - rect.left) / rect.width) * 2 - 1; ndc.y = -((e.clientY - rect.top) / rect.height) * 2 + 1; ray.setFromCamera(ndc, this.camera); const picks = Object.values(this.rigs) .map((r) => r.pick) .filter((m) => m.parent && m.parent.visible); const hits = ray.intersectObjects(picks, false); if (hits.length) this.snapTo(hits[0].object.userData.videoId); }); } _wireResize(parent) { const ro = new ResizeObserver(() => { const w = parent.clientWidth || 1; const h = parent.clientHeight || 1; this.renderer.setSize(w, h, false); // Aspect always follows the 3D canvas (never the video) so nothing stretches; snap-to- // camera only overrides the vertical fov + pose, matching the video's vertical framing. this.camera.aspect = w / h; this.camera.updateProjectionMatrix(); }); ro.observe(parent); } } export const scene3d = new Scene3D();