festifun/frontend/src/scene3d.js

// Three.js scene (spec M5): point cloud (/api/pointcloud via PLYLoader), each video's camera
// path as a line + current-pose frustum wireframe (colored per video), OrbitControls free roam,
// and snap-to-camera. ALL COLMAP->Three.js conversion goes through the FROZEN lib/pose.js
// helper (colmapToThreejs) — the diag(1,-1,-1) math is never reimplemented here.

import * as THREE from "three";
import { OrbitControls } from "three/addons/controls/OrbitControls.js";
import { PLYLoader } from "three/addons/loaders/PLYLoader.js";
import { colmapToThreejs } from "./lib/pose.js";
import { poseAt } from "./lib/poseTrack.js";
import { tVideoFromGlobal } from "./lib/timebase.js";
import { state, emit } from "./state.js";

// Per-video colors (match the frustum, path line, and grid accents).
export const VIDEO_COLORS = ["#ff5d73", "#4dd0ff", "#c9a2ff", "#ffcf5d", "#7dffb0", "#ff9d5d"];
export function videoColor(index) {
  return VIDEO_COLORS[index % VIDEO_COLORS.length];
}

const FRUSTUM_DEPTH = 1.2;
const DEFAULT_FOV = 50;
const STAGE_TARGET = new THREE.Vector3(0, 0.5, 0);

const _m4 = new THREE.Matrix4();
const _pos = new THREE.Vector3();
const _quat = new THREE.Quaternion();
const _scale = new THREE.Vector3();

export class Scene3D {
  constructor() {
    this.videoIndex = {}; // id -> index (for stable colors)
    this.rigs = {}; // id -> { frustum, marker, pick, path }
    this._tween = null;
    this._raf = null;
  }

  init(canvas) {
    this.canvas = canvas;
    const parent = canvas.parentElement;
    const w = parent.clientWidth || 1;
    const h = parent.clientHeight || 1;

    this.renderer = new THREE.WebGLRenderer({ canvas, antialias: true, alpha: false });
    this.renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2));
    this.renderer.setSize(w, h, false);
    this.renderer.setClearColor(0x0a0c11, 1);

    this.scene = new THREE.Scene();
    this.camera = new THREE.PerspectiveCamera(DEFAULT_FOV, w / h, 0.05, 3000);
    this.camera.position.set(12, 9, 15);
    this.camera.lookAt(STAGE_TARGET);

    this.controls = new OrbitControls(this.camera, this.renderer.domElement);
    this.controls.enableDamping = true;
    this.controls.dampingFactor = 0.08;
    this.controls.target.copy(STAGE_TARGET);
    this.controls.update();

    // Lights (points ignore lighting, but frusta/markers use MeshBasic so this is mostly cosmetic).
    this.scene.add(new THREE.AmbientLight(0xffffff, 0.9));
    const grid = new THREE.GridHelper(40, 40, 0x22384a, 0x161c26);
    grid.position.y = 0;
    this.scene.add(grid);
    const axes = new THREE.AxesHelper(1.5);
    this.scene.add(axes);

    this._buildRigs();
    this._loadPointCloud();
    this._wirePicking();
    this._wireResize(parent);
  }

  _buildRigs() {
    state.videos.forEach((v, i) => {
      this.videoIndex[v.id] = i;
      const color = new THREE.Color(videoColor(i));
      const group = new THREE.Group();

      // Frustum wireframe (built in Three.js camera-local space: looks down -z, +y up).
      const poses = state.poses[v.id] || [];
      const intr = poses[0]?.intrinsics;
      const W = v.width;
      const H = v.height;
      const hx = intr ? (FRUSTUM_DEPTH * W) / (2 * intr.fx) : FRUSTUM_DEPTH * 0.6;
      const hy = intr ? (FRUSTUM_DEPTH * H) / (2 * intr.fy) : FRUSTUM_DEPTH * 0.34;
      const d = FRUSTUM_DEPTH;
      const c = [
        [-hx, -hy, -d],
        [hx, -hy, -d],
        [hx, hy, -d],
        [-hx, hy, -d],
      ];
      const apex = [0, 0, 0];
      const segs = [
        apex, c[0], apex, c[1], apex, c[2], apex, c[3],
        c[0], c[1], c[1], c[2], c[2], c[3], c[3], c[0],
      ];
      const fg = new THREE.BufferGeometry();
      fg.setAttribute("position", new THREE.Float32BufferAttribute(segs.flat(), 3));
      const frustum = new THREE.LineSegments(
        fg,
        new THREE.LineBasicMaterial({ color })
      );
      frustum.matrixAutoUpdate = true;
      group.add(frustum);

      // Camera-center marker (visible) + an invisible larger sphere for click picking.
      const marker = new THREE.Mesh(
        new THREE.SphereGeometry(0.12, 12, 12),
        new THREE.MeshBasicMaterial({ color })
      );
      group.add(marker);
      this.scene.add(group);

      const pick = new THREE.Mesh(
        new THREE.SphereGeometry(0.55, 8, 8),
        new THREE.MeshBasicMaterial({ visible: false })
      );
      pick.userData.videoId = v.id;
      group.add(pick);

      // Camera path line (one vertex per stored pose center).
      let path = null;
      if (poses.length > 1) {
        const pts = [];
        for (const p of poses) {
          const { position } = colmapToThreejs(p.q, p.t);
          pts.push(position[0], position[1], position[2]);
        }
        const pg = new THREE.BufferGeometry();
        pg.setAttribute("position", new THREE.Float32BufferAttribute(pts, 3));
        path = new THREE.Line(
          pg,
          new THREE.LineBasicMaterial({ color, transparent: true, opacity: 0.5 })
        );
        this.scene.add(path);
      }

      this.rigs[v.id] = { group, frustum, marker, pick, path };
    });
  }

  _loadPointCloud() {
    const loader = new PLYLoader();
    loader.load(
      state.apiBase + "/api/pointcloud",
      (geometry) => {
        geometry.computeBoundingBox();
        const hasColor = !!geometry.getAttribute("color");
        const material = new THREE.PointsMaterial({
          size: 0.05,
          sizeAttenuation: true,
          vertexColors: hasColor,
          color: hasColor ? 0xffffff : 0x88ccff,
        });
        this.points = new THREE.Points(geometry, material);
        this.scene.add(this.points);
        emit("pointcloud-loaded", geometry.getAttribute("position")?.count ?? 0);
      },
      undefined,
      (err) => {
        console.warn("[scene3d] point cloud load failed", err);
      }
    );
  }

  // Live pose (COLMAP convention) for a video at the current master time, or null.
  _livePose(videoId) {
    const poses = state.poses[videoId];
    if (!poses || poses.length === 0) return null;
    const meta = state.videos.find((v) => v.id === videoId);
    const tv = tVideoFromGlobal(state.tGlobal, meta.offset_ms || 0, meta.drift_ppm || 0);
    return poseAt(poses, tv);
  }

  /** Snap the viewer camera to a video's pose and follow it while playing. */
  snapTo(videoId) {
    const pose = this._livePose(videoId);
    if (!pose) return;
    state.followCameraId = videoId;
    this.controls.enabled = false;
    const targetFov = (2 * Math.atan(state.videos.find((v) => v.id === videoId).height /
      (2 * pose.intrinsics.fy)) * 180) / Math.PI;
    this._tween = {
      start: performance.now(),
      dur: 600,
      fromPos: this.camera.position.clone(),
      fromQuat: this.camera.quaternion.clone(),
      fromFov: this.camera.fov,
      toFov: targetFov,
    };
    emit("follow", videoId);
  }

  /** Detach to free roam (OrbitControls). */
  freeRoam() {
    if (state.followCameraId == null) return;
    state.followCameraId = null;
    this._tween = null;
    this.camera.fov = DEFAULT_FOV;
    this.camera.updateProjectionMatrix();
    // Re-anchor OrbitControls to the stage, orbiting from the current position.
    this.controls.target.copy(STAGE_TARGET);
    this.controls.enabled = true;
    this.controls.update();
    emit("follow", null);
  }

  _applyPoseToCamera(pose, targetFov) {
    const { matrixWorld } = colmapToThreejs(pose.q, pose.t);
    _m4.fromArray(matrixWorld);
    _m4.decompose(_pos, _quat, _scale);
    const t = this._tween;
    if (t) {
      const k = Math.min(1, (performance.now() - t.start) / t.dur);
      const e = k * k * (3 - 2 * k); // smoothstep
      this.camera.position.lerpVectors(t.fromPos, _pos, e);
      this.camera.quaternion.copy(t.fromQuat).slerp(_quat, e);
      this.camera.fov = t.fromFov + (t.toFov - t.fromFov) * e;
      this.camera.updateProjectionMatrix();
      if (k >= 1) this._tween = null;
    } else {
      this.camera.position.copy(_pos);
      this.camera.quaternion.copy(_quat);
      if (Math.abs(this.camera.fov - targetFov) > 1e-3) {
        this.camera.fov = targetFov;
        this.camera.updateProjectionMatrix();
      }
    }
  }

  /** Per-frame update: move frusta to current poses, drive follow-cam, render. */
  update() {
    for (const v of state.videos) {
      const rig = this.rigs[v.id];
      if (!rig) continue;
      const pose = this._livePose(v.id);
      const on = pose && state.enabled[v.id] !== false;
      rig.group.visible = !!on;
      if (rig.path) rig.path.visible = state.enabled[v.id] !== false;
      if (!on) continue;
      const { matrixWorld } = colmapToThreejs(pose.q, pose.t);
      _m4.fromArray(matrixWorld);
      _m4.decompose(rig.group.position, rig.group.quaternion, rig.group.scale);
    }

    if (state.followCameraId != null) {
      const pose = this._livePose(state.followCameraId);
      if (pose) {
        const meta = state.videos.find((v) => v.id === state.followCameraId);
        const targetFov =
          (2 * Math.atan(meta.height / (2 * pose.intrinsics.fy)) * 180) / Math.PI;
        this._applyPoseToCamera(pose, targetFov);
      }
    } else {
      this.controls.update();
    }

    this.renderer.render(this.scene, this.camera);
  }

  _wirePicking() {
    const el = this.renderer.domElement;
    const ray = new THREE.Raycaster();
    const ndc = new THREE.Vector2();
    let downX = 0;
    let downY = 0;
    el.addEventListener("pointerdown", (e) => {
      downX = e.clientX;
      downY = e.clientY;
    });
    el.addEventListener("pointerup", (e) => {
      if (Math.hypot(e.clientX - downX, e.clientY - downY) > 5) return; // was a drag, not a click
      const rect = el.getBoundingClientRect();
      ndc.x = ((e.clientX - rect.left) / rect.width) * 2 - 1;
      ndc.y = -((e.clientY - rect.top) / rect.height) * 2 + 1;
      ray.setFromCamera(ndc, this.camera);
      const picks = Object.values(this.rigs)
        .map((r) => r.pick)
        .filter((m) => m.parent && m.parent.visible);
      const hits = ray.intersectObjects(picks, false);
      if (hits.length) this.snapTo(hits[0].object.userData.videoId);
    });
  }

  _wireResize(parent) {
    const ro = new ResizeObserver(() => {
      const w = parent.clientWidth || 1;
      const h = parent.clientHeight || 1;
      this.renderer.setSize(w, h, false);
      // Aspect always follows the 3D canvas (never the video) so nothing stretches; snap-to-
      // camera only overrides the vertical fov + pose, matching the video's vertical framing.
      this.camera.aspect = w / h;
      this.camera.updateProjectionMatrix();
    });
    ro.observe(parent);
  }
}

export const scene3d = new Scene3D();