festifun/frontend/src/overlays.js

// 3D -> 2D anchor projection onto each video's overlay canvas — the "x-ray" HUD (spec M6).
//
// For each visible video we build a Three.js PerspectiveCamera at that video's CURRENT pose
// (from the FROZEN colmapToThreejs helper, matrixWorld path) with an fov derived from the
// stored intrinsics, project every anchor, drop anchors behind the camera (camera-space z >= 0),
// convert NDC -> native video pixels -> canvas pixels through the letterbox content rect, and
// draw a dot + halo + label. Anchors are drawn regardless of real-world occlusion — that IS the
// x-ray feature. We consume pose.js; we never reimplement the COLMAP->Three.js conversion.

import * as THREE from "three";
import { colmapToThreejs } from "./lib/pose.js";
import { poseAt } from "./lib/poseTrack.js";
import { contentRect } from "./videoGrid.js";
import { state } from "./state.js";

// Reused across all videos/frames (every synthetic video shares intrinsics + resolution, and
// we fully overwrite the matrices each call, so a single scratch camera is safe and cheap).
const _cam = new THREE.PerspectiveCamera();
_cam.matrixAutoUpdate = false;
const _m4 = new THREE.Matrix4();
const _v = new THREE.Vector3();

function configureProjector(pose, W, H) {
  // Centered-principal-point pinhole -> PerspectiveCamera (spec M5/M6 prototype assumption;
  // the synthetic fixture has cx=W/2, cy=H/2 exactly).
  const fy = pose.intrinsics.fy;
  _cam.fov = (2 * Math.atan(H / (2 * fy)) * 180) / Math.PI;
  _cam.aspect = W / H;
  _cam.near = 0.01;
  _cam.far = 2000;
  _cam.updateProjectionMatrix();

  const { matrixWorld } = colmapToThreejs(pose.q, pose.t);
  _m4.fromArray(matrixWorld);
  _cam.matrixWorld.copy(_m4);
  _cam.matrixWorldInverse.copy(_m4).invert();
}

/** Project a world point to native video pixels, or null if behind the camera. */
function projectToVideoPx(x, y, z, W, H) {
  _v.set(x, y, z);
  // Behind-camera test in camera space: Three.js cameras look down -z, so a visible point has
  // camera-space z < 0. Do this BEFORE .project() (which mutates _v into NDC).
  _v.applyMatrix4(_cam.matrixWorldInverse);
  if (_v.z >= 0) return null;
  _v.applyMatrix4(_cam.projectionMatrix); // perspective divide happens inside .project(); do it manually
  // _v is now clip space already divided (applyMatrix4 on a Vector3 divides by w). NDC in [-1,1].
  const u = (_v.x * 0.5 + 0.5) * W;
  const vpx = (1 - (_v.y * 0.5 + 0.5)) * H; // flip Y: NDC +y is up, pixels grow downward
  return { u, v: vpx };
}

/**
 * Project a world point to native video pixels through the FROZEN pose helper for a given
 * COLMAP pose (as returned by poseAt) and video resolution. Returns {u, v} or null (behind).
 * Single source of truth for overlay projection — used by drawOverlay and by tests.
 */
export function projectWorldToVideoPx(pose, x, y, z, W, H) {
  configureProjector(pose, W, H);
  return projectToVideoPx(x, y, z, W, H);
}

function drawMarker(ctx, x, y, anchor) {
  const color = anchor.color || "#59d499";
  // halo
  ctx.beginPath();
  ctx.arc(x, y, 5.5, 0, Math.PI * 2);
  ctx.fillStyle = "rgba(255,255,255,0.85)";
  ctx.fill();
  // dot
  ctx.beginPath();
  ctx.arc(x, y, 4, 0, Math.PI * 2);
  ctx.fillStyle = color;
  ctx.fill();
  // label
  const text = anchor.label || "";
  if (text) {
    ctx.font = "600 11px -apple-system, system-ui, sans-serif";
    ctx.textBaseline = "middle";
    const tx = x + 9;
    const ty = y;
    const w = ctx.measureText(text).width;
    ctx.fillStyle = "rgba(6,8,14,0.72)";
    ctx.fillRect(tx - 3, ty - 8, w + 6, 16);
    ctx.fillStyle = "#e6e8ee";
    ctx.fillText(text, tx, ty + 0.5);
  }
}

/** Redraw one cell's overlay for the frame currently displayed by its <video>. */
export function drawOverlay(cell) {
  const { video, canvas } = cell;
  const ctx = canvas.getContext("2d");
  const dpr = window.devicePixelRatio || 1;
  const elW = video.clientWidth;
  const elH = video.clientHeight;
  if (elW === 0 || elH === 0) return;

  // Resolution-match the canvas backing store to the element (dpr-aware).
  const bw = Math.round(elW * dpr);
  const bh = Math.round(elH * dpr);
  if (canvas.width !== bw || canvas.height !== bh) {
    canvas.width = bw;
    canvas.height = bh;
    canvas.style.width = elW + "px";
    canvas.style.height = elH + "px";
  }
  ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
  ctx.clearRect(0, 0, elW, elH);

  if (!state.hasPoses || !state.inRange[cell.id]) return;
  if (video.videoWidth === 0) return;
  const poses = state.poses[cell.id];
  if (!poses || poses.length === 0 || state.anchors.length === 0) return;

  // Use the frame the <video> is ACTUALLY showing (currentTime), not the master target, so the
  // overlay tracks the displayed picture even mid-correction.
  const pose = poseAt(poses, video.currentTime);
  if (!pose) return;

  const W = video.videoWidth;
  const H = video.videoHeight;
  configureProjector(pose, W, H);
  const rect = contentRect(video);

  for (const a of state.anchors) {
    const p = projectToVideoPx(a.x, a.y, a.z, W, H);
    if (!p) continue;
    const cx = rect.ox + (p.u / W) * rect.cw;
    const cy = rect.oy + (p.v / H) * rect.ch;
    // Skip anchors that fall well outside the visible content rect.
    const m = 24;
    if (cx < rect.ox - m || cx > rect.ox + rect.cw + m) continue;
    if (cy < rect.oy - m || cy > rect.oy + rect.ch + m) continue;
    drawMarker(ctx, cx, cy, a);
  }
}