"""COLMAP orchestration, model parsing, scene normalization, pose export (spec M2, lane B). Pipeline (spec M2): - Sample frames (``frames.sample_frames``), then drive the COLMAP CLI via subprocess: ``feature_extractor`` (OPENCV model, single camera per folder) -> ``exhaustive_matcher`` (small frame counts; exhaustive covers within- and cross-video pairs) -> ``mapper`` -> ``image_undistorter`` + ``model_converter`` to a PINHOLE TXT model. Parse ``images.txt`` / ``cameras.txt`` / ``points3D.txt`` by hand (no pycolmap). - **Normalize** the scene: centroid -> origin, camera bounding sphere radius -> 10, world-up -> average camera up. The same similarity transform is applied to poses and points. - **Interpolate** poses for unregistered frames (slerp rotation, lerp translation, mark ``registered=False``); never extrapolate past the first/last registered frame of a video. - Export ``config.POINTS_PLY`` (binary little-endian, matching ``synthetic.write_ply``) and fill ``camera_poses`` via ``db.set_poses``. - **Failure handling** (spec M2): COLMAP optional at runtime (``shutil.which("colmap")``); if it registers < 60% of sampled frames or < 2 videos into one model, print a clear diagnostic and leave existing poses untouched — never corrupt the DB. The app degrades to "synced videos, no 3D". """ from __future__ import annotations import logging import functools import struct import subprocess from pathlib import Path import numpy as np from festival4d import config, db, frames from festival4d.geometry import mat_to_quat, quat_to_mat, slerp_pose log = logging.getLogger("festival4d.sfm") # Minimum share of sampled frames COLMAP must register, and minimum videos in one model, # for the reconstruction to be trusted (spec M2 failure handling). MIN_REGISTERED_FRACTION = 0.60 MIN_VIDEOS_IN_MODEL = 2 NORMALIZED_SPHERE_RADIUS = 10.0 # How COLMAP camera models lay out PARAMS -> (fx, fy, cx, cy). Distortion terms are ignored: # the stored intrinsics are the pinhole part (exact after image_undistorter's PINHOLE output, # approximate if a distorted model is parsed directly). Covers the models COLMAP emits here. _INTRINSICS_FROM_PARAMS = { "SIMPLE_PINHOLE": lambda p: (p[0], p[0], p[1], p[2]), "PINHOLE": lambda p: (p[0], p[1], p[2], p[3]), "SIMPLE_RADIAL": lambda p: (p[0], p[0], p[1], p[2]), "RADIAL": lambda p: (p[0], p[0], p[1], p[2]), "SIMPLE_RADIAL_FISHEYE": lambda p: (p[0], p[0], p[1], p[2]), "RADIAL_FISHEYE": lambda p: (p[0], p[0], p[1], p[2]), "OPENCV": lambda p: (p[0], p[1], p[2], p[3]), "OPENCV_FISHEYE": lambda p: (p[0], p[1], p[2], p[3]), "FULL_OPENCV": lambda p: (p[0], p[1], p[2], p[3]), "FOV": lambda p: (p[0], p[1], p[2], p[3]), "THIN_PRISM_FISHEYE": lambda p: (p[0], p[1], p[2], p[3]), } def colmap_available() -> bool: """Whether the COLMAP binary is on PATH (spec M2 optional-at-runtime rule).""" import shutil return shutil.which("colmap") is not None # --------------------------------------------------------------------------- # COLMAP TXT-model parsers (hand-written; no pycolmap dependency) # --------------------------------------------------------------------------- def _iter_data_lines(path: Path): """Yield non-empty, non-comment lines from a COLMAP TXT file.""" with open(path, "r", encoding="utf-8") as f: for line in f: line = line.strip() if line and not line.startswith("#"): yield line def parse_images_txt(path: Path) -> list[dict]: """Parse a COLMAP ``images.txt`` into per-image pose records (spec M2, lane B). Format: two lines per image. The first is ``IMAGE_ID QW QX QY QZ TX TY TZ CAMERA_ID NAME``; the second is the 2D keypoint list (ignored here). Returns one dict per image with the world->camera quaternion ``[w,x,y,z]``, translation, camera id, and image name. """ records: list[dict] = [] take_pose_line = True # image data alternates: pose line, then points2D line for line in _iter_data_lines(path): if take_pose_line: parts = line.split() if len(parts) < 10: raise ValueError(f"malformed images.txt pose line: {line!r}") records.append({ "image_id": int(parts[0]), "qw": float(parts[1]), "qx": float(parts[2]), "qy": float(parts[3]), "qz": float(parts[4]), "tx": float(parts[5]), "ty": float(parts[6]), "tz": float(parts[7]), "camera_id": int(parts[8]), "name": " ".join(parts[9:]), # NAME may itself contain spaces }) take_pose_line = not take_pose_line return records def parse_cameras_txt(path: Path) -> dict[int, dict]: """Parse a COLMAP ``cameras.txt`` into ``camera_id -> intrinsics`` (spec M2, lane B). Format per line: ``CAMERA_ID MODEL WIDTH HEIGHT PARAMS[]``. Returns, per camera id, a dict with ``model``, ``width``, ``height``, pinhole ``fx, fy, cx, cy`` and the raw ``params`` list. """ cameras: dict[int, dict] = {} for line in _iter_data_lines(path): parts = line.split() if len(parts) < 4: raise ValueError(f"malformed cameras.txt line: {line!r}") cam_id = int(parts[0]) model = parts[1] width, height = int(parts[2]), int(parts[3]) params = [float(x) for x in parts[4:]] if model not in _INTRINSICS_FROM_PARAMS: raise ValueError(f"unsupported COLMAP camera model {model!r}") try: fx, fy, cx, cy = _INTRINSICS_FROM_PARAMS[model](params) except IndexError: raise ValueError( f"cameras.txt: model {model} has too few PARAMS: {line!r}" ) from None cameras[cam_id] = { "model": model, "width": width, "height": height, "fx": float(fx), "fy": float(fy), "cx": float(cx), "cy": float(cy), "params": params, } return cameras def parse_points3d_txt(path: Path) -> tuple[np.ndarray, np.ndarray]: """Parse a COLMAP ``points3D.txt`` into ``(points Nx3 float32, colors Nx3 uint8)``. Format per line: ``POINT3D_ID X Y Z R G B ERROR TRACK[]`` (track ignored). """ xyz: list[tuple[float, float, float]] = [] rgb: list[tuple[int, int, int]] = [] for line in _iter_data_lines(path): parts = line.split() if len(parts) < 7: raise ValueError(f"malformed points3D.txt line: {line!r}") xyz.append((float(parts[1]), float(parts[2]), float(parts[3]))) rgb.append((int(parts[4]), int(parts[5]), int(parts[6]))) if not xyz: return (np.zeros((0, 3), dtype=np.float32), np.zeros((0, 3), dtype=np.uint8)) return (np.asarray(xyz, dtype=np.float32), np.asarray(rgb, dtype=np.uint8)) # --------------------------------------------------------------------------- # Scene normalization # --------------------------------------------------------------------------- def _rotation_aligning(a: np.ndarray, b: np.ndarray) -> np.ndarray: """Shortest-arc rotation matrix that maps unit vector ``a`` onto unit vector ``b``.""" a = a / np.linalg.norm(a) b = b / np.linalg.norm(b) v = np.cross(a, b) c = float(np.dot(a, b)) s = float(np.linalg.norm(v)) if s < 1e-12: if c > 0: # already aligned return np.eye(3) # antiparallel: rotate 180 deg about any axis perpendicular to a perp = np.array([1.0, 0.0, 0.0]) if abs(a[0]) > 0.9: perp = np.array([0.0, 1.0, 0.0]) axis = np.cross(a, perp) axis /= np.linalg.norm(axis) return 2.0 * np.outer(axis, axis) - np.eye(3) vx = np.array([[0.0, -v[2], v[1]], [v[2], 0.0, -v[0]], [-v[1], v[0], 0.0]]) return np.eye(3) + vx + vx @ vx * ((1.0 - c) / (s * s)) def _camera_center(pose: dict) -> np.ndarray: """World-space camera center ``C = -R^T t`` for a world->camera pose dict.""" R = quat_to_mat([pose["qw"], pose["qx"], pose["qy"], pose["qz"]]) t = np.array([pose["tx"], pose["ty"], pose["tz"]], dtype=np.float64) return -R.T @ t def _camera_up(pose: dict) -> np.ndarray: """World-space camera up vector. COLMAP camera +y is *down*, so up = R^T @ (0,-1,0).""" R = quat_to_mat([pose["qw"], pose["qx"], pose["qy"], pose["qz"]]) return R.T @ np.array([0.0, -1.0, 0.0]) def normalize_scene(points, poses): """Similarity transform: centroid->origin, camera sphere radius->10, up->+Y (spec M2). ``points`` is an ``(N,3)`` array; ``poses`` a list of world->camera pose dicts (keys ``qw,qx,qy,qz,tx,ty,tz`` plus any extras, which are preserved). The single similarity transform ``X' = s * Rn @ (X - c)`` (translate to the point centroid, rotate the average camera-up onto +Y, scale so the camera bounding sphere has radius 10) is applied to both the points and the poses. Returns ``(points_new float32 (N,3), poses_new list[dict])``. """ pts = np.asarray(points, dtype=np.float64).reshape(-1, 3) poses = list(poses) # 1. Translate to the point-cloud centroid. centroid = pts.mean(axis=0) if len(pts) else np.zeros(3) # 2. Rotate so the average camera-up aligns with +Y (phones held roughly upright). if poses: ups = np.array([_camera_up(p) for p in poses]) up_avg = ups.mean(axis=0) if np.linalg.norm(up_avg) > 1e-9: Rn = _rotation_aligning(up_avg, np.array([0.0, 1.0, 0.0])) else: Rn = np.eye(3) else: Rn = np.eye(3) # 3. Scale so the farthest camera center from the centroid sits at radius 10. if poses: centers = np.array([_camera_center(p) for p in poses]) max_dist = float(np.max(np.linalg.norm(centers - centroid, axis=1))) scale = NORMALIZED_SPHERE_RADIUS / max_dist if max_dist > 1e-9 else 1.0 else: scale = 1.0 # Apply X' = s * Rn @ (X - c) to points. points_new = (scale * (pts - centroid) @ Rn.T).astype(np.float32) # Apply the matching transform to each world->camera pose: # R' = R @ Rn^T , t' = s * (t + R @ c) (derived so projections are unchanged). poses_new: list[dict] = [] for p in poses: R = quat_to_mat([p["qw"], p["qx"], p["qy"], p["qz"]]) t = np.array([p["tx"], p["ty"], p["tz"]], dtype=np.float64) R_new = R @ Rn.T t_new = scale * (t + R @ centroid) q_new = mat_to_quat(R_new) out = dict(p) out["qw"], out["qx"], out["qy"], out["qz"] = ( float(q_new[0]), float(q_new[1]), float(q_new[2]), float(q_new[3])) out["tx"], out["ty"], out["tz"] = float(t_new[0]), float(t_new[1]), float(t_new[2]) poses_new.append(out) return points_new, poses_new # --------------------------------------------------------------------------- # Pose interpolation # --------------------------------------------------------------------------- def interpolate_poses(registered: list[dict], sampled: list[dict]) -> list[dict]: """Fill unregistered sampled frames of one video between registered ones (spec M2). ``registered`` are solved poses (with ``registered=True``); ``sampled`` are the frames that were fed to COLMAP as ``{frame_idx, t_video_s, ...}``. For each sampled frame that lies strictly between the first and last registered frame but wasn't solved, slerp the rotation and lerp the translation of the two bracketing registered poses (intrinsics inherited from the earlier neighbor), marking it ``registered=False``. Frames outside the registered span are dropped (no extrapolation). Returns all poses sorted by ``t_video_s``. """ reg = sorted(registered, key=lambda p: p["t_video_s"]) if not reg: return [] reg_frames = {int(p["frame_idx"]) for p in reg} t_first, t_last = reg[0]["t_video_s"], reg[-1]["t_video_s"] out: list[dict] = list(reg) for sf in sampled: fidx = int(sf["frame_idx"]) tv = float(sf["t_video_s"]) if fidx in reg_frames: continue if tv <= t_first or tv >= t_last: continue # no extrapolation beyond the registered span # bracketing registered neighbors lo = max((p for p in reg if p["t_video_s"] <= tv), key=lambda p: p["t_video_s"]) hi = min((p for p in reg if p["t_video_s"] >= tv), key=lambda p: p["t_video_s"]) span = hi["t_video_s"] - lo["t_video_s"] alpha = 0.0 if span <= 1e-12 else (tv - lo["t_video_s"]) / span q, t = slerp_pose( [lo["qw"], lo["qx"], lo["qy"], lo["qz"]], [lo["tx"], lo["ty"], lo["tz"]], [hi["qw"], hi["qx"], hi["qy"], hi["qz"]], [hi["tx"], hi["ty"], hi["tz"]], alpha, ) out.append({ "frame_idx": fidx, "t_video_s": tv, "qw": float(q[0]), "qx": float(q[1]), "qy": float(q[2]), "qz": float(q[3]), "tx": float(t[0]), "ty": float(t[1]), "tz": float(t[2]), "fx": lo["fx"], "fy": lo["fy"], "cx": lo["cx"], "cy": lo["cy"], "registered": False, }) out.sort(key=lambda p: p["t_video_s"]) return out # --------------------------------------------------------------------------- # COLMAP CLI orchestration # --------------------------------------------------------------------------- @functools.lru_cache(maxsize=None) def _colmap_help(command: str) -> str: """Cached ``colmap --help`` text, for build-aware option detection. COLMAP prints its option list to stderr, so both streams are captured. """ try: proc = subprocess.run( ["colmap", command, "--help"], capture_output=True, text=True ) return (proc.stdout or "") + "\n" + (proc.stderr or "") except Exception: return "" def _cpu_flag(command: str) -> list[str]: """Force CPU SIFT for ``command`` if this COLMAP build exposes a ``use_gpu`` option. The option prefix differs by version: COLMAP 3.x uses ``SiftExtraction`` / ``SiftMatching``, 4.x uses ``FeatureExtraction`` / ``FeatureMatching``. We read the option name from ``--help`` so the pipeline runs headless on either (and on CPU-only builds that omit the option entirely, we pass nothing). Returns e.g. ``["--FeatureExtraction.use_gpu", "0"]`` or ``[]``. """ for line in _colmap_help(command).splitlines(): line = line.strip() if line.startswith("--") and ".use_gpu" in line: return [line.split()[0], "0"] return [] def _run_colmap_step(args: list[str]) -> bool: """Run one ``colmap `` step; return True on success, False (logged) on failure.""" cmd = ["colmap", *args] log.info("colmap: %s", " ".join(args[:2])) try: proc = subprocess.run(cmd, capture_output=True, text=True) except FileNotFoundError: log.error("colmap binary vanished from PATH") return False if proc.returncode != 0: log.error("colmap %s failed (rc=%d): %s", args[0], proc.returncode, (proc.stderr or proc.stdout or "").strip()[-500:]) return False return True def _registered_image_count(model_dir: Path) -> int | None: """Number of registered images in a COLMAP model dir, or None if it has no image list. The mapper writes binary models by default; ``images.bin`` begins with a little-endian ``uint64`` giving the registered-image count, which is the exact metric we want (file size would instead track total keypoint observations and mis-rank multi-component runs). Falls back to counting the TXT model's two-lines-per-image list. """ images_txt = model_dir / "images.txt" images_bin = model_dir / "images.bin" if images_txt.exists(): return sum(1 for _ in _iter_data_lines(images_txt)) // 2 if images_bin.exists(): with open(images_bin, "rb") as f: header = f.read(8) return struct.unpack(" Path | None: """Return the reconstruction subdir (``0``, ``1``, ...) with the most registered images.""" candidates = [d for d in sparse_dir.iterdir() if d.is_dir()] if sparse_dir.exists() else [] best, best_n = None, -1 for d in candidates: n = _registered_image_count(d) if n is None: continue if n > best_n: best, best_n = d, n return best def run_colmap(frames_dir: Path, workspace: Path) -> Path | None: """Drive the COLMAP CLI and return the exported TXT-model directory (spec M2, lane B). ``frames_dir`` holds one subfolder of JPEGs per video (single camera per folder). Runs feature extraction (CPU SIFT, OPENCV model), exhaustive matching, incremental mapping, then undistortion + TXT conversion. Returns the directory containing ``images.txt`` / ``cameras.txt`` / ``points3D.txt``, or ``None`` if COLMAP produced no usable model (caller degrades gracefully). """ workspace = Path(workspace) # Start each run from a clean workspace: a stale database.db makes feature_extractor # fail ("images already exist"), and a leftover sparse/ model would poison the pick. if workspace.exists(): import shutil shutil.rmtree(workspace) workspace.mkdir(parents=True, exist_ok=True) database = workspace / "database.db" sparse = workspace / "sparse" sparse.mkdir(exist_ok=True) ok = _run_colmap_step([ "feature_extractor", "--database_path", str(database), "--image_path", str(frames_dir), "--ImageReader.single_camera_per_folder", "1", "--ImageReader.camera_model", "OPENCV", *_cpu_flag("feature_extractor"), ]) if not ok: return None # Exhaustive matching covers within- and cross-video pairs; frame counts are small. if not _run_colmap_step([ "exhaustive_matcher", "--database_path", str(database), *_cpu_flag("exhaustive_matcher"), ]): return None if not _run_colmap_step([ "mapper", "--database_path", str(database), "--image_path", str(frames_dir), "--output_path", str(sparse), ]): return None model = _largest_model_dir(sparse) if model is None: log.error("colmap mapper produced no reconstruction") return None # Undistort -> PINHOLE model, then convert to TXT. Fall back to converting the raw # (possibly distorted) sparse model directly if undistortion fails. txt_dir = workspace / "model_txt" txt_dir.mkdir(exist_ok=True) dense = workspace / "dense" undistorted = _run_colmap_step([ "image_undistorter", "--image_path", str(frames_dir), "--input_path", str(model), "--output_path", str(dense), "--output_type", "COLMAP", ]) convert_input = (dense / "sparse") if undistorted and (dense / "sparse").exists() else model if not _run_colmap_step([ "model_converter", "--input_path", str(convert_input), "--output_path", str(txt_dir), "--output_type", "TXT", ]): return None if not (txt_dir / "images.txt").exists(): log.error("colmap model_converter produced no images.txt") return None return txt_dir # --------------------------------------------------------------------------- # Full reconstruction entrypoint # --------------------------------------------------------------------------- def _video_id_and_frame(name: str) -> tuple[int, int] | None: """Recover ``(video_id, frame_idx)`` from a ``{video_id}_{frame_idx}.jpg`` image name.""" stem = Path(name).stem if "_" not in stem: return None vid_s, frame_s = stem.rsplit("_", 1) try: return int(vid_s), int(frame_s) except ValueError: return None def _diagnostic(msg: str) -> None: """Emit a clear operator-facing diagnostic (spec M2 graceful degradation).""" log.warning("reconstruct: %s", msg) print(f"[reconstruct] {msg}") def run_reconstruct() -> dict: """Full reconstruction: sample frames, run COLMAP, normalize, interpolate, export. Entrypoint for ``python -m festival4d reconstruct``. Degrades gracefully when COLMAP is absent or the reconstruction is too weak (spec M2 failure handling): existing poses are left untouched and the DB is never corrupted. Returns a summary dict with a ``status``. """ db.init_engine() db.init_db() videos = db.get_videos() if not videos: _diagnostic("no videos in the project — run `synthetic` or `ingest` first.") return {"status": "no_videos"} if not colmap_available(): _diagnostic( "COLMAP not found on PATH — skipping 3D reconstruction. Install it " "(`brew install colmap` on macOS) to enable pose/point-cloud solving. " "Existing (synthetic/previous) poses are left untouched; the app still runs as " "synced videos without 3D." ) return {"status": "skipped_no_colmap", "videos": len(videos)} # 1. Sample sharp frames per video into one subfolder each (single camera per folder). frames_root = config.FRAMES_DIR per_video_sampled: dict[int, list[dict]] = {} for v in videos: video_path = config.RAW_DIR / v.filename if not video_path.exists(): _diagnostic(f"video file missing, skipping: {video_path}") continue out_dir = frames_root / str(v.id) paths = frames.sample_frames(video_path, v.id, out_dir) fps = float(v.fps) if v.fps and v.fps > 0 else 30.0 sampled = [] for p in paths: parsed = _video_id_and_frame(p.name) if parsed is None: continue _, fidx = parsed sampled.append({"frame_idx": fidx, "t_video_s": fidx / fps}) per_video_sampled[v.id] = sorted(sampled, key=lambda s: s["frame_idx"]) total_sampled = sum(len(s) for s in per_video_sampled.values()) if total_sampled == 0: _diagnostic("no frames could be sampled from any video — nothing to reconstruct.") return {"status": "no_frames"} # 2. Run COLMAP. model_dir = run_colmap(frames_root, config.COLMAP_DIR) if model_dir is None: _diagnostic( "COLMAP produced no usable reconstruction. Likely causes: too-dark footage, " "motion blur, or insufficient view overlap between cameras. Existing poses left " "untouched — the app degrades to synced videos, no 3D." ) return {"status": "failed_no_model", "sampled": total_sampled} # 3. Parse the model. images = parse_images_txt(model_dir / "images.txt") cameras = parse_cameras_txt(model_dir / "cameras.txt") points, colors = parse_points3d_txt(model_dir / "points3D.txt") # 4. Attach each registered image to its video + intrinsics. registered_by_video: dict[int, list[dict]] = {} for img in images: parsed = _video_id_and_frame(img["name"]) if parsed is None: continue video_id, frame_idx = parsed cam = cameras.get(img["camera_id"]) if cam is None: continue fps = next((float(v.fps) for v in videos if v.id == video_id), 30.0) or 30.0 registered_by_video.setdefault(video_id, []).append({ "frame_idx": frame_idx, "t_video_s": frame_idx / fps, "qw": img["qw"], "qx": img["qx"], "qy": img["qy"], "qz": img["qz"], "tx": img["tx"], "ty": img["ty"], "tz": img["tz"], "fx": cam["fx"], "fy": cam["fy"], "cx": cam["cx"], "cy": cam["cy"], "registered": True, "video_id": video_id, }) registered_count = sum(len(v) for v in registered_by_video.values()) videos_in_model = len(registered_by_video) frac = registered_count / total_sampled if total_sampled else 0.0 if frac < MIN_REGISTERED_FRACTION or videos_in_model < MIN_VIDEOS_IN_MODEL: _diagnostic( f"weak reconstruction: {registered_count}/{total_sampled} frames " f"({frac:.0%}) registered across {videos_in_model} video(s); need " f">= {MIN_REGISTERED_FRACTION:.0%} of frames and >= {MIN_VIDEOS_IN_MODEL} " "videos. Likely causes: too-dark footage, motion blur, or too little view " "overlap. Existing poses left untouched — degrading to synced videos, no 3D." ) return { "status": "failed_weak", "sampled": total_sampled, "registered": registered_count, "videos_in_model": videos_in_model, } # 5. Normalize the whole scene (points + all registered poses) with one transform. all_registered = [p for v in registered_by_video.values() for p in v] norm_points, norm_registered = normalize_scene(points, all_registered) norm_by_video: dict[int, list[dict]] = {} for p in norm_registered: norm_by_video.setdefault(p["video_id"], []).append(p) # 6. Interpolate unregistered sampled frames, then write poses per video (atomic replace). interpolated_total = 0 for video_id, reg in norm_by_video.items(): sampled = per_video_sampled.get(video_id, []) final = interpolate_poses(reg, sampled) interpolated_total += sum(1 for p in final if not p.get("registered", True)) db.set_poses(video_id, final) # 7. Export the normalized point cloud in the frozen PLY format. from festival4d.synthetic import write_ply write_ply(config.POINTS_PLY, norm_points, colors) summary = { "status": "ok", "videos_in_model": videos_in_model, "registered": registered_count, "interpolated": interpolated_total, "points": int(len(norm_points)), "points_ply": str(config.POINTS_PLY), } log.info("reconstruct: done — %s", summary) print(f"[reconstruct] reconstructed {videos_in_model} videos, " f"{registered_count} registered + {interpolated_total} interpolated poses, " f"{len(norm_points)} points -> {config.POINTS_PLY}") return summary