bb542960c2
Implements spec M2 (frame sampling, COLMAP orchestration, TXT-model parsing, scene normalization, pose interpolation, PLY + DB export) and the M8 geometry functions. geometry.py (M8): slerp_pose (shortest-arc, double-cover), ray_from_pixel (COLMAP +y-down back-projection), triangulate_rays (closest-point + parallel guard), nearest_point_on_ray (in-front radius cylinder). Frozen colmap_to_threejs untouched. frames.py (M2): variance-of-Laplacian sharpness + windowed sharpest-frame sampling. sfm.py (M2): hand-written COLMAP images/cameras/points3D parsers; normalize_scene (centroid->0, camera sphere r->10, up->+Y) as one similarity transform over points+poses; interpolate_poses (slerp+lerp, no extrapolation); build-aware COLMAP CLI orchestration (3.x/4.x option detection, CPU SIFT, single-camera-per-folder, undistort->TXT, largest component by images.bin header count); run_reconstruct with full graceful degradation (COLMAP absent / <60% / <2 videos / no frames / missing files -> poses left untouched). Tests: 61 pass (24 foundation + 37 lane-B) with independent oracles (scipy Slerp, projection-inverse, closed-form geometry) and every degradation/DB-safety branch covered. Validated end-to-end against real COLMAP 4.1.0 (6 components, largest selected, weak -> graceful degradation, exit 0). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
609 lines
26 KiB
Python
609 lines
26 KiB
Python
"""COLMAP orchestration, model parsing, scene normalization, pose export (spec M2, lane B).
|
|
|
|
Pipeline (spec M2):
|
|
- Sample frames (``frames.sample_frames``), then drive the COLMAP CLI via subprocess:
|
|
``feature_extractor`` (OPENCV model, single camera per folder) -> ``exhaustive_matcher``
|
|
(small frame counts; exhaustive covers within- and cross-video pairs) -> ``mapper`` ->
|
|
``image_undistorter`` + ``model_converter`` to a PINHOLE TXT model. Parse ``images.txt`` /
|
|
``cameras.txt`` / ``points3D.txt`` by hand (no pycolmap).
|
|
- **Normalize** the scene: centroid -> origin, camera bounding sphere radius -> 10, world-up
|
|
-> average camera up. The same similarity transform is applied to poses and points.
|
|
- **Interpolate** poses for unregistered frames (slerp rotation, lerp translation, mark
|
|
``registered=False``); never extrapolate past the first/last registered frame of a video.
|
|
- Export ``config.POINTS_PLY`` (binary little-endian, matching ``synthetic.write_ply``) and
|
|
fill ``camera_poses`` via ``db.set_poses``.
|
|
- **Failure handling** (spec M2): COLMAP optional at runtime (``shutil.which("colmap")``); if
|
|
it registers < 60% of sampled frames or < 2 videos into one model, print a clear diagnostic
|
|
and leave existing poses untouched — never corrupt the DB. The app degrades to "synced
|
|
videos, no 3D".
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import functools
|
|
import struct
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
|
|
from festival4d import config, db, frames
|
|
from festival4d.geometry import mat_to_quat, quat_to_mat, slerp_pose
|
|
|
|
log = logging.getLogger("festival4d.sfm")
|
|
|
|
# Minimum share of sampled frames COLMAP must register, and minimum videos in one model,
|
|
# for the reconstruction to be trusted (spec M2 failure handling).
|
|
MIN_REGISTERED_FRACTION = 0.60
|
|
MIN_VIDEOS_IN_MODEL = 2
|
|
NORMALIZED_SPHERE_RADIUS = 10.0
|
|
|
|
# How COLMAP camera models lay out PARAMS -> (fx, fy, cx, cy). Distortion terms are ignored:
|
|
# the stored intrinsics are the pinhole part (exact after image_undistorter's PINHOLE output,
|
|
# approximate if a distorted model is parsed directly). Covers the models COLMAP emits here.
|
|
_INTRINSICS_FROM_PARAMS = {
|
|
"SIMPLE_PINHOLE": lambda p: (p[0], p[0], p[1], p[2]),
|
|
"PINHOLE": lambda p: (p[0], p[1], p[2], p[3]),
|
|
"SIMPLE_RADIAL": lambda p: (p[0], p[0], p[1], p[2]),
|
|
"RADIAL": lambda p: (p[0], p[0], p[1], p[2]),
|
|
"SIMPLE_RADIAL_FISHEYE": lambda p: (p[0], p[0], p[1], p[2]),
|
|
"RADIAL_FISHEYE": lambda p: (p[0], p[0], p[1], p[2]),
|
|
"OPENCV": lambda p: (p[0], p[1], p[2], p[3]),
|
|
"OPENCV_FISHEYE": lambda p: (p[0], p[1], p[2], p[3]),
|
|
"FULL_OPENCV": lambda p: (p[0], p[1], p[2], p[3]),
|
|
"FOV": lambda p: (p[0], p[1], p[2], p[3]),
|
|
"THIN_PRISM_FISHEYE": lambda p: (p[0], p[1], p[2], p[3]),
|
|
}
|
|
|
|
|
|
def colmap_available() -> bool:
|
|
"""Whether the COLMAP binary is on PATH (spec M2 optional-at-runtime rule)."""
|
|
import shutil
|
|
|
|
return shutil.which("colmap") is not None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# COLMAP TXT-model parsers (hand-written; no pycolmap dependency)
|
|
# ---------------------------------------------------------------------------
|
|
def _iter_data_lines(path: Path):
|
|
"""Yield non-empty, non-comment lines from a COLMAP TXT file."""
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line and not line.startswith("#"):
|
|
yield line
|
|
|
|
|
|
def parse_images_txt(path: Path) -> list[dict]:
|
|
"""Parse a COLMAP ``images.txt`` into per-image pose records (spec M2, lane B).
|
|
|
|
Format: two lines per image. The first is
|
|
``IMAGE_ID QW QX QY QZ TX TY TZ CAMERA_ID NAME``; the second is the 2D keypoint list
|
|
(ignored here). Returns one dict per image with the world->camera quaternion
|
|
``[w,x,y,z]``, translation, camera id, and image name.
|
|
"""
|
|
records: list[dict] = []
|
|
take_pose_line = True # image data alternates: pose line, then points2D line
|
|
for line in _iter_data_lines(path):
|
|
if take_pose_line:
|
|
parts = line.split()
|
|
if len(parts) < 10:
|
|
raise ValueError(f"malformed images.txt pose line: {line!r}")
|
|
records.append({
|
|
"image_id": int(parts[0]),
|
|
"qw": float(parts[1]), "qx": float(parts[2]),
|
|
"qy": float(parts[3]), "qz": float(parts[4]),
|
|
"tx": float(parts[5]), "ty": float(parts[6]), "tz": float(parts[7]),
|
|
"camera_id": int(parts[8]),
|
|
"name": " ".join(parts[9:]), # NAME may itself contain spaces
|
|
})
|
|
take_pose_line = not take_pose_line
|
|
return records
|
|
|
|
|
|
def parse_cameras_txt(path: Path) -> dict[int, dict]:
|
|
"""Parse a COLMAP ``cameras.txt`` into ``camera_id -> intrinsics`` (spec M2, lane B).
|
|
|
|
Format per line: ``CAMERA_ID MODEL WIDTH HEIGHT PARAMS[]``. Returns, per camera id, a
|
|
dict with ``model``, ``width``, ``height``, pinhole ``fx, fy, cx, cy`` and the raw
|
|
``params`` list.
|
|
"""
|
|
cameras: dict[int, dict] = {}
|
|
for line in _iter_data_lines(path):
|
|
parts = line.split()
|
|
if len(parts) < 4:
|
|
raise ValueError(f"malformed cameras.txt line: {line!r}")
|
|
cam_id = int(parts[0])
|
|
model = parts[1]
|
|
width, height = int(parts[2]), int(parts[3])
|
|
params = [float(x) for x in parts[4:]]
|
|
if model not in _INTRINSICS_FROM_PARAMS:
|
|
raise ValueError(f"unsupported COLMAP camera model {model!r}")
|
|
try:
|
|
fx, fy, cx, cy = _INTRINSICS_FROM_PARAMS[model](params)
|
|
except IndexError:
|
|
raise ValueError(
|
|
f"cameras.txt: model {model} has too few PARAMS: {line!r}"
|
|
) from None
|
|
cameras[cam_id] = {
|
|
"model": model, "width": width, "height": height,
|
|
"fx": float(fx), "fy": float(fy), "cx": float(cx), "cy": float(cy),
|
|
"params": params,
|
|
}
|
|
return cameras
|
|
|
|
|
|
def parse_points3d_txt(path: Path) -> tuple[np.ndarray, np.ndarray]:
|
|
"""Parse a COLMAP ``points3D.txt`` into ``(points Nx3 float32, colors Nx3 uint8)``.
|
|
|
|
Format per line: ``POINT3D_ID X Y Z R G B ERROR TRACK[]`` (track ignored).
|
|
"""
|
|
xyz: list[tuple[float, float, float]] = []
|
|
rgb: list[tuple[int, int, int]] = []
|
|
for line in _iter_data_lines(path):
|
|
parts = line.split()
|
|
if len(parts) < 7:
|
|
raise ValueError(f"malformed points3D.txt line: {line!r}")
|
|
xyz.append((float(parts[1]), float(parts[2]), float(parts[3])))
|
|
rgb.append((int(parts[4]), int(parts[5]), int(parts[6])))
|
|
if not xyz:
|
|
return (np.zeros((0, 3), dtype=np.float32), np.zeros((0, 3), dtype=np.uint8))
|
|
return (np.asarray(xyz, dtype=np.float32), np.asarray(rgb, dtype=np.uint8))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scene normalization
|
|
# ---------------------------------------------------------------------------
|
|
def _rotation_aligning(a: np.ndarray, b: np.ndarray) -> np.ndarray:
|
|
"""Shortest-arc rotation matrix that maps unit vector ``a`` onto unit vector ``b``."""
|
|
a = a / np.linalg.norm(a)
|
|
b = b / np.linalg.norm(b)
|
|
v = np.cross(a, b)
|
|
c = float(np.dot(a, b))
|
|
s = float(np.linalg.norm(v))
|
|
if s < 1e-12:
|
|
if c > 0: # already aligned
|
|
return np.eye(3)
|
|
# antiparallel: rotate 180 deg about any axis perpendicular to a
|
|
perp = np.array([1.0, 0.0, 0.0])
|
|
if abs(a[0]) > 0.9:
|
|
perp = np.array([0.0, 1.0, 0.0])
|
|
axis = np.cross(a, perp)
|
|
axis /= np.linalg.norm(axis)
|
|
return 2.0 * np.outer(axis, axis) - np.eye(3)
|
|
vx = np.array([[0.0, -v[2], v[1]], [v[2], 0.0, -v[0]], [-v[1], v[0], 0.0]])
|
|
return np.eye(3) + vx + vx @ vx * ((1.0 - c) / (s * s))
|
|
|
|
|
|
def _camera_center(pose: dict) -> np.ndarray:
|
|
"""World-space camera center ``C = -R^T t`` for a world->camera pose dict."""
|
|
R = quat_to_mat([pose["qw"], pose["qx"], pose["qy"], pose["qz"]])
|
|
t = np.array([pose["tx"], pose["ty"], pose["tz"]], dtype=np.float64)
|
|
return -R.T @ t
|
|
|
|
|
|
def _camera_up(pose: dict) -> np.ndarray:
|
|
"""World-space camera up vector. COLMAP camera +y is *down*, so up = R^T @ (0,-1,0)."""
|
|
R = quat_to_mat([pose["qw"], pose["qx"], pose["qy"], pose["qz"]])
|
|
return R.T @ np.array([0.0, -1.0, 0.0])
|
|
|
|
|
|
def normalize_scene(points, poses):
|
|
"""Similarity transform: centroid->origin, camera sphere radius->10, up->+Y (spec M2).
|
|
|
|
``points`` is an ``(N,3)`` array; ``poses`` a list of world->camera pose dicts (keys
|
|
``qw,qx,qy,qz,tx,ty,tz`` plus any extras, which are preserved). The single similarity
|
|
transform ``X' = s * Rn @ (X - c)`` (translate to the point centroid, rotate the average
|
|
camera-up onto +Y, scale so the camera bounding sphere has radius 10) is applied to both
|
|
the points and the poses.
|
|
|
|
Returns ``(points_new float32 (N,3), poses_new list[dict])``.
|
|
"""
|
|
pts = np.asarray(points, dtype=np.float64).reshape(-1, 3)
|
|
poses = list(poses)
|
|
|
|
# 1. Translate to the point-cloud centroid.
|
|
centroid = pts.mean(axis=0) if len(pts) else np.zeros(3)
|
|
|
|
# 2. Rotate so the average camera-up aligns with +Y (phones held roughly upright).
|
|
if poses:
|
|
ups = np.array([_camera_up(p) for p in poses])
|
|
up_avg = ups.mean(axis=0)
|
|
if np.linalg.norm(up_avg) > 1e-9:
|
|
Rn = _rotation_aligning(up_avg, np.array([0.0, 1.0, 0.0]))
|
|
else:
|
|
Rn = np.eye(3)
|
|
else:
|
|
Rn = np.eye(3)
|
|
|
|
# 3. Scale so the farthest camera center from the centroid sits at radius 10.
|
|
if poses:
|
|
centers = np.array([_camera_center(p) for p in poses])
|
|
max_dist = float(np.max(np.linalg.norm(centers - centroid, axis=1)))
|
|
scale = NORMALIZED_SPHERE_RADIUS / max_dist if max_dist > 1e-9 else 1.0
|
|
else:
|
|
scale = 1.0
|
|
|
|
# Apply X' = s * Rn @ (X - c) to points.
|
|
points_new = (scale * (pts - centroid) @ Rn.T).astype(np.float32)
|
|
|
|
# Apply the matching transform to each world->camera pose:
|
|
# R' = R @ Rn^T , t' = s * (t + R @ c) (derived so projections are unchanged).
|
|
poses_new: list[dict] = []
|
|
for p in poses:
|
|
R = quat_to_mat([p["qw"], p["qx"], p["qy"], p["qz"]])
|
|
t = np.array([p["tx"], p["ty"], p["tz"]], dtype=np.float64)
|
|
R_new = R @ Rn.T
|
|
t_new = scale * (t + R @ centroid)
|
|
q_new = mat_to_quat(R_new)
|
|
out = dict(p)
|
|
out["qw"], out["qx"], out["qy"], out["qz"] = (
|
|
float(q_new[0]), float(q_new[1]), float(q_new[2]), float(q_new[3]))
|
|
out["tx"], out["ty"], out["tz"] = float(t_new[0]), float(t_new[1]), float(t_new[2])
|
|
poses_new.append(out)
|
|
|
|
return points_new, poses_new
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pose interpolation
|
|
# ---------------------------------------------------------------------------
|
|
def interpolate_poses(registered: list[dict], sampled: list[dict]) -> list[dict]:
|
|
"""Fill unregistered sampled frames of one video between registered ones (spec M2).
|
|
|
|
``registered`` are solved poses (with ``registered=True``); ``sampled`` are the frames
|
|
that were fed to COLMAP as ``{frame_idx, t_video_s, ...}``. For each sampled frame that
|
|
lies strictly between the first and last registered frame but wasn't solved, slerp the
|
|
rotation and lerp the translation of the two bracketing registered poses (intrinsics
|
|
inherited from the earlier neighbor), marking it ``registered=False``. Frames outside the
|
|
registered span are dropped (no extrapolation). Returns all poses sorted by ``t_video_s``.
|
|
"""
|
|
reg = sorted(registered, key=lambda p: p["t_video_s"])
|
|
if not reg:
|
|
return []
|
|
reg_frames = {int(p["frame_idx"]) for p in reg}
|
|
t_first, t_last = reg[0]["t_video_s"], reg[-1]["t_video_s"]
|
|
|
|
out: list[dict] = list(reg)
|
|
for sf in sampled:
|
|
fidx = int(sf["frame_idx"])
|
|
tv = float(sf["t_video_s"])
|
|
if fidx in reg_frames:
|
|
continue
|
|
if tv <= t_first or tv >= t_last:
|
|
continue # no extrapolation beyond the registered span
|
|
# bracketing registered neighbors
|
|
lo = max((p for p in reg if p["t_video_s"] <= tv), key=lambda p: p["t_video_s"])
|
|
hi = min((p for p in reg if p["t_video_s"] >= tv), key=lambda p: p["t_video_s"])
|
|
span = hi["t_video_s"] - lo["t_video_s"]
|
|
alpha = 0.0 if span <= 1e-12 else (tv - lo["t_video_s"]) / span
|
|
q, t = slerp_pose(
|
|
[lo["qw"], lo["qx"], lo["qy"], lo["qz"]], [lo["tx"], lo["ty"], lo["tz"]],
|
|
[hi["qw"], hi["qx"], hi["qy"], hi["qz"]], [hi["tx"], hi["ty"], hi["tz"]],
|
|
alpha,
|
|
)
|
|
out.append({
|
|
"frame_idx": fidx, "t_video_s": tv,
|
|
"qw": float(q[0]), "qx": float(q[1]), "qy": float(q[2]), "qz": float(q[3]),
|
|
"tx": float(t[0]), "ty": float(t[1]), "tz": float(t[2]),
|
|
"fx": lo["fx"], "fy": lo["fy"], "cx": lo["cx"], "cy": lo["cy"],
|
|
"registered": False,
|
|
})
|
|
out.sort(key=lambda p: p["t_video_s"])
|
|
return out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# COLMAP CLI orchestration
|
|
# ---------------------------------------------------------------------------
|
|
@functools.lru_cache(maxsize=None)
|
|
def _colmap_help(command: str) -> str:
|
|
"""Cached ``colmap <command> --help`` text, for build-aware option detection.
|
|
|
|
COLMAP prints its option list to stderr, so both streams are captured.
|
|
"""
|
|
try:
|
|
proc = subprocess.run(
|
|
["colmap", command, "--help"], capture_output=True, text=True
|
|
)
|
|
return (proc.stdout or "") + "\n" + (proc.stderr or "")
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def _cpu_flag(command: str) -> list[str]:
|
|
"""Force CPU SIFT for ``command`` if this COLMAP build exposes a ``use_gpu`` option.
|
|
|
|
The option prefix differs by version: COLMAP 3.x uses ``SiftExtraction`` /
|
|
``SiftMatching``, 4.x uses ``FeatureExtraction`` / ``FeatureMatching``. We read the option
|
|
name from ``--help`` so the pipeline runs headless on either (and on CPU-only builds that
|
|
omit the option entirely, we pass nothing). Returns e.g. ``["--FeatureExtraction.use_gpu",
|
|
"0"]`` or ``[]``.
|
|
"""
|
|
for line in _colmap_help(command).splitlines():
|
|
line = line.strip()
|
|
if line.startswith("--") and ".use_gpu" in line:
|
|
return [line.split()[0], "0"]
|
|
return []
|
|
|
|
|
|
def _run_colmap_step(args: list[str]) -> bool:
|
|
"""Run one ``colmap <args>`` step; return True on success, False (logged) on failure."""
|
|
cmd = ["colmap", *args]
|
|
log.info("colmap: %s", " ".join(args[:2]))
|
|
try:
|
|
proc = subprocess.run(cmd, capture_output=True, text=True)
|
|
except FileNotFoundError:
|
|
log.error("colmap binary vanished from PATH")
|
|
return False
|
|
if proc.returncode != 0:
|
|
log.error("colmap %s failed (rc=%d): %s", args[0], proc.returncode,
|
|
(proc.stderr or proc.stdout or "").strip()[-500:])
|
|
return False
|
|
return True
|
|
|
|
|
|
def _registered_image_count(model_dir: Path) -> int | None:
|
|
"""Number of registered images in a COLMAP model dir, or None if it has no image list.
|
|
|
|
The mapper writes binary models by default; ``images.bin`` begins with a little-endian
|
|
``uint64`` giving the registered-image count, which is the exact metric we want (file
|
|
size would instead track total keypoint observations and mis-rank multi-component runs).
|
|
Falls back to counting the TXT model's two-lines-per-image list.
|
|
"""
|
|
images_txt = model_dir / "images.txt"
|
|
images_bin = model_dir / "images.bin"
|
|
if images_txt.exists():
|
|
return sum(1 for _ in _iter_data_lines(images_txt)) // 2
|
|
if images_bin.exists():
|
|
with open(images_bin, "rb") as f:
|
|
header = f.read(8)
|
|
return struct.unpack("<Q", header)[0] if len(header) == 8 else 0
|
|
return None
|
|
|
|
|
|
def _largest_model_dir(sparse_dir: Path) -> Path | None:
|
|
"""Return the reconstruction subdir (``0``, ``1``, ...) with the most registered images."""
|
|
candidates = [d for d in sparse_dir.iterdir() if d.is_dir()] if sparse_dir.exists() else []
|
|
best, best_n = None, -1
|
|
for d in candidates:
|
|
n = _registered_image_count(d)
|
|
if n is None:
|
|
continue
|
|
if n > best_n:
|
|
best, best_n = d, n
|
|
return best
|
|
|
|
|
|
def run_colmap(frames_dir: Path, workspace: Path) -> Path | None:
|
|
"""Drive the COLMAP CLI and return the exported TXT-model directory (spec M2, lane B).
|
|
|
|
``frames_dir`` holds one subfolder of JPEGs per video (single camera per folder). Runs
|
|
feature extraction (CPU SIFT, OPENCV model), exhaustive matching, incremental mapping,
|
|
then undistortion + TXT conversion. Returns the directory containing ``images.txt`` /
|
|
``cameras.txt`` / ``points3D.txt``, or ``None`` if COLMAP produced no usable model
|
|
(caller degrades gracefully).
|
|
"""
|
|
workspace = Path(workspace)
|
|
# Start each run from a clean workspace: a stale database.db makes feature_extractor
|
|
# fail ("images already exist"), and a leftover sparse/ model would poison the pick.
|
|
if workspace.exists():
|
|
import shutil
|
|
shutil.rmtree(workspace)
|
|
workspace.mkdir(parents=True, exist_ok=True)
|
|
database = workspace / "database.db"
|
|
sparse = workspace / "sparse"
|
|
sparse.mkdir(exist_ok=True)
|
|
|
|
ok = _run_colmap_step([
|
|
"feature_extractor",
|
|
"--database_path", str(database),
|
|
"--image_path", str(frames_dir),
|
|
"--ImageReader.single_camera_per_folder", "1",
|
|
"--ImageReader.camera_model", "OPENCV",
|
|
*_cpu_flag("feature_extractor"),
|
|
])
|
|
if not ok:
|
|
return None
|
|
|
|
# Exhaustive matching covers within- and cross-video pairs; frame counts are small.
|
|
if not _run_colmap_step([
|
|
"exhaustive_matcher",
|
|
"--database_path", str(database),
|
|
*_cpu_flag("exhaustive_matcher"),
|
|
]):
|
|
return None
|
|
|
|
if not _run_colmap_step([
|
|
"mapper",
|
|
"--database_path", str(database),
|
|
"--image_path", str(frames_dir),
|
|
"--output_path", str(sparse),
|
|
]):
|
|
return None
|
|
|
|
model = _largest_model_dir(sparse)
|
|
if model is None:
|
|
log.error("colmap mapper produced no reconstruction")
|
|
return None
|
|
|
|
# Undistort -> PINHOLE model, then convert to TXT. Fall back to converting the raw
|
|
# (possibly distorted) sparse model directly if undistortion fails.
|
|
txt_dir = workspace / "model_txt"
|
|
txt_dir.mkdir(exist_ok=True)
|
|
dense = workspace / "dense"
|
|
undistorted = _run_colmap_step([
|
|
"image_undistorter",
|
|
"--image_path", str(frames_dir),
|
|
"--input_path", str(model),
|
|
"--output_path", str(dense),
|
|
"--output_type", "COLMAP",
|
|
])
|
|
convert_input = (dense / "sparse") if undistorted and (dense / "sparse").exists() else model
|
|
if not _run_colmap_step([
|
|
"model_converter",
|
|
"--input_path", str(convert_input),
|
|
"--output_path", str(txt_dir),
|
|
"--output_type", "TXT",
|
|
]):
|
|
return None
|
|
if not (txt_dir / "images.txt").exists():
|
|
log.error("colmap model_converter produced no images.txt")
|
|
return None
|
|
return txt_dir
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Full reconstruction entrypoint
|
|
# ---------------------------------------------------------------------------
|
|
def _video_id_and_frame(name: str) -> tuple[int, int] | None:
|
|
"""Recover ``(video_id, frame_idx)`` from a ``{video_id}_{frame_idx}.jpg`` image name."""
|
|
stem = Path(name).stem
|
|
if "_" not in stem:
|
|
return None
|
|
vid_s, frame_s = stem.rsplit("_", 1)
|
|
try:
|
|
return int(vid_s), int(frame_s)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _diagnostic(msg: str) -> None:
|
|
"""Emit a clear operator-facing diagnostic (spec M2 graceful degradation)."""
|
|
log.warning("reconstruct: %s", msg)
|
|
print(f"[reconstruct] {msg}")
|
|
|
|
|
|
def run_reconstruct() -> dict:
|
|
"""Full reconstruction: sample frames, run COLMAP, normalize, interpolate, export.
|
|
|
|
Entrypoint for ``python -m festival4d reconstruct``. Degrades gracefully when COLMAP is
|
|
absent or the reconstruction is too weak (spec M2 failure handling): existing poses are
|
|
left untouched and the DB is never corrupted. Returns a summary dict with a ``status``.
|
|
"""
|
|
db.init_engine()
|
|
db.init_db()
|
|
videos = db.get_videos()
|
|
if not videos:
|
|
_diagnostic("no videos in the project — run `synthetic` or `ingest` first.")
|
|
return {"status": "no_videos"}
|
|
|
|
if not colmap_available():
|
|
_diagnostic(
|
|
"COLMAP not found on PATH — skipping 3D reconstruction. Install it "
|
|
"(`brew install colmap` on macOS) to enable pose/point-cloud solving. "
|
|
"Existing (synthetic/previous) poses are left untouched; the app still runs as "
|
|
"synced videos without 3D."
|
|
)
|
|
return {"status": "skipped_no_colmap", "videos": len(videos)}
|
|
|
|
# 1. Sample sharp frames per video into one subfolder each (single camera per folder).
|
|
frames_root = config.FRAMES_DIR
|
|
per_video_sampled: dict[int, list[dict]] = {}
|
|
for v in videos:
|
|
video_path = config.RAW_DIR / v.filename
|
|
if not video_path.exists():
|
|
_diagnostic(f"video file missing, skipping: {video_path}")
|
|
continue
|
|
out_dir = frames_root / str(v.id)
|
|
paths = frames.sample_frames(video_path, v.id, out_dir)
|
|
fps = float(v.fps) if v.fps and v.fps > 0 else 30.0
|
|
sampled = []
|
|
for p in paths:
|
|
parsed = _video_id_and_frame(p.name)
|
|
if parsed is None:
|
|
continue
|
|
_, fidx = parsed
|
|
sampled.append({"frame_idx": fidx, "t_video_s": fidx / fps})
|
|
per_video_sampled[v.id] = sorted(sampled, key=lambda s: s["frame_idx"])
|
|
|
|
total_sampled = sum(len(s) for s in per_video_sampled.values())
|
|
if total_sampled == 0:
|
|
_diagnostic("no frames could be sampled from any video — nothing to reconstruct.")
|
|
return {"status": "no_frames"}
|
|
|
|
# 2. Run COLMAP.
|
|
model_dir = run_colmap(frames_root, config.COLMAP_DIR)
|
|
if model_dir is None:
|
|
_diagnostic(
|
|
"COLMAP produced no usable reconstruction. Likely causes: too-dark footage, "
|
|
"motion blur, or insufficient view overlap between cameras. Existing poses left "
|
|
"untouched — the app degrades to synced videos, no 3D."
|
|
)
|
|
return {"status": "failed_no_model", "sampled": total_sampled}
|
|
|
|
# 3. Parse the model.
|
|
images = parse_images_txt(model_dir / "images.txt")
|
|
cameras = parse_cameras_txt(model_dir / "cameras.txt")
|
|
points, colors = parse_points3d_txt(model_dir / "points3D.txt")
|
|
|
|
# 4. Attach each registered image to its video + intrinsics.
|
|
registered_by_video: dict[int, list[dict]] = {}
|
|
for img in images:
|
|
parsed = _video_id_and_frame(img["name"])
|
|
if parsed is None:
|
|
continue
|
|
video_id, frame_idx = parsed
|
|
cam = cameras.get(img["camera_id"])
|
|
if cam is None:
|
|
continue
|
|
fps = next((float(v.fps) for v in videos if v.id == video_id), 30.0) or 30.0
|
|
registered_by_video.setdefault(video_id, []).append({
|
|
"frame_idx": frame_idx, "t_video_s": frame_idx / fps,
|
|
"qw": img["qw"], "qx": img["qx"], "qy": img["qy"], "qz": img["qz"],
|
|
"tx": img["tx"], "ty": img["ty"], "tz": img["tz"],
|
|
"fx": cam["fx"], "fy": cam["fy"], "cx": cam["cx"], "cy": cam["cy"],
|
|
"registered": True, "video_id": video_id,
|
|
})
|
|
|
|
registered_count = sum(len(v) for v in registered_by_video.values())
|
|
videos_in_model = len(registered_by_video)
|
|
frac = registered_count / total_sampled if total_sampled else 0.0
|
|
if frac < MIN_REGISTERED_FRACTION or videos_in_model < MIN_VIDEOS_IN_MODEL:
|
|
_diagnostic(
|
|
f"weak reconstruction: {registered_count}/{total_sampled} frames "
|
|
f"({frac:.0%}) registered across {videos_in_model} video(s); need "
|
|
f">= {MIN_REGISTERED_FRACTION:.0%} of frames and >= {MIN_VIDEOS_IN_MODEL} "
|
|
"videos. Likely causes: too-dark footage, motion blur, or too little view "
|
|
"overlap. Existing poses left untouched — degrading to synced videos, no 3D."
|
|
)
|
|
return {
|
|
"status": "failed_weak", "sampled": total_sampled,
|
|
"registered": registered_count, "videos_in_model": videos_in_model,
|
|
}
|
|
|
|
# 5. Normalize the whole scene (points + all registered poses) with one transform.
|
|
all_registered = [p for v in registered_by_video.values() for p in v]
|
|
norm_points, norm_registered = normalize_scene(points, all_registered)
|
|
norm_by_video: dict[int, list[dict]] = {}
|
|
for p in norm_registered:
|
|
norm_by_video.setdefault(p["video_id"], []).append(p)
|
|
|
|
# 6. Interpolate unregistered sampled frames, then write poses per video (atomic replace).
|
|
interpolated_total = 0
|
|
for video_id, reg in norm_by_video.items():
|
|
sampled = per_video_sampled.get(video_id, [])
|
|
final = interpolate_poses(reg, sampled)
|
|
interpolated_total += sum(1 for p in final if not p.get("registered", True))
|
|
db.set_poses(video_id, final)
|
|
|
|
# 7. Export the normalized point cloud in the frozen PLY format.
|
|
from festival4d.synthetic import write_ply
|
|
write_ply(config.POINTS_PLY, norm_points, colors)
|
|
|
|
summary = {
|
|
"status": "ok",
|
|
"videos_in_model": videos_in_model,
|
|
"registered": registered_count,
|
|
"interpolated": interpolated_total,
|
|
"points": int(len(norm_points)),
|
|
"points_ply": str(config.POINTS_PLY),
|
|
}
|
|
log.info("reconstruct: done — %s", summary)
|
|
print(f"[reconstruct] reconstructed {videos_in_model} videos, "
|
|
f"{registered_count} registered + {interpolated_total} interpolated poses, "
|
|
f"{len(norm_points)} points -> {config.POINTS_PLY}")
|
|
return summary
|