From bf8b1a35d5029ff3df22fda9e8a7ebc14a0d4785 Mon Sep 17 00:00:00 2001 From: m3ultra Date: Mon, 3 Aug 2026 20:39:45 +1000 Subject: [PATCH] Close all four open items: MoGe camera, manifold remesh, winding, UV bake THE DECIMATION FLOOR WAS MISDIAGNOSED. I attributed it to ~180k boundary edges. It is non-manifold edges. Measured on the shipped 500k mesh: boundary edges 32,370 NON-MANIFOLD 81,112 <- the actual blocker, 2.5x more Quadric decimation cannot collapse an edge shared by more than two faces. Upstream's own fix (fill_holes, via CUDA-only cumesh) targets boundaries and caps at max_hole_perimeter=3e-2, so it was never going to help: trimesh's equivalent moved boundaries 32,370 -> 30,990 and the floor only 214k -> 210k. That falsified it. --manifold: voxelise -> fill -> marching cubes. Removes BOTH classes at once and so closes three of the four items in one change: as shipped 499,984 faces bnd 32,370 nonmani 81,112 watertight=F IoU 0.969 remeshed 1,178,142 faces bnd 0 nonmani 0 watertight=T IoU 0.949 -> 20k 19,998 faces bnd 0 winding consistent IoU 0.956 25x smaller, fully manifold, consistent winding, for 1.3% silhouette IoU. Lossy by design - it gives up the dual grid's open-surface representation - so it is opt-in. UV BAKE is unblocked by the same change: its cost is driven by face count, not by remesh. 5.0s at 20k faces against >20min at 214k. No longer offline-only when paired with manifold. THE SCALING TRAP, worth knowing: marching_cubes returns vertices in VOXEL INDEX space. Translating without apply_scale(pitch) leaves the mesh ~292x too large. It still exports and renders as a plausible object; it silhouettes at IoU 0.08. That is how it was caught. MoGe-2 CAMERA is now wired and is the default, matching upstream; --fixed-fov keeps the old constant. It runs once per image in torch/MPS, ~0.4s after load. Reporting this one straight: it did NOT improve the samples. On 1_img, fixed 49.1 deg scored 0.893 and MoGe's 29.7 deg scored 0.883. Two caveats keep it as the default anyway - the silhouette metric projects with the SAME FOV used to generate, so a wrong-but-consistent camera can still score well and the metric cannot fully arbitrate camera correctness; and the bundled samples are synthetic renders, not the photographs MoGe reads. Real photos are the intended input here, and upstream estimates too. But the constant is one flag away and the measurement is on record rather than assumed. Operator gains manifold, divisions, fixed_fov. README and PROFILE.md corrected where they repeated the boundary-edge claim. Co-Authored-By: Claude Fable 5 --- PROFILE.md | 23 +++++++++++++ README.md | 39 ++++++++++++++++++---- pixal3d_mlx/camera.py | 70 ++++++++++++++++++++++++++++++++++++++++ pixal3d_mlx/cleanup.py | 43 ++++++++++++++++++++++-- scripts/image_to_mesh.py | 34 +++++++++++++++++-- 5 files changed, 198 insertions(+), 11 deletions(-) create mode 100644 pixal3d_mlx/camera.py diff --git a/PROFILE.md b/PROFILE.md index 51c44a7..3140a98 100644 --- a/PROFILE.md +++ b/PROFILE.md @@ -99,6 +99,29 @@ layout. Compiling the blocks would mean restructuring that, for a measured ~0%. 3. **The Metal spconv kernel — for memory only**, if this ever needs to run somewhere smaller than a Studio. +## Correction: the decimation floor was misdiagnosed + +An earlier version of this file blamed the ~214k floor on "~180,000 boundary edges". +That was wrong. Measured on the shipped 500k mesh: + +``` + boundary edges 32,370 + NON-MANIFOLD 81,112 <- the actual blocker, 2.5x more +``` + +Quadric decimation cannot collapse an edge shared by more than two faces. Filling +holes moved boundaries 32,370 → 30,990 and the floor only 214k → 210k, which is what +falsified the boundary theory. + +`--manifold` (voxel remesh) removes both classes: 0 boundary, 0 non-manifold, +watertight, winding consistent, and it decimates to **20k faces at IoU 0.956** against +0.969 for the 500k non-manifold original. It also drops the **UV bake from >20 minutes +to 5.0 seconds**, since xatlas was choking on face count. + +Watch for the scaling trap: `marching_cubes` returns vertices in VOXEL INDEX space. +Translating without `apply_scale(pitch)` leaves the mesh ~292x too large — it still +exports and renders as a plausible object and silhouettes at IoU 0.08. + ## m4pro cannot run this Peak is **37.1 GB**. The m4pro is 24 GB, so the full cascade will not fit, and the diff --git a/README.md b/README.md index e705368..eaf6b66 100644 --- a/README.md +++ b/README.md @@ -111,13 +111,40 @@ reconstruction that does not track the input has failed, even though nothing rai | raw decoder output | 7,996,876 | 0.969 | | 500,000 | 499,984 | **0.965** | | 200,000 / 100,000 / 20,000 | 214,322 (floor) | 0.823 | +| **`--manifold` → 20,000** | **19,998** | **0.956** | -**~214k is a hard floor.** The Flexible Dual Grid emits ~180,000 boundary edges for -open surfaces and quadric decimation will not collapse those — no `target_reduction` -or `agg` setting changes it, and a single `fast_simplification` call additionally -refuses to reduce past ~4.4% of its input (hence the iterative loop). Going lower -needs a remesh; o_voxel's ran >20 minutes on 214k faces before being killed, so it is -not currently a practical route. **500k is effectively lossless — use that.** +**Without `--manifold` there is a hard floor around 214k.** The cause is **non-manifold +edges (81,112)**, NOT boundary edges (32,370) — an earlier note here said the opposite. +Quadric decimation cannot collapse an edge shared by more than two faces, and no +`target_reduction` or `agg` setting changes that. Filling holes barely moved either +number (boundaries 32,370 → 30,990; floor 214k → 210k), which is what ruled the +boundary theory out. + +Those non-manifold edges are the Flexible Dual Grid working as designed — it +represents open and non-manifold surfaces deliberately. + +**`--manifold` voxel-remeshes past it**: watertight, zero non-manifold edges, +consistent winding, decimates to **20k faces for ~1.3% silhouette IoU** (0.969 → +0.956). Lossy — it gives up the open-surface representation and softens sharp +features — so it is opt-in. It also makes the **UV bake practical: 5.0s at 20k faces** +against >20 minutes at 214k. + +Use `--manifold --target-faces 20000` for game-ready assets; plain `--target-faces +500000` when you want maximum fidelity and will clean up in Blender. + +## Camera + +FOV is estimated per image with **MoGe-2** by default (upstream's behaviour, ~0.4s); +`--fixed-fov` uses the 0.8576 rad constant. Everything downstream is placed by this, +so a wrong FOV reconstructs at the wrong depth scale and fails silently. + +Measured honestly: on the bundled sample renders MoGe estimates 25–31° against the +49.1° constant, and scored **marginally WORSE** (0.883 vs 0.893 on `1_img`). Two +caveats on that comparison — the silhouette metric projects with the same FOV used to +generate, so a wrong-but-consistent camera can still score well; and the samples are +synthetic renders, not the photographs MoGe was trained to read. Estimation stays the +default for upstream parity and because real photos are the intended input, but the +constant is one flag away. ## Model status diff --git a/pixal3d_mlx/camera.py b/pixal3d_mlx/camera.py new file mode 100644 index 0000000..72de95f --- /dev/null +++ b/pixal3d_mlx/camera.py @@ -0,0 +1,70 @@ +"""Per-image camera estimation with MoGe-2. + +Everything downstream assumes the object exactly fills the frame at a known FOV: +`distance_from_fov` places the camera so a unit mesh spans the image, and the proj +grid back-projects through that camera. A fixed default FOV therefore does not just +change framing — it reconstructs the subject at the WRONG DEPTH SCALE, and the failure +is quiet: you get a clean, plausible mesh that is subtly the wrong shape. + +Upstream estimates FOV per image with MoGe-2 and only falls back to a constant when +`--fov` is passed explicitly. This does the same. MoGe runs once per image in torch on +MPS, like DINOv3 and NAF — outside any denoising loop, so parity beats a port. +""" + +from __future__ import annotations + +import math + +import numpy as np + +MODEL_NAME = "Ruicheng/moge-2-vitl" +_model = None + + +def load_moge(device: str = "mps"): + """Load MoGe-2 once per process.""" + global _model + if _model is None: + import torch + from moge.model.v2 import MoGeModel + + _model = MoGeModel.from_pretrained(MODEL_NAME).eval().to(device) + _model.requires_grad_(False) + elif str(next(_model.parameters()).device).split(":")[0] != device: + _model = _model.to(device) + return _model + + +def estimate_fov(image_path: str, device: str = "mps") -> float: + """Horizontal FOV in radians, from MoGe-2's predicted intrinsics. + + `intrinsics[0,0]` is fx NORMALISED by image width, so it must be multiplied back + up before the arctan — the normalisation is easy to miss and yields a plausible + but wrong angle if skipped. + """ + import torch + from PIL import Image + + img = Image.open(image_path).convert("RGB") + width, _ = img.size + arr = np.asarray(img, dtype=np.float32) / 255.0 + tensor = torch.from_numpy(arr).permute(2, 0, 1).to(device) + + model = load_moge(device) + with torch.no_grad(): + out = model.infer(tensor) + fx_norm = float(np.asarray(out["intrinsics"].squeeze().cpu())[0, 0]) + return 2.0 * math.atan(width / (2.0 * fx_norm * width)) + + +def camera_for(image_path: str, fov: float | None = None, mesh_scale: float = 1.0, + image_resolution: int = 512, device: str = "mps") -> dict: + """(camera_angle_x, distance) for an image. Estimates FOV when `fov` is None.""" + from .proj import distance_from_fov + + estimated = fov is None + if estimated: + fov = estimate_fov(image_path, device) + return {"camera_angle_x": float(fov), + "distance": distance_from_fov(fov, mesh_scale, image_resolution), + "estimated": estimated} diff --git a/pixal3d_mlx/cleanup.py b/pixal3d_mlx/cleanup.py index 5a3f025..4d48701 100644 --- a/pixal3d_mlx/cleanup.py +++ b/pixal3d_mlx/cleanup.py @@ -58,6 +58,35 @@ def largest_components(mesh, keep_ratio: float = 0.01, min_faces: int = 100): "faces_removed": removed, "threshold": threshold} +def manifold_remesh(mesh, divisions: int = 256): + """Voxelise -> fill -> marching cubes. Trades the dual grid for a manifold surface. + + WHY THIS EXISTS. The decoder's output cannot be decimated below ~214k faces, and + the cause is NOT boundary edges (32,370) as first assumed — it is **non-manifold + edges (81,112)**, which quadric decimation refuses to collapse. Filling holes + barely moved either number (boundaries 32,370 -> 30,990, floor 214k -> 210k). + Those edges are the Flexible Dual Grid working as designed: it represents open and + non-manifold surfaces on purpose. So getting past the floor means giving that up, + deliberately, rather than tuning a decimator that was never going to win. + + Measured on the sample: 500k non-manifold faces at IoU 0.969 becomes 20k + watertight, winding-consistent faces at **IoU 0.956** — 25x smaller for 1.3%. + + THE TRAP: `marching_cubes` returns vertices in VOXEL INDEX space (0..divisions), + not world units. Translating without scaling leaves the mesh ~292x too large, + which still exports and renders as a plausible object — it just silhouettes at + IoU 0.08. Scale by pitch first. + """ + import trimesh + + pitch = float(np.asarray(mesh.extents).max()) / divisions + out = mesh.voxelized(pitch=pitch).fill().marching_cubes + out.apply_scale(pitch) # index space -> world + out.apply_translation(np.asarray(mesh.bounds[0]) - np.asarray(out.bounds[0])) + out.fix_normals() + return out + + def decimate(mesh, target_faces: int, max_passes: int = 8, step: float = 0.7): """Quadric decimation to a face budget, iteratively. @@ -92,8 +121,13 @@ def decimate(mesh, target_faces: int, max_passes: int = 8, step: float = 0.7): def clean(vertices, faces, target_faces: int | None = 100_000, - keep_ratio: float = 0.01, min_faces: int = 100, log=print): - """The whole hygiene pass. Returns (mesh, stats).""" + keep_ratio: float = 0.01, min_faces: int = 100, manifold: bool = False, + divisions: int = 256, log=print): + """The whole hygiene pass. Returns (mesh, stats). + + `manifold=True` inserts a voxel remesh after floater removal, which is the only + way past the ~214k non-manifold decimation floor. It is lossy and opt-in. + """ import trimesh mesh = trimesh.Trimesh(np.asarray(vertices), np.asarray(faces), process=False) @@ -109,6 +143,11 @@ def clean(vertices, faces, target_faces: int | None = 100_000, log(f" floaters {comp['components']:6} components -> {comp['kept']} kept, " f"{comp['faces_removed']:,} faces dropped") + if manifold: + mesh = manifold_remesh(mesh, divisions) + log(f" remeshed manifold @{divisions}: {len(mesh.faces):,} faces, " + f"watertight={mesh.is_watertight}") + if target_faces: mesh = decimate(mesh, target_faces) log(f" decimated {before[1]:,} -> {len(mesh.faces):,} faces") diff --git a/scripts/image_to_mesh.py b/scripts/image_to_mesh.py index 8bc5c93..917d452 100644 --- a/scripts/image_to_mesh.py +++ b/scripts/image_to_mesh.py @@ -93,7 +93,16 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("image", nargs="?", default=str(DEFAULT_IMAGE)) ap.add_argument("-o", "--output", default=str(REPO / "output.glb")) - ap.add_argument("--fov", type=float, default=DEFAULT_FOV) + ap.add_argument("--fov", type=float, default=None, + help="camera FOV in radians; omitted = estimate per image with " + "MoGe-2 (upstream's behaviour)") + ap.add_argument("--fixed-fov", action="store_true", + help=f"skip MoGe and use the constant {DEFAULT_FOV:.4f} rad") + ap.add_argument("--manifold", action="store_true", + help="voxel-remesh to a watertight manifold surface — the only way " + "past the ~214k non-manifold decimation floor; lossy, opt-in") + ap.add_argument("--divisions", type=int, default=256, + help="voxel resolution for --manifold") ap.add_argument("--seed", type=int, default=0) ap.add_argument("--target-faces", type=int, default=100_000, help="face budget after cleanup; 0 disables decimation") @@ -114,10 +123,27 @@ def main(): models = load_all(with_texture=a.texture) print(f"loaded models ({time.time() - t:.1f}s, lazy — weights fault in on first use)") + # Camera FIRST — everything downstream is placed by it. A wrong FOV reconstructs + # at the wrong depth scale and fails silently, so estimation is the default. + if a.fov is None: + if a.fixed_fov: + a.fov = DEFAULT_FOV + print(f" camera fixed {a.fov:.4f} rad ({np.degrees(a.fov):.1f} deg)") + else: + from pixal3d_mlx.camera import camera_for + t = time.time() + cam = camera_for(a.image) + a.fov = cam["camera_angle_x"] + print(f" camera MoGe-2 {a.fov:.4f} rad ({np.degrees(a.fov):.1f} deg), " + f"distance {cam['distance']:.3f} {time.time() - t:5.1f}s") + info_fov = a.fov + v, f, info = image_to_mesh(a.image, models, camera_angle_x=a.fov, seed=a.seed, normalization=normalization("shape"), tex_normalization=normalization("tex") if a.texture else None, texture=a.texture) + info["fov"] = round(float(info_fov), 6) + info["fov_source"] = "fixed" if a.fixed_fov else "moge2" subs = info.pop("subs", None) info.pop("hr_slat", None) tex_voxels = info.pop("tex_voxels", None) @@ -133,7 +159,8 @@ def main(): t = time.time() pre, _ = clean(v.cpu().numpy(), f.cpu().numpy(), - target_faces=a.target_faces or 500_000) + target_faces=a.target_faces or 500_000, + manifold=a.manifold, divisions=a.divisions) print(f" pre-bake {len(pre.faces):,} faces {time.time() - t:6.1f}s") t = time.time() @@ -171,7 +198,8 @@ def main(): from pixal3d_mlx.cleanup import clean t = time.time() mesh, stats = clean(v.cpu().numpy(), f.cpu().numpy(), - target_faces=a.target_faces or None) + target_faces=a.target_faces or None, + manifold=a.manifold, divisions=a.divisions) print(f" cleanup {time.time() - t:6.1f}s") info["cleanup"] = stats