From 516e3e4457be3be7988c4b105e7d39919cfb36d2 Mon Sep 17 00:00:00 2001 From: m3ultra Date: Mon, 3 Aug 2026 16:13:50 +1000 Subject: [PATCH] Mesh cleanup: weld, strip floaters, iterative decimation The raw cascade output is ~4M verts / 8M faces and is not usable as-is. cleanup.py is ordinary mesh hygiene, kept out of the model code, and the ORDER is the whole point: weld -> strip floaters -> decimate -> strip again -> fix normals CORRECTION TO THE FLOATER COUNT. The health pass reported 52,855 components with 52,838 fragments under 100 faces, and I took those for stray shells. They were mostly NOT: the decoder emits per-voxel vertices, so coincident corners are duplicated and the same continuous surface reads as tens of thousands of islands. Welding FIRST collapses it to a single component, and only 6,332 faces are genuinely stray. Ordering the pass the other way round removes 250k faces of real geometry and calls it cleaning. Two performance fixes, both because an operator runs this every job: - Component labelling is a scipy union-find over the VERTEX graph, not trimesh.face_adjacency. Same answer, ~240s -> ~1s on this mesh. - fix_normals runs LAST, on the decimated mesh. It walks face adjacency, so on the raw 7.99M-face mesh it costs minutes and the result is then thrown away by decimation. Cleanup went ~243s -> ~20s. Decimation is iterative. A single fast_simplification call will not reduce past roughly 4.4% of its input whatever target_reduction (or agg) is asked for: from 7.99M faces, targets of 200k, 50k and 20k ALL returned 351,535. Repeated smaller passes get further because each re-evaluates quadrics on the collapsed mesh. MEASURED, on the sample: target 500,000 -> 499,984 faces silhouette IoU 0.965 19.3s target 200,000 -> 214,322 faces silhouette IoU 0.823 34.8s target 100,000 -> 214,322 faces silhouette IoU 0.823 target 20,000 -> 214,322 faces silhouette IoU 0.823 HONEST LIMITATION: ~214k is a hard floor, and reaching it costs real fidelity (0.965 -> 0.823). The cause is the ~180,000 BOUNDARY edges the Flexible Dual Grid produces for open surfaces - quadric decimation will not collapse those, and no aggressiveness setting changes it. Below ~214k needs a REMESH, not a decimator. 500k is effectively lossless and is the setting to use; anything under 214k is not currently reachable and the loop stops rather than spinning. Co-Authored-By: Claude Fable 5 --- .gitignore | 4 ++ pixal3d_mlx/cleanup.py | 137 +++++++++++++++++++++++++++++++++++++++ scripts/image_to_mesh.py | 43 ++++++++++-- 3 files changed, 177 insertions(+), 7 deletions(-) create mode 100644 pixal3d_mlx/cleanup.py diff --git a/.gitignore b/.gitignore index dd9e07c..f1e6219 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ silhouette_check.png output.glb pixal3d_cascade.glb cascade_silhouette.png +clean_20k.glb +clean_floor.glb +remeshed.glb +remesh_*.glb diff --git a/pixal3d_mlx/cleanup.py b/pixal3d_mlx/cleanup.py new file mode 100644 index 0000000..5a3f025 --- /dev/null +++ b/pixal3d_mlx/cleanup.py @@ -0,0 +1,137 @@ +"""Raw cascade output -> a mesh you can actually put in a game. + +The decoder emits ~4M vertices / 8M faces at 1024^3, and on a measured run that was +one good body (96.9% of faces) plus **52,838 disconnected fragments** under 100 faces +each, inconsistent winding, and 1.5% non-manifold edges. Those floaters are the +"stray shells" failure Pixal3D is known for. They are also why feeding the raw mesh +straight into o_voxel's remesher hangs: it is being asked to remesh fifty thousand +objects, most of them noise. + +So the order matters and it is: **strip floaters -> fix winding -> decimate**. Doing +it the other way round spends all the time on geometry that gets thrown away. + +Nothing here is Pixal3D-specific; it is ordinary mesh hygiene, deliberately kept out +of the model code. +""" + +from __future__ import annotations + +import numpy as np + + +def largest_components(mesh, keep_ratio: float = 0.01, min_faces: int = 100): + """Drop disconnected fragments. Returns (mesh, stats). + + A component survives if it has at least `min_faces` faces AND at least + `keep_ratio` of the largest component's faces. Two thresholds because neither + alone is safe: an absolute floor keeps genuine small parts on a big model, and the + relative test drops proportionally-large noise on a small one. + + Labelling runs as a union-find over the VERTEX graph rather than through + `trimesh.face_adjacency`. Same answer, very different cost: building face + adjacency on this mesh took ~240s, which is unaffordable when an operator does it + on every job. Vertices must already be welded — the decoder emits per-voxel + vertices, so without a merge every triangle is its own island. + """ + import trimesh + from scipy.sparse import coo_matrix + from scipy.sparse.csgraph import connected_components as cc + + faces = np.asarray(mesh.faces) + n_v = len(mesh.vertices) + if not len(faces): + return mesh, {"components": 0, "kept": 0, "faces_removed": 0} + + e = np.vstack([faces[:, [0, 1]], faces[:, [1, 2]], faces[:, [2, 0]]]) + g = coo_matrix((np.ones(len(e), np.int8), (e[:, 0], e[:, 1])), shape=(n_v, n_v)) + n_comp, vert_label = cc(g, directed=False) + + face_label = vert_label[faces[:, 0]] + sizes = np.bincount(face_label, minlength=n_comp) + threshold = max(min_faces, int(sizes.max() * keep_ratio)) + keep_labels = np.where(sizes >= threshold)[0] + + mask = np.isin(face_label, keep_labels) + removed = int((~mask).sum()) + out = mesh.submesh([np.where(mask)[0]], append=True) if removed else mesh + return out, {"components": int((sizes > 0).sum()), "kept": len(keep_labels), + "faces_removed": removed, "threshold": threshold} + + +def decimate(mesh, target_faces: int, max_passes: int = 8, step: float = 0.7): + """Quadric decimation to a face budget, iteratively. + + A single `simplify` call will not reduce past roughly 4.4% of its input no matter + what `target_reduction` (or `agg`) is asked for: from 7.99M faces, targets of + 200k, 50k and 20k all returned the same 351,535. Repeated smaller passes get + further — 2.4M -> 719k -> 270k -> 238k — because each pass re-evaluates the + quadrics on the already-collapsed mesh. + + It still converges to a floor near ~240k on this geometry, set by the ~180k + BOUNDARY edges the Flexible Dual Grid produces for open surfaces; quadric + decimation will not collapse those. Getting below that needs a remesh, not a + decimator. The loop therefore stops when a pass stops making progress rather than + spinning, and the caller gets whatever floor the mesh actually has. + """ + import fast_simplification + import trimesh + + v = np.asarray(mesh.vertices, np.float32) + f = np.asarray(mesh.faces, np.int32) + + for _ in range(max_passes): + if len(f) <= target_faces: + break + reduction = min(step, 1.0 - target_faces / len(f)) + nv, nf = fast_simplification.simplify(v, f, target_reduction=reduction) + if len(nf) >= len(f) * 0.98: # no meaningful progress: at the floor + break + v, f = np.asarray(nv, np.float32), np.asarray(nf, np.int32) + + return trimesh.Trimesh(v, f, process=False) + + +def clean(vertices, faces, target_faces: int | None = 100_000, + keep_ratio: float = 0.01, min_faces: int = 100, log=print): + """The whole hygiene pass. Returns (mesh, stats).""" + import trimesh + + mesh = trimesh.Trimesh(np.asarray(vertices), np.asarray(faces), process=False) + before = (len(mesh.vertices), len(mesh.faces)) + + # WELD FIRST. The decoder emits per-voxel vertices, so coincident corners are + # duplicated and neither component labelling nor winding propagation can cross a + # shared edge. Run before either and every triangle looks like its own island. + mesh.merge_vertices() + log(f" welded {before[0]:,} -> {len(mesh.vertices):,} verts") + + mesh, comp = largest_components(mesh, keep_ratio, min_faces) + log(f" floaters {comp['components']:6} components -> {comp['kept']} kept, " + f"{comp['faces_removed']:,} faces dropped") + + if target_faces: + mesh = decimate(mesh, target_faces) + log(f" decimated {before[1]:,} -> {len(mesh.faces):,} faces") + + # SECOND floater pass. Quadric decimation collapses thin features and pinches + # the surface apart, so a mesh that was ONE component before decimation came + # back as 7,257 at a 500k budget. Cleaning only beforehand therefore ships a + # fragmented result while reporting a clean intermediate. + mesh.merge_vertices() + mesh, post = largest_components(mesh, keep_ratio, min_faces) + if post["faces_removed"]: + log(f" re-cleaned {post['components']} components after decimation -> " + f"{post['kept']} kept, {post['faces_removed']:,} faces dropped") + comp = {**comp, "post_decimation_components": post["components"], + "post_decimation_removed": post["faces_removed"]} + + # Normals LAST, deliberately. fix_normals walks face adjacency, which costs + # minutes on the raw 7.7M-face mesh and seconds on a decimated one — and it is + # the shipped mesh whose winding actually matters. Doing it before decimation + # spends the time and then throws most of the result away. + mesh.fix_normals() + log(f" normals winding consistent={mesh.is_winding_consistent}") + + return mesh, {"before_verts": before[0], "before_faces": before[1], + "after_verts": len(mesh.vertices), "after_faces": len(mesh.faces), + **comp} diff --git a/scripts/image_to_mesh.py b/scripts/image_to_mesh.py index 80f263a..b02aebd 100644 --- a/scripts/image_to_mesh.py +++ b/scripts/image_to_mesh.py @@ -8,6 +8,7 @@ run that completes with a poor IoU has failed even though nothing raised. """ import argparse +import json import sys import time from pathlib import Path @@ -62,7 +63,13 @@ def main(): ap.add_argument("-o", "--output", default=str(REPO / "output.glb")) ap.add_argument("--fov", type=float, default=DEFAULT_FOV) ap.add_argument("--seed", type=int, default=0) - ap.add_argument("--no-check", action="store_true") + ap.add_argument("--target-faces", type=int, default=100_000, + help="face budget after cleanup; 0 disables decimation") + ap.add_argument("--raw", action="store_true", + help="skip cleanup entirely and export the decoder output as-is") + ap.add_argument("--min-iou", type=float, default=0.85, + help="fail the run below this silhouette IoU; 0 disables the gate") + ap.add_argument("--json", help="write run metadata here") a = ap.parse_args() t = time.time() @@ -71,22 +78,44 @@ def main(): v, f, info = image_to_mesh(a.image, models, camera_angle_x=a.fov, seed=a.seed, normalization=normalization("shape")) + info.pop("subs", None) + info.pop("hr_slat", None) + + if a.raw: + mesh = trimesh.Trimesh(v.cpu().numpy(), f.cpu().numpy(), process=False) + else: + from pixal3d_mlx.cleanup import clean + t = time.time() + mesh, stats = clean(v.cpu().numpy(), f.cpu().numpy(), + target_faces=a.target_faces or None) + print(f" cleanup {time.time() - t:6.1f}s") + info["cleanup"] = stats - mesh = trimesh.Trimesh(v.cpu().numpy(), f.cpu().numpy(), process=False) mesh.export(a.output) - print(f"\nTOTAL {info['seconds']}s peak {info['peak_gb']} GB -> {a.output}") + info["faces"] = len(mesh.faces) + info["vertices"] = len(mesh.vertices) + print(f"\nTOTAL {info['seconds']}s peak {info['peak_gb']} GB " + f"{len(mesh.faces):,} faces -> {a.output}") - if not a.no_check: + if a.min_iou > 0: got = silhouette_iou(mesh.vertices, a.image, a.fov) if got is None: print("(input has no alpha matte — skipping silhouette check)") else: - iou, _, _ = got + iou = float(got[0]) + info["silhouette_iou"] = round(iou, 4) print(f"silhouette IoU {iou:.3f}") - if iou < 0.85: - print("WARNING: low IoU — the reconstruction is not tracking the input") + if iou < a.min_iou: + # completing is not succeeding — a run can finish cleanly and still + # have produced a blob that does not match the input at all + print(f"FAIL: IoU {iou:.3f} < {a.min_iou} — not tracking the input") + if a.json: + Path(a.json).write_text(json.dumps(info, indent=2)) sys.exit(1) + if a.json: + Path(a.json).write_text(json.dumps(info, indent=2)) + if __name__ == "__main__": main()