From 5f14517553547752c8da8cec1dc5e388f071e0e1 Mon Sep 17 00:00:00 2001 From: m3ultra Date: Mon, 3 Aug 2026 18:38:36 +1000 Subject: [PATCH] Fix the silhouette gate: sample the surface, not the vertex list Two bugs in the acceptance test, both found by running it for real. 1. THE GATE DID NOT APPLY TO TEXTURED RUNS. The texture path returned before the check, so the one path most likely to be used for real assets was the only one that could ship a blob silently. The check is now a shared gate() called from both. 2. THE METRIC WAS TESSELLATION-DEPENDENT. It projected the VERTEX LIST, so a decimated mesh sampled its own silhouette more sparsely and scored lower for an identical shape - holes appear inside the outline and count as misses. Measured on two real assets: asset verts vertex-proj surface-sampled 1_img (gate FAILED) 133,842 0.790 0.911 0_img (gate passed) 227,546 0.965 0.969 The dense mesh barely moves; the sparse one jumps 0.12. That is the metric measuring tessellation, not accuracy - and at min_iou 0.85 it had just rejected a good reconstruction. Overlay confirmed it: the "missing" region was speckle inside the silhouette, not a wrong shape. Now samples 3M points uniformly over the surface, so density is a constant of the metric rather than a property of the mesh. Worth stating plainly: the gate caught a real problem on its first live failure - just not the one it reported. A quality gate that is itself unvalidated is a liability, and this one needed the same "measure it, do not reason about it" treatment as the model. Co-Authored-By: Claude Fable 5 --- scripts/image_to_mesh.py | 60 ++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/scripts/image_to_mesh.py b/scripts/image_to_mesh.py index ab8b398..8bc5c93 100644 --- a/scripts/image_to_mesh.py +++ b/scripts/image_to_mesh.py @@ -27,10 +27,17 @@ from pixal3d_mlx.pipeline import DEFAULT_FOV, image_to_mesh # noqa: E402 DEFAULT_IMAGE = REPO / "upstream" / "Pixal3D" / "assets" / "images" / "0_img.png" -def silhouette_iou(vertices, image_path, fov, res=512): +def silhouette_iou(mesh_or_vertices, image_path, fov, res=512, samples=3_000_000): """Re-project the mesh through the generating camera; IoU against the input matte. - Vertices MUST be rotated into the camera frame first — o_voxel returns them in the + Points are SAMPLED UNIFORMLY OVER THE SURFACE, not taken from the vertex list. + Projecting vertices makes the score depend on tessellation: the same shape scored + 0.965 at 227k vertices and 0.790 at 134k, purely because a sparser point cloud + leaves holes inside its own silhouette. That would fail good assets for the crime + of being decimated. Fixed-count surface sampling makes density a constant of the + metric instead of a property of the mesh. + + Points MUST be rotated into the camera frame — o_voxel returns geometry in the voxel-grid frame while ProjGrid rotates its lattice before projecting. """ from PIL import Image @@ -44,9 +51,14 @@ def silhouette_iou(vertices, image_path, fov, res=512): return None matte = np.asarray(preprocess_image(img).convert("L").resize((res, res))) > 8 + if isinstance(mesh_or_vertices, trimesh.Trimesh): + pts3, _ = trimesh.sample.sample_surface(mesh_or_vertices, samples) + else: + pts3 = np.asarray(mesh_or_vertices) # bare vertices: caller accepts the bias + tm = _FRONT_VIEW.copy() tm[1, 3] = -distance_from_fov(fov, 1.0, res) - pts = to_camera_frame(vertices).astype(np.float32)[None] + pts = to_camera_frame(pts3).astype(np.float32)[None] px, _, _ = project_points(mx.array(pts), mx.array(tm[None]), fov, res) px = np.asarray(px)[0].astype(int) @@ -57,6 +69,26 @@ def silhouette_iou(vertices, image_path, fov, res=512): return (proj & matte).sum() / (proj | matte).sum(), proj, matte +def gate(mesh, a, info): + """Silhouette check + non-zero exit. Shared by the textured and geometry paths.""" + if a.min_iou <= 0: + return + got = silhouette_iou(mesh, a.image, a.fov) + if got is None: + print("(input has no alpha matte — skipping silhouette check)") + return + iou = float(got[0]) + info["silhouette_iou"] = round(iou, 4) + print(f"silhouette IoU {iou:.3f}") + if iou < a.min_iou: + # completing is not succeeding — a run can finish cleanly and still have + # produced a blob that does not match the input at all + print(f"FAIL: IoU {iou:.3f} < {a.min_iou} — not tracking the input") + if a.json: + Path(a.json).write_text(json.dumps(info, indent=2)) + sys.exit(1) + + def main(): ap = argparse.ArgumentParser() ap.add_argument("image", nargs="?", default=str(DEFAULT_IMAGE)) @@ -124,6 +156,11 @@ def main(): mesh.export(a.output) info["faces"], info["vertices"] = len(mesh.faces), len(mesh.vertices) print(f"\nTOTAL {info['seconds']}s peak {info['peak_gb']} GB -> {a.output}") + + # The gate applies to TEXTURED runs too. Skipping it here would mean the + # textured path — the one most likely to be used for real assets — is the only + # one that can ship a blob silently. + gate(mesh, a, info) if a.json: Path(a.json).write_text(json.dumps(info, indent=2)) return @@ -144,25 +181,12 @@ def main(): print(f"\nTOTAL {info['seconds']}s peak {info['peak_gb']} GB " f"{len(mesh.faces):,} faces -> {a.output}") - if a.min_iou > 0: - got = silhouette_iou(mesh.vertices, a.image, a.fov) - if got is None: - print("(input has no alpha matte — skipping silhouette check)") - else: - iou = float(got[0]) - info["silhouette_iou"] = round(iou, 4) - print(f"silhouette IoU {iou:.3f}") - if iou < a.min_iou: - # completing is not succeeding — a run can finish cleanly and still - # have produced a blob that does not match the input at all - print(f"FAIL: IoU {iou:.3f} < {a.min_iou} — not tracking the input") - if a.json: - Path(a.json).write_text(json.dumps(info, indent=2)) - sys.exit(1) + gate(mesh, a, info) if a.json: Path(a.json).write_text(json.dumps(info, indent=2)) + if __name__ == "__main__": main()