From f12b6e0847fe536f72d01cbfd179cf61671419d4 Mon Sep 17 00:00:00 2001 From: m3ultra Date: Thu, 16 Jul 2026 22:32:01 +1000 Subject: [PATCH] Splat upgrade (ideas.md #1) + MODELBEAST crossover - GET /api/splat serves data/work/splat.ply; manifest reports has_splat - scene3d: lazy-load @mkkellogg/gaussian-splats-3d DropInViewer when a splat exists (rigs/anchors/paths overlay as before); point-cloud fallback intact - scripts/splat_via_modelbeast.sh: train the splat on the MODELBEAST fleet (colmap_poses -> brush_train via the mb queue) and drop it in place - docs/modelbeast-crossover.md: full crossover map (fleet compute, volumetric capture w/ CorridorKey + audio sync, per-moment splats) Verified in-browser: 32MB room splat rendered via /api/splat with overlays live. --- .claude/launch.json | 19 ++++++++++++++++ backend/festival4d/api.py | 18 +++++++++++++++ backend/festival4d/config.py | 1 + docs/modelbeast-crossover.md | 24 ++++++++++++++++++++ frontend/package-lock.json | 10 +++++++++ frontend/package.json | 1 + frontend/src/main.js | 1 + frontend/src/scene3d.js | 28 +++++++++++++++++++++++ frontend/src/state.js | 1 + scripts/splat_via_modelbeast.sh | 40 +++++++++++++++++++++++++++++++++ 10 files changed, 143 insertions(+) create mode 100644 .claude/launch.json create mode 100644 docs/modelbeast-crossover.md create mode 100755 scripts/splat_via_modelbeast.sh diff --git a/.claude/launch.json b/.claude/launch.json new file mode 100644 index 0000000..09f1751 --- /dev/null +++ b/.claude/launch.json @@ -0,0 +1,19 @@ +{ + "version": "0.0.1", + "configurations": [ + { + "name": "festifun-api", + "runtimeExecutable": "uv", + "runtimeArgs": ["run", "python", "-m", "festival4d", "serve"], + "port": 8000, + "autoPort": false + }, + { + "name": "festifun-web", + "runtimeExecutable": "npm", + "runtimeArgs": ["run", "dev", "--prefix", "frontend"], + "port": 5173, + "autoPort": false + } + ] +} diff --git a/backend/festival4d/api.py b/backend/festival4d/api.py index 9e9d6bf..1d11158 100644 --- a/backend/festival4d/api.py +++ b/backend/festival4d/api.py @@ -146,6 +146,7 @@ def manifest() -> dict: "videos": [_video_dict(v) for v in videos], "t_global_max": t_global_max, "has_poses": db.has_poses(), + "has_splat": config.SPLAT_PLY.exists(), } @@ -167,6 +168,23 @@ def pointcloud() -> FileResponse: ) +@app.get("/api/splat") +def splat() -> FileResponse: + """Optional 3DGS splat of the scene (see docs/modelbeast-crossover.md). + + Trained externally from the same COLMAP reconstruction (e.g. MODELBEAST's + brush_train) and dropped at ``data/work/splat.ply``. The viewer prefers it + over the sparse point cloud when present. + """ + if not config.SPLAT_PLY.exists(): + raise HTTPException(status_code=404, detail="no splat (train one — see docs/modelbeast-crossover.md)") + return FileResponse( + config.SPLAT_PLY, + media_type="application/octet-stream", + filename="splat.ply", + ) + + @app.get("/api/anchors") def get_anchors() -> list[dict]: return [_anchor_dict(a) for a in db.get_anchors()] diff --git a/backend/festival4d/config.py b/backend/festival4d/config.py index 90af265..492bd09 100644 --- a/backend/festival4d/config.py +++ b/backend/festival4d/config.py @@ -40,6 +40,7 @@ COLMAP_DIR = WORK_DIR / "colmap" # COLMAP workspace DB_PATH = DATA_DIR / "project.db" # SQLite (gitignored) POINTS_PLY = WORK_DIR / "points.ply" # reconstructed / synthetic point cloud +SPLAT_PLY = WORK_DIR / "splat.ply" # optional 3DGS splat (trained externally, e.g. via MODELBEAST — see docs/modelbeast-crossover.md); viewer prefers it over the point cloud SYNC_JSON = WORK_DIR / "sync.json" # exported sync solution (lane A) GROUND_TRUTH_JSON = WORK_DIR / "ground_truth.json" # synthetic fixture ground truth diff --git a/docs/modelbeast-crossover.md b/docs/modelbeast-crossover.md new file mode 100644 index 0000000..191c5d9 --- /dev/null +++ b/docs/modelbeast-crossover.md @@ -0,0 +1,24 @@ +# Festival 4D × MODELBEAST — crossover map + +*2026-07-16. Both projects live on the same fleet (M3 Ultra 256GB primary, M1 Ultra 128GB, M4 Pro 24GB — see MODELBEAST's `CLUSTER.md`). They stay separate products — asset factory vs. experience viewer — but share infrastructure and pipelines.* + +## Shipped: the splat upgrade (ideas.md #1) ✅ + +`docs/ideas.md`'s top item ("replace the sparse point cloud with 3D Gaussian Splatting") is now wired: + +- **Backend:** `GET /api/splat` serves `data/work/splat.ply` when present; `/api/manifest` reports `has_splat`. +- **Viewer:** `scene3d.js` lazy-loads `@mkkellogg/gaussian-splats-3d` and renders the splat as a `DropInViewer` in the existing scene (camera rigs, anchors, and fly-paths overlay as before). Falls back to the point cloud when absent or on load error. +- **Training:** `scripts/splat_via_modelbeast.sh` sends this project's frames to the MODELBEAST queue (`colmap_poses` → `brush_train` on whichever GPU node is free) and drops the trained `.ply` into place. No local GPU blocking; the farm handles it. + +## The bigger crossover map + +1. **Fleet compute for every heavy step.** MODELBEAST's queue pools gpu (1 job/Mac) and cpu lanes across all three nodes. Festival 4D's COLMAP/splat/classification steps can submit there (`mb` CLI + token) instead of hogging the local machine. +2. **Poor-man's volumetric capture** (with CorridorKey): several phones film one subject on a green screen → *our GCC-PHAT audio sync* aligns them with no clapboard → CorridorKey (neural keyer, tuned on this fleet — see MODELBEAST's `CORRIDORKEY.md`) extracts true-edge mattes per synced frame → per-instant multi-view cutouts → 3D-gen or per-moment splats. DIY volumetric video from phones; every part exists in-house today. +3. **Per-moment splats = pragmatic "4D".** True temporal 4DGS has no credible Mac port yet (honest limit). But we already detect *moments*: pool all cameras' frames in a ±1s window around each moment → COLMAP → splat → one splat per moment → scrub between them on the timeline. Static splats, time-indexed — feels 4D where it counts. Caveat: needs decent multi-camera coverage; 3 fan phones is thin. +4. **Back-flow to MODELBEAST:** our audio-sync module is the missing piece for any multi-camera capture rig there; our moment detection could auto-pick hero frames for video→3D. +5. **GVM wildcard:** CorridorKey's ~80GB auto-matting module (untested on Metal, fits only the M3 256GB) would key *real concert footage* — performer/crowd separation without green screens — feeding use case 2 with actual festival material. + +## Practical notes +- The `mb` CLI needs `MB_HOST` + `MB_TOKEN` (MODELBEAST Settings → Users). The M3 primary queue is at `100.89.131.57:8777` on the tailnet. +- Splat training cost: a ~40s 4K room clip → ~190 frames → COLMAP (~1 min, GLOMAP) → Brush 30k steps (~15 min on the M3). Concert scenes with crowds will be harder than rooms — expect floaters; SuperSplat can crop/clean. +- License note for use case 2: CorridorKey is CC BY-NC-SA (non-commercial). diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 9320e82..ae038f8 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,6 +8,7 @@ "name": "festival4d-frontend", "version": "0.1.0", "dependencies": { + "@mkkellogg/gaussian-splats-3d": "^0.4.7", "three": "^0.170.0" }, "devDependencies": { @@ -456,6 +457,15 @@ "node": ">=18" } }, + "node_modules/@mkkellogg/gaussian-splats-3d": { + "version": "0.4.7", + "resolved": "https://registry.npmjs.org/@mkkellogg/gaussian-splats-3d/-/gaussian-splats-3d-0.4.7.tgz", + "integrity": "sha512-0vy9/i9sJLFH/v3WJZ4axCsqjkToe8UsV3xY7bvK5EUC0akiRsWZODoCiSzpxhTLNyzSKTsyQKozIFeNA5RWRA==", + "license": "MIT", + "peerDependencies": { + "three": ">=0.160.0" + } + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.62.2", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index c615b81..622b23a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -9,6 +9,7 @@ "preview": "vite preview" }, "dependencies": { + "@mkkellogg/gaussian-splats-3d": "^0.4.7", "three": "^0.170.0" }, "devDependencies": { diff --git a/frontend/src/main.js b/frontend/src/main.js index fafe631..a21ba18 100644 --- a/frontend/src/main.js +++ b/frontend/src/main.js @@ -42,6 +42,7 @@ async function boot() { state.videos = manifest.videos; state.tGlobalMax = manifest.t_global_max; state.hasPoses = manifest.has_poses; + state.hasSplat = !!manifest.has_splat; for (const v of state.videos) state.enabled[v.id] = true; state.audioSourceId = referenceVideoId(); diff --git a/frontend/src/scene3d.js b/frontend/src/scene3d.js index a5e990c..172f2a3 100644 --- a/frontend/src/scene3d.js +++ b/frontend/src/scene3d.js @@ -235,6 +235,13 @@ export class Scene3D { } _loadPointCloud() { + // Prefer a trained 3DGS splat when the backend has one (docs/modelbeast-crossover.md); + // fall back to the sparse COLMAP point cloud. The splat library is heavy, so it's + // lazy-imported only when actually needed. + if (state.hasSplat) { + this._loadSplat(); + return; + } const loader = new PLYLoader(); loader.load( state.apiBase + "/api/pointcloud", @@ -258,6 +265,27 @@ export class Scene3D { ); } + async _loadSplat() { + try { + const GS = await import("@mkkellogg/gaussian-splats-3d"); + // DropInViewer is a THREE.Object3D — it joins the existing scene and + // renders with our camera/controls, so rigs/anchors/paths overlay as usual. + const splatViewer = new GS.DropInViewer({ sharedMemoryForWorkers: false }); + await splatViewer.addSplatScene(state.apiBase + "/api/splat", { + showLoadingUI: false, + progressiveLoad: true, + format: GS.SceneFormat.Ply, + }); + this.splat = splatViewer; + this.scene.add(splatViewer); + emit("pointcloud-loaded", -1); // -1 = splat (no discrete point count) + } catch (err) { + console.warn("[scene3d] splat load failed — falling back to point cloud", err); + state.hasSplat = false; + this._loadPointCloud(); + } + } + // Live pose (COLMAP convention) for a video at the current master time, or null. _livePose(videoId) { const poses = state.poses[videoId]; diff --git a/frontend/src/state.js b/frontend/src/state.js index 8961f3b..748189f 100644 --- a/frontend/src/state.js +++ b/frontend/src/state.js @@ -17,6 +17,7 @@ export const state = { videos: [], // [{id, filename, url, duration_s, fps, width, height, offset_ms, drift_ppm}] tGlobalMax: 0, hasPoses: false, + hasSplat: false, // optional 3DGS splat at /api/splat (preferred over the point cloud) poses: {}, // id -> [{frame_idx, t_video_s, q:[w,x,y,z], t:[x,y,z], intrinsics, registered}] anchors: [], // [{id, label, x, y, z, color}] events: [], // [{id, t_global_s, duration_s, event_type, confidence, description, source}] diff --git a/scripts/splat_via_modelbeast.sh b/scripts/splat_via_modelbeast.sh new file mode 100755 index 0000000..c66a8df --- /dev/null +++ b/scripts/splat_via_modelbeast.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Train a 3DGS splat of this project's scene on the MODELBEAST fleet, and drop +# it where the viewer looks for it (data/work/splat.ply). +# +# Uses the sampled SfM frames (data/work/frames) as input: uploads them to +# MODELBEAST, runs colmap_poses + brush_train on the render farm's queue, and +# downloads the trained splat back. Requires the `mb` CLI env: +# export MB_HOST=http://100.89.131.57:8777 # M3 primary +# export MB_TOKEN=... # from MODELBEAST Settings → Users +# +# (v1 re-runs COLMAP on the farm rather than shipping our local reconstruction — +# simpler and uses the farm's known-good chain; direct dataset handoff is a +# future optimization. See docs/modelbeast-crossover.md.) +set -euo pipefail +cd "$(dirname "$0")/.." + +MB=${MB:-/Users/m3ultra/Documents/MODELBEAST/mb} +FRAMES_DIR=${FESTIVAL4D_DATA_DIR:-data}/work/frames +OUT=${FESTIVAL4D_DATA_DIR:-data}/work + +[ -d "$FRAMES_DIR" ] || { echo "no frames at $FRAMES_DIR — run the reconstruct step first"; exit 1; } +command -v "$MB" >/dev/null || { echo "mb CLI not found (set MB=/path/to/mb)"; exit 1; } + +echo "[splat] uploading frames..." +mapfile -t frames < <(find "$FRAMES_DIR" -name '*.jpg' -o -name '*.png' | sort) +echo "[splat] ${#frames[@]} frames" +ids=$("$MB" upload "${frames[@]}" | awk '{print $1}') +first=$(echo "$ids" | head -1) + +echo "[splat] running colmap_poses on the farm..." +cj=$("$MB" run colmap_poses --asset "$first" $(echo "$ids" | tail -n +2 | sed 's/^/--asset /') | grep -oE '[0-9a-f]{10,}' | head -1) +"$MB" wait "$cj" +ds=$("$MB" assets | awk '/colmap_dataset/ {print $1; exit}') +echo "[splat] dataset: $ds — training splat (brush)..." +bj=$("$MB" run brush_train --asset "$ds" | grep -oE '[0-9a-f]{10,}' | head -1) +"$MB" wait "$bj" --download "$OUT/_splat_dl" +ply=$(find "$OUT/_splat_dl" -name '*.ply' | head -1) +[ -n "$ply" ] || { echo "no .ply came back"; exit 1; } +mv "$ply" "$OUT/splat.ply" && rm -rf "$OUT/_splat_dl" +echo "[splat] done → $OUT/splat.ply (restart/reload the app; the 3D view now renders the splat)"