From cf6a6fb2c83c272a3eea46f0ce58989dd168af70 Mon Sep 17 00:00:00 2001 From: m3ultra Date: Thu, 16 Jul 2026 00:45:55 +1000 Subject: [PATCH] Foundation (M0 + M3 + frozen contracts): scaffold, synthetic fixtures, full API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit What now works: - M0 scaffold: pyproject (all spec deps), uv/py3.12 env, `python -m festival4d` CLI registering synthetic|ingest|sync|reconstruct|events|serve. Vite hello page. - Synthetic fixture (synthetic.py): 3 shifted-audio videos (offsets 0/+1370/-842 ms), camera-arc poses, stage point cloud -> points.ply, seeded events + anchors, ground_truth.json. `python -m festival4d synthetic` populates data/ + DB. - DB schema exactly per spec §2 (db.py) + CRUD helpers all lanes use. - M3 API (api.py) full against synthetic data: manifest/poses/pointcloud/anchors/ events/detect/annotations; Range-capable video serving (206 verified); CORS for any localhost origin. - Frozen geometry contract: geometry.colmap_to_threejs (M5 math) + unit test (3 known vectors, random round-trip, scipy oracle); mirrored frontend/src/lib/pose.js with identical POSE_TEST_VECTORS. Lane-B stubs: slerp_pose, ray_from_pixel, triangulate_rays, nearest_point_on_ray. - Classifier contract (events_ai.py): MomentClassification model + MomentClassifier protocol + Gemini/Claude/Local provider stubs. - Lane-owned modules stubbed with final signatures (ingest, audio_sync, frames, sfm, events_ai); cli/api catch NotImplementedError and degrade gracefully. - plan/CHANGE_REQUESTS.md created; plan/status/foundation.md updated. Acceptance: pytest 24 passed; serve endpoints verified via curl + browser (video seek, manifest fetch cross-origin, pose.js self-test, 0 console errors). Co-Authored-By: Claude Opus 4.8 --- README.md | 57 + backend/festival4d/__init__.py | 8 + backend/festival4d/__main__.py | 6 + backend/festival4d/api.py | 209 ++++ backend/festival4d/audio_sync.py | 66 + backend/festival4d/cli.py | 130 ++ backend/festival4d/config.py | 78 ++ backend/festival4d/db.py | 397 ++++++ backend/festival4d/events_ai.py | 144 +++ backend/festival4d/frames.py | 28 + backend/festival4d/geometry.py | 217 ++++ backend/festival4d/ingest.py | 31 + backend/festival4d/sfm.py | 71 ++ backend/festival4d/synthetic.py | 556 +++++++++ backend/tests/conftest.py | 13 + backend/tests/test_api.py | 106 ++ backend/tests/test_geometry.py | 110 ++ backend/tests/test_synthetic.py | 118 ++ frontend/index.html | 43 + frontend/package-lock.json | 1152 +++++++++++++++++ frontend/package.json | 17 + frontend/src/annotate.js | 6 + frontend/src/camPath.js | 6 + frontend/src/lib/pose.js | 143 +++ frontend/src/main.js | 91 ++ frontend/src/overlays.js | 6 + frontend/src/scene3d.js | 6 + frontend/src/state.js | 9 + frontend/src/timeline.js | 6 + frontend/src/transport.js | 8 + frontend/src/videoGrid.js | 5 + plan/CHANGE_REQUESTS.md | 20 + plan/status/foundation.md | 31 + pyproject.toml | 38 + uv.lock | 1998 ++++++++++++++++++++++++++++++ 35 files changed, 5930 insertions(+) create mode 100644 README.md create mode 100644 backend/festival4d/__init__.py create mode 100644 backend/festival4d/__main__.py create mode 100644 backend/festival4d/api.py create mode 100644 backend/festival4d/audio_sync.py create mode 100644 backend/festival4d/cli.py create mode 100644 backend/festival4d/config.py create mode 100644 backend/festival4d/db.py create mode 100644 backend/festival4d/events_ai.py create mode 100644 backend/festival4d/frames.py create mode 100644 backend/festival4d/geometry.py create mode 100644 backend/festival4d/ingest.py create mode 100644 backend/festival4d/sfm.py create mode 100644 backend/festival4d/synthetic.py create mode 100644 backend/tests/conftest.py create mode 100644 backend/tests/test_api.py create mode 100644 backend/tests/test_geometry.py create mode 100644 backend/tests/test_synthetic.py create mode 100644 frontend/index.html create mode 100644 frontend/package-lock.json create mode 100644 frontend/package.json create mode 100644 frontend/src/annotate.js create mode 100644 frontend/src/camPath.js create mode 100644 frontend/src/lib/pose.js create mode 100644 frontend/src/main.js create mode 100644 frontend/src/overlays.js create mode 100644 frontend/src/scene3d.js create mode 100644 frontend/src/state.js create mode 100644 frontend/src/timeline.js create mode 100644 frontend/src/transport.js create mode 100644 frontend/src/videoGrid.js create mode 100644 plan/CHANGE_REQUESTS.md create mode 100644 plan/status/foundation.md create mode 100644 pyproject.toml create mode 100644 uv.lock diff --git a/README.md b/README.md new file mode 100644 index 0000000..6658709 --- /dev/null +++ b/README.md @@ -0,0 +1,57 @@ +# Festival 4D + +Turn multiple fan-shot smartphone videos of the same concert into a synchronized, +explorable **4D experience**: time-aligned multi-video playback, 3D scene reconstruction +with camera poses, a free-roam "god's eye" viewer, AR-style overlays projected onto each +video, and AI-tagged moments on a shared timeline. + +> **Status: foundation phase.** The scaffold, database, frozen contracts (API, pose math, +> DB schema), and a synthetic fixture generator are in place. Feature lanes (media sync, +> reconstruction, viewer, AI events) build on top. The full README with the real-footage +> workflow is written in the integration phase (M9). See +> [`OPUS_BUILD_INSTRUCTIONS.md`](OPUS_BUILD_INSTRUCTIONS.md) for the canonical spec and +> [`plan/`](plan/) for the execution plan. + +## Prerequisites + +- **Python 3.11+** and [`uv`](https://docs.astral.sh/uv/) (or venv + pip) +- **ffmpeg** / **ffprobe** on your `PATH` (required) +- **COLMAP** (optional — reconstruction degrades gracefully without it) +- A classifier API key (optional — `GEMINI_API_KEY` by default; see `events_ai.py`) +- **Node 18+** for the frontend + +## Quickstart (synthetic demo — no footage needed) + +```bash +# 1. backend env +uv venv --python 3.12 +uv pip install -e ".[dev]" + +# 2. generate the synthetic fixture project (fake videos + poses + point cloud + events) +uv run python -m festival4d synthetic + +# 3. serve the API (http://127.0.0.1:8000) +uv run python -m festival4d serve + +# 4. in another terminal, the frontend +cd frontend +npm install +npm run dev # http://localhost:5173 +``` + +## Backend CLI + +``` +python -m festival4d synthetic # generate the synthetic fixture (M0) +python -m festival4d ingest # probe videos + extract audio (lane A / M1) +python -m festival4d sync # GCC-PHAT audio alignment (lane A / M1) +python -m festival4d reconstruct # COLMAP SfM + pose export (lane B / M2) +python -m festival4d events # audio candidates + AI classify (lane D / M7) +python -m festival4d serve # FastAPI app (M3) +``` + +## Tests + +```bash +uv run pytest +``` diff --git a/backend/festival4d/__init__.py b/backend/festival4d/__init__.py new file mode 100644 index 0000000..25abc48 --- /dev/null +++ b/backend/festival4d/__init__.py @@ -0,0 +1,8 @@ +"""Festival 4D — synchronized, explorable 4D concert replay. + +See ``OPUS_BUILD_INSTRUCTIONS.md`` for the canonical spec and ``plan/`` for the +parallel execution plan. This package is the backend: CLI, database, synthetic +fixture generator, media/reconstruction/AI pipelines, and the FastAPI app. +""" + +__version__ = "0.1.0" diff --git a/backend/festival4d/__main__.py b/backend/festival4d/__main__.py new file mode 100644 index 0000000..dc1a5ba --- /dev/null +++ b/backend/festival4d/__main__.py @@ -0,0 +1,6 @@ +"""Enable ``python -m festival4d ``.""" + +from festival4d.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/festival4d/api.py b/backend/festival4d/api.py new file mode 100644 index 0000000..dbaf02b --- /dev/null +++ b/backend/festival4d/api.py @@ -0,0 +1,209 @@ +"""FastAPI app (spec M3). FROZEN after foundation — lanes never edit routes or shapes. + +Serves the synthetic project fully so lane C can treat this API as finished: + + GET /api/manifest -> {videos:[{id,filename,url,duration_s,fps,width,height, + offset_ms,drift_ppm}], t_global_max, has_poses} + GET /api/videos/{id}/poses -> [{frame_idx,t_video_s,q:[w,x,y,z],t:[x,y,z], + intrinsics:{fx,fy,cx,cy},registered}] + GET /api/pointcloud -> points.ply (binary; Range-capable) + GET /api/anchors -> [...] POST /api/anchors + GET /api/events -> [...] + POST /api/events/detect {t_global_s?, window_s?} -> runs M7 detection + POST /api/annotations {video_id, t_video_s, bbox:[x0,y0,x1,y1]} -> {anchor_id?, point?} + +Source videos in ``data/raw`` are mounted at ``/media`` via Starlette ``StaticFiles``, +which supports HTTP Range (required for ``