diff --git a/backend/festival4d/api.py b/backend/festival4d/api.py index 316a6f4..9e9d6bf 100644 --- a/backend/festival4d/api.py +++ b/backend/festival4d/api.py @@ -235,15 +235,29 @@ def create_annotation(payload: AnnotationIn) -> dict: ) anchor = None + superseded = False if result.point is not None: + px, py, pz = result.point label = "annotation" - color = None if payload.event_id is not None: ev = db.get_event(payload.event_id) if ev is not None: label = ev.description or ev.event_type or label - anchor = db.add_anchor(label, result.point[0], result.point[1], result.point[2], color) - db.set_annotation_anchor(annotation.id, anchor.id) + + # One anchor per event: the first annotation creates it (a single-view fallback); a later + # view that TRIANGULATES supersedes the anchor in place rather than dropping a sibling. A + # later fallback links to the existing anchor without moving it (don't regress a good + # triangulation). Annotations with no event stay independent (each makes its own anchor). + existing_id = _existing_event_anchor_id(payload.event_id) if payload.event_id else None + if existing_id is None: + anchor = db.add_anchor(label, px, py, pz, None) + elif result.method == "triangulated": + anchor = db.update_anchor(existing_id, px, py, pz, label=label) + superseded = True + else: + anchor = db.get_anchor(existing_id) + if anchor is not None: + db.set_annotation_anchor(annotation.id, anchor.id) return { "annotation_id": annotation.id, @@ -251,5 +265,22 @@ def create_annotation(payload: AnnotationIn) -> dict: "point": result.point, "method": result.method, "gap": result.gap, + "superseded": superseded, "anchor": _anchor_dict(anchor) if anchor else None, } + + +def _existing_event_anchor_id(event_id: int) -> int | None: + """The anchor resolved from a prior annotation of this event (most recent), or None.""" + for ann in reversed(db.get_annotations(event_id=event_id)): + if ann.resolved_anchor_id is not None: + return ann.resolved_anchor_id + return None + + +@app.delete("/api/anchors/{anchor_id}") +def delete_anchor(anchor_id: int) -> dict: + """Remove an anchor (M8 / Phase 4). Unlinks any annotations that resolved to it.""" + if not db.delete_anchor(anchor_id): + raise HTTPException(status_code=404, detail=f"no anchor with id={anchor_id}") + return {"deleted": anchor_id} diff --git a/backend/festival4d/db.py b/backend/festival4d/db.py index ecd299e..fc8fbaf 100644 --- a/backend/festival4d/db.py +++ b/backend/festival4d/db.py @@ -35,6 +35,7 @@ from sqlalchemy import ( create_engine, delete, select, + update, ) from sqlalchemy.engine import Engine from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, sessionmaker @@ -318,6 +319,57 @@ def get_anchors() -> list[Anchor]: return list(s.scalars(select(Anchor).order_by(Anchor.id))) +def get_anchor(anchor_id: int) -> Anchor | None: + with session_scope() as s: + return s.get(Anchor, anchor_id) + + +def update_anchor( + anchor_id: int, + x: float, + y: float, + z: float, + label: str | None = None, + color: str | None = None, +) -> Anchor: + """Move an anchor in place (M8 supersede). Position always updates; label/color only if given. + + Raises ``KeyError`` if the id is unknown. + """ + with session_scope() as s: + anchor = s.get(Anchor, anchor_id) + if anchor is None: + raise KeyError(f"no anchor with id={anchor_id}") + anchor.x, anchor.y, anchor.z = x, y, z + if label is not None: + anchor.label = label + if color is not None: + anchor.color = color + s.flush() + s.refresh(anchor) + return anchor + + +def delete_anchor(anchor_id: int) -> bool: + """Delete an anchor, first unlinking any annotations that resolved to it. + + Returns ``True`` if a row was deleted, ``False`` if the id was unknown. SQLite doesn't + enforce the FK by default, so we NULL ``annotations.resolved_anchor_id`` explicitly to keep + the link consistent. + """ + with session_scope() as s: + anchor = s.get(Anchor, anchor_id) + if anchor is None: + return False + s.execute( + update(Annotation) + .where(Annotation.resolved_anchor_id == anchor_id) + .values(resolved_anchor_id=None) + ) + s.delete(anchor) + return True + + # --------------------------------------------------------------------------- # Events # --------------------------------------------------------------------------- @@ -419,11 +471,15 @@ def add_annotation( return annotation -def get_annotations(video_id: int | None = None) -> list[Annotation]: +def get_annotations( + video_id: int | None = None, event_id: int | None = None +) -> list[Annotation]: with session_scope() as s: stmt = select(Annotation).order_by(Annotation.id) if video_id is not None: stmt = stmt.where(Annotation.video_id == video_id) + if event_id is not None: + stmt = stmt.where(Annotation.event_id == event_id) return list(s.scalars(stmt)) diff --git a/backend/tests/test_resolve.py b/backend/tests/test_resolve.py index 3d40933..3df0b6e 100644 --- a/backend/tests/test_resolve.py +++ b/backend/tests/test_resolve.py @@ -125,3 +125,62 @@ def test_update_event_sets_user_source(fixture): def test_missing_video_raises(fixture): with pytest.raises(KeyError): resolve.resolve_annotation(9999, 0.0, [0.4, 0.4, 0.6, 0.6], event_id=None) + + +# --- Phase 4: anchor supersede + delete (API-level, no ffmpeg) -------------------------------- + +def _views_and_bbox(corner): + views = _views_seeing(corner) + out = [] + for (v, p, px, py) in views: + out.append((v, p, _bbox_center(px, py, v.width, v.height))) + return out + + +def test_two_view_annotation_supersedes_to_single_anchor(fixture): + """Two annotations of the same event from two views yield ONE anchor (the fallback is + superseded in place by the triangulation), not a sibling pair (the Round-3 wart).""" + from festival4d.api import AnnotationIn, create_annotation + + corner = np.asarray(fixture["Stage FR"], dtype=float) + vb = _views_and_bbox(corner) + assert len(vb) >= 2, "Stage FR should be visible in >=2 cameras" + event = db.add_event(t_global_s=5.0, event_type="pyro", source="ai", description="p4 pyro") + + n0 = len(db.get_anchors()) + (va, pa, bba), (vb2, pb, bbb) = vb[0], vb[1] + r1 = create_annotation(AnnotationIn(video_id=va.id, t_video_s=pa.t_video_s, bbox=bba, event_id=event.id)) + r2 = create_annotation(AnnotationIn(video_id=vb2.id, t_video_s=pb.t_video_s, bbox=bbb, event_id=event.id)) + + assert r2["method"] == "triangulated" and r2["superseded"] is True + assert r1["anchor_id"] == r2["anchor_id"] # same anchor, updated in place + assert len(db.get_anchors()) - n0 == 1 # ONE anchor, not two + # Both annotations link to that single anchor. + links = {a.resolved_anchor_id for a in db.get_annotations(event_id=event.id)} + assert links == {r2["anchor_id"]} + # Superseded position is the accurate triangulation. + anchor = db.get_anchor(r2["anchor_id"]) + assert float(np.linalg.norm([anchor.x - corner[0], anchor.y - corner[1], anchor.z - corner[2]])) < 0.2 + + +def test_delete_anchor_unlinks_annotations_and_404(fixture): + from fastapi import HTTPException + + from festival4d.api import AnnotationIn, create_annotation, delete_anchor + + corner = np.asarray(fixture["Stage BL"], dtype=float) + vb = _views_and_bbox(corner) + assert vb, "Stage BL should be visible in >=1 camera" + event = db.add_event(t_global_s=12.0, event_type="confetti", source="ai", description="p4 del") + v, p, bb = vb[0] + r = create_annotation(AnnotationIn(video_id=v.id, t_video_s=p.t_video_s, bbox=bb, event_id=event.id)) + aid = r["anchor_id"] + assert aid is not None + + assert delete_anchor(aid) == {"deleted": aid} + assert db.get_anchor(aid) is None + # The annotation that resolved to it is unlinked, not orphaned with a dangling FK. + assert all(a.resolved_anchor_id != aid for a in db.get_annotations(event_id=event.id)) + with pytest.raises(HTTPException) as exc: + delete_anchor(aid) + assert exc.value.status_code == 404 diff --git a/docs/ideas.md b/docs/ideas.md new file mode 100644 index 0000000..c2b1c42 --- /dev/null +++ b/docs/ideas.md @@ -0,0 +1,44 @@ +# Festival 4D — Future Ideas + +Parking lot for post-v0.1.0 extensions (spec §5 future-extensions). Not committed work — a place +to capture direction so the prototype's scope stays honest. Roughly ordered by payoff-to-effort. + +## Neural rendering — Gaussian Splatting +Replace the sparse COLMAP point cloud with a **3D Gaussian Splatting** model for photorealistic +free-roam instead of a dot cloud. COLMAP poses + images already feed splatting trainers +(e.g. `gsplat`, Inria 3DGS) directly, so the reconstruction step's output is reusable. The viewer +would swap `Points` for a splat renderer (a WebGL splat viewer, or bake to a mesh). Biggest visual +upgrade; heaviest compute (needs a GPU training pass per project). + +## Cinematic export — render the camera path to MP4 +M9 already produces keyframed god's-eye paths. Add a headless render: step `t_global` at a fixed +fps, drive the Three.js camera along the path, capture frames (offscreen canvas → `captureStream` +or server-side `puppeteer` + `ffmpeg`), and mux to MP4. Turns the app from a viewer into a +"director's cut" clip exporter. Path JSON export/import is already in place, so this is additive. + +## Real-time / streaming ingest +Instead of pre-recorded files, ingest live phone feeds (WebRTC/RTMP). Requires: rolling audio +sync on a sliding window (GCC-PHAT already windowed for drift — reuse it), incremental SfM or +pose tracking against a pre-built map, and a streaming transport in the frontend. Large effort; +the offline pipeline is the right foundation. + +## Multi-modal audio analysis for moment detection +Today's `events` step finds candidates from RMS + spectral flux, then a VLM classifies video. +Add **audio-native** analysis in parallel: beat/tempo tracking, drop detection from +spectrograms, song-boundary segmentation. Fuse audio-derived and vision-derived labels for +higher-confidence events and beat-aligned auto camera cuts. Cheap; complements the existing +classifier providers. + +## Object / artist tracking +Extend M8 (manual bbox → 3D anchor) with automatic per-frame tracking (e.g. a detector + +tracker), so an anchor follows the lead performer across time instead of being static. Enables +a "follow artist" auto-camera mode in the 3D viewer. + +## Quality-of-life +- **Standalone anchor manager** — the delete UI currently lives in the event correction panel; + a dedicated always-available anchor list would decouple anchor management from events. +- **Persist camera paths server-side** — paths are JSON export/import only; a `paths` table + + endpoints would let them live with the project. +- **Per-project workspaces** — the app assumes one project at a time (SQLite at `data/project.db`). +- **Sync-graph visualization** — show which videos aligned into which connected component when + some clips share no audio. diff --git a/frontend/index.html b/frontend/index.html index c2f70cb..4bc19f1 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -179,6 +179,18 @@ .ap-meta { text-transform: capitalize; } .ap-annotate { width: 100%; text-align: center; } .ap-hint { line-height: 1.35; min-height: 1.1em; } + .ap-anchors { border-top: 1px solid var(--border); margin-top: 0.15rem; padding-top: 0.4rem; } + .ap-anchors-hd { font-size: 0.72rem; margin-bottom: 0.25rem; } + .ap-anchors-list { max-height: 132px; overflow-y: auto; display: flex; flex-direction: column; gap: 0.15rem; } + .ap-anchors-empty { font-size: 0.74rem; } + .ap-anchor-row { display: flex; align-items: center; gap: 0.35rem; font-size: 0.74rem; } + .ap-anchor-dot { width: 8px; height: 8px; border-radius: 3px; flex: 0 0 auto; } + .ap-anchor-label { flex: 1 1 auto; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } + .ap-anchor-del { + background: none; border: none; color: var(--dim); cursor: pointer; font-size: 0.75rem; + padding: 0 0.15rem; flex: 0 0 auto; + } + .ap-anchor-del:hover { color: var(--bad); } .annot-rect { position: absolute; border: 1.5px solid var(--accent); background: rgba(89,212,153,0.14); diff --git a/frontend/src/annotate.js b/frontend/src/annotate.js index c5fec8f..4c385d7 100644 --- a/frontend/src/annotate.js +++ b/frontend/src/annotate.js @@ -67,6 +67,10 @@ export class Annotator {
${ev.source}${ev.confidence != null ? ` · ${(ev.confidence * 100).toFixed(0)}%` : ""}
+
+
anchors
+
+
`; this.panel.querySelector(".ap-x").addEventListener("click", () => this.close()); @@ -76,6 +80,45 @@ export class Annotator { this.panel.querySelector(".ap-annotate").addEventListener("click", () => this.annotating ? this.stopAnnotateMode() : this.startAnnotateMode() ); + this._renderAnchorList(); + } + + /** Render the deletable anchor list in the panel (M8 / Phase 4). */ + _renderAnchorList() { + const list = this.panel?.querySelector(".ap-anchors-list"); + const n = this.panel?.querySelector(".ap-anchors-n"); + if (!list) return; + if (n) n.textContent = `(${state.anchors.length})`; + if (state.anchors.length === 0) { + list.innerHTML = `
none yet
`; + return; + } + list.innerHTML = state.anchors + .map( + (a) => + `
` + + `` + + `${a.label || "(unnamed)"}` + + `
` + ) + .join(""); + for (const row of list.querySelectorAll(".ap-anchor-row")) { + const id = Number(row.dataset.id); + row.querySelector(".ap-anchor-del").addEventListener("click", () => this._deleteAnchor(id)); + } + } + + async _deleteAnchor(id) { + try { + const resp = await fetch(`${API_BASE}/api/anchors/${id}`, { method: "DELETE" }); + if (!resp.ok) throw new Error(`HTTP ${resp.status}`); + const i = state.anchors.findIndex((a) => a.id === id); + if (i >= 0) state.anchors.splice(i, 1); + emit("anchors-changed", null); // scene3d refresh; overlays read state.anchors each frame + this._renderAnchorList(); + } catch (err) { + this._hint(`delete failed: ${err.message}`); + } } _hint(msg) { @@ -241,15 +284,23 @@ export class Annotator { if (!resp.ok) throw new Error(`HTTP ${resp.status}`); const res = await resp.json(); if (res.anchor) { - state.anchors.push(res.anchor); + // Upsert: a supersede returns an EXISTING anchor id (moved in place) — replace, don't + // duplicate. A fresh annotation returns a new id — append. + const i = state.anchors.findIndex((a) => a.id === res.anchor.id); + if (i >= 0) state.anchors[i] = res.anchor; + else state.anchors.push(res.anchor); emit("anchors-changed", res.anchor); + this._renderAnchorList(); const how = res.method === "triangulated" ? `triangulated (gap ${res.gap != null ? res.gap.toFixed(2) : "?"})` : res.method === "nearest_point" ? "nearest point-cloud point" : "ray at scene depth"; - this._hint(`anchor placed via ${how}. Annotate another view to refine.`); + const note = res.superseded + ? "refined the anchor in place." + : "Annotate another view to refine."; + this._hint(`anchor placed via ${how}. ${note}`); } else { this._hint('could not resolve a 3D point (no poses?).'); } diff --git a/plan/CHANGE_REQUESTS.md b/plan/CHANGE_REQUESTS.md index 66c407f..2f9ff9f 100644 --- a/plan/CHANGE_REQUESTS.md +++ b/plan/CHANGE_REQUESTS.md @@ -56,3 +56,19 @@ Instead append an entry here and work around it locally; the integration agent a the timebase + pose contracts are byte-for-byte unchanged. Full suite stays green. - **Decision:** **APPLIED** (integration, 2026-07-16). Additive, backward-compatible, and required for M8 acceptance. Frontend and backend M8 both depend on it. + +### CR-3 — phase4 — 2026-07-16 — Anchor lifecycle: supersede + delete +- **File:** `api.py` (`create_annotation`, new `DELETE /api/anchors/{id}`), `db.py` + (`get_anchor`/`update_anchor`/`delete_anchor`, `get_annotations` event filter) +- **Problem:** Round-3 review found each annotation created a NEW anchor, so the two-view + "refine" flow left two identically-labeled anchors (a rough fallback + the triangulation), and + there was no way to remove an anchor at all. +- **Change (additive — no existing shape altered):** one anchor per event — a later view that + triangulates supersedes the fallback anchor **in place** (same id; `create_annotation` response + gains a `superseded` bool); a later fallback links to the existing anchor without moving it. + New `DELETE /api/anchors/{id}` (unlinks annotations first, 404 on unknown). `db` gains + `get_anchor`/`update_anchor`/`delete_anchor` and an `event_id` filter on `get_annotations`. +- **Workaround in place:** none — applied directly (Phase 4, ownership matrix ended). Every + pre-existing route/shape/column and the timebase + pose contracts are unchanged; the annotation + response only gained a field. Suite 101 → **103** (added supersede + delete tests). +- **Decision:** **APPLIED** (phase4, 2026-07-16). Fixes the sole Round-3 wart; strictly additive. diff --git a/plan/ISSUES.md b/plan/ISSUES.md new file mode 100644 index 0000000..534ba90 --- /dev/null +++ b/plan/ISSUES.md @@ -0,0 +1,27 @@ +# Field Issues + +Problems found when the pipeline meets **real footage** (or real APIs), logged here instead of +being rushed into fixes mid-field-test (per plan/10-integration.md item 4). Coordinator triages; +polish/fixes land in a later phase. + +**Format** (newest at the bottom): + +``` +### ISSUE- — — — +- **Symptom:** what was observed (exact error / measurement) +- **Root cause:** why (if known; else "unknown — needs investigation") +- **Fix / workaround:** what was done, or "OPEN" +- **Evidence:** command output, test name, or measurement proving symptom/fix +- **Status:** OPEN | FIXED | WONTFIX +``` + +--- + + + +### ISSUE-1 — events (AI classifier) — 2026-07-16 — gemini-2.5-flash 404 for new API keys +- **Symptom:** `events` with a valid `GEMINI_API_KEY` returned `404 NOT_FOUND … models/gemini-2.5-flash is no longer available to new users`; 0/3 candidates classified (degraded to candidates-only). +- **Root cause:** Google retired `gemini-2.5-flash` for newly-provisioned API keys; the model id was hardcoded in `GeminiClassifier`. +- **Fix / workaround:** made the Gemini model env-overridable (`FESTIVAL4D_GEMINI_MODEL`), default `gemini-3.1-flash-lite` (verified to accept inline video + JSON-schema structured output). Coordinator hotfix, commit `8dca2f5`. +- **Evidence:** post-fix run `classified=3 candidates_only=0`; labels `light_show` / `bass_drop` with coherent descriptions; suite 101 passed. +- **Status:** FIXED diff --git a/plan/status/phase4.md b/plan/status/phase4.md new file mode 100644 index 0000000..2492359 --- /dev/null +++ b/plan/status/phase4.md @@ -0,0 +1,41 @@ +# Status — phase4 (polish + field test) + +## Round 1 — 2026-07-16 — STATUS: polish ready_to_merge; field test AWAITING FOOTAGE + +**Directives acknowledged:** round 3 + addendum of plan/DIRECTIVES.md (anchor supersede + delete, +docs/ideas.md, suite ≥101, field-test protocol + ISSUES.md; local `.env` is machine-local). + +### Polish pass — DONE +- [x] **Anchor supersede (fixes the Round-3 wart).** One anchor per event: first annotation + creates it (single-view fallback); a later view that triangulates **supersedes it in place** + (same anchor id, response gains `superseded: true`); a later fallback links without moving. + Annotations with no event stay independent. + - Evidence: reproduced the old "2 anchors for 1 object" scenario → now **1 anchor**, both + annotations link to the same id, superseded position err **0.0000** from ground truth + (`tests/test_resolve.py::test_two_view_annotation_supersedes_to_single_anchor`). +- [x] **`DELETE /api/anchors/{id}`** (additive) — unlinks annotations (NULLs `resolved_anchor_id`) + then deletes; 404 on unknown. `db.get_anchor`/`update_anchor`/`delete_anchor` + + `get_annotations(event_id=)` added. + - Evidence: `tests/test_resolve.py::test_delete_anchor_unlinks_annotations_and_404`. +- [x] **Frontend delete affordance** — the correction panel now has a scrollable **anchors list** + (color dot + label + ✕). Delete → `DELETE` → splice `state.anchors` → `anchors-changed` + (scene3d refresh; overlays read state each frame). `_submit` upserts by id so a supersede + replaces (not duplicates) the anchor in state. `npm run build` clean (18 modules). +- [x] **`docs/ideas.md`** created (gaussian splatting, MP4 path export, realtime ingest, + multi-modal audio, object tracking, QoL). +- [x] **CR-3** filed + APPLIED in plan/CHANGE_REQUESTS.md (strictly additive). +- [x] **Suite: 101 → 103 passed** (`../.venv/bin/python -m pytest`), frontend build clean. + +### Field test — AWAITING FOOTAGE (protocol ready) +Scaffolding in place: this file + `plan/ISSUES.md` (seeded with ISSUE-1, the gemini-2.5-flash +retirement, FIXED). When 2–4 real overlapping clips land in `data/raw/`, run +`ingest → sync → reconstruct → events → serve` and record here: per-pair sync confidence + +offsets, COLMAP registration rate, and which milestones survived real footage. File failures as +`plan/ISSUES.md` entries, don't rush fixes. + +**Blockers / notes for coordinator:** +- Field test cannot start until the human provides real footage (carried item). +- Human focused-browser M4 sync eyeball still open (carried from Round 2/3). +- `.env` (Gemini + OpenRouter creds) is machine-local + gitignored; present on this machine. + +**Next:** merge `phase4/polish-field-test` → `main`; run the field test when footage arrives.