diff --git a/scenegod/server.py b/scenegod/server.py index 55a0393..6613ae6 100644 --- a/scenegod/server.py +++ b/scenegod/server.py @@ -7,6 +7,7 @@ formats. See PLAN.md §4.3 for the HTTP contract. uvicorn scenegod.server:app --port 8020 # binds 127.0.0.1 """ +import base64 import json import os import re @@ -14,6 +15,8 @@ import shutil import subprocess import threading import time +import urllib.error +import urllib.request import uuid from pathlib import Path @@ -43,6 +46,23 @@ SCENES.mkdir(parents=True, exist_ok=True) RENDERS.mkdir(parents=True, exist_ok=True) HAVE_FFMPEG = shutil.which("ffmpeg") is not None +# MODELBEAST proxy (M4) — off unless SCENEGOD_MB=1. Token read from disk at +# request time, never logged. Contract mirrors MESHGOD meshgod/ops.py. +MB_HOST = os.environ.get("MB_HOST", "http://100.89.131.57:8777") +MB_ENV = Path(os.environ.get("MB_ENV", "~/Documents/backnforth/.env")).expanduser() + + +def _mb_token() -> str: + if os.environ.get("MB_TOKEN"): + return os.environ["MB_TOKEN"] + try: + for line in MB_ENV.read_text().splitlines(): + if line.startswith("MB_TOKEN"): + return line.split("=", 1)[1].strip().strip('"').strip("'") + except OSError: + pass + raise HTTPException(503, f"no MB token (set MB_TOKEN or provide {MB_ENV})") + def reap_renders(max_age_s=48 * 3600): """Delete render dirs older than max_age_s. Called at boot (C5).""" @@ -229,11 +249,22 @@ def _render_dir(rid: str) -> Path: return safe_under(RENDERS, rid) # rid is a hex uuid; guard anyway -def _encode(rid: str, fps: int): +def _encode(rid: str, fps: int, audio: list[dict]): + """audio = [{file: abs path, start: sec, gain: float}] (paths pre-resolved).""" d = _render_dir(rid) _renders[rid] = {"state": "encoding", "log": ""} - cmd = ["ffmpeg", "-y", "-framerate", str(fps), "-i", str(d / "frames" / "%06d.png"), - "-c:v", "libx264", "-pix_fmt", "yuv420p", "-crf", "18", str(d / "out.mp4")] + cmd = ["ffmpeg", "-y", "-framerate", str(fps), "-i", str(d / "frames" / "%06d.png")] + for a in audio: + cmd += ["-i", a["file"]] + cmd += ["-c:v", "libx264", "-pix_fmt", "yuv420p", "-crf", "18"] + if audio: # delay each track to its start, apply gain, mix down to one stereo stream + parts = [f"[{i}:a]adelay={int(a['start']*1000)}:all=1,volume={a['gain']}[a{i}]" + for i, a in enumerate(audio, start=1)] + parts.append("".join(f"[a{i}]" for i in range(1, len(audio) + 1)) + + f"amix=inputs={len(audio)}:normalize=0[aout]") + cmd += ["-filter_complex", ";".join(parts), + "-map", "0:v", "-map", "[aout]", "-c:a", "aac", "-shortest"] + cmd += [str(d / "out.mp4")] try: p = subprocess.run(cmd, capture_output=True, text=True) ok = p.returncode == 0 and (d / "out.mp4").is_file() @@ -270,7 +301,7 @@ async def render_frame(rid: str, n: int, request: Request): @app.post("/render/{rid}/end") -async def render_end(rid: str): +async def render_end(rid: str, request: Request): d = _render_dir(rid) if not (d / "frames").is_dir(): raise HTTPException(404, "unknown renderId") @@ -280,7 +311,18 @@ async def render_end(rid: str): fps = int(json.loads((d / "meta.json").read_text()).get("fps", 30)) except (OSError, json.JSONDecodeError): fps = 30 - threading.Thread(target=_encode, args=(rid, fps), daemon=True).start() + try: + body = json.loads(await request.body() or b"{}") + except json.JSONDecodeError: + body = {} + audio = [] + for a in body.get("audio") or []: # scene JSON audio[] (M4) + f = safe_under(ASSETS, a["path"]) # path-guard every audio input + if not f.is_file(): + raise HTTPException(400, f"audio not found: {a['path']}") + audio.append({"file": str(f), "start": float(a.get("start", 0)), + "gain": float(a.get("gain", 1.0))}) + threading.Thread(target=_encode, args=(rid, fps, audio), daemon=True).start() return {"ok": True} @@ -305,6 +347,46 @@ def render_out(rid: str): return FileResponse(out, media_type="video/mp4") +# ---- MODELBEAST proxy (M4, gated) ---- +def _mb_post(path: str, data: bytes, content_type: str) -> dict: + req = urllib.request.Request(MB_HOST + path, data=data, method="POST", + headers={"Authorization": "Bearer " + _mb_token(), "Content-Type": content_type}) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + return json.load(resp) + except urllib.error.HTTPError as e: + raise HTTPException(502, f"MB {path}: {e.code} {e.read()[:200].decode('utf-8', 'replace')}") + except OSError as e: # never surface the token in the error + raise HTTPException(502, f"MB {path} unreachable: {e.__class__.__name__}") + + +@app.post("/mb/submit") +async def mb_submit(request: Request): + """Submit a MODELBEAST job (tts/voice-clone/lipsync). Optional input file → + /api/assets first, then /api/jobs — the meshgod/ops.py contract.""" + if os.environ.get("SCENEGOD_MB") != "1": + raise HTTPException(503, "MODELBEAST proxy disabled (set SCENEGOD_MB=1)") + body = await request.json() + op = (body.get("operator") or "").strip() + if not op: + raise HTTPException(400, "operator required") + asset_id = None + if body.get("file_b64"): + raw = base64.b64decode(body["file_b64"].split(",")[-1]) + name = os.path.basename(body.get("file_name") or "input.bin") or "input.bin" + bnd = uuid.uuid4().hex + mp = ((f'--{bnd}\r\nContent-Disposition: form-data; name="file"; filename="{name}"\r\n' + f"Content-Type: application/octet-stream\r\n\r\n").encode() + + raw + f"\r\n--{bnd}--\r\n".encode()) + a = _mb_post("/api/assets", mp, f"multipart/form-data; boundary={bnd}") + asset_id = a.get("id") or (a.get("items") or [a])[0].get("id") + payload = {"operator": op, "params": body.get("params") or {}} + if asset_id: + payload["asset_id"] = asset_id + j = _mb_post("/api/jobs", json.dumps(payload).encode(), "application/json") + return {"job_id": j.get("id"), "asset_id": asset_id} + + # ---- statics (last: catch-all /web is defined after API routes) ---- @app.get("/", response_class=HTMLResponse) def index(): diff --git a/scenegod/web/render.js b/scenegod/web/render.js index a0c0b3f..2c88a1d 100644 --- a/scenegod/web/render.js +++ b/scenegod/web/render.js @@ -26,7 +26,7 @@ export function draftRecord(stage, timeline, { fps = 30, mimeType = 'video/webm' // --- final: step frame-by-frame (deterministic), upload PNGs, ffmpeg encode --- export async function finalRender(stage, timeline, opts = {}) { - const { fps = 30, width = 1920, height = 1080, name = 'scene', onProgress } = opts; + const { fps = 30, width = 1920, height = 1080, name = 'scene', audio = [], onProgress } = opts; const canvas = stage.renderer.domElement; const prev = { w: canvas.width, h: canvas.height, dpr: stage.renderer.getPixelRatio() }; stage.renderer.setPixelRatio(1); @@ -50,7 +50,7 @@ export async function finalRender(stage, timeline, opts = {}) { } await Promise.all(inflight); - await postJSON(`/render/${renderId}/end`, {}); + await postJSON(`/render/${renderId}/end`, { audio }); // scene audio[] muxed by ffmpeg let st; do { await sleep(500); st = await (await fetch(`/render/${renderId}/status`)).json(); } while (st.state === 'queued' || st.state === 'encoding'); diff --git a/scripts/graft_limb.py b/scripts/graft_limb.py new file mode 100644 index 0000000..487193b --- /dev/null +++ b/scripts/graft_limb.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""graft_limb.py — graft a rigged limb GLB onto a rigged body GLB (headless Blender). + + blender -b -P scripts/graft_limb.py -- \ + --body body.glb --limb hand.glb --bone mixamorig:RightHand --out grafted.glb + blender -b -P scripts/graft_limb.py -- --selftest # builds synthetic fixtures + +What it does (C-server.md §C6): import both; align the limb's root bone head to +the target bone's world head; join the two armatures; parent the limb root under +the target bone keeping its offset; join the meshes (vertex-group weights survive +because the limb's bone names now live in the merged armature); export GLB. + +Refusal: if the body armature ALREADY has the bones the limb provides, this is a +mitten (the body is skinned there already) — exit 2 with a re-skin message +instead of duplicating bones. + +Rig contract: the grafted output NEVER goes through a join/decimate "finish" +path — that spawns phantom joint meshes and mauls the armature. Export as-is. +""" +import sys + +import bpy +from mathutils import Vector + + +class GraftRefused(Exception): + """Body already carries the limb's bones — re-skin, don't graft.""" + + +def _clear(): + bpy.ops.object.select_all(action="SELECT") + bpy.ops.object.delete() + for coll in (bpy.data.meshes, bpy.data.armatures): + for d in list(coll): + if d.users == 0: + coll.remove(d) + + +def _import(path): + """Import a GLB; return (armature_obj, [skinned mesh_objs]). Meshes not bound + to the armature are dropped — the Blender 5 glTF importer injects a stray + unparented icosphere, and a graft only cares about rig-deformed geometry.""" + before = set(bpy.data.objects) + bpy.ops.import_scene.gltf(filepath=path) + new = [o for o in bpy.data.objects if o not in before] + arms = [o for o in new if o.type == "ARMATURE"] + if not arms: + sys.exit(f"[graft] no armature in {path}") + arm = arms[0] + meshes = [o for o in new if o.type == "MESH" and ( + o.parent is arm or o.vertex_groups or any(m.type == "ARMATURE" for m in o.modifiers))] + for o in new: # remove strays (phantom icosphere, lights/cameras aren't MESH) + if o.type == "MESH" and o not in meshes: + bpy.data.objects.remove(o, do_unlink=True) + return arm, meshes + + +def _root_bone(arm): + for b in arm.data.bones: + if b.parent is None: + return b.name + return arm.data.bones[0].name + + +def _world_head(arm, bone_name): + return arm.matrix_world @ arm.data.bones[bone_name].head_local + + +def graft(body_path, limb_path, target_bone, out_path): + _clear() + body_arm, body_meshes = _import(body_path) + limb_arm, limb_meshes = _import(limb_path) + + if target_bone not in body_arm.data.bones: + sys.exit(f"[graft] target bone {target_bone!r} not in body armature " + f"(have: {[b.name for b in body_arm.data.bones][:20]})") + + limb_root = _root_bone(limb_arm) + body_bones = {b.name for b in body_arm.data.bones} + provides = {b.name for b in limb_arm.data.bones} - {limb_root} + clash = provides & body_bones + if clash: + raise GraftRefused(f"bones exist; mitten weights — re-skin, don't graft " + f"(overlap: {sorted(clash)})") + + # 1. align limb root head -> target bone head (world space) + delta = _world_head(body_arm, target_bone) - _world_head(limb_arm, limb_root) + limb_arm.matrix_world.translation += delta + bpy.context.view_layer.update() + + # 2. remember which meshes the limb armature deforms, then join armatures + for m in limb_meshes: + for mod in m.modifiers: + if mod.type == "ARMATURE": + mod.object = None # detach before join deletes limb_arm; re-point below + bpy.ops.object.select_all(action="DESELECT") + limb_arm.select_set(True) + body_arm.select_set(True) + bpy.context.view_layer.objects.active = body_arm + bpy.ops.object.join() # limb bones now live inside body_arm; limb_arm object gone + + # 3. re-point limb meshes at the merged armature + parent limb root under target + for m in limb_meshes: + had = any(mod.type == "ARMATURE" for mod in m.modifiers) + for mod in m.modifiers: + if mod.type == "ARMATURE": + mod.object = body_arm + if not had: + mod = m.modifiers.new("Armature", "ARMATURE") + mod.object = body_arm + m.parent = body_arm + bpy.context.view_layer.objects.active = body_arm + bpy.ops.object.mode_set(mode="EDIT") + eb = body_arm.data.edit_bones + eb[limb_root].parent = eb[target_bone] + eb[limb_root].use_connect = False # keep offset + bpy.ops.object.mode_set(mode="OBJECT") + + # 4. join meshes (vertex groups merge by bone name -> weights survive) + all_meshes = body_meshes + limb_meshes + if len(all_meshes) > 1: + bpy.ops.object.select_all(action="DESELECT") + for m in all_meshes: + m.select_set(True) + bpy.context.view_layer.objects.active = all_meshes[0] + bpy.ops.object.join() + + # 5. export as-is — NO finish/decimate (rig contract) + bpy.ops.object.select_all(action="SELECT") + bpy.ops.export_scene.gltf(filepath=out_path, export_format="GLB", use_selection=True) + print(f"[graft] wrote {out_path}") + + +# --- selftest: build synthetic fixtures and exercise both paths ------------- +def _make_rig(name, bones, out): + """bones = [(name, head, tail, parent_or_None)]; builds a skinned cube, exports GLB.""" + _clear() + bpy.ops.object.armature_add(enter_editmode=True, location=(0, 0, 0)) + arm = bpy.context.object + arm.name = name + eb = arm.data.edit_bones + eb.remove(eb[0]) # drop the default bone + for bn, head, tail, parent in bones: + b = eb.new(bn) + b.head, b.tail = Vector(head), Vector(tail) + if parent: + b.parent = eb[parent] + bpy.ops.object.mode_set(mode="OBJECT") + bpy.ops.mesh.primitive_cube_add(size=0.4, location=tuple(bones[-1][1])) + cube = bpy.context.object + bpy.ops.object.select_all(action="DESELECT") + cube.select_set(True) + arm.select_set(True) + bpy.context.view_layer.objects.active = arm + bpy.ops.object.parent_set(type="ARMATURE_AUTO") + bpy.ops.object.select_all(action="SELECT") + bpy.ops.export_scene.gltf(filepath=out, export_format="GLB", use_selection=True) + + +def selftest(): + import os + import tempfile + d = tempfile.mkdtemp(prefix="graft_selftest_") + body = os.path.join(d, "body.glb") + limb_ok = os.path.join(d, "limb_ok.glb") + limb_bad = os.path.join(d, "limb_bad.glb") + out = os.path.join(d, "grafted.glb") + # full-rig body: root -> RightHand -> RightHandIndex1 + _make_rig("Body", [ + ("root", (0, 0, 0), (0, 0, 1), None), + ("mixamorig:RightHand", (0, 0, 1), (0, 0, 1.3), "root"), + ("mixamorig:RightHandIndex1", (0, 0, 1.3), (0, 0, 1.5), "mixamorig:RightHand"), + ], body) + # good limb: custom bone names, no overlap with body + _make_rig("LimbOK", [ + ("nub_root", (0, 0, 0), (0, 0, 0.3), None), + ("nub_tip", (0, 0, 0.3), (0, 0, 0.5), "nub_root"), + ], limb_ok) + # bad limb: provides a bone the body already has -> mitten + _make_rig("LimbBad", [ + ("bad_root", (0, 0, 0), (0, 0, 0.2), None), + ("mixamorig:RightHandIndex1", (0, 0, 0.2), (0, 0, 0.4), "bad_root"), + ], limb_bad) + + # refusal path + try: + graft(body, limb_bad, "mixamorig:RightHand", out) + sys.exit("[selftest] FAIL: expected GraftRefused for overlapping bones") + except GraftRefused as e: + print(f"[selftest] refusal OK: {e}") + + # graft path + graft(body, limb_ok, "mixamorig:RightHand", out) + assert os.path.isfile(out) and os.path.getsize(out) > 0, "output GLB missing" + _clear() + arm, meshes = _import(out) + names = {b.name for b in arm.data.bones} + assert {"mixamorig:RightHand", "nub_root", "nub_tip"} <= names, f"merged bones wrong: {names}" + assert arm.data.bones["nub_root"].parent.name == "mixamorig:RightHand", "root not parented" + assert len(meshes) == 1, f"meshes not joined: {len(meshes)}" + print("[selftest] graft OK: merged armature + single skinned mesh") + + +def main(): + argv = sys.argv[sys.argv.index("--") + 1:] if "--" in sys.argv else [] + if "--selftest" in argv: + selftest() + return + + def opt(flag): + return argv[argv.index(flag) + 1] if flag in argv else None + body, limb, bone, out = opt("--body"), opt("--limb"), opt("--bone"), opt("--out") + if not all((body, limb, bone, out)): + sys.exit("usage: -- --body B.glb --limb L.glb --bone NAME --out O.glb") + try: + graft(body, limb, bone, out) + except GraftRefused as e: + print(f"[graft] REFUSED: {e}", file=sys.stderr) + sys.exit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_server.py b/scripts/test_server.py index f541a11..80811ab 100644 --- a/scripts/test_server.py +++ b/scripts/test_server.py @@ -7,16 +7,19 @@ Spins up the server against throwaway asset/scene dirs, then asserts tree grouping, file streaming, path-traversal rejection, scene round-trip, and validation failures. """ +import base64 import json import os import shutil import subprocess import sys import tempfile +import threading import time import urllib.error import urllib.parse import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path ROOT = Path(__file__).resolve().parent.parent @@ -142,7 +145,13 @@ def main(): for i, png in enumerate(sorted(fr.glob("*.png"))): st, _ = req("POST", f"/render/{rid}/frame/{i}", png.read_bytes()) assert st == 200, st - assert req("POST", f"/render/{rid}/end", "{}")[0] == 200 + # M4: mux a tone from assets/audio/test/ via the scene audio[] contract + (assets / "audio" / "test").mkdir(parents=True) + subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "sine=frequency=440:duration=1", + str(assets / "audio" / "test" / "tone.wav")], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + end_body = json.dumps({"audio": [{"path": "audio/test/tone.wav", "start": 0, "gain": 1.0}]}) + assert req("POST", f"/render/{rid}/end", end_body)[0] == 200 for _ in range(100): # poll up to ~20s state = json.loads(req("GET", f"/render/{rid}/status")[1])["state"] if state in ("done", "error"): @@ -151,15 +160,77 @@ def main(): assert state == "done", state st, mp4 = req("GET", f"/render/{rid}/out.mp4") assert st == 200 and mp4[:4] != b"", st + out = str(renders / rid / "out.mp4") dur = subprocess.check_output( ["ffprobe", "-v", "error", "-show_entries", "format=duration", - "-of", "default=nw=1:nokey=1", str(renders / rid / "out.mp4")], - text=True).strip() + "-of", "default=nw=1:nokey=1", out], text=True).strip() assert abs(float(dur) - 1.0) < 0.2, f"mp4 duration {dur}s, expected ~1s" + codecs = subprocess.check_output( + ["ffprobe", "-v", "error", "-select_streams", "a", "-show_entries", + "stream=codec_type", "-of", "default=nw=1:nokey=1", out], text=True).strip() + assert "audio" in codecs, f"no audio stream muxed: {codecs!r}" + # bad audio path rejected before encode + assert req("POST", f"/render/{rid}/end", + json.dumps({"audio": [{"path": "audio/nope.wav"}]}))[0] == 400 n += 1 else: print("(render test skipped — ffmpeg/ffprobe not found)") + # --- MODELBEAST proxy: gating + two-step contract (mock MB, no farm jobs) --- + assert req("POST", "/mb/submit", json.dumps({"operator": "tts"}))[0] == 503 # gated off + n += 1 + + captured = [] + + class MB(BaseHTTPRequestHandler): + def do_POST(self): + data = self.rfile.read(int(self.headers.get("Content-Length", 0))) + captured.append({"path": self.path, "auth": self.headers.get("Authorization"), + "ctype": self.headers.get("Content-Type", ""), "body": data}) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps( + {"id": "asset123" if self.path == "/api/assets" else "job456"}).encode()) + + def log_message(self, *a): + pass + + mbport, port2 = PORT + 1, PORT + 2 + srv = HTTPServer(("127.0.0.1", mbport), MB) + threading.Thread(target=srv.serve_forever, daemon=True).start() + env2 = {**env, "SCENEGOD_MB": "1", "MB_HOST": f"http://127.0.0.1:{mbport}", + "MB_TOKEN": "secret"} + p2 = subprocess.Popen( + [str(PY), "-m", "uvicorn", "scenegod.server:app", "--port", str(port2)], + cwd=ROOT, env=env2, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + for _ in range(50): + try: + urllib.request.urlopen(f"http://127.0.0.1:{port2}/scenes") + break + except urllib.error.URLError: + time.sleep(0.2) + sub = {"operator": "voice-clone", "params": {"text": "hi"}, + "file_b64": base64.b64encode(b"WAVDATA").decode(), "file_name": "in.wav"} + r = urllib.request.urlopen(urllib.request.Request( + f"http://127.0.0.1:{port2}/mb/submit", + data=json.dumps(sub).encode(), method="POST", + headers={"Content-Type": "application/json"})) + out = json.loads(r.read()) + assert out == {"job_id": "job456", "asset_id": "asset123"}, out + assert [c["path"] for c in captured] == ["/api/assets", "/api/jobs"], captured + assert all(c["auth"] == "Bearer secret" for c in captured), "token not forwarded" + assert captured[0]["ctype"].startswith("multipart/form-data"), captured[0]["ctype"] + assert b"WAVDATA" in captured[0]["body"], "file bytes not uploaded" + job = json.loads(captured[1]["body"]) + assert job["asset_id"] == "asset123" and job["operator"] == "voice-clone", job + n += 1 + finally: + p2.terminate() + p2.wait() + srv.shutdown() + print(f"OK: {n} test groups passed") finally: proc.terminate()