diff --git a/tools/factory.py b/tools/factory.py index dd9695f..3984468 100644 --- a/tools/factory.py +++ b/tools/factory.py @@ -124,8 +124,11 @@ for iid, phrase, tpl, view, want3d in ITEMS: break if not os.path.exists(cut): if not os.path.exists(png): - step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl, - "view": view, "backend": "klein"}), 900) + # use the SERVER-reported output path: wardrobegod truncates + # long slugs, so predicting the filename fails on long phrases + png = step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl, + "view": view, "backend": "klein"}), 900) or png + cut = png.rsplit(".", 1)[0] + "-cut.png" else: row["steps"]["gen"] = "skipped" step(row, "rmbg", api("/api/rmbg", {"path": png}), 900) @@ -133,7 +136,10 @@ for iid, phrase, tpl, view, want3d in ITEMS: row["steps"]["gen"] = row["steps"]["rmbg"] = "skipped" row["cut"] = cut if want3d: - out = step(row, "to3d", api("/api/to3d", {"path": cut}), 2400) + # the farm GPU lane is shared fleet-wide and queues run deep + # (measured 50+ min just to reach the front) — wait 2h, don't + # give up and resubmit into the same queue + out = step(row, "to3d", api("/api/to3d", {"path": cut}), 7200) row["glb"] = out state["done"] += 1 row.pop("error", None) @@ -141,8 +147,8 @@ for iid, phrase, tpl, view, want3d in ITEMS: except Exception as e: msg = str(e)[:300] row["error"] = msg - retryable = ("429", "Too Many", "timed out", "401", "Unauthorized", - "500", "502", "503") + retryable = ("429", "Too Many", "timed out", "timeout", "401", + "Unauthorized", "500", "502", "503") if any(t in msg for t in retryable) and attempt < 3: row["steps"][f"retry{attempt+1}"] = "waiting 300s (farm busy)" hb(state) diff --git a/tools/factory7.py b/tools/factory7.py new file mode 100644 index 0000000..4acc17c --- /dev/null +++ b/tools/factory7.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""wardrobegod batch factory v2 — build-first catalogue items via the live API. + +v2 fixes (learned from batch1's carnage): +- 429 "too many active jobs" is EXPECTED under farm contention -> exponential + backoff up to ~20 min instead of instant failure +- to3d waits 40 min (TRELLIS queues behind other users' GPU jobs; 20 was not enough) +- idempotent: skips any step whose output already exists on disk, so re-running + after a partial batch only does the missing work +- one item fully finishes before the next starts (never more than 1 farm job + from us in flight -> the active-jobs cap cannot cascade) + +Heartbeat: ~/.jobs/wardrobe-batch7.json +""" +import json, os, time, urllib.request, urllib.error + +API = "http://100.91.239.7:8150" +LIB = os.path.expanduser("~/Documents/wardrobegod/library") +HB = os.path.expanduser("~/.jobs/wardrobe-batch7.json") +os.makedirs(os.path.dirname(HB), exist_ok=True) + +# id, phrase, template, view, to3d? +ITEMS = [ + # flats + ("coat-overcoat-wool", "long charcoal wool overcoat", "flat3d", "top", False), + ("hoodie-grey", "plain grey pullover hoodie with drawstrings", "flat3d", "top", False), + ("jacket-varsity-flat","varsity jacket with cream sleeves and navy body, plain letter patch", "flat3d", "top", False), + ("uniform-busdriver", "light blue bus driver uniform shirt with epaulettes", "flat3d", "top", False), + ("pyjamas-flannel", "red tartan flannel pyjama set", "flat3d", "top", False), + ("robe-dressing-gown", "burgundy quilted dressing gown with cord belt", "flat3d", "top", False), + ("nightie-cotton", "white cotton nightie with lace collar", "flat3d", "top", False), + ("skirt-pencil", "black knee length pencil skirt", "flat3d", "bottom", False), + ("dress-maxi-floral", "long floral maxi dress with tiered skirt", "flat3d", "top", False), + ("top-corset-boned", "black satin boned corset top with lacing", "flat3d", "top", False), + ("waistcoat", "grey tweed waistcoat with buttons", "flat3d", "top", False), + ("shorts-board", "long floral board shorts", "flat3d", "bottom", False), + # headwear round 3 + ("fedora", "dark grey felt fedora with black band", "object3d", "hat", True), + ("trilby", "brown straw trilby hat", "object3d", "hat", True), + ("visor-sports", "white sports sun visor", "object3d", "hat", True), + ("sun-hat-wide", "cream wide brim woven sun hat with ribbon", "object3d", "hat", True), + ("headscarf-tied", "red polka dot headscarf tied in a knot", "object3d", "hat", True), + ("bandana-tied", "folded navy paisley bandana tied in a ring", "object3d", "hat", True), + ("crown-simple", "simple gold crown with five points", "object3d", "hat", True), + ("tiara", "silver tiara with crystal peaks", "object3d", "hat", True), + ("party-hat", "striped cardboard cone party hat with pom pom", "object3d", "hat", True), + ("durag-black", "black satin durag with long tails", "object3d", "hat", True), + # footwear round 2 + ("sneaker-high-canvas","pair of red high top canvas sneakers", "object3d", "shoes", True), + ("sneaker-runner-90s", "pair of chunky 90s running shoes, white and neon", "object3d", "shoes", True), + ("boot-cowboy", "pair of tooled leather cowboy boots", "object3d", "shoes", True), + ("boot-hiking", "pair of brown leather hiking boots with red laces", "object3d", "shoes", True), + ("boot-wellington", "pair of green rubber wellington boots", "object3d", "shoes", True), + ("flat-loafer", "pair of burgundy penny loafers", "object3d", "shoes", True), + ("flat-mary-jane", "pair of black patent mary jane shoes", "object3d", "shoes", True), + ("heel-stiletto", "pair of red stiletto high heels", "object3d", "shoes", True), + # props + ("umbrella-closed", "closed black umbrella with hook handle", "object3d", "hat", True), + ("umbrella-open", "open rainbow striped umbrella", "object3d", "bag", True), + ("cane-walking", "wooden walking cane with curved handle", "object3d", "hat", True), + ("skate-roller-quad", "pair of retro quad roller skates with orange wheels", "object3d", "shoes", True), + ("sign-handheld", "rectangular handheld cardboard sign, blank", "object3d", "hat", True), + ("traffic-wand", "orange traffic control light wand baton", "object3d", "hat", True), + ("wallet-chain", "black leather wallet with silver chain", "object3d", "hat", True), + ("keys-ring", "bunch of keys on a metal key ring", "object3d", "hat", True), + # jewellery trials + ("chain-figaro", "gold figaro chain necklace laid in a circle", "object3d", "hat", True), + ("earring-hoop-large", "pair of large gold hoop earrings", "object3d", "hat", True), +] + + +def slug(s): + import re + return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-") + + +def api(path, payload=None, timeout=30): + """POST/GET with backoff on 429/5xx. 429 here means the farm's active-job + cap or lane contention — wait it out, don't fail the item.""" + delay = 30 + for attempt in range(12): + try: + if payload is None: + req = urllib.request.Request(API + path) + else: + req = urllib.request.Request(API + path, data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.load(r) + except urllib.error.HTTPError as e: + body = "" + try: + body = e.read().decode()[:120] + except Exception: + pass + if e.code in (429, 500, 502, 503, 404) and attempt < 11: + time.sleep(delay) + delay = min(delay * 1.6, 180) + continue + raise RuntimeError(f"HTTP {e.code} {body}") + except OSError: + if attempt < 11: + time.sleep(delay) + continue + raise + raise RuntimeError("retries exhausted") + + +def wait(jid, timeout): + t0 = time.time() + while time.time() - t0 < timeout: + j = api(f"/api/job/{jid}") + if j.get("status") in ("done", "error"): + return j + time.sleep(8) + return {"status": "error", "log": "poll timeout"} + + +def step(row, name, jid_resp, timeout): + j = wait(jid_resp["job"], timeout) + row["steps"][name] = j.get("status") + if j.get("status") != "done": + raise RuntimeError(f"{name}: " + str(j.get("log", j.get("note", "")))[:200]) + return j.get("out") + + +def hb(state): + state["updated"] = time.strftime("%Y-%m-%d %H:%M:%S") + json.dump(state, open(HB, "w"), indent=1) + + +results = [] +state = {"job": "wardrobe-batch7", "status": "running", "total": len(ITEMS), + "done": 0, "failed": 0, "skipped": 0, "current": "", "results": results} + +for iid, phrase, tpl, view, want3d in ITEMS: + row = {"id": iid, "steps": {}} + results.append(row) + state["current"] = iid + hb(state) + s = slug(phrase) + png = os.path.join(LIB, "gen", f"{s}-v0.png") + cut = os.path.join(LIB, "gen", f"{s}-v0-cut.png") + glb = os.path.join(LIB, "garments", f"{slug(s + '-v0-cut')}-rigid.glb") + # farm 429s surface as *job errors* inside wardrobegod's worker thread (its + # fx() has no retry), so backoff must live at ITEM level, not HTTP level. + for attempt in range(4): + try: + if want3d and os.path.exists(glb): + row["steps"]["all"] = "skipped (glb exists)" + row["glb"] = glb + state["skipped"] += 1 + break + if not os.path.exists(cut): + if not os.path.exists(png): + # use the SERVER-reported output path: wardrobegod truncates + # long slugs, so predicting the filename fails on long phrases + png = step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl, + "view": view, "backend": "klein"}), 900) or png + cut = png.rsplit(".", 1)[0] + "-cut.png" + else: + row["steps"]["gen"] = "skipped" + step(row, "rmbg", api("/api/rmbg", {"path": png}), 900) + else: + row["steps"]["gen"] = row["steps"]["rmbg"] = "skipped" + row["cut"] = cut + if want3d: + # the farm GPU lane is shared fleet-wide and queues run deep + # (measured 50+ min just to reach the front) — wait 2h, don't + # give up and resubmit into the same queue + out = step(row, "to3d", api("/api/to3d", {"path": cut}), 7200) + row["glb"] = out + state["done"] += 1 + row.pop("error", None) + break + except Exception as e: + msg = str(e)[:300] + row["error"] = msg + retryable = ("429", "Too Many", "timed out", "timeout", "401", + "Unauthorized", "500", "502", "503") + if any(t in msg for t in retryable) and attempt < 3: + row["steps"][f"retry{attempt+1}"] = "waiting 300s (farm busy)" + hb(state) + time.sleep(300) + continue + state["failed"] += 1 + break + hb(state) + +state["status"] = "done" +state["current"] = "" +hb(state) +print(json.dumps({k: state[k] for k in ("done", "failed", "skipped")}, indent=1))