#!/usr/bin/env python3 """wardrobegod batch factory v2 — build-first catalogue items via the live API. v2 fixes (learned from batch1's carnage): - 429 "too many active jobs" is EXPECTED under farm contention -> exponential backoff up to ~20 min instead of instant failure - to3d waits 40 min (TRELLIS queues behind other users' GPU jobs; 20 was not enough) - idempotent: skips any step whose output already exists on disk, so re-running after a partial batch only does the missing work - one item fully finishes before the next starts (never more than 1 farm job from us in flight -> the active-jobs cap cannot cascade) Heartbeat: ~/.jobs/wardrobe-batch7.json """ import json, os, time, urllib.request, urllib.error API = "http://100.91.239.7:8150" LIB = os.path.expanduser("~/Documents/wardrobegod/library") HB = os.path.expanduser("~/.jobs/wardrobe-batch7.json") os.makedirs(os.path.dirname(HB), exist_ok=True) # id, phrase, template, view, to3d? ITEMS = [ # flats ("coat-overcoat-wool", "long charcoal wool overcoat", "flat3d", "top", False), ("hoodie-grey", "plain grey pullover hoodie with drawstrings", "flat3d", "top", False), ("jacket-varsity-flat","varsity jacket with cream sleeves and navy body, plain letter patch", "flat3d", "top", False), ("uniform-busdriver", "light blue bus driver uniform shirt with epaulettes", "flat3d", "top", False), ("pyjamas-flannel", "red tartan flannel pyjama set", "flat3d", "top", False), ("robe-dressing-gown", "burgundy quilted dressing gown with cord belt", "flat3d", "top", False), ("nightie-cotton", "white cotton nightie with lace collar", "flat3d", "top", False), ("skirt-pencil", "black knee length pencil skirt", "flat3d", "bottom", False), ("dress-maxi-floral", "long floral maxi dress with tiered skirt", "flat3d", "top", False), ("top-corset-boned", "black satin boned corset top with lacing", "flat3d", "top", False), ("waistcoat", "grey tweed waistcoat with buttons", "flat3d", "top", False), ("shorts-board", "long floral board shorts", "flat3d", "bottom", False), # headwear round 3 ("fedora", "dark grey felt fedora with black band", "object3d", "hat", True), ("trilby", "brown straw trilby hat", "object3d", "hat", True), ("visor-sports", "white sports sun visor", "object3d", "hat", True), ("sun-hat-wide", "cream wide brim woven sun hat with ribbon", "object3d", "hat", True), ("headscarf-tied", "red polka dot headscarf tied in a knot", "object3d", "hat", True), ("bandana-tied", "folded navy paisley bandana tied in a ring", "object3d", "hat", True), ("crown-simple", "simple gold crown with five points", "object3d", "hat", True), ("tiara", "silver tiara with crystal peaks", "object3d", "hat", True), ("party-hat", "striped cardboard cone party hat with pom pom", "object3d", "hat", True), ("durag-black", "black satin durag with long tails", "object3d", "hat", True), # footwear round 2 ("sneaker-high-canvas","pair of red high top canvas sneakers", "object3d", "shoes", True), ("sneaker-runner-90s", "pair of chunky 90s running shoes, white and neon", "object3d", "shoes", True), ("boot-cowboy", "pair of tooled leather cowboy boots", "object3d", "shoes", True), ("boot-hiking", "pair of brown leather hiking boots with red laces", "object3d", "shoes", True), ("boot-wellington", "pair of green rubber wellington boots", "object3d", "shoes", True), ("flat-loafer", "pair of burgundy penny loafers", "object3d", "shoes", True), ("flat-mary-jane", "pair of black patent mary jane shoes", "object3d", "shoes", True), ("heel-stiletto", "pair of red stiletto high heels", "object3d", "shoes", True), # props ("umbrella-closed", "closed black umbrella with hook handle", "object3d", "hat", True), ("umbrella-open", "open rainbow striped umbrella", "object3d", "bag", True), ("cane-walking", "wooden walking cane with curved handle", "object3d", "hat", True), ("skate-roller-quad", "pair of retro quad roller skates with orange wheels", "object3d", "shoes", True), ("sign-handheld", "rectangular handheld cardboard sign, blank", "object3d", "hat", True), ("traffic-wand", "orange traffic control light wand baton", "object3d", "hat", True), ("wallet-chain", "black leather wallet with silver chain", "object3d", "hat", True), ("keys-ring", "bunch of keys on a metal key ring", "object3d", "hat", True), # jewellery trials ("chain-figaro", "gold figaro chain necklace laid in a circle", "object3d", "hat", True), ("earring-hoop-large", "pair of large gold hoop earrings", "object3d", "hat", True), ] def slug(s): import re return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-") def api(path, payload=None, timeout=30): """POST/GET with backoff on 429/5xx. 429 here means the farm's active-job cap or lane contention — wait it out, don't fail the item.""" delay = 30 for attempt in range(12): try: if payload is None: req = urllib.request.Request(API + path) else: req = urllib.request.Request(API + path, data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"}) with urllib.request.urlopen(req, timeout=timeout) as r: return json.load(r) except urllib.error.HTTPError as e: body = "" try: body = e.read().decode()[:120] except Exception: pass if e.code in (429, 500, 502, 503, 404) and attempt < 11: time.sleep(delay) delay = min(delay * 1.6, 180) continue raise RuntimeError(f"HTTP {e.code} {body}") except OSError: if attempt < 11: time.sleep(delay) continue raise raise RuntimeError("retries exhausted") def wait(jid, timeout): t0 = time.time() while time.time() - t0 < timeout: j = api(f"/api/job/{jid}") if j.get("status") in ("done", "error"): return j time.sleep(8) return {"status": "error", "log": "poll timeout"} def step(row, name, jid_resp, timeout): j = wait(jid_resp["job"], timeout) row["steps"][name] = j.get("status") if j.get("status") != "done": raise RuntimeError(f"{name}: " + str(j.get("log", j.get("note", "")))[:200]) return j.get("out") def hb(state): state["updated"] = time.strftime("%Y-%m-%d %H:%M:%S") json.dump(state, open(HB, "w"), indent=1) results = [] state = {"job": "wardrobe-batch7", "status": "running", "total": len(ITEMS), "done": 0, "failed": 0, "skipped": 0, "current": "", "results": results} for iid, phrase, tpl, view, want3d in ITEMS: row = {"id": iid, "steps": {}} results.append(row) state["current"] = iid hb(state) s = slug(phrase) png = os.path.join(LIB, "gen", f"{s}-v0.png") cut = os.path.join(LIB, "gen", f"{s}-v0-cut.png") glb = os.path.join(LIB, "garments", f"{slug(s + '-v0-cut')}-rigid.glb") # farm 429s surface as *job errors* inside wardrobegod's worker thread (its # fx() has no retry), so backoff must live at ITEM level, not HTTP level. for attempt in range(4): try: if want3d and os.path.exists(glb): row["steps"]["all"] = "skipped (glb exists)" row["glb"] = glb state["skipped"] += 1 break if not os.path.exists(cut): if not os.path.exists(png): # use the SERVER-reported output path: wardrobegod truncates # long slugs, so predicting the filename fails on long phrases png = step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl, "view": view, "backend": "klein"}), 900) or png cut = png.rsplit(".", 1)[0] + "-cut.png" else: row["steps"]["gen"] = "skipped" step(row, "rmbg", api("/api/rmbg", {"path": png}), 900) else: row["steps"]["gen"] = row["steps"]["rmbg"] = "skipped" row["cut"] = cut if want3d: # the farm GPU lane is shared fleet-wide and queues run deep # (measured 50+ min just to reach the front) — wait 2h, don't # give up and resubmit into the same queue out = step(row, "to3d", api("/api/to3d", {"path": cut}), 7200) row["glb"] = out state["done"] += 1 row.pop("error", None) break except Exception as e: msg = str(e)[:300] row["error"] = msg retryable = ("429", "Too Many", "timed out", "timeout", "401", "Unauthorized", "500", "502", "503") if any(t in msg for t in retryable) and attempt < 3: row["steps"][f"retry{attempt+1}"] = "waiting 300s (farm busy)" hb(state) time.sleep(300) continue state["failed"] += 1 break hb(state) state["status"] = "done" state["current"] = "" hb(state) print(json.dumps({k: state[k] for k in ("done", "failed", "skipped")}, indent=1))