#!/usr/bin/env python3 """wardrobegod batch factory v2 — build-first catalogue items via the live API. v2 fixes (learned from batch1's carnage): - 429 "too many active jobs" is EXPECTED under farm contention -> exponential backoff up to ~20 min instead of instant failure - to3d waits 40 min (TRELLIS queues behind other users' GPU jobs; 20 was not enough) - idempotent: skips any step whose output already exists on disk, so re-running after a partial batch only does the missing work - one item fully finishes before the next starts (never more than 1 farm job from us in flight -> the active-jobs cap cannot cascade) Heartbeat: ~/.jobs/wardrobe-batch1.json """ import json, os, time, urllib.request, urllib.error API = "http://100.91.239.7:8150" LIB = os.path.expanduser("~/Documents/wardrobegod/library") HB = os.path.expanduser("~/.jobs/wardrobe-batch1.json") os.makedirs(os.path.dirname(HB), exist_ok=True) # id, phrase, template, view, to3d? ITEMS = [ ("cap-baseball-fwd", "faded navy blue baseball cap", "object3d", "hat", True), ("hardhat-construction", "yellow construction hard hat", "object3d", "hat", True), ("headphones-dj-overear","black over-ear DJ headphones with a coiled cable", "object3d", "hat", True), ("record-vinyl-12in", "12 inch black vinyl record with a plain orange centre label", "object3d", "hat", True), ("sunglasses-wayfarer-generic", "black chunky rectangular-frame sunglasses", "object3d", "hat", True), ("bag-record-crossbody", "black canvas record bag with a wide shoulder strap", "object3d", "bag", True), ("shopping-basket", "red plastic shopping basket with two metal handles", "object3d", "bag", True), ("sneaker-low-canvas", "pair of plain white low-top canvas sneakers", "object3d", "shoes", True), ("boot-work-steelcap", "pair of tan leather steel-cap work boots", "object3d", "shoes", True), ("chain-necklace-curb", "chunky silver curb chain necklace laid in a circle", "object3d", "hat", True), ("tee-plain-white", "plain white crew-neck t-shirt", "flat3d", "top", False), ("tee-print-smiley", "black t-shirt with a large orange smiley face print", "flat3d", "top", False), ("tee-print-sunset", "white t-shirt with a retro sunset stripes print", "flat3d", "top", False), ("hivis-vest-yellow", "yellow hi-vis safety vest with silver reflective stripes", "flat3d", "top", False), ("tights-opaque-black", "pair of black opaque tights", "flat3d", "bottom", False), ("shorts-denim-cutoff", "light blue denim cutoff shorts", "flat3d", "bottom", False), ] def slug(s): import re return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-") def api(path, payload=None, timeout=30): """POST/GET with backoff on 429/5xx. 429 here means the farm's active-job cap or lane contention — wait it out, don't fail the item.""" delay = 30 for attempt in range(12): try: if payload is None: req = urllib.request.Request(API + path) else: req = urllib.request.Request(API + path, data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"}) with urllib.request.urlopen(req, timeout=timeout) as r: return json.load(r) except urllib.error.HTTPError as e: body = "" try: body = e.read().decode()[:120] except Exception: pass if e.code in (429, 500, 502, 503, 404) and attempt < 11: time.sleep(delay) delay = min(delay * 1.6, 180) continue raise RuntimeError(f"HTTP {e.code} {body}") except OSError: if attempt < 11: time.sleep(delay) continue raise raise RuntimeError("retries exhausted") def wait(jid, timeout): t0 = time.time() while time.time() - t0 < timeout: j = api(f"/api/job/{jid}") if j.get("status") in ("done", "error"): return j time.sleep(8) return {"status": "error", "log": "poll timeout"} def step(row, name, jid_resp, timeout): j = wait(jid_resp["job"], timeout) row["steps"][name] = j.get("status") if j.get("status") != "done": raise RuntimeError(f"{name}: " + str(j.get("log", j.get("note", "")))[:200]) return j.get("out") def hb(state): state["updated"] = time.strftime("%Y-%m-%d %H:%M:%S") json.dump(state, open(HB, "w"), indent=1) results = [] state = {"job": "wardrobe-batch1", "status": "running", "total": len(ITEMS), "done": 0, "failed": 0, "skipped": 0, "current": "", "results": results} for iid, phrase, tpl, view, want3d in ITEMS: row = {"id": iid, "steps": {}} results.append(row) state["current"] = iid hb(state) s = slug(phrase) png = os.path.join(LIB, "gen", f"{s}-v0.png") cut = os.path.join(LIB, "gen", f"{s}-v0-cut.png") glb = os.path.join(LIB, "garments", f"{slug(s + '-v0-cut')}-rigid.glb") # farm 429s surface as *job errors* inside wardrobegod's worker thread (its # fx() has no retry), so backoff must live at ITEM level, not HTTP level. for attempt in range(4): try: if want3d and os.path.exists(glb): row["steps"]["all"] = "skipped (glb exists)" row["glb"] = glb state["skipped"] += 1 break if not os.path.exists(cut): if not os.path.exists(png): step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl, "view": view, "backend": "klein"}), 900) else: row["steps"]["gen"] = "skipped" step(row, "rmbg", api("/api/rmbg", {"path": png}), 900) else: row["steps"]["gen"] = row["steps"]["rmbg"] = "skipped" row["cut"] = cut if want3d: out = step(row, "to3d", api("/api/to3d", {"path": cut}), 2400) row["glb"] = out state["done"] += 1 row.pop("error", None) break except Exception as e: msg = str(e)[:300] row["error"] = msg retryable = ("429", "Too Many", "timed out", "401", "Unauthorized", "500", "502", "503") if any(t in msg for t in retryable) and attempt < 3: row["steps"][f"retry{attempt+1}"] = "waiting 300s (farm busy)" hb(state) time.sleep(300) continue state["failed"] += 1 break hb(state) state["status"] = "done" state["current"] = "" hb(state) print(json.dumps({k: state[k] for k in ("done", "failed", "skipped")}, indent=1))