to3d: use live trellis2_mlx operator (trellis_mac has no worker); 2400s farm wait; factory v5 retries

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

server.py

@@ -1099,8 +1099,8 @@ class H(BaseHTTPRequestHandler):
 
             def fx(jid):
                 aid = mb_upload(p)
-                j = mb_req('/api/jobs', {'operator': 'trellis_mac', 'asset_id': aid, 'params': {}})
+                j = mb_req('/api/jobs', {'operator': 'trellis2_mlx', 'asset_id': aid, 'params': {}})
-                mb_wait(j.get('id') or j.get('job_id'), jid, timeout=1200)
+                mb_wait(j.get('id') or j.get('job_id'), jid, timeout=2400)
                 outs = mb_outputs(j.get('id') or j.get('job_id'))
                 glbs = [o for o in outs if str(o.get('name', o.get('filename', ''))).lower().endswith('.glb')] or outs
                 if not glbs:

tools/factory.py

@@ -1,21 +1,26 @@
 #!/usr/bin/env python3
-"""wardrobegod batch factory — build-first catalogue items, serially, via the
+"""wardrobegod batch factory v2 — build-first catalogue items via the live API.
-live wardrobegod API. Heartbeat at ~/.jobs/wardrobe-batch1.json.
 
-Per item: /api/gen (klein on the farm) -> /api/rmbg (RMBG-2.0 cutout) ->
+v2 fixes (learned from batch1's carnage):
-[3D items] /api/to3d (TRELLIS, ~5 min serial GPU lane).
+- 429 "too many active jobs" is EXPECTED under farm contention -> exponential
-Cutout PNGs and GLBs land in library/ automatically — that is the point of
+  backoff up to ~20 min instead of instant failure
-driving the product API instead of the farm directly.
+- to3d waits 40 min (TRELLIS queues behind other users' GPU jobs; 20 was not enough)
+- idempotent: skips any step whose output already exists on disk, so re-running
+  after a partial batch only does the missing work
+- one item fully finishes before the next starts (never more than 1 farm job
+  from us in flight -> the active-jobs cap cannot cascade)
+
+Heartbeat: ~/.jobs/wardrobe-batch1.json
 """
-import json, os, time, urllib.request
+import json, os, time, urllib.request, urllib.error
 
 API = "http://100.91.239.7:8150"
+LIB = os.path.expanduser("~/Documents/wardrobegod/library")
 HB = os.path.expanduser("~/.jobs/wardrobe-batch1.json")
 os.makedirs(os.path.dirname(HB), exist_ok=True)
 
 # id, phrase, template, view, to3d?
 ITEMS = [
-    # --- R2 rigid props/hats -> full 3D chain ---
     ("cap-baseball-fwd",     "faded navy blue baseball cap", "object3d", "hat", True),
     ("hardhat-construction", "yellow construction hard hat", "object3d", "hat", True),
     ("headphones-dj-overear","black over-ear DJ headphones with a coiled cable", "object3d", "hat", True),
@@ -26,7 +31,6 @@ ITEMS = [
     ("sneaker-low-canvas",   "pair of plain white low-top canvas sneakers", "object3d", "shoes", True),
     ("boot-work-steelcap",   "pair of tan leather steel-cap work boots", "object3d", "shoes", True),
     ("chain-necklace-curb",  "chunky silver curb chain necklace laid in a circle", "object3d", "hat", True),
-    # --- R1 garment cutouts (gen + rmbg only) ---
     ("tee-plain-white",      "plain white crew-neck t-shirt", "flat3d", "top", False),
     ("tee-print-smiley",     "black t-shirt with a large orange smiley face print", "flat3d", "top", False),
     ("tee-print-sunset",     "white t-shirt with a retro sunset stripes print", "flat3d", "top", False),
@@ -36,7 +40,17 @@ ITEMS = [
 ]
 
 
+def slug(s):
+    import re
+    return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")
+
+
 def api(path, payload=None, timeout=30):
+    """POST/GET with backoff on 429/5xx. 429 here means the farm's active-job
+    cap or lane contention — wait it out, don't fail the item."""
+    delay = 30
+    for attempt in range(12):
+        try:
             if payload is None:
                 req = urllib.request.Request(API + path)
             else:
@@ -44,18 +58,43 @@ def api(path, payload=None, timeout=30):
                                              headers={"Content-Type": "application/json"})
             with urllib.request.urlopen(req, timeout=timeout) as r:
                 return json.load(r)
+        except urllib.error.HTTPError as e:
+            body = ""
+            try:
+                body = e.read().decode()[:120]
+            except Exception:
+                pass
+            if e.code in (429, 500, 502, 503, 404) and attempt < 11:
+                time.sleep(delay)
+                delay = min(delay * 1.6, 180)
+                continue
+            raise RuntimeError(f"HTTP {e.code} {body}")
+        except OSError:
+            if attempt < 11:
+                time.sleep(delay)
+                continue
+            raise
+    raise RuntimeError("retries exhausted")
 
 
-def wait(jid, timeout=1500):
+def wait(jid, timeout):
     t0 = time.time()
     while time.time() - t0 < timeout:
         j = api(f"/api/job/{jid}")
         if j.get("status") in ("done", "error"):
             return j
-        time.sleep(6)
+        time.sleep(8)
     return {"status": "error", "log": "poll timeout"}
 
 
+def step(row, name, jid_resp, timeout):
+    j = wait(jid_resp["job"], timeout)
+    row["steps"][name] = j.get("status")
+    if j.get("status") != "done":
+        raise RuntimeError(f"{name}: " + str(j.get("log", j.get("note", "")))[:200])
+    return j.get("out")
+
+
 def hb(state):
     state["updated"] = time.strftime("%Y-%m-%d %H:%M:%S")
     json.dump(state, open(HB, "w"), indent=1)
@@ -63,45 +102,57 @@ def hb(state):
 
 results = []
 state = {"job": "wardrobe-batch1", "status": "running", "total": len(ITEMS),
-         "done": 0, "failed": 0, "current": "", "results": results}
+         "done": 0, "failed": 0, "skipped": 0, "current": "", "results": results}
-hb(state)
 
-for i, (iid, phrase, tpl, view, want3d) in enumerate(ITEMS):
+for iid, phrase, tpl, view, want3d in ITEMS:
     row = {"id": iid, "steps": {}}
     results.append(row)
     state["current"] = iid
     hb(state)
+    s = slug(phrase)
+    png = os.path.join(LIB, "gen", f"{s}-v0.png")
+    cut = os.path.join(LIB, "gen", f"{s}-v0-cut.png")
+    glb = os.path.join(LIB, "garments", f"{slug(s + '-v0-cut')}-rigid.glb")
+    # farm 429s surface as *job errors* inside wardrobegod's worker thread (its
+    # fx() has no retry), so backoff must live at ITEM level, not HTTP level.
+    for attempt in range(4):
         try:
-        g = api("/api/gen", {"phrase": phrase, "template": tpl, "view": view,
+            if want3d and os.path.exists(glb):
-                             "backend": "klein"})
+                row["steps"]["all"] = "skipped (glb exists)"
-        j = wait(g["job"], 300)
+                row["glb"] = glb
-        row["steps"]["gen"] = j.get("status")
+                state["skipped"] += 1
-        if j.get("status") != "done":
+                break
-            raise RuntimeError("gen: " + str(j.get("log", ""))[:200])
+            if not os.path.exists(cut):
-        png = j["out"]
+                if not os.path.exists(png):
-
+                    step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl,
-        c = api("/api/rmbg", {"path": png})
+                                                      "view": view, "backend": "klein"}), 900)
-        j = wait(c["job"], 300)
+                else:
-        row["steps"]["rmbg"] = j.get("status")
+                    row["steps"]["gen"] = "skipped"
-        if j.get("status") != "done":
+                step(row, "rmbg", api("/api/rmbg", {"path": png}), 900)
-            raise RuntimeError("rmbg: " + str(j.get("log", ""))[:200])
+            else:
-        cut = j["out"]
+                row["steps"]["gen"] = row["steps"]["rmbg"] = "skipped"
             row["cut"] = cut
-
             if want3d:
-            t = api("/api/to3d", {"path": cut})
+                out = step(row, "to3d", api("/api/to3d", {"path": cut}), 2400)
-            j = wait(t["job"], 1500)
+                row["glb"] = out
-            row["steps"]["to3d"] = j.get("status")
-            if j.get("status") != "done":
-                raise RuntimeError("to3d: " + str(j.get("log", ""))[:200])
-            row["glb"] = j.get("out")
             state["done"] += 1
+            row.pop("error", None)
+            break
         except Exception as e:
-        row["error"] = str(e)[:300]
+            msg = str(e)[:300]
+            row["error"] = msg
+            retryable = ("429", "Too Many", "timed out", "401", "Unauthorized",
+                         "500", "502", "503")
+            if any(t in msg for t in retryable) and attempt < 3:
+                row["steps"][f"retry{attempt+1}"] = "waiting 300s (farm busy)"
+                hb(state)
+                time.sleep(300)
+                continue
             state["failed"] += 1
+            break
     hb(state)
 
 state["status"] = "done"
 state["current"] = ""
 hb(state)
-print(json.dumps(state, indent=1))
+print(json.dumps({k: state[k] for k in ("done", "failed", "skipped")}, indent=1))