monster/wardrobegod
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

to3d: use live trellis2_mlx operator (trellis_mac has no worker); 2400s farm wait; factory v5 retries

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
type-two 2026-07-28 23:14:22 +10:00
parent 9c51ed1b58
commit b0188a41f0
2 changed files with 103 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server.py
View File

@ -1099,8 +1099,8 @@ class H(BaseHTTPRequestHandler):
def fx(jid):
aid = mb_upload(p)
j = mb_req('/api/jobs', {'operator': 'trellis_mac', 'asset_id': aid, 'params': {}})
mb_wait(j.get('id') or j.get('job_id'), jid, timeout=1200)
j = mb_req('/api/jobs', {'operator': 'trellis2_mlx', 'asset_id': aid, 'params': {}})
mb_wait(j.get('id') or j.get('job_id'), jid, timeout=2400)
outs = mb_outputs(j.get('id') or j.get('job_id'))
glbs = [o for o in outs if str(o.get('name', o.get('filename', ''))).lower().endswith('.glb')] or outs
if not glbs:

151
tools/factory.py
View File

@ -1,21 +1,26 @@
#!/usr/bin/env python3
"""wardrobegod batch factory — build-first catalogue items, serially, via the
live wardrobegod API. Heartbeat at ~/.jobs/wardrobe-batch1.json.
"""wardrobegod batch factory v2 — build-first catalogue items via the live API.
Per item: /api/gen (klein on the farm) -> /api/rmbg (RMBG-2.0 cutout) ->
[3D items] /api/to3d (TRELLIS, ~5 min serial GPU lane).
Cutout PNGs and GLBs land in library/ automatically — that is the point of
driving the product API instead of the farm directly.
v2 fixes (learned from batch1's carnage):
- 429 "too many active jobs" is EXPECTED under farm contention -> exponential
backoff up to ~20 min instead of instant failure
- to3d waits 40 min (TRELLIS queues behind other users' GPU jobs; 20 was not enough)
- idempotent: skips any step whose output already exists on disk, so re-running
after a partial batch only does the missing work
- one item fully finishes before the next starts (never more than 1 farm job
from us in flight -> the active-jobs cap cannot cascade)
Heartbeat: ~/.jobs/wardrobe-batch1.json
"""
import json, os, time, urllib.request
import json, os, time, urllib.request, urllib.error
API = "http://100.91.239.7:8150"
LIB = os.path.expanduser("~/Documents/wardrobegod/library")
HB = os.path.expanduser("~/.jobs/wardrobe-batch1.json")
os.makedirs(os.path.dirname(HB), exist_ok=True)
# id, phrase, template, view, to3d?
ITEMS = [
# --- R2 rigid props/hats -> full 3D chain ---
("cap-baseball-fwd", "faded navy blue baseball cap", "object3d", "hat", True),
("hardhat-construction", "yellow construction hard hat", "object3d", "hat", True),
("headphones-dj-overear","black over-ear DJ headphones with a coiled cable", "object3d", "hat", True),
@ -26,7 +31,6 @@ ITEMS = [
("sneaker-low-canvas", "pair of plain white low-top canvas sneakers", "object3d", "shoes", True),
("boot-work-steelcap", "pair of tan leather steel-cap work boots", "object3d", "shoes", True),
("chain-necklace-curb", "chunky silver curb chain necklace laid in a circle", "object3d", "hat", True),
# --- R1 garment cutouts (gen + rmbg only) ---
("tee-plain-white", "plain white crew-neck t-shirt", "flat3d", "top", False),
("tee-print-smiley", "black t-shirt with a large orange smiley face print", "flat3d", "top", False),
("tee-print-sunset", "white t-shirt with a retro sunset stripes print", "flat3d", "top", False),
@ -36,26 +40,61 @@ ITEMS = [
]
def slug(s):
import re
return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")
def api(path, payload=None, timeout=30):
if payload is None:
req = urllib.request.Request(API + path)
else:
req = urllib.request.Request(API + path, data=json.dumps(payload).encode(),
headers={"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.load(r)
"""POST/GET with backoff on 429/5xx. 429 here means the farm's active-job
cap or lane contention — wait it out, don't fail the item."""
delay = 30
for attempt in range(12):
try:
if payload is None:
req = urllib.request.Request(API + path)
else:
req = urllib.request.Request(API + path, data=json.dumps(payload).encode(),
headers={"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.load(r)
except urllib.error.HTTPError as e:
body = ""
try:
body = e.read().decode()[:120]
except Exception:
pass
if e.code in (429, 500, 502, 503, 404) and attempt < 11:
time.sleep(delay)
delay = min(delay * 1.6, 180)
continue
raise RuntimeError(f"HTTP {e.code} {body}")
except OSError:
if attempt < 11:
time.sleep(delay)
continue
raise
raise RuntimeError("retries exhausted")
def wait(jid, timeout=1500):
def wait(jid, timeout):
t0 = time.time()
while time.time() - t0 < timeout:
j = api(f"/api/job/{jid}")
if j.get("status") in ("done", "error"):
return j
time.sleep(6)
time.sleep(8)
return {"status": "error", "log": "poll timeout"}
def step(row, name, jid_resp, timeout):
j = wait(jid_resp["job"], timeout)
row["steps"][name] = j.get("status")
if j.get("status") != "done":
raise RuntimeError(f"{name}: " + str(j.get("log", j.get("note", "")))[:200])
return j.get("out")
def hb(state):
state["updated"] = time.strftime("%Y-%m-%d %H:%M:%S")
json.dump(state, open(HB, "w"), indent=1)
@ -63,45 +102,57 @@ def hb(state):
results = []
state = {"job": "wardrobe-batch1", "status": "running", "total": len(ITEMS),
"done": 0, "failed": 0, "current": "", "results": results}
hb(state)
"done": 0, "failed": 0, "skipped": 0, "current": "", "results": results}
for i, (iid, phrase, tpl, view, want3d) in enumerate(ITEMS):
for iid, phrase, tpl, view, want3d in ITEMS:
row = {"id": iid, "steps": {}}
results.append(row)
state["current"] = iid
hb(state)
try:
g = api("/api/gen", {"phrase": phrase, "template": tpl, "view": view,
"backend": "klein"})
j = wait(g["job"], 300)
row["steps"]["gen"] = j.get("status")
if j.get("status") != "done":
raise RuntimeError("gen: " + str(j.get("log", ""))[:200])
png = j["out"]
c = api("/api/rmbg", {"path": png})
j = wait(c["job"], 300)
row["steps"]["rmbg"] = j.get("status")
if j.get("status") != "done":
raise RuntimeError("rmbg: " + str(j.get("log", ""))[:200])
cut = j["out"]
row["cut"] = cut
if want3d:
t = api("/api/to3d", {"path": cut})
j = wait(t["job"], 1500)
row["steps"]["to3d"] = j.get("status")
if j.get("status") != "done":
raise RuntimeError("to3d: " + str(j.get("log", ""))[:200])
row["glb"] = j.get("out")
state["done"] += 1
except Exception as e:
row["error"] = str(e)[:300]
state["failed"] += 1
s = slug(phrase)
png = os.path.join(LIB, "gen", f"{s}-v0.png")
cut = os.path.join(LIB, "gen", f"{s}-v0-cut.png")
glb = os.path.join(LIB, "garments", f"{slug(s + '-v0-cut')}-rigid.glb")
# farm 429s surface as *job errors* inside wardrobegod's worker thread (its
# fx() has no retry), so backoff must live at ITEM level, not HTTP level.
for attempt in range(4):
try:
if want3d and os.path.exists(glb):
row["steps"]["all"] = "skipped (glb exists)"
row["glb"] = glb
state["skipped"] += 1
break
if not os.path.exists(cut):
if not os.path.exists(png):
step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl,
"view": view, "backend": "klein"}), 900)
else:
row["steps"]["gen"] = "skipped"
step(row, "rmbg", api("/api/rmbg", {"path": png}), 900)
else:
row["steps"]["gen"] = row["steps"]["rmbg"] = "skipped"
row["cut"] = cut
if want3d:
out = step(row, "to3d", api("/api/to3d", {"path": cut}), 2400)
row["glb"] = out
state["done"] += 1
row.pop("error", None)
break
except Exception as e:
msg = str(e)[:300]
row["error"] = msg
retryable = ("429", "Too Many", "timed out", "401", "Unauthorized",
"500", "502", "503")
if any(t in msg for t in retryable) and attempt < 3:
row["steps"][f"retry{attempt+1}"] = "waiting 300s (farm busy)"
hb(state)
time.sleep(300)
continue
state["failed"] += 1
break
hb(state)
state["status"] = "done"
state["current"] = ""
hb(state)
print(json.dumps(state, indent=1))
print(json.dumps({k: state[k] for k in ("done", "failed", "skipped")}, indent=1))