165 lines
7.2 KiB
Python
165 lines
7.2 KiB
Python
#!/usr/bin/env python3
|
|
"""wardrobegod batch factory v2 — build-first catalogue items via the live API.
|
|
|
|
v2 fixes (learned from batch1's carnage):
|
|
- 429 "too many active jobs" is EXPECTED under farm contention -> exponential
|
|
backoff up to ~20 min instead of instant failure
|
|
- to3d waits 40 min (TRELLIS queues behind other users' GPU jobs; 20 was not enough)
|
|
- idempotent: skips any step whose output already exists on disk, so re-running
|
|
after a partial batch only does the missing work
|
|
- one item fully finishes before the next starts (never more than 1 farm job
|
|
from us in flight -> the active-jobs cap cannot cascade)
|
|
|
|
Heartbeat: ~/.jobs/wardrobe-batch1.json
|
|
"""
|
|
import json, os, time, urllib.request, urllib.error
|
|
|
|
API = "http://100.91.239.7:8150"
|
|
LIB = os.path.expanduser("~/Documents/wardrobegod/library")
|
|
HB = os.path.expanduser("~/.jobs/wardrobe-batch1.json")
|
|
os.makedirs(os.path.dirname(HB), exist_ok=True)
|
|
|
|
# id, phrase, template, view, to3d?
|
|
ITEMS = [
|
|
("cap-baseball-fwd", "faded navy blue baseball cap", "object3d", "hat", True),
|
|
("hardhat-construction", "yellow construction hard hat", "object3d", "hat", True),
|
|
("headphones-dj-overear","black over-ear DJ headphones with a coiled cable", "object3d", "hat", True),
|
|
("record-vinyl-12in", "12 inch black vinyl record with a plain orange centre label", "object3d", "hat", True),
|
|
("sunglasses-wayfarer-generic", "black chunky rectangular-frame sunglasses", "object3d", "hat", True),
|
|
("bag-record-crossbody", "black canvas record bag with a wide shoulder strap", "object3d", "bag", True),
|
|
("shopping-basket", "red plastic shopping basket with two metal handles", "object3d", "bag", True),
|
|
("sneaker-low-canvas", "pair of plain white low-top canvas sneakers", "object3d", "shoes", True),
|
|
("boot-work-steelcap", "pair of tan leather steel-cap work boots", "object3d", "shoes", True),
|
|
("chain-necklace-curb", "chunky silver curb chain necklace laid in a circle", "object3d", "hat", True),
|
|
("tee-plain-white", "plain white crew-neck t-shirt", "flat3d", "top", False),
|
|
("tee-print-smiley", "black t-shirt with a large orange smiley face print", "flat3d", "top", False),
|
|
("tee-print-sunset", "white t-shirt with a retro sunset stripes print", "flat3d", "top", False),
|
|
("hivis-vest-yellow", "yellow hi-vis safety vest with silver reflective stripes", "flat3d", "top", False),
|
|
("tights-opaque-black", "pair of black opaque tights", "flat3d", "bottom", False),
|
|
("shorts-denim-cutoff", "light blue denim cutoff shorts", "flat3d", "bottom", False),
|
|
]
|
|
|
|
|
|
def slug(s):
|
|
import re
|
|
return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")
|
|
|
|
|
|
def api(path, payload=None, timeout=30):
|
|
"""POST/GET with backoff on 429/5xx. 429 here means the farm's active-job
|
|
cap or lane contention — wait it out, don't fail the item."""
|
|
delay = 30
|
|
for attempt in range(12):
|
|
try:
|
|
if payload is None:
|
|
req = urllib.request.Request(API + path)
|
|
else:
|
|
req = urllib.request.Request(API + path, data=json.dumps(payload).encode(),
|
|
headers={"Content-Type": "application/json"})
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return json.load(r)
|
|
except urllib.error.HTTPError as e:
|
|
body = ""
|
|
try:
|
|
body = e.read().decode()[:120]
|
|
except Exception:
|
|
pass
|
|
if e.code in (429, 500, 502, 503, 404) and attempt < 11:
|
|
time.sleep(delay)
|
|
delay = min(delay * 1.6, 180)
|
|
continue
|
|
raise RuntimeError(f"HTTP {e.code} {body}")
|
|
except OSError:
|
|
if attempt < 11:
|
|
time.sleep(delay)
|
|
continue
|
|
raise
|
|
raise RuntimeError("retries exhausted")
|
|
|
|
|
|
def wait(jid, timeout):
|
|
t0 = time.time()
|
|
while time.time() - t0 < timeout:
|
|
j = api(f"/api/job/{jid}")
|
|
if j.get("status") in ("done", "error"):
|
|
return j
|
|
time.sleep(8)
|
|
return {"status": "error", "log": "poll timeout"}
|
|
|
|
|
|
def step(row, name, jid_resp, timeout):
|
|
j = wait(jid_resp["job"], timeout)
|
|
row["steps"][name] = j.get("status")
|
|
if j.get("status") != "done":
|
|
raise RuntimeError(f"{name}: " + str(j.get("log", j.get("note", "")))[:200])
|
|
return j.get("out")
|
|
|
|
|
|
def hb(state):
|
|
state["updated"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
json.dump(state, open(HB, "w"), indent=1)
|
|
|
|
|
|
results = []
|
|
state = {"job": "wardrobe-batch1", "status": "running", "total": len(ITEMS),
|
|
"done": 0, "failed": 0, "skipped": 0, "current": "", "results": results}
|
|
|
|
for iid, phrase, tpl, view, want3d in ITEMS:
|
|
row = {"id": iid, "steps": {}}
|
|
results.append(row)
|
|
state["current"] = iid
|
|
hb(state)
|
|
s = slug(phrase)
|
|
png = os.path.join(LIB, "gen", f"{s}-v0.png")
|
|
cut = os.path.join(LIB, "gen", f"{s}-v0-cut.png")
|
|
glb = os.path.join(LIB, "garments", f"{slug(s + '-v0-cut')}-rigid.glb")
|
|
# farm 429s surface as *job errors* inside wardrobegod's worker thread (its
|
|
# fx() has no retry), so backoff must live at ITEM level, not HTTP level.
|
|
for attempt in range(4):
|
|
try:
|
|
if want3d and os.path.exists(glb):
|
|
row["steps"]["all"] = "skipped (glb exists)"
|
|
row["glb"] = glb
|
|
state["skipped"] += 1
|
|
break
|
|
if not os.path.exists(cut):
|
|
if not os.path.exists(png):
|
|
# use the SERVER-reported output path: wardrobegod truncates
|
|
# long slugs, so predicting the filename fails on long phrases
|
|
png = step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl,
|
|
"view": view, "backend": "klein"}), 900) or png
|
|
cut = png.rsplit(".", 1)[0] + "-cut.png"
|
|
else:
|
|
row["steps"]["gen"] = "skipped"
|
|
step(row, "rmbg", api("/api/rmbg", {"path": png}), 900)
|
|
else:
|
|
row["steps"]["gen"] = row["steps"]["rmbg"] = "skipped"
|
|
row["cut"] = cut
|
|
if want3d:
|
|
# the farm GPU lane is shared fleet-wide and queues run deep
|
|
# (measured 50+ min just to reach the front) — wait 2h, don't
|
|
# give up and resubmit into the same queue
|
|
out = step(row, "to3d", api("/api/to3d", {"path": cut}), 7200)
|
|
row["glb"] = out
|
|
state["done"] += 1
|
|
row.pop("error", None)
|
|
break
|
|
except Exception as e:
|
|
msg = str(e)[:300]
|
|
row["error"] = msg
|
|
retryable = ("429", "Too Many", "timed out", "timeout", "401",
|
|
"Unauthorized", "500", "502", "503")
|
|
if any(t in msg for t in retryable) and attempt < 3:
|
|
row["steps"][f"retry{attempt+1}"] = "waiting 300s (farm busy)"
|
|
hb(state)
|
|
time.sleep(300)
|
|
continue
|
|
state["failed"] += 1
|
|
break
|
|
hb(state)
|
|
|
|
state["status"] = "done"
|
|
state["current"] = ""
|
|
hb(state)
|
|
print(json.dumps({k: state[k] for k in ("done", "failed", "skipped")}, indent=1))
|