194 lines
9.4 KiB
Python
194 lines
9.4 KiB
Python
#!/usr/bin/env python3
|
|
"""wardrobegod batch factory v2 — build-first catalogue items via the live API.
|
|
|
|
v2 fixes (learned from batch1's carnage):
|
|
- 429 "too many active jobs" is EXPECTED under farm contention -> exponential
|
|
backoff up to ~20 min instead of instant failure
|
|
- to3d waits 40 min (TRELLIS queues behind other users' GPU jobs; 20 was not enough)
|
|
- idempotent: skips any step whose output already exists on disk, so re-running
|
|
after a partial batch only does the missing work
|
|
- one item fully finishes before the next starts (never more than 1 farm job
|
|
from us in flight -> the active-jobs cap cannot cascade)
|
|
|
|
Heartbeat: ~/.jobs/wardrobe-batch7.json
|
|
"""
|
|
import json, os, time, urllib.request, urllib.error
|
|
|
|
API = "http://100.91.239.7:8150"
|
|
LIB = os.path.expanduser("~/Documents/wardrobegod/library")
|
|
HB = os.path.expanduser("~/.jobs/wardrobe-batch7.json")
|
|
os.makedirs(os.path.dirname(HB), exist_ok=True)
|
|
|
|
# id, phrase, template, view, to3d?
|
|
ITEMS = [
|
|
# flats
|
|
("coat-overcoat-wool", "long charcoal wool overcoat", "flat3d", "top", False),
|
|
("hoodie-grey", "plain grey pullover hoodie with drawstrings", "flat3d", "top", False),
|
|
("jacket-varsity-flat","varsity jacket with cream sleeves and navy body, plain letter patch", "flat3d", "top", False),
|
|
("uniform-busdriver", "light blue bus driver uniform shirt with epaulettes", "flat3d", "top", False),
|
|
("pyjamas-flannel", "red tartan flannel pyjama set", "flat3d", "top", False),
|
|
("robe-dressing-gown", "burgundy quilted dressing gown with cord belt", "flat3d", "top", False),
|
|
("nightie-cotton", "white cotton nightie with lace collar", "flat3d", "top", False),
|
|
("skirt-pencil", "black knee length pencil skirt", "flat3d", "bottom", False),
|
|
("dress-maxi-floral", "long floral maxi dress with tiered skirt", "flat3d", "top", False),
|
|
("top-corset-boned", "black satin boned corset top with lacing", "flat3d", "top", False),
|
|
("waistcoat", "grey tweed waistcoat with buttons", "flat3d", "top", False),
|
|
("shorts-board", "long floral board shorts", "flat3d", "bottom", False),
|
|
# headwear round 3
|
|
("fedora", "dark grey felt fedora with black band", "object3d", "hat", True),
|
|
("trilby", "brown straw trilby hat", "object3d", "hat", True),
|
|
("visor-sports", "white sports sun visor", "object3d", "hat", True),
|
|
("sun-hat-wide", "cream wide brim woven sun hat with ribbon", "object3d", "hat", True),
|
|
("headscarf-tied", "red polka dot headscarf tied in a knot", "object3d", "hat", True),
|
|
("bandana-tied", "folded navy paisley bandana tied in a ring", "object3d", "hat", True),
|
|
("crown-simple", "simple gold crown with five points", "object3d", "hat", True),
|
|
("tiara", "silver tiara with crystal peaks", "object3d", "hat", True),
|
|
("party-hat", "striped cardboard cone party hat with pom pom", "object3d", "hat", True),
|
|
("durag-black", "black satin durag with long tails", "object3d", "hat", True),
|
|
# footwear round 2
|
|
("sneaker-high-canvas","pair of red high top canvas sneakers", "object3d", "shoes", True),
|
|
("sneaker-runner-90s", "pair of chunky 90s running shoes, white and neon", "object3d", "shoes", True),
|
|
("boot-cowboy", "pair of tooled leather cowboy boots", "object3d", "shoes", True),
|
|
("boot-hiking", "pair of brown leather hiking boots with red laces", "object3d", "shoes", True),
|
|
("boot-wellington", "pair of green rubber wellington boots", "object3d", "shoes", True),
|
|
("flat-loafer", "pair of burgundy penny loafers", "object3d", "shoes", True),
|
|
("flat-mary-jane", "pair of black patent mary jane shoes", "object3d", "shoes", True),
|
|
("heel-stiletto", "pair of red stiletto high heels", "object3d", "shoes", True),
|
|
# props
|
|
("umbrella-closed", "closed black umbrella with hook handle", "object3d", "hat", True),
|
|
("umbrella-open", "open rainbow striped umbrella", "object3d", "bag", True),
|
|
("cane-walking", "wooden walking cane with curved handle", "object3d", "hat", True),
|
|
("skate-roller-quad", "pair of retro quad roller skates with orange wheels", "object3d", "shoes", True),
|
|
("sign-handheld", "rectangular handheld cardboard sign, blank", "object3d", "hat", True),
|
|
("traffic-wand", "orange traffic control light wand baton", "object3d", "hat", True),
|
|
("wallet-chain", "black leather wallet with silver chain", "object3d", "hat", True),
|
|
("keys-ring", "bunch of keys on a metal key ring", "object3d", "hat", True),
|
|
# jewellery trials
|
|
("chain-figaro", "gold figaro chain necklace laid in a circle", "object3d", "hat", True),
|
|
("earring-hoop-large", "pair of large gold hoop earrings", "object3d", "hat", True),
|
|
]
|
|
|
|
|
|
def slug(s):
|
|
import re
|
|
return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")
|
|
|
|
|
|
def api(path, payload=None, timeout=30):
|
|
"""POST/GET with backoff on 429/5xx. 429 here means the farm's active-job
|
|
cap or lane contention — wait it out, don't fail the item."""
|
|
delay = 30
|
|
for attempt in range(12):
|
|
try:
|
|
if payload is None:
|
|
req = urllib.request.Request(API + path)
|
|
else:
|
|
req = urllib.request.Request(API + path, data=json.dumps(payload).encode(),
|
|
headers={"Content-Type": "application/json"})
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return json.load(r)
|
|
except urllib.error.HTTPError as e:
|
|
body = ""
|
|
try:
|
|
body = e.read().decode()[:120]
|
|
except Exception:
|
|
pass
|
|
if e.code in (429, 500, 502, 503, 404) and attempt < 11:
|
|
time.sleep(delay)
|
|
delay = min(delay * 1.6, 180)
|
|
continue
|
|
raise RuntimeError(f"HTTP {e.code} {body}")
|
|
except OSError:
|
|
if attempt < 11:
|
|
time.sleep(delay)
|
|
continue
|
|
raise
|
|
raise RuntimeError("retries exhausted")
|
|
|
|
|
|
def wait(jid, timeout):
|
|
t0 = time.time()
|
|
while time.time() - t0 < timeout:
|
|
j = api(f"/api/job/{jid}")
|
|
if j.get("status") in ("done", "error"):
|
|
return j
|
|
time.sleep(8)
|
|
return {"status": "error", "log": "poll timeout"}
|
|
|
|
|
|
def step(row, name, jid_resp, timeout):
|
|
j = wait(jid_resp["job"], timeout)
|
|
row["steps"][name] = j.get("status")
|
|
if j.get("status") != "done":
|
|
raise RuntimeError(f"{name}: " + str(j.get("log", j.get("note", "")))[:200])
|
|
return j.get("out")
|
|
|
|
|
|
def hb(state):
|
|
state["updated"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
json.dump(state, open(HB, "w"), indent=1)
|
|
|
|
|
|
results = []
|
|
state = {"job": "wardrobe-batch7", "status": "running", "total": len(ITEMS),
|
|
"done": 0, "failed": 0, "skipped": 0, "current": "", "results": results}
|
|
|
|
for iid, phrase, tpl, view, want3d in ITEMS:
|
|
row = {"id": iid, "steps": {}}
|
|
results.append(row)
|
|
state["current"] = iid
|
|
hb(state)
|
|
s = slug(phrase)
|
|
png = os.path.join(LIB, "gen", f"{s}-v0.png")
|
|
cut = os.path.join(LIB, "gen", f"{s}-v0-cut.png")
|
|
glb = os.path.join(LIB, "garments", f"{slug(s + '-v0-cut')}-rigid.glb")
|
|
# farm 429s surface as *job errors* inside wardrobegod's worker thread (its
|
|
# fx() has no retry), so backoff must live at ITEM level, not HTTP level.
|
|
for attempt in range(4):
|
|
try:
|
|
if want3d and os.path.exists(glb):
|
|
row["steps"]["all"] = "skipped (glb exists)"
|
|
row["glb"] = glb
|
|
state["skipped"] += 1
|
|
break
|
|
if not os.path.exists(cut):
|
|
if not os.path.exists(png):
|
|
# use the SERVER-reported output path: wardrobegod truncates
|
|
# long slugs, so predicting the filename fails on long phrases
|
|
png = step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl,
|
|
"view": view, "backend": "klein"}), 900) or png
|
|
cut = png.rsplit(".", 1)[0] + "-cut.png"
|
|
else:
|
|
row["steps"]["gen"] = "skipped"
|
|
step(row, "rmbg", api("/api/rmbg", {"path": png}), 900)
|
|
else:
|
|
row["steps"]["gen"] = row["steps"]["rmbg"] = "skipped"
|
|
row["cut"] = cut
|
|
if want3d:
|
|
# the farm GPU lane is shared fleet-wide and queues run deep
|
|
# (measured 50+ min just to reach the front) — wait 2h, don't
|
|
# give up and resubmit into the same queue
|
|
out = step(row, "to3d", api("/api/to3d", {"path": cut}), 7200)
|
|
row["glb"] = out
|
|
state["done"] += 1
|
|
row.pop("error", None)
|
|
break
|
|
except Exception as e:
|
|
msg = str(e)[:300]
|
|
row["error"] = msg
|
|
retryable = ("429", "Too Many", "timed out", "timeout", "401",
|
|
"Unauthorized", "500", "502", "503")
|
|
if any(t in msg for t in retryable) and attempt < 3:
|
|
row["steps"][f"retry{attempt+1}"] = "waiting 300s (farm busy)"
|
|
hb(state)
|
|
time.sleep(300)
|
|
continue
|
|
state["failed"] += 1
|
|
break
|
|
hb(state)
|
|
|
|
state["status"] = "done"
|
|
state["current"] = ""
|
|
hb(state)
|
|
print(json.dumps({k: state[k] for k in ("done", "failed", "skipped")}, indent=1))
|