factory: use server-reported gen output path — wardrobegod truncates long slugs
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
type-two
parent
d4cd8e7ad7
commit
9c172fcf70
@ -124,8 +124,11 @@ for iid, phrase, tpl, view, want3d in ITEMS:
|
||||
break
|
||||
if not os.path.exists(cut):
|
||||
if not os.path.exists(png):
|
||||
step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl,
|
||||
"view": view, "backend": "klein"}), 900)
|
||||
# use the SERVER-reported output path: wardrobegod truncates
|
||||
# long slugs, so predicting the filename fails on long phrases
|
||||
png = step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl,
|
||||
"view": view, "backend": "klein"}), 900) or png
|
||||
cut = png.rsplit(".", 1)[0] + "-cut.png"
|
||||
else:
|
||||
row["steps"]["gen"] = "skipped"
|
||||
step(row, "rmbg", api("/api/rmbg", {"path": png}), 900)
|
||||
@ -133,7 +136,10 @@ for iid, phrase, tpl, view, want3d in ITEMS:
|
||||
row["steps"]["gen"] = row["steps"]["rmbg"] = "skipped"
|
||||
row["cut"] = cut
|
||||
if want3d:
|
||||
out = step(row, "to3d", api("/api/to3d", {"path": cut}), 2400)
|
||||
# the farm GPU lane is shared fleet-wide and queues run deep
|
||||
# (measured 50+ min just to reach the front) — wait 2h, don't
|
||||
# give up and resubmit into the same queue
|
||||
out = step(row, "to3d", api("/api/to3d", {"path": cut}), 7200)
|
||||
row["glb"] = out
|
||||
state["done"] += 1
|
||||
row.pop("error", None)
|
||||
@ -141,8 +147,8 @@ for iid, phrase, tpl, view, want3d in ITEMS:
|
||||
except Exception as e:
|
||||
msg = str(e)[:300]
|
||||
row["error"] = msg
|
||||
retryable = ("429", "Too Many", "timed out", "401", "Unauthorized",
|
||||
"500", "502", "503")
|
||||
retryable = ("429", "Too Many", "timed out", "timeout", "401",
|
||||
"Unauthorized", "500", "502", "503")
|
||||
if any(t in msg for t in retryable) and attempt < 3:
|
||||
row["steps"][f"retry{attempt+1}"] = "waiting 300s (farm busy)"
|
||||
hb(state)
|
||||
|
||||
193
tools/factory7.py
Normal file
193
tools/factory7.py
Normal file
@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""wardrobegod batch factory v2 — build-first catalogue items via the live API.
|
||||
|
||||
v2 fixes (learned from batch1's carnage):
|
||||
- 429 "too many active jobs" is EXPECTED under farm contention -> exponential
|
||||
backoff up to ~20 min instead of instant failure
|
||||
- to3d waits 40 min (TRELLIS queues behind other users' GPU jobs; 20 was not enough)
|
||||
- idempotent: skips any step whose output already exists on disk, so re-running
|
||||
after a partial batch only does the missing work
|
||||
- one item fully finishes before the next starts (never more than 1 farm job
|
||||
from us in flight -> the active-jobs cap cannot cascade)
|
||||
|
||||
Heartbeat: ~/.jobs/wardrobe-batch7.json
|
||||
"""
|
||||
import json, os, time, urllib.request, urllib.error
|
||||
|
||||
API = "http://100.91.239.7:8150"
|
||||
LIB = os.path.expanduser("~/Documents/wardrobegod/library")
|
||||
HB = os.path.expanduser("~/.jobs/wardrobe-batch7.json")
|
||||
os.makedirs(os.path.dirname(HB), exist_ok=True)
|
||||
|
||||
# id, phrase, template, view, to3d?
|
||||
ITEMS = [
|
||||
# flats
|
||||
("coat-overcoat-wool", "long charcoal wool overcoat", "flat3d", "top", False),
|
||||
("hoodie-grey", "plain grey pullover hoodie with drawstrings", "flat3d", "top", False),
|
||||
("jacket-varsity-flat","varsity jacket with cream sleeves and navy body, plain letter patch", "flat3d", "top", False),
|
||||
("uniform-busdriver", "light blue bus driver uniform shirt with epaulettes", "flat3d", "top", False),
|
||||
("pyjamas-flannel", "red tartan flannel pyjama set", "flat3d", "top", False),
|
||||
("robe-dressing-gown", "burgundy quilted dressing gown with cord belt", "flat3d", "top", False),
|
||||
("nightie-cotton", "white cotton nightie with lace collar", "flat3d", "top", False),
|
||||
("skirt-pencil", "black knee length pencil skirt", "flat3d", "bottom", False),
|
||||
("dress-maxi-floral", "long floral maxi dress with tiered skirt", "flat3d", "top", False),
|
||||
("top-corset-boned", "black satin boned corset top with lacing", "flat3d", "top", False),
|
||||
("waistcoat", "grey tweed waistcoat with buttons", "flat3d", "top", False),
|
||||
("shorts-board", "long floral board shorts", "flat3d", "bottom", False),
|
||||
# headwear round 3
|
||||
("fedora", "dark grey felt fedora with black band", "object3d", "hat", True),
|
||||
("trilby", "brown straw trilby hat", "object3d", "hat", True),
|
||||
("visor-sports", "white sports sun visor", "object3d", "hat", True),
|
||||
("sun-hat-wide", "cream wide brim woven sun hat with ribbon", "object3d", "hat", True),
|
||||
("headscarf-tied", "red polka dot headscarf tied in a knot", "object3d", "hat", True),
|
||||
("bandana-tied", "folded navy paisley bandana tied in a ring", "object3d", "hat", True),
|
||||
("crown-simple", "simple gold crown with five points", "object3d", "hat", True),
|
||||
("tiara", "silver tiara with crystal peaks", "object3d", "hat", True),
|
||||
("party-hat", "striped cardboard cone party hat with pom pom", "object3d", "hat", True),
|
||||
("durag-black", "black satin durag with long tails", "object3d", "hat", True),
|
||||
# footwear round 2
|
||||
("sneaker-high-canvas","pair of red high top canvas sneakers", "object3d", "shoes", True),
|
||||
("sneaker-runner-90s", "pair of chunky 90s running shoes, white and neon", "object3d", "shoes", True),
|
||||
("boot-cowboy", "pair of tooled leather cowboy boots", "object3d", "shoes", True),
|
||||
("boot-hiking", "pair of brown leather hiking boots with red laces", "object3d", "shoes", True),
|
||||
("boot-wellington", "pair of green rubber wellington boots", "object3d", "shoes", True),
|
||||
("flat-loafer", "pair of burgundy penny loafers", "object3d", "shoes", True),
|
||||
("flat-mary-jane", "pair of black patent mary jane shoes", "object3d", "shoes", True),
|
||||
("heel-stiletto", "pair of red stiletto high heels", "object3d", "shoes", True),
|
||||
# props
|
||||
("umbrella-closed", "closed black umbrella with hook handle", "object3d", "hat", True),
|
||||
("umbrella-open", "open rainbow striped umbrella", "object3d", "bag", True),
|
||||
("cane-walking", "wooden walking cane with curved handle", "object3d", "hat", True),
|
||||
("skate-roller-quad", "pair of retro quad roller skates with orange wheels", "object3d", "shoes", True),
|
||||
("sign-handheld", "rectangular handheld cardboard sign, blank", "object3d", "hat", True),
|
||||
("traffic-wand", "orange traffic control light wand baton", "object3d", "hat", True),
|
||||
("wallet-chain", "black leather wallet with silver chain", "object3d", "hat", True),
|
||||
("keys-ring", "bunch of keys on a metal key ring", "object3d", "hat", True),
|
||||
# jewellery trials
|
||||
("chain-figaro", "gold figaro chain necklace laid in a circle", "object3d", "hat", True),
|
||||
("earring-hoop-large", "pair of large gold hoop earrings", "object3d", "hat", True),
|
||||
]
|
||||
|
||||
|
||||
def slug(s):
|
||||
import re
|
||||
return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")
|
||||
|
||||
|
||||
def api(path, payload=None, timeout=30):
|
||||
"""POST/GET with backoff on 429/5xx. 429 here means the farm's active-job
|
||||
cap or lane contention — wait it out, don't fail the item."""
|
||||
delay = 30
|
||||
for attempt in range(12):
|
||||
try:
|
||||
if payload is None:
|
||||
req = urllib.request.Request(API + path)
|
||||
else:
|
||||
req = urllib.request.Request(API + path, data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return json.load(r)
|
||||
except urllib.error.HTTPError as e:
|
||||
body = ""
|
||||
try:
|
||||
body = e.read().decode()[:120]
|
||||
except Exception:
|
||||
pass
|
||||
if e.code in (429, 500, 502, 503, 404) and attempt < 11:
|
||||
time.sleep(delay)
|
||||
delay = min(delay * 1.6, 180)
|
||||
continue
|
||||
raise RuntimeError(f"HTTP {e.code} {body}")
|
||||
except OSError:
|
||||
if attempt < 11:
|
||||
time.sleep(delay)
|
||||
continue
|
||||
raise
|
||||
raise RuntimeError("retries exhausted")
|
||||
|
||||
|
||||
def wait(jid, timeout):
|
||||
t0 = time.time()
|
||||
while time.time() - t0 < timeout:
|
||||
j = api(f"/api/job/{jid}")
|
||||
if j.get("status") in ("done", "error"):
|
||||
return j
|
||||
time.sleep(8)
|
||||
return {"status": "error", "log": "poll timeout"}
|
||||
|
||||
|
||||
def step(row, name, jid_resp, timeout):
|
||||
j = wait(jid_resp["job"], timeout)
|
||||
row["steps"][name] = j.get("status")
|
||||
if j.get("status") != "done":
|
||||
raise RuntimeError(f"{name}: " + str(j.get("log", j.get("note", "")))[:200])
|
||||
return j.get("out")
|
||||
|
||||
|
||||
def hb(state):
|
||||
state["updated"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
json.dump(state, open(HB, "w"), indent=1)
|
||||
|
||||
|
||||
results = []
|
||||
state = {"job": "wardrobe-batch7", "status": "running", "total": len(ITEMS),
|
||||
"done": 0, "failed": 0, "skipped": 0, "current": "", "results": results}
|
||||
|
||||
for iid, phrase, tpl, view, want3d in ITEMS:
|
||||
row = {"id": iid, "steps": {}}
|
||||
results.append(row)
|
||||
state["current"] = iid
|
||||
hb(state)
|
||||
s = slug(phrase)
|
||||
png = os.path.join(LIB, "gen", f"{s}-v0.png")
|
||||
cut = os.path.join(LIB, "gen", f"{s}-v0-cut.png")
|
||||
glb = os.path.join(LIB, "garments", f"{slug(s + '-v0-cut')}-rigid.glb")
|
||||
# farm 429s surface as *job errors* inside wardrobegod's worker thread (its
|
||||
# fx() has no retry), so backoff must live at ITEM level, not HTTP level.
|
||||
for attempt in range(4):
|
||||
try:
|
||||
if want3d and os.path.exists(glb):
|
||||
row["steps"]["all"] = "skipped (glb exists)"
|
||||
row["glb"] = glb
|
||||
state["skipped"] += 1
|
||||
break
|
||||
if not os.path.exists(cut):
|
||||
if not os.path.exists(png):
|
||||
# use the SERVER-reported output path: wardrobegod truncates
|
||||
# long slugs, so predicting the filename fails on long phrases
|
||||
png = step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl,
|
||||
"view": view, "backend": "klein"}), 900) or png
|
||||
cut = png.rsplit(".", 1)[0] + "-cut.png"
|
||||
else:
|
||||
row["steps"]["gen"] = "skipped"
|
||||
step(row, "rmbg", api("/api/rmbg", {"path": png}), 900)
|
||||
else:
|
||||
row["steps"]["gen"] = row["steps"]["rmbg"] = "skipped"
|
||||
row["cut"] = cut
|
||||
if want3d:
|
||||
# the farm GPU lane is shared fleet-wide and queues run deep
|
||||
# (measured 50+ min just to reach the front) — wait 2h, don't
|
||||
# give up and resubmit into the same queue
|
||||
out = step(row, "to3d", api("/api/to3d", {"path": cut}), 7200)
|
||||
row["glb"] = out
|
||||
state["done"] += 1
|
||||
row.pop("error", None)
|
||||
break
|
||||
except Exception as e:
|
||||
msg = str(e)[:300]
|
||||
row["error"] = msg
|
||||
retryable = ("429", "Too Many", "timed out", "timeout", "401",
|
||||
"Unauthorized", "500", "502", "503")
|
||||
if any(t in msg for t in retryable) and attempt < 3:
|
||||
row["steps"][f"retry{attempt+1}"] = "waiting 300s (farm busy)"
|
||||
hb(state)
|
||||
time.sleep(300)
|
||||
continue
|
||||
state["failed"] += 1
|
||||
break
|
||||
hb(state)
|
||||
|
||||
state["status"] = "done"
|
||||
state["current"] = ""
|
||||
hb(state)
|
||||
print(json.dumps({k: state[k] for k in ("done", "failed", "skipped")}, indent=1))
|
||||
Loading…
Reference in New Issue
Block a user