diff --git a/server.py b/server.py index cce6f3d..be15df2 100644 --- a/server.py +++ b/server.py @@ -1099,8 +1099,8 @@ class H(BaseHTTPRequestHandler): def fx(jid): aid = mb_upload(p) - j = mb_req('/api/jobs', {'operator': 'trellis_mac', 'asset_id': aid, 'params': {}}) - mb_wait(j.get('id') or j.get('job_id'), jid, timeout=1200) + j = mb_req('/api/jobs', {'operator': 'trellis2_mlx', 'asset_id': aid, 'params': {}}) + mb_wait(j.get('id') or j.get('job_id'), jid, timeout=2400) outs = mb_outputs(j.get('id') or j.get('job_id')) glbs = [o for o in outs if str(o.get('name', o.get('filename', ''))).lower().endswith('.glb')] or outs if not glbs: diff --git a/tools/factory.py b/tools/factory.py index 46fc4bb..dd9695f 100644 --- a/tools/factory.py +++ b/tools/factory.py @@ -1,21 +1,26 @@ #!/usr/bin/env python3 -"""wardrobegod batch factory — build-first catalogue items, serially, via the -live wardrobegod API. Heartbeat at ~/.jobs/wardrobe-batch1.json. +"""wardrobegod batch factory v2 — build-first catalogue items via the live API. -Per item: /api/gen (klein on the farm) -> /api/rmbg (RMBG-2.0 cutout) -> -[3D items] /api/to3d (TRELLIS, ~5 min serial GPU lane). -Cutout PNGs and GLBs land in library/ automatically — that is the point of -driving the product API instead of the farm directly. +v2 fixes (learned from batch1's carnage): +- 429 "too many active jobs" is EXPECTED under farm contention -> exponential + backoff up to ~20 min instead of instant failure +- to3d waits 40 min (TRELLIS queues behind other users' GPU jobs; 20 was not enough) +- idempotent: skips any step whose output already exists on disk, so re-running + after a partial batch only does the missing work +- one item fully finishes before the next starts (never more than 1 farm job + from us in flight -> the active-jobs cap cannot cascade) + +Heartbeat: ~/.jobs/wardrobe-batch1.json """ -import json, os, time, urllib.request +import json, os, time, urllib.request, urllib.error API = "http://100.91.239.7:8150" +LIB = os.path.expanduser("~/Documents/wardrobegod/library") HB = os.path.expanduser("~/.jobs/wardrobe-batch1.json") os.makedirs(os.path.dirname(HB), exist_ok=True) # id, phrase, template, view, to3d? ITEMS = [ - # --- R2 rigid props/hats -> full 3D chain --- ("cap-baseball-fwd", "faded navy blue baseball cap", "object3d", "hat", True), ("hardhat-construction", "yellow construction hard hat", "object3d", "hat", True), ("headphones-dj-overear","black over-ear DJ headphones with a coiled cable", "object3d", "hat", True), @@ -26,7 +31,6 @@ ITEMS = [ ("sneaker-low-canvas", "pair of plain white low-top canvas sneakers", "object3d", "shoes", True), ("boot-work-steelcap", "pair of tan leather steel-cap work boots", "object3d", "shoes", True), ("chain-necklace-curb", "chunky silver curb chain necklace laid in a circle", "object3d", "hat", True), - # --- R1 garment cutouts (gen + rmbg only) --- ("tee-plain-white", "plain white crew-neck t-shirt", "flat3d", "top", False), ("tee-print-smiley", "black t-shirt with a large orange smiley face print", "flat3d", "top", False), ("tee-print-sunset", "white t-shirt with a retro sunset stripes print", "flat3d", "top", False), @@ -36,26 +40,61 @@ ITEMS = [ ] +def slug(s): + import re + return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-") + + def api(path, payload=None, timeout=30): - if payload is None: - req = urllib.request.Request(API + path) - else: - req = urllib.request.Request(API + path, data=json.dumps(payload).encode(), - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=timeout) as r: - return json.load(r) + """POST/GET with backoff on 429/5xx. 429 here means the farm's active-job + cap or lane contention — wait it out, don't fail the item.""" + delay = 30 + for attempt in range(12): + try: + if payload is None: + req = urllib.request.Request(API + path) + else: + req = urllib.request.Request(API + path, data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.load(r) + except urllib.error.HTTPError as e: + body = "" + try: + body = e.read().decode()[:120] + except Exception: + pass + if e.code in (429, 500, 502, 503, 404) and attempt < 11: + time.sleep(delay) + delay = min(delay * 1.6, 180) + continue + raise RuntimeError(f"HTTP {e.code} {body}") + except OSError: + if attempt < 11: + time.sleep(delay) + continue + raise + raise RuntimeError("retries exhausted") -def wait(jid, timeout=1500): +def wait(jid, timeout): t0 = time.time() while time.time() - t0 < timeout: j = api(f"/api/job/{jid}") if j.get("status") in ("done", "error"): return j - time.sleep(6) + time.sleep(8) return {"status": "error", "log": "poll timeout"} +def step(row, name, jid_resp, timeout): + j = wait(jid_resp["job"], timeout) + row["steps"][name] = j.get("status") + if j.get("status") != "done": + raise RuntimeError(f"{name}: " + str(j.get("log", j.get("note", "")))[:200]) + return j.get("out") + + def hb(state): state["updated"] = time.strftime("%Y-%m-%d %H:%M:%S") json.dump(state, open(HB, "w"), indent=1) @@ -63,45 +102,57 @@ def hb(state): results = [] state = {"job": "wardrobe-batch1", "status": "running", "total": len(ITEMS), - "done": 0, "failed": 0, "current": "", "results": results} -hb(state) + "done": 0, "failed": 0, "skipped": 0, "current": "", "results": results} -for i, (iid, phrase, tpl, view, want3d) in enumerate(ITEMS): +for iid, phrase, tpl, view, want3d in ITEMS: row = {"id": iid, "steps": {}} results.append(row) state["current"] = iid hb(state) - try: - g = api("/api/gen", {"phrase": phrase, "template": tpl, "view": view, - "backend": "klein"}) - j = wait(g["job"], 300) - row["steps"]["gen"] = j.get("status") - if j.get("status") != "done": - raise RuntimeError("gen: " + str(j.get("log", ""))[:200]) - png = j["out"] - - c = api("/api/rmbg", {"path": png}) - j = wait(c["job"], 300) - row["steps"]["rmbg"] = j.get("status") - if j.get("status") != "done": - raise RuntimeError("rmbg: " + str(j.get("log", ""))[:200]) - cut = j["out"] - row["cut"] = cut - - if want3d: - t = api("/api/to3d", {"path": cut}) - j = wait(t["job"], 1500) - row["steps"]["to3d"] = j.get("status") - if j.get("status") != "done": - raise RuntimeError("to3d: " + str(j.get("log", ""))[:200]) - row["glb"] = j.get("out") - state["done"] += 1 - except Exception as e: - row["error"] = str(e)[:300] - state["failed"] += 1 + s = slug(phrase) + png = os.path.join(LIB, "gen", f"{s}-v0.png") + cut = os.path.join(LIB, "gen", f"{s}-v0-cut.png") + glb = os.path.join(LIB, "garments", f"{slug(s + '-v0-cut')}-rigid.glb") + # farm 429s surface as *job errors* inside wardrobegod's worker thread (its + # fx() has no retry), so backoff must live at ITEM level, not HTTP level. + for attempt in range(4): + try: + if want3d and os.path.exists(glb): + row["steps"]["all"] = "skipped (glb exists)" + row["glb"] = glb + state["skipped"] += 1 + break + if not os.path.exists(cut): + if not os.path.exists(png): + step(row, "gen", api("/api/gen", {"phrase": phrase, "template": tpl, + "view": view, "backend": "klein"}), 900) + else: + row["steps"]["gen"] = "skipped" + step(row, "rmbg", api("/api/rmbg", {"path": png}), 900) + else: + row["steps"]["gen"] = row["steps"]["rmbg"] = "skipped" + row["cut"] = cut + if want3d: + out = step(row, "to3d", api("/api/to3d", {"path": cut}), 2400) + row["glb"] = out + state["done"] += 1 + row.pop("error", None) + break + except Exception as e: + msg = str(e)[:300] + row["error"] = msg + retryable = ("429", "Too Many", "timed out", "401", "Unauthorized", + "500", "502", "503") + if any(t in msg for t in retryable) and attempt < 3: + row["steps"][f"retry{attempt+1}"] = "waiting 300s (farm busy)" + hb(state) + time.sleep(300) + continue + state["failed"] += 1 + break hb(state) state["status"] = "done" state["current"] = "" hb(state) -print(json.dumps(state, indent=1)) +print(json.dumps({k: state[k] for k in ("done", "failed", "skipped")}, indent=1))