monster/wardrobegod
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mb_req retry on transient farm 404/5xx; mb_wait default 3600s (GPU lane queues run deep)

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
type-two 2026-07-29 08:14:29 +10:00
parent b0188a41f0
commit 5b63aee88d
1 changed files with 25 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
server.py
View File

@ -415,20 +415,36 @@ def job_thread(jid, fn):
# ---------- MODELBEAST client (token stays server-side, never in the page) ---------- # ---------- MODELBEAST client (token stays server-side, never in the page) ----------
def mb_req(path, data=None, raw=None, ctype='application/json'): def mb_req(path, data=None, raw=None, ctype='application/json', tries=4):
# the farm intermittently 404s/5xxs valid requests under load (seen three
# times 2026-07-29: a job poll, a gen, a reskin — each one blip killed a
# multi-minute pipeline). Retry briefly; job-create 404s created nothing,
# so the retry cannot double-submit.
last = None
for attempt in range(tries):
req = urllib.request.Request(MB + path, data=raw if raw is not None else (json.dumps(data).encode() if data else None)) req = urllib.request.Request(MB + path, data=raw if raw is not None else (json.dumps(data).encode() if data else None))
req.add_header('Authorization', 'Bearer ' + MB_TOKEN) req.add_header('Authorization', 'Bearer ' + MB_TOKEN)
if data is not None or raw is not None: if data is not None or raw is not None:
req.add_header('Content-Type', ctype) req.add_header('Content-Type', ctype)
try:
with urllib.request.urlopen(req, timeout=120) as r: with urllib.request.urlopen(req, timeout=120) as r:
body = r.read() body = r.read()
break
except urllib.error.HTTPError as e:
if e.code in (404, 500, 502, 503) and attempt < tries - 1:
last = e
time.sleep(5 * (attempt + 1))
continue
raise
else:
raise last
try: try:
return json.loads(body) return json.loads(body)
except ValueError: except ValueError:
return body return body
def mb_wait(job_id, jid, timeout=900): def mb_wait(job_id, jid, timeout=3600):
t0 = time.time() t0 = time.time()
while time.time() - t0 < timeout: while time.time() - t0 < timeout:
j = mb_req(f'/api/jobs/{job_id}') j = mb_req(f'/api/jobs/{job_id}')
@ -1100,7 +1116,7 @@ class H(BaseHTTPRequestHandler):
def fx(jid): def fx(jid):
aid = mb_upload(p) aid = mb_upload(p)
j = mb_req('/api/jobs', {'operator': 'trellis2_mlx', 'asset_id': aid, 'params': {}}) j = mb_req('/api/jobs', {'operator': 'trellis2_mlx', 'asset_id': aid, 'params': {}})
mb_wait(j.get('id') or j.get('job_id'), jid, timeout=2400) mb_wait(j.get('id') or j.get('job_id'), jid, timeout=7200)
outs = mb_outputs(j.get('id') or j.get('job_id')) outs = mb_outputs(j.get('id') or j.get('job_id'))
glbs = [o for o in outs if str(o.get('name', o.get('filename', ''))).lower().endswith('.glb')] or outs glbs = [o for o in outs if str(o.get('name', o.get('filename', ''))).lower().endswith('.glb')] or outs
if not glbs: if not glbs: