diff --git a/server.py b/server.py index be15df2..0e6c289 100644 --- a/server.py +++ b/server.py @@ -415,20 +415,36 @@ def job_thread(jid, fn): # ---------- MODELBEAST client (token stays server-side, never in the page) ---------- -def mb_req(path, data=None, raw=None, ctype='application/json'): - req = urllib.request.Request(MB + path, data=raw if raw is not None else (json.dumps(data).encode() if data else None)) - req.add_header('Authorization', 'Bearer ' + MB_TOKEN) - if data is not None or raw is not None: - req.add_header('Content-Type', ctype) - with urllib.request.urlopen(req, timeout=120) as r: - body = r.read() +def mb_req(path, data=None, raw=None, ctype='application/json', tries=4): + # the farm intermittently 404s/5xxs valid requests under load (seen three + # times 2026-07-29: a job poll, a gen, a reskin — each one blip killed a + # multi-minute pipeline). Retry briefly; job-create 404s created nothing, + # so the retry cannot double-submit. + last = None + for attempt in range(tries): + req = urllib.request.Request(MB + path, data=raw if raw is not None else (json.dumps(data).encode() if data else None)) + req.add_header('Authorization', 'Bearer ' + MB_TOKEN) + if data is not None or raw is not None: + req.add_header('Content-Type', ctype) + try: + with urllib.request.urlopen(req, timeout=120) as r: + body = r.read() + break + except urllib.error.HTTPError as e: + if e.code in (404, 500, 502, 503) and attempt < tries - 1: + last = e + time.sleep(5 * (attempt + 1)) + continue + raise + else: + raise last try: return json.loads(body) except ValueError: return body -def mb_wait(job_id, jid, timeout=900): +def mb_wait(job_id, jid, timeout=3600): t0 = time.time() while time.time() - t0 < timeout: j = mb_req(f'/api/jobs/{job_id}') @@ -1100,7 +1116,7 @@ class H(BaseHTTPRequestHandler): def fx(jid): aid = mb_upload(p) j = mb_req('/api/jobs', {'operator': 'trellis2_mlx', 'asset_id': aid, 'params': {}}) - mb_wait(j.get('id') or j.get('job_id'), jid, timeout=2400) + mb_wait(j.get('id') or j.get('job_id'), jid, timeout=7200) outs = mb_outputs(j.get('id') or j.get('job_id')) glbs = [o for o in outs if str(o.get('name', o.get('filename', ''))).lower().endswith('.glb')] or outs if not glbs: