mb_req retry on transient farm 404/5xx; mb_wait default 3600s (GPU lane queues run deep)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

server.py

@@ -415,20 +415,36 @@ def job_thread(jid, fn):
 
 
 # ---------- MODELBEAST client (token stays server-side, never in the page) ----------
-def mb_req(path, data=None, raw=None, ctype='application/json'):
-    req = urllib.request.Request(MB + path, data=raw if raw is not None else (json.dumps(data).encode() if data else None))
-    req.add_header('Authorization', 'Bearer ' + MB_TOKEN)
-    if data is not None or raw is not None:
-        req.add_header('Content-Type', ctype)
-    with urllib.request.urlopen(req, timeout=120) as r:
-        body = r.read()
+def mb_req(path, data=None, raw=None, ctype='application/json', tries=4):
+    # the farm intermittently 404s/5xxs valid requests under load (seen three
+    # times 2026-07-29: a job poll, a gen, a reskin — each one blip killed a
+    # multi-minute pipeline). Retry briefly; job-create 404s created nothing,
+    # so the retry cannot double-submit.
+    last = None
+    for attempt in range(tries):
+        req = urllib.request.Request(MB + path, data=raw if raw is not None else (json.dumps(data).encode() if data else None))
+        req.add_header('Authorization', 'Bearer ' + MB_TOKEN)
+        if data is not None or raw is not None:
+            req.add_header('Content-Type', ctype)
+        try:
+            with urllib.request.urlopen(req, timeout=120) as r:
+                body = r.read()
+            break
+        except urllib.error.HTTPError as e:
+            if e.code in (404, 500, 502, 503) and attempt < tries - 1:
+                last = e
+                time.sleep(5 * (attempt + 1))
+                continue
+            raise
+    else:
+        raise last
     try:
         return json.loads(body)
     except ValueError:
         return body
 
 
-def mb_wait(job_id, jid, timeout=900):
+def mb_wait(job_id, jid, timeout=3600):
     t0 = time.time()
     while time.time() - t0 < timeout:
         j = mb_req(f'/api/jobs/{job_id}')
@@ -1100,7 +1116,7 @@ class H(BaseHTTPRequestHandler):
             def fx(jid):
                 aid = mb_upload(p)
                 j = mb_req('/api/jobs', {'operator': 'trellis2_mlx', 'asset_id': aid, 'params': {}})
-                mb_wait(j.get('id') or j.get('job_id'), jid, timeout=2400)
+                mb_wait(j.get('id') or j.get('job_id'), jid, timeout=7200)
                 outs = mb_outputs(j.get('id') or j.get('job_id'))
                 glbs = [o for o in outs if str(o.get('name', o.get('filename', ''))).lower().endswith('.glb')] or outs
                 if not glbs: