diff --git a/content.js b/content.js index 08124eb..898fdfa 100644 --- a/content.js +++ b/content.js @@ -35,6 +35,8 @@ let pollTimeout = null; let generationInProgress = false; let successCount = 0; let consecutiveRateLimits = 0; +let consecutiveFails = 0; +const FAIL_STREAK_STOP = 3; // when Flow goes quiet, it stays quiet. Stop; don't eat the bank. function init() { chrome.storage.local.get({ ...settings, pendingHarvest: null }, (data) => { @@ -480,17 +482,28 @@ async function executeTask(task) { else if (tiles.length) log('info', `New tile detected but no direct media URL — reporting the tile link (recover later). ${gridCensus()}`); await reportCompletion(task.id, 'success', null, fresh); chrome.storage.local.remove('requeue_' + task.id); - successCount++; consecutiveRateLimits = 0; + successCount++; consecutiveRateLimits = 0; consecutiveFails = 0; if (settings.restEvery && successCount % settings.restEvery === 0) { setCooldown(settings.restMs || 15 * 60 * 1000, `Scheduled rest after ${successCount} gens`); } } catch (err) { log('error', `Task ${task.id} failed: ${err.message}`); await reportCompletion(task.id, 'failed', err.message, []); + // CIRCUIT BREAKER. 2026-07-08: Flow went quiet after two images and this loop + // politely fed it 13 more prompts over 45 minutes, marking every one failed. + // When Flow stops producing it does not spontaneously start again — STOP, and + // leave the queue intact so the run can be resumed by hand. + if (++consecutiveFails >= FAIL_STREAK_STOP) { + settings.isRunning = false; + chrome.storage.local.set({ isRunning: false }); + log('system', `⛔ ${consecutiveFails} failures in a row — STOPPING. ` + + `Flow isn't generating. Check the tab (credits? signed out? queue jammed?), ` + + `then re-queue the failed ids and press Start.`); + } } finally { generationInProgress = false; chrome.storage.local.set({ status: settings.isRunning ? 'running' : 'idle' }); - scheduleNext(); + scheduleNext(); // no-ops when the breaker above cleared isRunning } } @@ -530,7 +543,7 @@ async function runPostReloadHarvest(ph) { chrome.storage.local.remove('pendingHarvest'); chrome.storage.local.remove('requeue_' + ph.id); await reportCompletion(ph.id, 'success', null, fresh); - successCount++; consecutiveRateLimits = 0; + successCount++; consecutiveRateLimits = 0; consecutiveFails = 0; } catch (err) { log('error', `Post-refresh harvest failed: ${err.message}`); chrome.storage.local.remove('pendingHarvest'); diff --git a/local_server_example.py b/local_server_example.py index 7a2cb79..e1d5d01 100755 --- a/local_server_example.py +++ b/local_server_example.py @@ -82,13 +82,24 @@ def load_queue(): return rows def done_ids(): + """Ids that must NOT be handed out again. + + 'ratelimited' is deliberately absent. handleRateLimit() writes that status meaning + "report the task back as pending" — no gen ran, the abuse flag just froze us mid-task. + Counting it as done silently binned the task forever, contradicting its own comment. + + 'failed' IS counted: a timeout may have burned credits invisibly, so nothing is retried + automatically. To re-run a failed task, delete its line from results.jsonl (requeue.py). + """ ids = set() if os.path.exists(RESULTS_FILE): with open(RESULTS_FILE) as f: for line in f: line = line.strip() if line: - ids.add(json.loads(line).get("id")) + row = json.loads(line) + if row.get("status") != "ratelimited": + ids.add(row.get("id")) return ids class Handler(BaseHTTPRequestHandler): diff --git a/requeue.py b/requeue.py new file mode 100644 index 0000000..99d2d17 --- /dev/null +++ b/requeue.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Make finished tasks pending again by removing their rows from results.jsonl. + +The queue server treats any id in results.jsonl as done (except 'ratelimited'), so a task +that failed is never retried automatically — a timeout may have burned credits invisibly +and we don't re-bill by accident. This is the deliberate, explicit way to retry. + + python3 requeue.py --list # what's failed, grouped + python3 requeue.py --dry cal79_ # what WOULD be requeued (prefix match) + python3 requeue.py cal79_ # do it (writes a .bak first) + python3 requeue.py --failed # every failed task + python3 requeue.py cal79_disco cal79_tmi # exact ids + +Stop the queue server first, or it may rewrite the file under you. +""" +import collections +import json +import os +import shutil +import sys + +HERE = os.path.expanduser('~/Library/Application Support/flowrinse') +RESULTS = os.path.join(HERE, 'results.jsonl') + + +def rows(): + with open(RESULTS) as f: + return [json.loads(l) for l in f if l.strip()] + + +def main(): + args = [a for a in sys.argv[1:]] + if not args: + print(__doc__) + return 1 + dry = '--dry' in args + only_failed = '--failed' in args + pats = [a for a in args if not a.startswith('--')] + rs = rows() + + if '--list' in args: + c = collections.Counter((r['status'], r['id'].rsplit('_', 1)[0]) for r in rs) + for (st, grp), n in sorted(c.items()): + print(f'{st:<12} {grp:<24} {n}') + return 0 + + def hit(r): + if r['status'] == 'success': + return False # never re-run a success; that's just burning credits + if only_failed and r['status'] == 'failed': + return True + return any(r['id'] == p or r['id'].startswith(p) for p in pats) + + doomed = [r for r in rs if hit(r)] + if not doomed: + print('nothing matched (successes are never requeued)') + return 1 + for r in doomed: + print(f" requeue {r['id']:<24} {r['status']:<10} {str(r.get('error',''))[:46]}") + print(f"\n{len(doomed)} task(s){' [DRY RUN]' if dry else ''}") + if dry: + return 0 + + shutil.copy2(RESULTS, RESULTS + '.bak') + keep = [r for r in rs if r not in doomed] + with open(RESULTS + '.tmp', 'w') as f: + for r in keep: + f.write(json.dumps(r) + '\n') + os.replace(RESULTS + '.tmp', RESULTS) # a kill mid-write can't truncate the real file + print(f'wrote {len(keep)} rows (backup: results.jsonl.bak). Restart the queue server; ' + 'they are pending again.') + return 0 + + +if __name__ == '__main__': + sys.exit(main())