fix: promptEchoed matched a 60-char PREFIX — 13 prompts silently skipped
THE ROOT CAUSE of the djsim_phonewall run, and it was never Flow. promptEchoed() answered "has this prompt already been submitted?" by searching the chat pane for prompt.slice(0, 60). All 15 calendar prompts open with the same 60 characters: "The illustrated top picture-panel of a cheap 1979 promotiona" So once Alien's prompt was echoed in the chat, every subsequent calendar looked already-submitted. The driver logged "Prompt already echoed in the session — not re-typing", typed nothing, and waited out the full 3-minute image timeout. Thirteen times, at a metronomic 3m30s apart. Flow was working the whole time. No fixed slice is safe: djsim_phonewall needs 160 chars of PREFIX to disambiguate its prompts; djsim_batch3 needs 190 chars of SUFFIX. So match the whole normalized prompt, or (if the DOM mangles the middle) require both ends in the same element. Whitespace-normalized, with the container guard kept. test_prompt_echo.mjs pins it: asserts the old impl DOES false-positive, the new one doesn't, that it still fires as the "it stuck" signal, survives DOM whitespace reflow, ignores a page-sized container, and that every prompt in every bank is distinguishable from all its neighbours (0 missed, 0 cross-matched). The circuit breaker from the previous commit stands: it would have stopped this after 3 instead of 13. But the driver should never have been silent in the first place. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
3cff97df93
commit
82d5ff494c
26
content.js
26
content.js
@ -240,12 +240,28 @@ function countDoneMsgs() {
|
||||
|
||||
// The chat panel echoes the submitted prompt — the strongest "it stuck" signal, and
|
||||
// on a retry it means the PREVIOUS submit registered (so don't type it again).
|
||||
//
|
||||
// It must identify THIS prompt, not "a prompt that starts the same way". 2026-07-08: this
|
||||
// matched prompt.slice(0,60), and 13 of the 15 djsim_phonewall prompts opened with an
|
||||
// identical 60 chars ("The illustrated top picture-panel of a cheap 1979 promotiona"). The
|
||||
// previous task's echo was still on screen, so the driver decided every one of them had
|
||||
// already been submitted, never typed them, and sat out a 3-minute timeout. Thirteen times.
|
||||
//
|
||||
// No fixed slice is safe: djsim_phonewall needs 160 chars of PREFIX to disambiguate,
|
||||
// djsim_batch3 needs 190 chars of SUFFIX. So match the whole thing — or, if the DOM has
|
||||
// mangled the middle, require both ends in the same element.
|
||||
const normText = (s) => String(s).replace(/\s+/g, ' ').trim();
|
||||
|
||||
function promptEchoed(prompt) {
|
||||
const needle = String(prompt).slice(0, 60).trim();
|
||||
if (!needle) return false;
|
||||
return [...document.querySelectorAll('div, p, span')].some(e => visible(e) &&
|
||||
!e.closest('[contenteditable="true"], textarea') &&
|
||||
(e.textContent || '').includes(needle) && (e.textContent || '').length < needle.length + 500);
|
||||
const full = normText(prompt);
|
||||
if (full.length < 12) return false;
|
||||
const head = full.slice(0, 100), tail = full.slice(-100);
|
||||
return [...document.querySelectorAll('div, p, span')].some(e => {
|
||||
if (!visible(e) || e.closest('[contenteditable="true"], textarea')) return false;
|
||||
const t = normText(e.textContent || '');
|
||||
if (t.length > full.length + 500) return false; // that's a container, not the message
|
||||
return t.includes(full) || (t.includes(head) && t.includes(tail));
|
||||
});
|
||||
}
|
||||
|
||||
// Flow's "Try again" button, shown beside the "Something went wrong" banner.
|
||||
|
||||
91
test_prompt_echo.mjs
Normal file
91
test_prompt_echo.mjs
Normal file
@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env node
|
||||
// Regression test for promptEchoed(). Run: node test_prompt_echo.mjs
|
||||
//
|
||||
// 2026-07-08: promptEchoed matched prompt.slice(0,60). 13 of the 15 djsim_phonewall prompts
|
||||
// open with an identical 60 chars, so once Alien's prompt was echoed in the chat pane, every
|
||||
// later calendar looked "already submitted". The driver never typed them and sat out a 3-minute
|
||||
// timeout, thirteen times. This test pins the fix and documents the bug it replaced.
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const BANK = path.join(HERE, 'banks', 'djsim_phonewall.jsonl');
|
||||
const prompts = Object.fromEntries(fs.readFileSync(BANK, 'utf8').trim().split('\n')
|
||||
.map(l => { const r = JSON.parse(l); return [r.id, r.prompt]; }));
|
||||
|
||||
const makeDoc = (msgs) => ({ querySelectorAll: () => msgs.map(m => ({ textContent: m, _editor: false })) });
|
||||
const visible = () => true;
|
||||
|
||||
// The shipped implementation, kept in sync with content.js by hand (no bundler here).
|
||||
const normText = (s) => String(s).replace(/\s+/g, ' ').trim();
|
||||
function promptEchoed(prompt, document) {
|
||||
const full = normText(prompt);
|
||||
if (full.length < 12) return false;
|
||||
const head = full.slice(0, 100), tail = full.slice(-100);
|
||||
return [...document.querySelectorAll('div, p, span')].some(e => {
|
||||
if (!visible(e) || e._editor) return false;
|
||||
const t = normText(e.textContent || '');
|
||||
if (t.length > full.length + 500) return false; // that's a container, not the message
|
||||
return t.includes(full) || (t.includes(head) && t.includes(tail));
|
||||
});
|
||||
}
|
||||
|
||||
// What it replaced — kept only to assert the bug is real, and stays fixed.
|
||||
function promptEchoedBuggy(prompt, document) {
|
||||
const needle = String(prompt).slice(0, 60).trim();
|
||||
if (!needle) return false;
|
||||
return [...document.querySelectorAll('div, p, span')].some(e => visible(e) && !e._editor &&
|
||||
(e.textContent || '').includes(needle) && (e.textContent || '').length < needle.length + 500);
|
||||
}
|
||||
|
||||
let fails = 0;
|
||||
const check = (name, got, want) => {
|
||||
if (got !== want) { fails++; console.log(` FAIL ${name} (got ${got}, want ${want})`); }
|
||||
else console.log(` ok ${name}`);
|
||||
};
|
||||
|
||||
// The exact scene: skylab and alien have been sent; their prompts sit in the chat pane.
|
||||
const chat = makeDoc([prompts.cal79_skylab, prompts.cal79_alien,
|
||||
"I've created that 1979 Alien calendar illustration for you."]);
|
||||
|
||||
console.log('the bug is real (old impl mistakes a shared 60-char opening for a match):');
|
||||
check('old impl false-positives joydivision', promptEchoedBuggy(prompts.cal79_joydivision, chat), true);
|
||||
check('old impl false-positives afghanistan', promptEchoedBuggy(prompts.cal79_afghanistan, chat), true);
|
||||
|
||||
console.log('\nthe fix:');
|
||||
check('alien reads as echoed', promptEchoed(prompts.cal79_alien, chat), true);
|
||||
check('skylab reads as echoed', promptEchoed(prompts.cal79_skylab, chat), true);
|
||||
const unsent = Object.keys(prompts).filter(k => !['cal79_skylab', 'cal79_alien'].includes(k));
|
||||
const falsePos = unsent.filter(k => promptEchoed(prompts[k], chat));
|
||||
check(`all ${unsent.length} unsent prompts read as NOT echoed`, falsePos.length, 0);
|
||||
if (falsePos.length) console.log(' false positives:', falsePos);
|
||||
|
||||
console.log('\nit still works as the "it stuck" signal:');
|
||||
const chat2 = makeDoc([prompts.cal79_alien, prompts.cal79_disco]);
|
||||
check('disco echoed once present', promptEchoed(prompts.cal79_disco, chat2), true);
|
||||
check('tmi still not echoed', promptEchoed(prompts.cal79_tmi, chat2), false);
|
||||
|
||||
console.log('\nrobustness:');
|
||||
check('survives DOM whitespace reflow',
|
||||
promptEchoed(prompts.cal79_disco, makeDoc([prompts.cal79_disco.replace(/ /g, '\n ')])), true);
|
||||
check('a page-sized container does not count',
|
||||
promptEchoed(prompts.cal79_tmi, makeDoc([Object.values(prompts).join(' ') + ' '.repeat(2000)])), false);
|
||||
check('empty prompt is never echoed', promptEchoed('', chat), false);
|
||||
|
||||
// every bank must be self-distinguishable under the new rule
|
||||
console.log('\nevery bank:');
|
||||
for (const f of fs.readdirSync(path.join(HERE, 'banks')).filter(f => f.endsWith('.jsonl'))) {
|
||||
const ps = fs.readFileSync(path.join(HERE, 'banks', f), 'utf8').trim().split('\n')
|
||||
.map(l => JSON.parse(l).prompt);
|
||||
const doc = makeDoc(ps); // pretend the whole bank was sent
|
||||
const missed = ps.filter(p => !promptEchoed(p, doc)).length;
|
||||
// and each prompt must NOT match a chat containing only the OTHER prompts
|
||||
let cross = 0;
|
||||
ps.forEach((p, i) => { if (promptEchoed(p, makeDoc(ps.filter((_, j) => j !== i)))) cross++; });
|
||||
check(`${f}: ${ps.length} prompts, ${missed} missed, ${cross} cross-matched`, missed + cross, 0);
|
||||
}
|
||||
|
||||
console.log(fails ? `\n${fails} FAILURE(S)` : '\nALL GREEN');
|
||||
process.exit(fails ? 1 : 0);
|
||||
Loading…
Reference in New Issue
Block a user