7962887057
- bench.py now generates every README table row (ANSI CI, pip/tqdm, pytest, pretty JSON added) and refuses to print chars/4 estimates — reproduced numbers can no longer silently diverge from the claims - eval.py: the receipt ratios can't give — 8 seeded failure logs with planted root causes and red herrings, asked raw vs compressed, graded by deterministic keyword check. First run (haiku 4.5): raw 6/8, compressed 8/8 at 3.1x fewer tokens; both raw misses were lost-in-the-middle - README: eval section, related-work section (rtk, Headroom, Drain3, LLMLingua-2, arXiv 2604.13066), prompt-cache claim sharpened to the rebuilt-prompt scenario, agent-harness integration (--run, --hook), re-measured receipts table Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
101 lines
5.3 KiB
Python
101 lines
5.3 KiB
Python
"""Reproduce the README benchmark numbers — every row of the receipts table.
|
|
|
|
Requires tiktoken (pip install tiktoken): the README numbers are o200k counts,
|
|
and printing chars/4 estimates here would let the receipts silently drift from
|
|
the claims. Pass --estimate if you really want the guesses.
|
|
"""
|
|
import json
|
|
import random
|
|
import sys
|
|
|
|
from lessismore import compress, count_tokens, _encoder
|
|
|
|
if _encoder() is None and "--estimate" not in sys.argv:
|
|
sys.exit("bench.py: tiktoken not installed — these would be chars/4 GUESSES,\n"
|
|
"not the o200k-measured numbers the README claims.\n"
|
|
" pip install tiktoken (or pass --estimate to see the guesses anyway)")
|
|
|
|
|
|
def bench(name, text, level):
|
|
b, a = count_tokens(text), count_tokens(compress(text, level))
|
|
print(f"{name:22} level {level} {b:>7,} -> {a:>6,} ({b / a:4.1f}x, {1 - a / b:.0%} saved)")
|
|
|
|
|
|
# mixed log: 85% repeated ERROR line, 15% INFO lines differing only in numbers, one JWT
|
|
random.seed(1)
|
|
lines = []
|
|
for i in range(2000):
|
|
ts = f"2026-07-07T10:{i // 60 % 60:02d}:{i % 60:02d}.{random.randint(100, 999)}Z"
|
|
if random.random() < 0.85:
|
|
lines.append(f"{ts} ERROR [pool-3] psycopg2.OperationalError: connection to "
|
|
f"server at 10.0.0.5 failed: Connection refused")
|
|
else:
|
|
lines.append(f"{ts} INFO [worker-{random.randint(1, 4)}] processed batch {i} ok")
|
|
lines.insert(500, "2026-07-07T10:08:20.123Z DEBUG auth token=" + "eyJhbGciOiJIUzI1NiJ9" * 8)
|
|
bench("mixed error log", "\n".join(lines), 2)
|
|
|
|
# interleaved log: three services alternating — zero consecutive runs, dedupe-proof
|
|
msgs = ["ERROR [pool-3] psycopg2.OperationalError: connection to server at "
|
|
"10.0.0.5 failed: Connection refused",
|
|
"WARN [redis-1] retry queue depth exceeded threshold, backing off 500ms",
|
|
"ERROR [worker-2] TimeoutError: upstream api.example.com did not respond within 30s"]
|
|
bench("interleaved log", "\n".join(
|
|
f"2026-07-07T10:{i // 60 % 60:02d}:{i % 60:02d}Z " + msgs[i % 3] for i in range(1500)), 2)
|
|
|
|
# ANSI CI log: color codes make identical lines look distinct to any dedupe
|
|
random.seed(3)
|
|
ci = []
|
|
for step in range(1, 13):
|
|
ci.append(f"\x1b[1m\x1b[34mStep {step}/12\x1b[0m : RUN pip install -r requirements.txt")
|
|
for _ in range(random.randint(20, 60)):
|
|
ci.append("\x1b[36m ---> Using cache\x1b[0m")
|
|
for pkg in range(random.randint(5, 15)):
|
|
ci.append(f"\x1b[90m2026-07-07T10:0{step % 10}:00Z\x1b[0m \x1b[32m✓\x1b[0m "
|
|
f"Collecting dependency {pkg} (cached wheel, {random.randint(10, 900)} kB)")
|
|
bench("ANSI CI log", "\n".join(ci), 2)
|
|
|
|
# captured pip/tqdm: hundreds of \r-overwritten progress frames, invisible on screen
|
|
frames = []
|
|
for pkg in ("requests", "urllib3", "charset_normalizer", "idna", "certifi"):
|
|
frames.append(f"Collecting {pkg}\n Downloading {pkg}-2.0.0-py3-none-any.whl (150 kB)\n")
|
|
frames.append("".join(f"\r |{'█' * (i // 4)}{' ' * (25 - i // 4)}| "
|
|
f"{i}% {i * 15 // 10} kB {random.randint(100, 999)} kB/s"
|
|
for i in range(1, 101)) + "\n")
|
|
bench("pip/tqdm capture", "".join(frames) + "Successfully installed requests-2.0.0\n", 2)
|
|
|
|
# pytest failure dump: one venv path prefix repeated across every traceback frame
|
|
venv = "/Users/dev/project/.venv/lib/python3.12/site-packages"
|
|
tb = ["=================================== FAILURES ==================================",
|
|
"_________________________________ test_checkout _______________________________"]
|
|
for mod, ln in [("httpx/_client", 1054), ("httpx/_transports/default", 249),
|
|
("httpcore/_sync/connection_pool", 216), ("httpcore/_sync/connection", 99),
|
|
("httpcore/_sync/http11", 136), ("httpcore/_backends/sync", 126)] * 3:
|
|
tb += [f' File "{venv}/{mod}.py", line {ln}, in request',
|
|
" raise exc from None"]
|
|
tb += ["E httpcore.ConnectTimeout: timed out",
|
|
"=========================== short test summary info ==========================",
|
|
"FAILED tests/test_checkout.py::test_checkout - httpcore.ConnectTimeout"]
|
|
bench("pytest failure dump", "\n".join(tb), 2)
|
|
|
|
# pretty-printed JSON API response: indentation + UUIDs are the tax
|
|
random.seed(4)
|
|
resp = {"data": [{"id": f"{random.getrandbits(32):08x}-{random.getrandbits(16):04x}-"
|
|
f"4{random.getrandbits(12):03x}-a{random.getrandbits(12):03x}-"
|
|
f"{random.getrandbits(48):012x}",
|
|
"status": random.choice(["active", "pending"]),
|
|
"amount": random.randint(100, 99999),
|
|
"currency": "USD",
|
|
"created_at": "2026-07-07T10:00:00Z"} for _ in range(120)],
|
|
"page": 1, "total": 120}
|
|
bench("pretty JSON response", json.dumps(resp, indent=2), 2)
|
|
|
|
# chatty prose: levels 3 vs 4 show where filler stripping ends and caveman starts
|
|
prose = ("So basically what happened was that the team had been trying to get the "
|
|
"deployment pipeline working for about three weeks, and it turned out that the "
|
|
"main problem was that the configuration file was pointing to the wrong database "
|
|
"server. I think we should also mention that the staging environment has been "
|
|
"flaky and there are a couple of monitoring alerts that have not been "
|
|
"investigated yet. ") * 6
|
|
bench("chatty prose", prose, 3)
|
|
bench("chatty prose", prose, 4)
|