From 9f9ea1a11b08f77ca6c4964aff9d00fa2e8057c0 Mon Sep 17 00:00:00 2001 From: type-two Date: Tue, 7 Jul 2026 13:52:17 +1000 Subject: [PATCH] README: test log, dead ends, adversarial review results, honest efficacy table + reproducible bench.py Co-Authored-By: Claude Fable 5 --- README.md | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ bench.py | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 bench.py diff --git a/README.md b/README.md index 444556c..7b0aca5 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,72 @@ tiktoken` for exact token counts (falls back to chars/4). native mechanism, and `grunts.py` mines your own transcript history for what you actually retype (`--emit` writes the command stubs). +## How it was tested, what won, what died + +Development was measurement-first: every pass had to beat the real o200k +tokenizer (tiktoken) on a realistic sample before it earned its lines, and an +adversarial review then hunted for inputs that corrupt meaning. `python3 +bench.py` reproduces the headline numbers; `python3 test_lessismore.py` runs +the regression suite (26 asserts, stdlib only, covers every fixed bug). + +### Wins (real o200k counts) + +| Sample | Tokens | Why it works | +|---|---|---| +| Mixed error log, 2000 lines (85% one repeated ERROR, 15% INFO differing only in numbers) | 86,914 → 7,895 (**11.0x**) | timestamp strip → dedupe; similar-line collapse catches the numbered INFO lines | +| Interleaved 3-service log, zero consecutive repeats | 55,999 → 4,587 (**12.2x**) | `alias_repeats` dictionary-codes scattered duplicates consecutive dedupe can't touch (alone: 36,524 → 8,795 where plain dedupe managed ~0%) | +| ANSI-colored CI/docker build log | 21,198 → 2,387 (**8.9x**) | ANSI strip (colors make identical lines differ) + similar-line collapse | +| Captured pip/tqdm output with `\r` redraws | 14,483 → 291 (**98%**) | every overwritten progress frame is invisible in a terminal but real tokens in a capture | +| pytest failure dump | 2,223 → 1,673 (25%) | venv path spam: one site-packages prefix = 25 tokens → 7 | +| Pretty-printed JSON API response | 11,741 → 6,833 (42%) | minify (lossless) + UUID→8-hex squash | +| Chatty prose, gist mode (level 4) | 427 → 264 (38%) | every dropped function word is a whole token; level 3 managed only 2% on the same text | + +### Dead ends (measured so they stay dead) + +- **Shorthand codebooks**: `[fmt:md_tbl+hdr]` = 8 tokens vs the plain-English + sentence = 10. BPE already compresses common English; brackets shred. +- **Word→2-3-letter-code recoding**: common words are already 1 token, codes + cost 2, plus ~4 tokens/entry codebook tax. Can't beat a 200k-entry codebook + from inside its own encoding. +- **Known abbreviations** (db, fn, env, auth): 1 → 1 tokens. Zero. +- **Personalized shorthand skill** (SwiftKey-style): scanned 2,917 real typed + prompts across 654 transcripts — 2,762 were unique, and the repeats were + already grunts ("yes", "go", "continue"). Model-side decode skills cost + ~1k tokens/turn to save ~10. `grunts.py` keeps the useful half: mine your + history, emit client-side slash-command stubs (free). +- **Digit-masked template dedupe**: 0.0% — real progress bars differ in bar + glyphs, not digits; the alpha-skeleton key in `dedupe_similar` is what works. +- **Separator-line shortening**: a 78-char `----` rule is already 1 token. +- **Common-prefix hoisting**: 0.0% — one stray line kills the common prefix. +- **JSON→TSV, float truncation, hex-pointer squash**: real savings, but each + paid in corruption risk or lines-of-code for marginal gains over minify. + +### Adversarial review: 14 confirmed breaks → 9 fixed, 4 documented, 1 wontfix + +Worst finds, all reproduced then fixed with regression tests: filler stripping +inverted negations ("was **not just** the database" → "was **not** the +database"); URLs eaten as base64 blobs (`/` was in the character class); +crashes on empty and non-UTF-8 stdin; two different SHA-256s squashing to the +same stub and falsely merging as "repeated"; `two_sticks` eating "IT" and "US" +as function words; dedupe markers longer than the short lines they replaced. +The four survivors are documented under Known tradeoffs below. + +### Overall efficacy — the honest version + +Savings are proportional to **redundancy**, not size. This tool removes +repetition and machine noise; it cannot compress information. + +| Content | Expect | Verdict | +|---|---|---| +| Repetitive machine output (logs, CI, installs, captured progress) | 5–12x, up to 50x on pathological repeats | the sweet spot — use level 2 by default | +| Structured data (JSON, tracebacks) | 25–40% | worthwhile, lossless where it fires | +| Varied prose | ~2% (level 3) / 38% lossy (level 4) | only worth it in gist mode | +| Clean code, unique dense text | ~0% by design | don't bother — that's what `--ml` or truncation is for | + +Cost side: pure regex/stdlib, measured ~6 MB/s single-core at level 2 (a 28 MB +log in 4.4s) — no model, no network, no deps. The break-even input size is +effectively zero; it just never *wins* on non-redundant text. + ## Fleet setup (stupendo / m3 air) ```bash diff --git a/bench.py b/bench.py new file mode 100644 index 0000000..395fe86 --- /dev/null +++ b/bench.py @@ -0,0 +1,42 @@ +"""Reproduce the README benchmark numbers. pip install tiktoken for exact counts.""" +import random + +from lessismore import compress, count_tokens + + +def bench(name, text, level): + b, a = count_tokens(text), count_tokens(compress(text, level)) + print(f"{name:22} level {level} {b:>7,} -> {a:>6,} ({b / a:4.1f}x, {1 - a / b:.0%} saved)") + + +# mixed log: 85% repeated ERROR line, 15% INFO lines differing only in numbers, one JWT +random.seed(1) +lines = [] +for i in range(2000): + ts = f"2026-07-07T10:{i // 60 % 60:02d}:{i % 60:02d}.{random.randint(100, 999)}Z" + if random.random() < 0.85: + lines.append(f"{ts} ERROR [pool-3] psycopg2.OperationalError: connection to " + f"server at 100.94.195.115 failed: Connection refused") + else: + lines.append(f"{ts} INFO [worker-{random.randint(1, 4)}] processed batch {i} ok") +lines.insert(500, "2026-07-07T10:08:20.123Z DEBUG auth token=" + "eyJhbGciOiJIUzI1NiJ9" * 8) +bench("mixed error log", "\n".join(lines), 2) + +# interleaved log: three services alternating — zero consecutive runs, dedupe-proof +random.seed(2) +msgs = ["ERROR [pool-3] psycopg2.OperationalError: connection to server at " + "100.94.195.115 failed: Connection refused", + "WARN [redis-1] retry queue depth exceeded threshold, backing off 500ms", + "ERROR [worker-2] TimeoutError: upstream api.dealgod.pro did not respond within 30s"] +bench("interleaved log", "\n".join( + f"2026-07-07T10:{i // 60 % 60:02d}:{i % 60:02d}Z " + msgs[i % 3] for i in range(1500)), 2) + +# chatty prose: levels 3 vs 4 show where filler stripping ends and caveman starts +prose = ("So basically what happened was that the team had been trying to get the " + "deployment pipeline working for about three weeks, and it turned out that the " + "main problem was that the configuration file was pointing to the wrong database " + "server. I think we should also mention that the staging environment has been " + "flaky and there are a couple of monitoring alerts that have not been " + "investigated yet. ") * 6 +bench("chatty prose", prose, 3) +bench("chatty prose", prose, 4)