From 34c91840655d0b9adde4f324cca661d2fab8b771 Mon Sep 17 00:00:00 2001 From: type-two Date: Tue, 7 Jul 2026 13:43:00 +1000 Subject: [PATCH] lessismore: deterministic prompt compression + grunts history miner Regex/stdlib passes (dedupe, alias, ANSI/timestamp/blob strip, JSON minify, caveman word-drop) measured at 8.9-12.2x on real logs with o200k counts. Optional LLMLingua-2 wrap. grunts.py mines Claude Code transcripts for retyped prompts -> slash-command stubs. Co-Authored-By: Claude Fable 5 --- .claude/scheduled_tasks.lock | 1 + .gitignore | 1 + README.md | 106 ++++++++++++++ grunts.py | 81 +++++++++++ lessismore.py | 274 +++++++++++++++++++++++++++++++++++ test_lessismore.py | 91 ++++++++++++ 6 files changed, 554 insertions(+) create mode 100644 .claude/scheduled_tasks.lock create mode 100644 .gitignore create mode 100644 README.md create mode 100644 grunts.py create mode 100644 lessismore.py create mode 100644 test_lessismore.py diff --git a/.claude/scheduled_tasks.lock b/.claude/scheduled_tasks.lock new file mode 100644 index 0000000..8f5981c --- /dev/null +++ b/.claude/scheduled_tasks.lock @@ -0,0 +1 @@ +{"sessionId":"b1485757-c727-45be-be51-c0147d9825ad","pid":63486,"procStart":"Tue Jul 7 02:45:21 2026","acquiredAt":1783393961121} \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..444556c --- /dev/null +++ b/README.md @@ -0,0 +1,106 @@ +# lessismore + +Squeeze text before it hits an LLM. Deterministic passes first (free, safe, +reproducible); optional perplexity pruning via LLMLingua-2 for the last mile. + +```bash +python3 lessismore.py dump.log -l 2 > small.log # file in, file out +tail -5000 app.log | python3 lessismore.py -l 2 # pipe filter +python3 lessismore.py big.txt -l 2 --ml 0.5 # + LLMLingua (pip install llmlingua) +``` + +```python +from lessismore import compress, count_tokens +small = compress(big_log, level=2) +``` + +Token stats print to stderr, so pipes stay clean. Measured at level 2 with the +real o200k tokenizer (no ML, no deps): a 2000-line mixed error log went +**86,914 → 7,895 tokens (11x)**; an interleaved three-service log went +**55,999 → 4,587 (12.2x)**; an ANSI-colored CI build log went **21,198 → 2,387 +(8.9x)**; captured pip/tqdm output with `\r` redraws compressed **98%**. + +| Level | Passes | Safe for | +|---|---|---| +| 1 | whitespace collapse, duplicate-line runs | code structure & indentation (whitespace runs *inside string literals* still collapse) | +| 2 | + minify whole-input JSON, keep final state of `\r` progress redraws, strip ANSI + ISO timestamps, squash blobs/UUIDs/venv paths, alias scattered duplicate lines, collapse same-words-different-numbers runs | logs, dumps, tool output | +| 3 | + filler-word stripping | prose only (eats "just" everywhere) | +| 4 | + `two_sticks` caveman word-dropping | gist-only prose — never instructions (38% measured on chatty prose) | + +`--ml RATE` adds LLMLingua-2 perplexity pruning after the deterministic passes +(RATE = fraction of tokens kept). Needs `pip install llmlingua`; `pip install +tiktoken` for exact token counts (falls back to chars/4). + +## Design notes — where the wins actually are + +1. **Compress the bulk, not the question.** A 30-token prompt isn't worth + touching; the 50k-token log dump, retrieved doc, or file dump is. That's why + this is a pipe filter, not a chat-prompt rewriter. + +2. **Characters ≠ tokens.** Measured with o200k: `[fmt:md_tbl+hdr]` = 8 tokens + vs `"Format the output as a markdown table with headers."` = 10. + `[out:json_strict]` vs `"Respond only with valid JSON."` = 6 vs 6. + Tokenizers already compress common English; bracket-shorthand gets shredded + into fragments *and* adds ambiguity. Shorthand codebooks: skipped, measured, dead. + +3. **Stop-word stripping: instructions no, gist yes.** Mangled grammar hurts + instruction-following, so levels 1–3 never touch structure words. But every + dropped word is a whole token, so for content you only need the gist of, + level 4 (`two_sticks`) drops articles/copulas/auxiliaries — measured 38% on + chatty prose. Negations, modals, and order words are never dropped: "do not + delete" must stay "not delete", never "delete". + + The two-sticks *recoding* idea (map words to 2–3 letter codes) is dead on + arrival, measured: common words are already 1 token (` database` = 1) while + off-distribution codes cost 2 (` qx` = 2), plus a ~4-token/entry codebook + tax. BPE already is a 200k-entry compression codebook; you can't beat it + with an 18k-entry codebook written inside its own encoding. Zipping a zip. + + The salvage: codes DO pay when one code replaces a repeated *multi-token + sequence* — that's `alias_repeats`, dictionary compression that composes + with BPE instead of fighting it (12.2x measured on interleaved logs). + +4. **Perplexity pruning is a dependency, not a project.** Microsoft's + `llmlingua` already does the budget-model math. We wrap it in six lines + behind `--ml` instead of reimplementing it. Deterministic passes run first + so you're not paying a classifier to delete duplicate log lines. + +5. **Determinism matters for prompt caching.** Same input → same output keeps + cache prefixes stable. Corollary: never compress a stable cached system + prompt — you'd bust the cache to save tokens that were already free. + +6. **The compressor must cost less than it saves.** Regex passes are ~free at + any size. The ML pass loads a model, so it only pays off on multi-KB inputs. + +7. **The information-loss dial** from the original idea is the `-l` level + + `--ml` rate: level 1 is lossless-ish and code-safe, `--ml 0.3` is maximum + squeeze for prose you only need the gist of. + +8. **Personal shorthand belongs on the keyboard, not in the model.** Your typed + prompts are the cheapest tokens in the context (~10 each); a skill teaching + the model your abbreviations costs more per turn than it can ever save. + Expand shorthand client-side instead — Claude Code slash commands are the + native mechanism, and `grunts.py` mines your own transcript history for + what you actually retype (`--emit` writes the command stubs). + +## Fleet setup (stupendo / m3 air) + +```bash +git clone ssh://git@100.71.119.27:222/monster/lessismore.git ~/Documents/lessismore +printf '#!/bin/sh\nexec python3 ~/Documents/lessismore/lessismore.py "$@"\n' | sudo tee /opt/homebrew/bin/lm >/dev/null +sudo chmod +x /opt/homebrew/bin/lm +``` + +`lm` is the pipe shim: `docker logs dealgod | lm -l 2`. Optional: `pip3 install +tiktoken` for exact token stats (chars/4 heuristic otherwise). + +## Known tradeoffs + +- Levels 2+ assume line *order* matters but wall-clock timing doesn't. When + gaps and deltas are the signal (hang hunting), stay on level 1. +- Output is for LLM consumption, not round-tripping: markdown hard breaks + (trailing double-space) and diff context lines don't survive even level 1. +- At levels 3–4, dedupe counts describe the post-stripped text — five + differently-phrased "retry the job" lines can legitimately merge. +- In-band markers can collide with input that already contains them; + `alias_repeats` bails out if its own `@N` markers already appear as lines. diff --git a/grunts.py b/grunts.py new file mode 100644 index 0000000..b2684cb --- /dev/null +++ b/grunts.py @@ -0,0 +1,81 @@ +"""grunts — mine your Claude Code prompt history for your most-retyped grunts. + +Your typed prompts are the cheapest tokens in the context, so shorthand the +MODEL decodes can never pay for its codebook (see README). But shorthand your +KEYBOARD expands is free: this finds what you actually retype, so the long +ones can become slash commands (~/.claude/commands/.md). + + python3 grunts.py # report + python3 grunts.py --emit # also write ./grunts/.md command stubs +""" +import json +import re +import sys +from collections import Counter +from pathlib import Path + +_SKIP = ("", "", "") + + +def user_messages(root=Path.home() / ".claude" / "projects"): + for f in root.glob("*/*.jsonl"): + for line in open(f, encoding="utf-8", errors="replace"): + # ponytail: substring prefilter before json.loads keeps 1GB scans fast + if '"type":"user"' not in line or '"isMeta":true' in line: + continue + if len(line) > 20000: # giant rows are pastes/images, never typed grunts + continue + try: + c = json.loads(line)["message"]["content"] + except (KeyError, TypeError, json.JSONDecodeError): + continue + if isinstance(c, list): + if any(p.get("type") == "tool_result" for p in c if isinstance(p, dict)): + continue + c = " ".join(p.get("text", "") for p in c if isinstance(p, dict)) + if not isinstance(c, str): + continue + t = c.strip() + if t and len(t) <= 400 and not any(s in t for s in _SKIP): + yield t + + +def main(): + msgs = list(user_messages()) + norm = Counter(re.sub(r"\s+", " ", m.lower()) for m in msgs) + grams = Counter() + for m, n in norm.items(): + words = m.split() + for k in (4, 5, 6): + for i in range(len(words) - k + 1): + grams[" ".join(words[i:i + k])] += n + + print(f"{len(msgs):,} typed prompts scanned across {len(norm):,} distinct\n") + top = [(m, c) for m, c in norm.most_common(30) if c >= 3] + print("== most retyped whole prompts ==") + for m, c in top[:20]: + print(f"{c:5d}x {m[:90]}") + print("\n== most retyped phrases (4-6 words, not already above) ==") + shown = 0 + for g, c in grams.most_common(300): + if c < 5 or any(g in m for m, _ in top[:20]): + continue + print(f"{c:5d}x {g}") + shown += 1 + if shown == 15: + break + + if "--emit" in sys.argv: + out = Path("grunts") + out.mkdir(exist_ok=True) + for m, c in top: + if len(m) < 25: # a short grunt is already faster to type than /slash + continue + slug = "-".join(re.findall(r"[a-z0-9]+", m)[:4]) + (out / f"{slug}.md").write_text(m + "\n") + print("\nwrote stubs to ./grunts/ — copy keepers to ~/.claude/commands/") + + +if __name__ == "__main__": + main() diff --git a/lessismore.py b/lessismore.py new file mode 100644 index 0000000..fa87da0 --- /dev/null +++ b/lessismore.py @@ -0,0 +1,274 @@ +"""lessismore — squeeze text before it hits an LLM. + +Deterministic passes first (free, safe, cacheable). Optional ML pruning +(LLMLingua-2) only when installed and only worth it on big inputs. + + from lessismore import compress + small = compress(big_log, level=2) + + $ python3 lessismore.py dump.log -l 2 > small.log + $ tail -5000 app.log | python3 lessismore.py -l 2 | llm ... +""" + +import json +import re +import sys +from collections import Counter +from functools import lru_cache + +# ---------------------------------------------------------------- tokens + +@lru_cache(maxsize=1) +def _encoder(): + try: + import tiktoken + return tiktoken.get_encoding("o200k_base") + except ImportError: + return None + + +def count_tokens(text: str) -> int: + enc = _encoder() + if enc: + return len(enc.encode(text)) + return max(1, len(text) // 4) # ponytail: chars/4 heuristic; pip install tiktoken for real counts + + +# ---------------------------------------------------------------- passes +# Each pass is (str) -> str. Order matters: whitespace before dedupe. + +def collapse_whitespace(text: str) -> str: + text = re.sub(r"[ \t]+$", "", text, flags=re.M) # trailing whitespace + text = re.sub(r"(?<=\S)[ \t]{2,}", " ", text) # interior runs (leading indent kept: code-safe) + text = re.sub(r"\n{3,}", "\n\n", text) # blank-line runs + return text + + +def dedupe_lines(text: str) -> str: + """Collapse runs of identical lines — the classic log killer.""" + lines = text.split("\n") + out, i = [], 0 + while i < len(lines): + j = i + while j < len(lines) and lines[j] == lines[i]: + j += 1 + run = j - i + # marker must actually be shorter than the lines it replaces + if run >= 4 and lines[i].strip() and (run - 1) * (len(lines[i]) + 1) > 45: + out.append(lines[i]) + out.append(f"[previous line repeated {run - 1} more times]") + else: + out.extend(lines[i:j]) + i = j + # ponytail: only consecutive repeats; add block-level dedupe (repeated stack traces) if logs demand it + return "\n".join(out) + + +# no '/' in the class: URL paths are ≥64-char alnum+slash runs and they ARE the content +_BLOB = re.compile(r"\b(?:[A-Za-z0-9+]{64,}={0,2}|[0-9a-fA-F]{48,})\b") + + +def squash_blobs(text: str) -> str: + """Base64/hex runs are token-dense and semantically opaque — keep head and tail. + + The tail suffix keeps two different blobs from squashing to the same stub + and then being falsely merged as "repeated" by dedupe_lines. + """ + return _BLOB.sub(lambda m: f"{m.group()[:12]}[+{len(m.group()) - 16} chars]{m.group()[-4:]}", text) + + +_TIMESTAMP = re.compile( + r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?\b ?" +) + + +def strip_timestamps(text: str) -> str: + """Line order already encodes sequence; per-line ISO timestamps are ~8 tokens each.""" + # ponytail: ISO-8601 only; add syslog/other formats when a real log needs them + return _TIMESTAMP.sub("", text) + + +def minify_json(text: str) -> str: + """Whole-input pretty JSON → minified. Lossless when it fires, untouched when not.""" + try: + return json.dumps(json.loads(text), separators=(",", ":"), ensure_ascii=False) + except ValueError: + return text + + +def collapse_cr(text: str) -> str: + """Keep only the final state of \\r-overwritten progress lines (pip/tqdm/wget).""" + text = text.replace("\r\n", "\n") + return "\n".join(l.rsplit("\r", 1)[-1] for l in text.split("\n")) + + +_ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]") # ponytail: CSI only; add OSC if titles show up + + +def strip_ansi(text: str) -> str: + """Color codes carry nothing for an LLM, and they make identical lines differ.""" + return _ANSI.sub("", text) + + +_UUID = re.compile(r"\b[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}\b") + + +def squash_uuids(text: str) -> str: + """36 chars → 8-hex prefix, git-short-hash style; cross-references still resolve.""" + return _UUID.sub(lambda m: m.group()[:8] + "…", text) + + +_PKGPATH = re.compile(r'[^\s"\']+/(?:site-packages|dist-packages|lib/python3\.\d+)/') + + +def squash_pkgpaths(text: str) -> str: + """Traceback path spam: …/httpx/_client.py:1054 is still unique without the venv prefix.""" + return _PKGPATH.sub("…/", text) + + +# the lookbehinds keep "not just X" from becoming "not X" — meaning inversion +_FILLER = re.compile( + r"(? str: + """Aggressive: eats words like 'just' anywhere, including inside strings. Prose only.""" + return _FILLER.sub("", text) + + +# caveman-style word dropping: every function word is a whole token. +# NEVER add negations (not/no/never), modals (must/should), or order words +# (before/after) — dropping those changes meaning, not just style. +_STICKS = re.compile( + r"\b(?:the|a|an|is|are|was|were|be|been|being|am|i|we|you|they|it|" + r"that|which|who|have|has|had|do|does|did|there)\b ?", + re.I, +) + + +def two_sticks(text: str) -> str: + """Gist-only: strips text to caveman. Fine for articles/transcripts, never instructions.""" + # keep IT/US-style acronyms that case-insensitively collide with function words + return _STICKS.sub(lambda m: m.group() if m.group().strip().isupper() + and len(m.group().strip()) > 1 else "", text) + + +_SKEL = re.compile(r"[^A-Za-z]+") + + +def dedupe_similar(text: str) -> str: + """Collapse runs of lines identical after masking non-letters — 'same words, + different numbers' (progress lines, per-item CI steps). Keeps first and last + so progression endpoints survive.""" + lines = text.split("\n") + out, i = [], 0 + while i < len(lines): + k, j = _SKEL.sub(" ", lines[i]).strip(), i + while j < len(lines) and _SKEL.sub(" ", lines[j]).strip() == k: + j += 1 + if j - i >= 4 and k: + out += [lines[i], f"[{j - i - 2} similar lines omitted]", lines[j - 1]] + else: + out.extend(lines[i:j]) + i = j + return "\n".join(out) + + +_REF = re.compile(r"@\d+") + + +def alias_repeats(text: str, min_count: int = 4, min_len: int = 30) -> str: + """Dictionary-code scattered duplicate lines that consecutive dedupe can't reach. + + Lossless — the legend keeps every line verbatim. A code only pays when it + replaces a repeated multi-token sequence; single words are already 1 BPE + token each, so word-level codebooks lose (measured, see README). + """ + lines = text.split("\n") + if any(_REF.fullmatch(l) for l in lines): # already aliased, or real @N content — bail + return text + counts = Counter(l for l in lines if len(l) >= min_len) + # ponytail: c*len>200 chars is the payoff heuristic; tune if legends ever dominate + worth = [l for l, c in counts.items() if c >= min_count and c * len(l) > 200] + if not worth: + return text + ref = {l: f"@{i}" for i, l in enumerate(worth, 1)} + legend = [f"@{i} = {l}" for i, l in enumerate(worth, 1)] + return "\n".join(["[repeated lines aliased:]"] + legend + [""] + + [ref.get(l, l) for l in lines]) + + +# order: strippers leave doubled spaces, so collapse_whitespace runs after them; +# normalized lines then match better in alias_repeats/dedupe_lines/dedupe_similar +_STRIP2 = [minify_json, collapse_cr, strip_ansi, strip_timestamps, + squash_blobs, squash_uuids, squash_pkgpaths] +_MERGE = [collapse_whitespace, alias_repeats, dedupe_lines, dedupe_similar] +LEVELS = { + 1: [collapse_whitespace, dedupe_lines], # code-safe + 2: _STRIP2 + _MERGE, # logs/dumps/tool output + 3: _STRIP2 + [strip_filler] + _MERGE, # prose + 4: _STRIP2 + [strip_filler, two_sticks] + _MERGE, # gist-only +} + + +def compress(text: str, level: int = 1) -> str: + for f in LEVELS[level]: + text = f(text) + return text + + +# ---------------------------------------------------------------- optional ML pass + +@lru_cache(maxsize=1) +def _llmlingua(): + from llmlingua import PromptCompressor # pip install llmlingua + return PromptCompressor( + model_name="microsoft/llmlingua-2-xlm-roberta-large-meetingbank", + use_llmlingua2=True, + ) + + +def compress_ml(text: str, rate: float = 0.5) -> str: + """Perplexity-based token pruning (LLMLingua-2). + + Runs a local classifier model — only pays for itself on multi-KB inputs. + Run the deterministic passes first; never feed it code you need verbatim. + """ + return _llmlingua().compress_prompt(text, rate=rate)["compressed_prompt"] + + +# ---------------------------------------------------------------- CLI + +def main(): + import argparse + + p = argparse.ArgumentParser(prog="lessismore", description=__doc__.splitlines()[0]) + p.add_argument("file", nargs="?", help="input file (default: stdin)") + p.add_argument("-l", "--level", type=int, default=1, choices=sorted(LEVELS), + help="1=code-safe 2=logs/dumps 3=prose 4=gist-only caveman (default 1)") + p.add_argument("--ml", type=float, metavar="RATE", + help="also run LLMLingua-2 keeping RATE of tokens (needs: pip install llmlingua)") + a = p.parse_args() + + # newline="" / buffer.read(): keep \r intact for collapse_cr + raw = (open(a.file, encoding="utf-8", errors="replace", newline="").read() if a.file + else sys.stdin.buffer.read().decode("utf-8", "replace")) + out = compress(raw, a.level) + if a.ml: + try: + out = compress_ml(out, a.ml) + except ImportError: + sys.exit("--ml needs: pip install llmlingua") + sys.stdout.write(out) + before, after = count_tokens(raw), count_tokens(out) + print(f"lessismore: {before} → {after} tokens ({1 - after / max(before, 1):.0%} saved)", + file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/test_lessismore.py b/test_lessismore.py new file mode 100644 index 0000000..f6868c8 --- /dev/null +++ b/test_lessismore.py @@ -0,0 +1,91 @@ +import subprocess +import sys + +from lessismore import (compress, count_tokens, dedupe_lines, squash_blobs, + strip_filler, strip_timestamps, two_sticks) + + +def test(): + # dedupe: long-line runs collapse, short runs where the marker wouldn't pay stay + line = "ERROR connection refused by upstream database" + assert dedupe_lines((line + "\n") * 5 + "b") == \ + line + "\n[previous line repeated 4 more times]\nb" + assert dedupe_lines("a\n" * 5 + "b") == "a\n" * 5 + "b" + assert dedupe_lines(line + "\n" + line + "\nb") == line + "\n" + line + "\nb" + + # blobs squash with head+tail stub; URLs are not blobs + assert "[+84 chars]" in squash_blobs("token=" + "A" * 100) + url = ("GET https://api.dealgod.pro/api/v1/users/12345/orders/67890/" + "items/status/pending/verbose HTTP/1.1") + assert squash_blobs(url) == url + + # distinct blobs keep distinct stubs — no false "repeated" merge at level 2 + blobs = "\n".join("stored blob deadbeefcafe" + f"{i:052x}" for i in range(4)) + assert "repeated" not in compress(blobs, 2) + + # timestamps + assert strip_timestamps("2026-07-07T10:00:00.123Z boom") == "boom" + + # filler stripping never inverts negation + assert strip_filler("This is not just a warning.") == "This is not just a warning." + assert strip_filler("just do it") == "do it" + + # level 1 is code-safe: indentation and content untouched + code = "def f(x):\n return x + 1\n" + assert compress(code, 1) == code + + # two_sticks drops function words, never negations, and spares acronyms + assert two_sticks("The server is down") == "server down" + assert "not" in two_sticks("do not delete the backup") + assert two_sticks("The IT team said it is a US issue") == "IT team said US issue" + + # interleaved repeats defeat consecutive dedupe but get dictionary-aliased + inter = ("db connection refused on host alpha-primary\n" + "redis queue depth exceeded on host beta-cache\n") * 30 + out = compress(inter, 2) + assert "[repeated lines aliased:]" in out and "@1" in out + assert count_tokens(out) < count_tokens(inter) / 2 + + # a repetitive log shrinks hard, and stripping leaves no doubled spaces + log = "2026-07-07T10:00:01Z ERROR conn refused db=dealgod retry\n" * 500 + out = compress(log, 2) + assert count_tokens(out) < count_tokens(log) / 50 + assert " " not in out + + # \r progress redraws keep only the final state + assert compress("downloading 1%\rdownloading 99%\rdone\n", 2) == "done\n" + + # ANSI escapes stripped + assert compress("\x1b[32mok\x1b[0m\n", 2) == "ok\n" + + # venv path spam squashed, file:line still unique + tb = 'File "/Users/j/venv/lib/python3.14/site-packages/httpx/_client.py", line 5\n' + assert compress(tb, 2) == 'File "…/httpx/_client.py", line 5\n' + + # uuids shorten to 8-hex prefix + assert compress("id=550e8400-e29b-41d4-a716-446655440000 done\n", 2) == "id=550e8400… done\n" + + # same-words-different-numbers runs collapse, endpoints kept + sim = "\n".join(f"Downloading chunk {i} of 50 at {i * 3}kbps" for i in range(50)) + out = compress(sim, 2) + assert "[48 similar lines omitted]" in out + assert "chunk 0 of" in out and "chunk 49 of" in out + + # whole-input pretty JSON minifies losslessly + import json + pretty = json.dumps({"items": [{"id": i, "name": f"x{i}"} for i in range(10)]}, indent=2) + out = compress(pretty, 2) + assert json.loads(out) == json.loads(pretty) + assert len(out) < len(pretty) / 1.5 + + # CLI survives empty and non-UTF-8 stdin + for stdin in (b"", b"ok\n\xff\xfe\n"): + r = subprocess.run([sys.executable, "lessismore.py"], + input=stdin, capture_output=True) + assert r.returncode == 0, r.stderr + + print("ok") + + +if __name__ == "__main__": + test()