Hygiene: untrack build/, egg-info, .claude lock; gitignore them
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jing
parent
493c3e912a
commit
dcdf720fbf
@ -1 +0,0 @@
|
||||
{"sessionId":"b1485757-c727-45be-be51-c0147d9825ad","pid":63486,"procStart":"Tue Jul 7 02:45:21 2026","acquiredAt":1783393961121}
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@ -1,2 +1,5 @@
|
||||
__pycache__/
|
||||
.claude/
|
||||
build/
|
||||
*.egg-info/
|
||||
grunts/
|
||||
|
||||
@ -1,362 +0,0 @@
|
||||
"""lessismore — squeeze text before it hits an LLM.
|
||||
|
||||
Deterministic passes first (free, safe, cacheable). Optional ML pruning
|
||||
(LLMLingua-2) only when installed and only worth it on big inputs.
|
||||
|
||||
from lessismore import compress
|
||||
small = compress(big_log, level=2)
|
||||
|
||||
$ python3 lessismore.py dump.log -l 2 > small.log
|
||||
$ tail -5000 app.log | python3 lessismore.py -l 2 | llm ...
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from functools import lru_cache
|
||||
|
||||
# ---------------------------------------------------------------- tokens
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _encoder():
|
||||
try:
|
||||
import tiktoken
|
||||
return tiktoken.get_encoding("o200k_base")
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
|
||||
def count_tokens(text: str) -> int:
|
||||
enc = _encoder()
|
||||
if enc:
|
||||
return len(enc.encode(text))
|
||||
return max(1, len(text) // 4) # ponytail: chars/4 heuristic; pip install tiktoken for real counts
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- passes
|
||||
# Each pass is (str) -> str. Order matters: whitespace before dedupe.
|
||||
|
||||
def collapse_whitespace(text: str) -> str:
|
||||
text = re.sub(r"[ \t]+$", "", text, flags=re.M) # trailing whitespace
|
||||
text = re.sub(r"(?<=\S)[ \t]{2,}", " ", text) # interior runs (leading indent kept: code-safe)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text) # blank-line runs
|
||||
return text
|
||||
|
||||
|
||||
def dedupe_lines(text: str) -> str:
|
||||
"""Collapse runs of identical lines — the classic log killer."""
|
||||
lines = text.split("\n")
|
||||
out, i = [], 0
|
||||
while i < len(lines):
|
||||
j = i
|
||||
while j < len(lines) and lines[j] == lines[i]:
|
||||
j += 1
|
||||
run = j - i
|
||||
# marker must actually be shorter than the lines it replaces
|
||||
if run >= 4 and lines[i].strip() and (run - 1) * (len(lines[i]) + 1) > 45:
|
||||
out.append(lines[i])
|
||||
out.append(f"[previous line repeated {run - 1} more times]")
|
||||
else:
|
||||
out.extend(lines[i:j])
|
||||
i = j
|
||||
# ponytail: only consecutive repeats; add block-level dedupe (repeated stack traces) if logs demand it
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
# no '/' in the class: URL paths are ≥64-char alnum+slash runs and they ARE the content
|
||||
_BLOB = re.compile(r"\b(?:[A-Za-z0-9+]{64,}={0,2}|[0-9a-fA-F]{48,})\b")
|
||||
|
||||
|
||||
def squash_blobs(text: str) -> str:
|
||||
"""Base64/hex runs are token-dense and semantically opaque — keep head and tail.
|
||||
|
||||
The tail suffix keeps two different blobs from squashing to the same stub
|
||||
and then being falsely merged as "repeated" by dedupe_lines.
|
||||
"""
|
||||
return _BLOB.sub(lambda m: f"{m.group()[:12]}[+{len(m.group()) - 16} chars]{m.group()[-4:]}", text)
|
||||
|
||||
|
||||
_TIMESTAMP = re.compile(
|
||||
r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?\b ?"
|
||||
)
|
||||
|
||||
|
||||
def strip_timestamps(text: str) -> str:
|
||||
"""Line order already encodes sequence; per-line ISO timestamps are ~8 tokens each."""
|
||||
# ponytail: ISO-8601 only; add syslog/other formats when a real log needs them
|
||||
return _TIMESTAMP.sub("", text)
|
||||
|
||||
|
||||
def minify_json(text: str) -> str:
|
||||
"""Whole-input pretty JSON → minified. Lossless when it fires, untouched when not."""
|
||||
try:
|
||||
return json.dumps(json.loads(text), separators=(",", ":"), ensure_ascii=False)
|
||||
except ValueError:
|
||||
return text
|
||||
|
||||
|
||||
def collapse_cr(text: str) -> str:
|
||||
"""Keep only the final state of \\r-overwritten progress lines (pip/tqdm/wget)."""
|
||||
text = text.replace("\r\n", "\n")
|
||||
return "\n".join(l.rsplit("\r", 1)[-1] for l in text.split("\n"))
|
||||
|
||||
|
||||
_ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]") # ponytail: CSI only; add OSC if titles show up
|
||||
|
||||
|
||||
def strip_ansi(text: str) -> str:
|
||||
"""Color codes carry nothing for an LLM, and they make identical lines differ."""
|
||||
return _ANSI.sub("", text)
|
||||
|
||||
|
||||
_UUID = re.compile(r"\b[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}\b")
|
||||
|
||||
|
||||
def squash_uuids(text: str) -> str:
|
||||
"""36 chars → 8-hex prefix, git-short-hash style; cross-references still resolve."""
|
||||
return _UUID.sub(lambda m: m.group()[:8] + "…", text)
|
||||
|
||||
|
||||
_PKGPATH = re.compile(r'[^\s"\']+/(?:site-packages|dist-packages|lib/python3\.\d+)/')
|
||||
|
||||
|
||||
def squash_pkgpaths(text: str) -> str:
|
||||
"""Traceback path spam: …/httpx/_client.py:1054 is still unique without the venv prefix."""
|
||||
return _PKGPATH.sub("…/", text)
|
||||
|
||||
|
||||
# the lookbehinds keep "not just X" from becoming "not X" — meaning inversion
|
||||
_FILLER = re.compile(
|
||||
r"(?<!\bnot )(?<!n't )"
|
||||
r"\b(?:could you please|can you please|i would like you to|i want you to|"
|
||||
r"go ahead and|hey there,?|please|kindly|basically|actually|currently|"
|
||||
r"really|simply|just|very|quite)\b ?",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def strip_filler(text: str) -> str:
|
||||
"""Aggressive: eats words like 'just' anywhere, including inside strings. Prose only."""
|
||||
return _FILLER.sub("", text)
|
||||
|
||||
|
||||
# caveman-style word dropping: every function word is a whole token.
|
||||
# NEVER add negations (not/no/never), modals (must/should), or order words
|
||||
# (before/after) — dropping those changes meaning, not just style.
|
||||
_STICKS = re.compile(
|
||||
r"\b(?:the|a|an|is|are|was|were|be|been|being|am|i|we|you|they|it|"
|
||||
r"that|which|who|have|has|had|do|does|did|there)\b ?",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def two_sticks(text: str) -> str:
|
||||
"""Gist-only: strips text to caveman. Fine for articles/transcripts, never instructions."""
|
||||
# keep IT/US-style acronyms that case-insensitively collide with function words
|
||||
return _STICKS.sub(lambda m: m.group() if m.group().strip().isupper()
|
||||
and len(m.group().strip()) > 1 else "", text)
|
||||
|
||||
|
||||
_SKEL = re.compile(r"[^A-Za-z]+")
|
||||
|
||||
|
||||
def dedupe_similar(text: str) -> str:
|
||||
"""Collapse runs of lines identical after masking non-letters — 'same words,
|
||||
different numbers' (progress lines, per-item CI steps). Keeps first and last
|
||||
so progression endpoints survive."""
|
||||
lines = text.split("\n")
|
||||
out, i = [], 0
|
||||
while i < len(lines):
|
||||
k, j = _SKEL.sub(" ", lines[i]).strip(), i
|
||||
while j < len(lines) and _SKEL.sub(" ", lines[j]).strip() == k:
|
||||
j += 1
|
||||
if j - i >= 4 and k:
|
||||
out += [lines[i], f"[{j - i - 2} similar lines omitted]", lines[j - 1]]
|
||||
else:
|
||||
out.extend(lines[i:j])
|
||||
i = j
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
_REF = re.compile(r"@\d+")
|
||||
|
||||
|
||||
def alias_repeats(text: str, min_count: int = 4, min_len: int = 30) -> str:
|
||||
"""Dictionary-code scattered duplicate lines that consecutive dedupe can't reach.
|
||||
|
||||
Lossless — the legend keeps every line verbatim. A code only pays when it
|
||||
replaces a repeated multi-token sequence; single words are already 1 BPE
|
||||
token each, so word-level codebooks lose (measured, see README).
|
||||
"""
|
||||
lines = text.split("\n")
|
||||
if any(_REF.fullmatch(l) for l in lines): # already aliased, or real @N content — bail
|
||||
return text
|
||||
counts = Counter(l for l in lines if len(l) >= min_len)
|
||||
# ponytail: c*len>200 chars is the payoff heuristic; tune if legends ever dominate
|
||||
worth = [l for l, c in counts.items() if c >= min_count and c * len(l) > 200]
|
||||
if not worth:
|
||||
return text
|
||||
ref = {l: f"@{i}" for i, l in enumerate(worth, 1)}
|
||||
legend = [f"@{i} = {l}" for i, l in enumerate(worth, 1)]
|
||||
return "\n".join(["[repeated lines aliased:]"] + legend + [""] +
|
||||
[ref.get(l, l) for l in lines])
|
||||
|
||||
|
||||
# order: strippers leave doubled spaces, so collapse_whitespace runs after them;
|
||||
# normalized lines then match better in alias_repeats/dedupe_lines/dedupe_similar
|
||||
_STRIP2 = [minify_json, collapse_cr, strip_ansi, strip_timestamps,
|
||||
squash_blobs, squash_uuids, squash_pkgpaths]
|
||||
_MERGE = [collapse_whitespace, alias_repeats, dedupe_lines, dedupe_similar]
|
||||
LEVELS = {
|
||||
1: [collapse_whitespace, dedupe_lines], # code-safe
|
||||
2: _STRIP2 + _MERGE, # logs/dumps/tool output
|
||||
3: _STRIP2 + [strip_filler] + _MERGE, # prose
|
||||
4: _STRIP2 + [strip_filler, two_sticks] + _MERGE, # gist-only
|
||||
}
|
||||
|
||||
|
||||
def compress(text: str, level: int = 1) -> str:
|
||||
for f in LEVELS[level]:
|
||||
text = f(text)
|
||||
return text
|
||||
|
||||
|
||||
def budget(text: str, max_tokens: int) -> str:
|
||||
"""Hard cap: keep head and tail lines, drop the middle with a marker.
|
||||
|
||||
The only pass with GUARANTEED bounded output — run it after compression,
|
||||
as the backstop, never instead of it.
|
||||
"""
|
||||
total = count_tokens(text)
|
||||
if total <= max_tokens:
|
||||
return text
|
||||
lines = text.split("\n")
|
||||
limit = max(0, max_tokens - 8) # reserve for the marker line
|
||||
head, tail = [], []
|
||||
h_tok = t_tok = 0
|
||||
hi, ti = 0, len(lines) - 1
|
||||
while hi <= ti:
|
||||
take_head = h_tok <= t_tok
|
||||
line = lines[hi] if take_head else lines[ti]
|
||||
tok = count_tokens(line) + 1
|
||||
if h_tok + t_tok + tok > limit:
|
||||
break
|
||||
if take_head:
|
||||
head.append(line); h_tok += tok; hi += 1
|
||||
else:
|
||||
tail.append(line); t_tok += tok; ti -= 1
|
||||
if not head and not tail: # one giant line (e.g. minified JSON): slice by chars
|
||||
keep = limit * 2 # ~4 chars/token, half per side
|
||||
head, tail = [text[:keep]], [text[-keep:]]
|
||||
h_tok = count_tokens(head[0]); t_tok = count_tokens(tail[0])
|
||||
omitted = max(0, total - h_tok - t_tok)
|
||||
return "\n".join(head + ["[~%d tokens omitted]" % omitted] + list(reversed(tail)))
|
||||
|
||||
|
||||
def serve(port=7777):
|
||||
"""Paste-in demo page on localhost. Zero deps, binds 127.0.0.1 only."""
|
||||
import html
|
||||
import http.server
|
||||
import urllib.parse
|
||||
|
||||
page = ("<!doctype html><meta charset=utf-8><title>lessismore</title>"
|
||||
"<style>body{font-family:system-ui;max-width:900px;margin:2rem auto;padding:0 1rem;"
|
||||
"background:#10141a;color:#e6edf3}textarea{width:100%;height:35vh;font-family:monospace;"
|
||||
"background:#1a2028;color:#e6edf3;border:1px solid #2d3743;border-radius:8px;padding:8px}"
|
||||
"select,button{font-size:16px;padding:8px 14px;border-radius:8px;border:1px solid #2d3743;"
|
||||
"background:#1f6feb;color:#fff;cursor:pointer}select{background:#1a2028}</style>"
|
||||
"<h1>lessismore \U0001F4C9</h1>"
|
||||
"<form method=post><textarea name=text placeholder='paste your ugliest log here'></textarea>"
|
||||
"<p>level <select name=level><option>1<option selected>2<option>3<option>4</select> "
|
||||
"<button>squeeze</button></p></form>{result}")
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def _page(self, result=""):
|
||||
body = page.replace("{result}", result).encode("utf-8")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_GET(self):
|
||||
self._page()
|
||||
|
||||
def do_POST(self):
|
||||
n = int(self.headers.get("Content-Length", 0))
|
||||
q = urllib.parse.parse_qs(self.rfile.read(n).decode("utf-8", "replace"))
|
||||
text = q.get("text", [""])[0]
|
||||
level = min(4, max(1, int(q.get("level", ["2"])[0])))
|
||||
out = compress(text, level)
|
||||
b, a = count_tokens(text), count_tokens(out)
|
||||
self._page("<p><b>{:,} → {:,} tokens ({}% saved)</b></p>"
|
||||
"<textarea readonly>{}</textarea>".format(
|
||||
b, a, round(100 * (1 - a / max(b, 1))), html.escape(out)))
|
||||
|
||||
print("lessismore UI on http://localhost:%d (Ctrl+C to stop)" % port)
|
||||
http.server.ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- optional ML pass
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _llmlingua():
|
||||
from llmlingua import PromptCompressor # pip install llmlingua
|
||||
return PromptCompressor(
|
||||
model_name="microsoft/llmlingua-2-xlm-roberta-large-meetingbank",
|
||||
use_llmlingua2=True,
|
||||
)
|
||||
|
||||
|
||||
def compress_ml(text: str, rate: float = 0.5) -> str:
|
||||
"""Perplexity-based token pruning (LLMLingua-2).
|
||||
|
||||
Runs a local classifier model — only pays for itself on multi-KB inputs.
|
||||
Run the deterministic passes first; never feed it code you need verbatim.
|
||||
"""
|
||||
return _llmlingua().compress_prompt(text, rate=rate)["compressed_prompt"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- CLI
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
|
||||
p = argparse.ArgumentParser(prog="lessismore", description=__doc__.splitlines()[0])
|
||||
p.add_argument("file", nargs="?", help="input file (default: stdin)")
|
||||
p.add_argument("-l", "--level", type=int, default=1, choices=sorted(LEVELS),
|
||||
help="1=code-safe 2=logs/dumps 3=prose 4=gist-only caveman (default 1)")
|
||||
p.add_argument("--ml", type=float, metavar="RATE",
|
||||
help="also run LLMLingua-2 keeping RATE of tokens (needs: pip install llmlingua)")
|
||||
p.add_argument("--budget", type=int, metavar="N",
|
||||
help="hard cap output at ~N tokens: keep head+tail, drop the middle")
|
||||
p.add_argument("--serve", nargs="?", const=7777, type=int, metavar="PORT",
|
||||
help="serve a paste-in demo page on localhost (default port 7777)")
|
||||
a = p.parse_args()
|
||||
|
||||
if a.serve:
|
||||
return serve(a.serve)
|
||||
|
||||
# newline="" / buffer.read(): keep \r intact for collapse_cr
|
||||
raw = (open(a.file, encoding="utf-8", errors="replace", newline="").read() if a.file
|
||||
else sys.stdin.buffer.read().decode("utf-8", "replace"))
|
||||
out = compress(raw, a.level)
|
||||
if a.ml:
|
||||
try:
|
||||
out = compress_ml(out, a.ml)
|
||||
except ImportError:
|
||||
sys.exit("--ml needs: pip install llmlingua")
|
||||
if a.budget:
|
||||
out = budget(out, a.budget)
|
||||
sys.stdout.write(out)
|
||||
before, after = count_tokens(raw), count_tokens(out)
|
||||
print(f"lessismore: {before} → {after} tokens ({1 - after / max(before, 1):.0%} saved)",
|
||||
file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,200 +0,0 @@
|
||||
Metadata-Version: 2.4
|
||||
Name: lessismore
|
||||
Version: 0.1.0
|
||||
Summary: Squeeze text before it hits an LLM — deterministic prompt compression, measured in real tokens
|
||||
Author: John King (monsterrobotsoft)
|
||||
License: MIT
|
||||
Keywords: llm,prompt,compression,tokens,context
|
||||
Requires-Python: >=3.8
|
||||
Description-Content-Type: text/markdown
|
||||
License-File: LICENSE
|
||||
Provides-Extra: tokens
|
||||
Requires-Dist: tiktoken; extra == "tokens"
|
||||
Provides-Extra: ml
|
||||
Requires-Dist: llmlingua; extra == "ml"
|
||||
Dynamic: license-file
|
||||
|
||||
# lessismore 📉
|
||||
|
||||
**Turn 50,000 tokens of log spam into 4,500 tokens of pure signal — before it
|
||||
ever hits your LLM.**
|
||||
|
||||
lessismore is a deterministic compression filter for the bulk text that
|
||||
actually fills context windows: logs, CI output, tracebacks, JSON dumps,
|
||||
captured tool output. Pure stdlib Python. Zero dependencies. Every claim below
|
||||
was measured against the real o200k tokenizer — and every idea that failed the
|
||||
measurement is documented at the bottom, so you know exactly what you're
|
||||
getting.
|
||||
|
||||
```bash
|
||||
tail -5000 app.log | python3 lessismore.py -l 2 | llm "why did this crash?"
|
||||
```
|
||||
|
||||
**11x** on mixed error logs · **12x** on interleaved service logs ·
|
||||
**8.9x** on ANSI CI logs · **98%** on captured pip/tqdm output ·
|
||||
**~6 MB/s** single-core · **0** dependencies
|
||||
|
||||
## Why this exists
|
||||
|
||||
Three facts about LLM context, and the gap between them is this tool:
|
||||
|
||||
1. **The expensive part isn't your prompt.** Your typed question is ~30
|
||||
tokens. The log dump you attach is 50,000. Compress the wall, not the
|
||||
question.
|
||||
2. **Machine output is mostly redundancy.** ANSI color codes, `\r` progress
|
||||
redraws, timestamps, venv path spam, the same error line 400 times — noise
|
||||
that looks small on a terminal screen but is real tokens in a capture.
|
||||
3. **BPE tokenizers already compress English** (common words are 1 token —
|
||||
you can't out-abbreviate them, we checked). What BPE *can't* see is
|
||||
repetition across a file or terminal noise. That's the seam this tool
|
||||
mines.
|
||||
|
||||
Three payoffs when you pipe through it:
|
||||
|
||||
- **Context capacity** — an hour of log history fits where two minutes did.
|
||||
On a local model, that's the difference between full speed and crawling.
|
||||
- **Prompt-cache longevity** — every pass is deterministic: same input, same
|
||||
output, byte for byte. Follow-up questions re-hit the provider cache
|
||||
instead of re-paying for the logs. An ML compressor in the loop would bust
|
||||
the cache on every subtle variation; the regex passes never do.
|
||||
- **Model attention** — LLMs lose things in the middle of walls of text. Feed
|
||||
signal, not noise, and the first answer is the right one more often.
|
||||
|
||||
## Quickstart
|
||||
|
||||
```bash
|
||||
pip install . # from a clone — installs the `lessismore` and `lm` commands
|
||||
pip install '.[tokens]' # + tiktoken for exact token stats (else chars/4 estimate)
|
||||
python3 test_lessismore.py # should print "ok"
|
||||
```
|
||||
|
||||
```bash
|
||||
lm dump.log -l 2 > small.log # file in, file out
|
||||
docker logs myapp 2>&1 | lm -l 2 # pipe filter
|
||||
lm big.txt -l 2 --budget 4000 # compress, then hard-cap at ~4k tokens
|
||||
lm big.txt -l 2 --ml 0.5 # + LLMLingua last mile
|
||||
lm --serve # paste-in demo page on localhost:7777
|
||||
```
|
||||
|
||||
No install needed either — `python3 lessismore.py` works the same from a bare
|
||||
clone; it's one stdlib-only file.
|
||||
|
||||
```python
|
||||
from lessismore import compress, count_tokens, budget
|
||||
small = budget(compress(big_log, level=2), 8000)
|
||||
```
|
||||
|
||||
Token stats print to stderr, so pipes stay clean.
|
||||
|
||||
## The dial
|
||||
|
||||
Four levels, from byte-cautious to caveman. Pick by content, not by greed.
|
||||
|
||||
| Level | What it eats | Point it at | Profile |
|
||||
|---|---|---|---|
|
||||
| **1** | whitespace runs, consecutive duplicate lines | code, scripts, anything | structure-safe: indentation and content untouched (whitespace *inside string literals* still collapses) |
|
||||
| **2** | + JSON minify, `\r` redraw collapse, ANSI strip, ISO timestamps, base64/hex blobs, UUIDs, venv paths, scattered-duplicate aliasing, similar-line collapse | logs, CI dumps, traces, tool output | **the sweet spot** — destroys machine noise, keeps every distinct fact |
|
||||
| **3** | + filler-phrase stripping ("could you please", "just") | prose, chat history | fine for text, never for strict logic |
|
||||
| **4** | + `two_sticks` caveman mode: drops articles/copulas/auxiliaries, never negations or modals | gist-only prose, transcripts | lossy on style, protective of meaning — "do **not** delete" keeps its *not* |
|
||||
|
||||
The crown jewel at level 2 is `alias_repeats`: ordinary dedupe only sees
|
||||
*consecutive* repeats, so interleaved multi-service logs sail straight through
|
||||
it. `alias_repeats` hunts scattered duplicates across the whole file and
|
||||
dictionary-codes them (`@1 = ERROR [pool-3] psycopg2...` once in a legend,
|
||||
2-token `@1` everywhere else). It's lossless — the legend keeps every line
|
||||
verbatim — and on the interleaved benchmark it's the difference between
|
||||
54,999 → 34,999 (dedupe alone, 1.6x) and 54,999 → 4,585 (**12x**).
|
||||
|
||||
`--budget N` is the backstop, not the compressor: after the passes run, it
|
||||
keeps head and tail lines and drops the middle with a `[~N tokens omitted]`
|
||||
marker. It's the only pass with *guaranteed* bounded output — use it when the
|
||||
context limit is a hard wall.
|
||||
|
||||
## The receipts
|
||||
|
||||
All reproducible: `python3 bench.py` (o200k counts via tiktoken).
|
||||
|
||||
| Sample | Tokens | Why it wins |
|
||||
|---|---|---|
|
||||
| Mixed error log, 2000 lines | 86,914 → 7,895 (**11.0x**) | timestamp strip unmasks identical lines → dedupe; similar-line collapse catches the numbered stragglers |
|
||||
| Interleaved 3-service log, zero consecutive repeats | 54,999 → 4,585 (**12.0x**) | `alias_repeats` — consecutive dedupe alone managed 1.6x on this input |
|
||||
| ANSI-colored CI/docker build log | 21,198 → 2,387 (**8.9x**) | color codes make identical lines look different; strip them and the log collapses |
|
||||
| Captured pip/tqdm output | 14,483 → 291 (**98%**) | every overwritten `\r` progress frame is invisible on screen but real tokens in a capture |
|
||||
| pytest failure dump | 2,223 → 1,673 (25%) | one site-packages path prefix = 25 tokens → 7 |
|
||||
| Pretty-printed JSON API response | 11,741 → 6,833 (42%) | minify (round-trip verified lossless) + UUID→8-hex squash |
|
||||
| Chatty prose, level 4 | 427 → 264 (38%) | every dropped function word is a whole token; level 3 got 2% on the same text |
|
||||
|
||||
Throughput: measured **~6 MB/s single-core** at level 2 (28 MB log in 4.4s).
|
||||
No model, no network — break-even input size is effectively zero.
|
||||
|
||||
## What we refused to build (measured so it stays dead)
|
||||
|
||||
The graveyard is a feature. Each of these looks clever on paper and loses
|
||||
against a real tokenizer:
|
||||
|
||||
- **Shorthand codebooks** — `[fmt:md_tbl+hdr]` costs **8 tokens**; "Format
|
||||
the output as a markdown table with headers." costs **10**. BPE already has
|
||||
English baked in; bracket syntax shreds into off-distribution fragments.
|
||||
- **Word→code recoding** ("database" → `qx`) — common words are already 1
|
||||
token, random codes cost 2, plus ~4 tokens/entry of codebook tax. You
|
||||
cannot beat a 200k-entry codebook from inside its own encoding. Zipping a
|
||||
zip.
|
||||
- **Known abbreviations** (db, fn, env, auth) — measured 1 → 1 tokens. Zero.
|
||||
- **Personalized shorthand skills** — scanned 2,917 real typed prompts across
|
||||
654 transcripts: 2,762 unique, and the repeats were already grunts ("yes",
|
||||
"go"). A model-side decode skill costs ~1k tokens/turn to save ~10.
|
||||
`grunts.py` keeps the half that works: mine your own history, emit
|
||||
*client-side* slash-command stubs — expansion before the model sees it is
|
||||
free.
|
||||
- **Digit-masked dedupe** (0.0% — progress bars differ in glyphs, not
|
||||
digits), **separator shortening** (a 78-char `----` is already 1 token),
|
||||
**prefix hoisting** (one stray line kills it), **JSON→TSV / float
|
||||
truncation / pointer squash** (real savings, worse trade).
|
||||
|
||||
## Battle-tested
|
||||
|
||||
An adversarial review agent was told to break it and confirmed 14 real
|
||||
failure modes — negation-inverting filler stripping ("was **not just** the
|
||||
db" → "was **not** the db"), URLs eaten as base64, crashes on empty and
|
||||
non-UTF-8 stdin, distinct SHA-256s falsely merging as "repeated",
|
||||
`two_sticks` eating "IT" and "US" as function words. Nine fixed with
|
||||
regression tests, four documented below, one wontfix (adversarial in-band
|
||||
marker collision). `python3 test_lessismore.py` — 30 asserts, no framework.
|
||||
|
||||
## Where it does nothing (on purpose)
|
||||
|
||||
Savings are proportional to **redundancy, not size**. This removes repetition
|
||||
and machine noise; it cannot compress information, and doesn't pretend to.
|
||||
|
||||
| Content | Expect | Verdict |
|
||||
|---|---|---|
|
||||
| Repetitive machine output | 5–12x, up to 50x on pathological repeats | the reason this exists |
|
||||
| Structured data (JSON, tracebacks) | 25–42% | worthwhile, lossless where it fires |
|
||||
| Varied prose | ~2% (level 3) / 38% lossy (level 4) | gist mode only |
|
||||
| Clean code, unique dense text | ~0% by design | that's what `--ml` or truncation is for |
|
||||
|
||||
`--ml RATE` bolts on Microsoft's LLMLingua-2 for perplexity pruning as a
|
||||
last mile (`pip install llmlingua`) — runs *after* the deterministic passes so
|
||||
you're not paying a classifier model to delete duplicate log lines. Only
|
||||
worth it on multi-KB inputs, and it forfeits the cache-stability guarantee.
|
||||
|
||||
## Try it in a browser
|
||||
|
||||
`lm --serve` runs a paste-in demo page on `http://localhost:7777` — paste your
|
||||
ugliest log, pick a level, watch the token count drop. Stdlib only, binds
|
||||
localhost only.
|
||||
|
||||
## License
|
||||
|
||||
MIT — see [LICENSE](LICENSE).
|
||||
|
||||
## Known tradeoffs
|
||||
|
||||
- Levels 2+ assume line *order* matters but wall-clock timing doesn't. When
|
||||
gaps and deltas are the signal (hang hunting), stay on level 1.
|
||||
- Output is for LLM consumption, not round-tripping: markdown hard breaks
|
||||
(trailing double-space) and diff context lines don't survive even level 1.
|
||||
- At levels 3–4, dedupe counts describe the post-stripped text — five
|
||||
differently-phrased "retry the job" lines can legitimately merge.
|
||||
- In-band markers can collide with input that already contains them;
|
||||
`alias_repeats` bails out if its own `@N` markers already appear as lines.
|
||||
@ -1,10 +0,0 @@
|
||||
LICENSE
|
||||
README.md
|
||||
lessismore.py
|
||||
pyproject.toml
|
||||
lessismore.egg-info/PKG-INFO
|
||||
lessismore.egg-info/SOURCES.txt
|
||||
lessismore.egg-info/dependency_links.txt
|
||||
lessismore.egg-info/entry_points.txt
|
||||
lessismore.egg-info/requires.txt
|
||||
lessismore.egg-info/top_level.txt
|
||||
@ -1 +0,0 @@
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
[console_scripts]
|
||||
lessismore = lessismore:main
|
||||
lm = lessismore:main
|
||||
@ -1,6 +0,0 @@
|
||||
|
||||
[ml]
|
||||
llmlingua
|
||||
|
||||
[tokens]
|
||||
tiktoken
|
||||
@ -1 +0,0 @@
|
||||
lessismore
|
||||
Loading…
Reference in New Issue
Block a user