diff --git a/.claude/scheduled_tasks.lock b/.claude/scheduled_tasks.lock deleted file mode 100644 index 8f5981c..0000000 --- a/.claude/scheduled_tasks.lock +++ /dev/null @@ -1 +0,0 @@ -{"sessionId":"b1485757-c727-45be-be51-c0147d9825ad","pid":63486,"procStart":"Tue Jul 7 02:45:21 2026","acquiredAt":1783393961121} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 82b31f9..d1fea1e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ __pycache__/ .claude/ +build/ +*.egg-info/ +grunts/ diff --git a/build/lib/lessismore.py b/build/lib/lessismore.py deleted file mode 100644 index d8c2c72..0000000 --- a/build/lib/lessismore.py +++ /dev/null @@ -1,362 +0,0 @@ -"""lessismore — squeeze text before it hits an LLM. - -Deterministic passes first (free, safe, cacheable). Optional ML pruning -(LLMLingua-2) only when installed and only worth it on big inputs. - - from lessismore import compress - small = compress(big_log, level=2) - - $ python3 lessismore.py dump.log -l 2 > small.log - $ tail -5000 app.log | python3 lessismore.py -l 2 | llm ... -""" - -import json -import re -import sys -from collections import Counter -from functools import lru_cache - -# ---------------------------------------------------------------- tokens - -@lru_cache(maxsize=1) -def _encoder(): - try: - import tiktoken - return tiktoken.get_encoding("o200k_base") - except ImportError: - return None - - -def count_tokens(text: str) -> int: - enc = _encoder() - if enc: - return len(enc.encode(text)) - return max(1, len(text) // 4) # ponytail: chars/4 heuristic; pip install tiktoken for real counts - - -# ---------------------------------------------------------------- passes -# Each pass is (str) -> str. Order matters: whitespace before dedupe. - -def collapse_whitespace(text: str) -> str: - text = re.sub(r"[ \t]+$", "", text, flags=re.M) # trailing whitespace - text = re.sub(r"(?<=\S)[ \t]{2,}", " ", text) # interior runs (leading indent kept: code-safe) - text = re.sub(r"\n{3,}", "\n\n", text) # blank-line runs - return text - - -def dedupe_lines(text: str) -> str: - """Collapse runs of identical lines — the classic log killer.""" - lines = text.split("\n") - out, i = [], 0 - while i < len(lines): - j = i - while j < len(lines) and lines[j] == lines[i]: - j += 1 - run = j - i - # marker must actually be shorter than the lines it replaces - if run >= 4 and lines[i].strip() and (run - 1) * (len(lines[i]) + 1) > 45: - out.append(lines[i]) - out.append(f"[previous line repeated {run - 1} more times]") - else: - out.extend(lines[i:j]) - i = j - # ponytail: only consecutive repeats; add block-level dedupe (repeated stack traces) if logs demand it - return "\n".join(out) - - -# no '/' in the class: URL paths are ≥64-char alnum+slash runs and they ARE the content -_BLOB = re.compile(r"\b(?:[A-Za-z0-9+]{64,}={0,2}|[0-9a-fA-F]{48,})\b") - - -def squash_blobs(text: str) -> str: - """Base64/hex runs are token-dense and semantically opaque — keep head and tail. - - The tail suffix keeps two different blobs from squashing to the same stub - and then being falsely merged as "repeated" by dedupe_lines. - """ - return _BLOB.sub(lambda m: f"{m.group()[:12]}[+{len(m.group()) - 16} chars]{m.group()[-4:]}", text) - - -_TIMESTAMP = re.compile( - r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?\b ?" -) - - -def strip_timestamps(text: str) -> str: - """Line order already encodes sequence; per-line ISO timestamps are ~8 tokens each.""" - # ponytail: ISO-8601 only; add syslog/other formats when a real log needs them - return _TIMESTAMP.sub("", text) - - -def minify_json(text: str) -> str: - """Whole-input pretty JSON → minified. Lossless when it fires, untouched when not.""" - try: - return json.dumps(json.loads(text), separators=(",", ":"), ensure_ascii=False) - except ValueError: - return text - - -def collapse_cr(text: str) -> str: - """Keep only the final state of \\r-overwritten progress lines (pip/tqdm/wget).""" - text = text.replace("\r\n", "\n") - return "\n".join(l.rsplit("\r", 1)[-1] for l in text.split("\n")) - - -_ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]") # ponytail: CSI only; add OSC if titles show up - - -def strip_ansi(text: str) -> str: - """Color codes carry nothing for an LLM, and they make identical lines differ.""" - return _ANSI.sub("", text) - - -_UUID = re.compile(r"\b[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}\b") - - -def squash_uuids(text: str) -> str: - """36 chars → 8-hex prefix, git-short-hash style; cross-references still resolve.""" - return _UUID.sub(lambda m: m.group()[:8] + "…", text) - - -_PKGPATH = re.compile(r'[^\s"\']+/(?:site-packages|dist-packages|lib/python3\.\d+)/') - - -def squash_pkgpaths(text: str) -> str: - """Traceback path spam: …/httpx/_client.py:1054 is still unique without the venv prefix.""" - return _PKGPATH.sub("…/", text) - - -# the lookbehinds keep "not just X" from becoming "not X" — meaning inversion -_FILLER = re.compile( - r"(? str: - """Aggressive: eats words like 'just' anywhere, including inside strings. Prose only.""" - return _FILLER.sub("", text) - - -# caveman-style word dropping: every function word is a whole token. -# NEVER add negations (not/no/never), modals (must/should), or order words -# (before/after) — dropping those changes meaning, not just style. -_STICKS = re.compile( - r"\b(?:the|a|an|is|are|was|were|be|been|being|am|i|we|you|they|it|" - r"that|which|who|have|has|had|do|does|did|there)\b ?", - re.I, -) - - -def two_sticks(text: str) -> str: - """Gist-only: strips text to caveman. Fine for articles/transcripts, never instructions.""" - # keep IT/US-style acronyms that case-insensitively collide with function words - return _STICKS.sub(lambda m: m.group() if m.group().strip().isupper() - and len(m.group().strip()) > 1 else "", text) - - -_SKEL = re.compile(r"[^A-Za-z]+") - - -def dedupe_similar(text: str) -> str: - """Collapse runs of lines identical after masking non-letters — 'same words, - different numbers' (progress lines, per-item CI steps). Keeps first and last - so progression endpoints survive.""" - lines = text.split("\n") - out, i = [], 0 - while i < len(lines): - k, j = _SKEL.sub(" ", lines[i]).strip(), i - while j < len(lines) and _SKEL.sub(" ", lines[j]).strip() == k: - j += 1 - if j - i >= 4 and k: - out += [lines[i], f"[{j - i - 2} similar lines omitted]", lines[j - 1]] - else: - out.extend(lines[i:j]) - i = j - return "\n".join(out) - - -_REF = re.compile(r"@\d+") - - -def alias_repeats(text: str, min_count: int = 4, min_len: int = 30) -> str: - """Dictionary-code scattered duplicate lines that consecutive dedupe can't reach. - - Lossless — the legend keeps every line verbatim. A code only pays when it - replaces a repeated multi-token sequence; single words are already 1 BPE - token each, so word-level codebooks lose (measured, see README). - """ - lines = text.split("\n") - if any(_REF.fullmatch(l) for l in lines): # already aliased, or real @N content — bail - return text - counts = Counter(l for l in lines if len(l) >= min_len) - # ponytail: c*len>200 chars is the payoff heuristic; tune if legends ever dominate - worth = [l for l, c in counts.items() if c >= min_count and c * len(l) > 200] - if not worth: - return text - ref = {l: f"@{i}" for i, l in enumerate(worth, 1)} - legend = [f"@{i} = {l}" for i, l in enumerate(worth, 1)] - return "\n".join(["[repeated lines aliased:]"] + legend + [""] + - [ref.get(l, l) for l in lines]) - - -# order: strippers leave doubled spaces, so collapse_whitespace runs after them; -# normalized lines then match better in alias_repeats/dedupe_lines/dedupe_similar -_STRIP2 = [minify_json, collapse_cr, strip_ansi, strip_timestamps, - squash_blobs, squash_uuids, squash_pkgpaths] -_MERGE = [collapse_whitespace, alias_repeats, dedupe_lines, dedupe_similar] -LEVELS = { - 1: [collapse_whitespace, dedupe_lines], # code-safe - 2: _STRIP2 + _MERGE, # logs/dumps/tool output - 3: _STRIP2 + [strip_filler] + _MERGE, # prose - 4: _STRIP2 + [strip_filler, two_sticks] + _MERGE, # gist-only -} - - -def compress(text: str, level: int = 1) -> str: - for f in LEVELS[level]: - text = f(text) - return text - - -def budget(text: str, max_tokens: int) -> str: - """Hard cap: keep head and tail lines, drop the middle with a marker. - - The only pass with GUARANTEED bounded output — run it after compression, - as the backstop, never instead of it. - """ - total = count_tokens(text) - if total <= max_tokens: - return text - lines = text.split("\n") - limit = max(0, max_tokens - 8) # reserve for the marker line - head, tail = [], [] - h_tok = t_tok = 0 - hi, ti = 0, len(lines) - 1 - while hi <= ti: - take_head = h_tok <= t_tok - line = lines[hi] if take_head else lines[ti] - tok = count_tokens(line) + 1 - if h_tok + t_tok + tok > limit: - break - if take_head: - head.append(line); h_tok += tok; hi += 1 - else: - tail.append(line); t_tok += tok; ti -= 1 - if not head and not tail: # one giant line (e.g. minified JSON): slice by chars - keep = limit * 2 # ~4 chars/token, half per side - head, tail = [text[:keep]], [text[-keep:]] - h_tok = count_tokens(head[0]); t_tok = count_tokens(tail[0]) - omitted = max(0, total - h_tok - t_tok) - return "\n".join(head + ["[~%d tokens omitted]" % omitted] + list(reversed(tail))) - - -def serve(port=7777): - """Paste-in demo page on localhost. Zero deps, binds 127.0.0.1 only.""" - import html - import http.server - import urllib.parse - - page = ("
{:,} → {:,} tokens ({}% saved)
" - "".format( - b, a, round(100 * (1 - a / max(b, 1))), html.escape(out))) - - print("lessismore UI on http://localhost:%d (Ctrl+C to stop)" % port) - http.server.ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever() - - -# ---------------------------------------------------------------- optional ML pass - -@lru_cache(maxsize=1) -def _llmlingua(): - from llmlingua import PromptCompressor # pip install llmlingua - return PromptCompressor( - model_name="microsoft/llmlingua-2-xlm-roberta-large-meetingbank", - use_llmlingua2=True, - ) - - -def compress_ml(text: str, rate: float = 0.5) -> str: - """Perplexity-based token pruning (LLMLingua-2). - - Runs a local classifier model — only pays for itself on multi-KB inputs. - Run the deterministic passes first; never feed it code you need verbatim. - """ - return _llmlingua().compress_prompt(text, rate=rate)["compressed_prompt"] - - -# ---------------------------------------------------------------- CLI - -def main(): - import argparse - - p = argparse.ArgumentParser(prog="lessismore", description=__doc__.splitlines()[0]) - p.add_argument("file", nargs="?", help="input file (default: stdin)") - p.add_argument("-l", "--level", type=int, default=1, choices=sorted(LEVELS), - help="1=code-safe 2=logs/dumps 3=prose 4=gist-only caveman (default 1)") - p.add_argument("--ml", type=float, metavar="RATE", - help="also run LLMLingua-2 keeping RATE of tokens (needs: pip install llmlingua)") - p.add_argument("--budget", type=int, metavar="N", - help="hard cap output at ~N tokens: keep head+tail, drop the middle") - p.add_argument("--serve", nargs="?", const=7777, type=int, metavar="PORT", - help="serve a paste-in demo page on localhost (default port 7777)") - a = p.parse_args() - - if a.serve: - return serve(a.serve) - - # newline="" / buffer.read(): keep \r intact for collapse_cr - raw = (open(a.file, encoding="utf-8", errors="replace", newline="").read() if a.file - else sys.stdin.buffer.read().decode("utf-8", "replace")) - out = compress(raw, a.level) - if a.ml: - try: - out = compress_ml(out, a.ml) - except ImportError: - sys.exit("--ml needs: pip install llmlingua") - if a.budget: - out = budget(out, a.budget) - sys.stdout.write(out) - before, after = count_tokens(raw), count_tokens(out) - print(f"lessismore: {before} → {after} tokens ({1 - after / max(before, 1):.0%} saved)", - file=sys.stderr) - - -if __name__ == "__main__": - main() diff --git a/lessismore.egg-info/PKG-INFO b/lessismore.egg-info/PKG-INFO deleted file mode 100644 index e345bc2..0000000 --- a/lessismore.egg-info/PKG-INFO +++ /dev/null @@ -1,200 +0,0 @@ -Metadata-Version: 2.4 -Name: lessismore -Version: 0.1.0 -Summary: Squeeze text before it hits an LLM — deterministic prompt compression, measured in real tokens -Author: John King (monsterrobotsoft) -License: MIT -Keywords: llm,prompt,compression,tokens,context -Requires-Python: >=3.8 -Description-Content-Type: text/markdown -License-File: LICENSE -Provides-Extra: tokens -Requires-Dist: tiktoken; extra == "tokens" -Provides-Extra: ml -Requires-Dist: llmlingua; extra == "ml" -Dynamic: license-file - -# lessismore 📉 - -**Turn 50,000 tokens of log spam into 4,500 tokens of pure signal — before it -ever hits your LLM.** - -lessismore is a deterministic compression filter for the bulk text that -actually fills context windows: logs, CI output, tracebacks, JSON dumps, -captured tool output. Pure stdlib Python. Zero dependencies. Every claim below -was measured against the real o200k tokenizer — and every idea that failed the -measurement is documented at the bottom, so you know exactly what you're -getting. - -```bash -tail -5000 app.log | python3 lessismore.py -l 2 | llm "why did this crash?" -``` - -**11x** on mixed error logs · **12x** on interleaved service logs · -**8.9x** on ANSI CI logs · **98%** on captured pip/tqdm output · -**~6 MB/s** single-core · **0** dependencies - -## Why this exists - -Three facts about LLM context, and the gap between them is this tool: - -1. **The expensive part isn't your prompt.** Your typed question is ~30 - tokens. The log dump you attach is 50,000. Compress the wall, not the - question. -2. **Machine output is mostly redundancy.** ANSI color codes, `\r` progress - redraws, timestamps, venv path spam, the same error line 400 times — noise - that looks small on a terminal screen but is real tokens in a capture. -3. **BPE tokenizers already compress English** (common words are 1 token — - you can't out-abbreviate them, we checked). What BPE *can't* see is - repetition across a file or terminal noise. That's the seam this tool - mines. - -Three payoffs when you pipe through it: - -- **Context capacity** — an hour of log history fits where two minutes did. - On a local model, that's the difference between full speed and crawling. -- **Prompt-cache longevity** — every pass is deterministic: same input, same - output, byte for byte. Follow-up questions re-hit the provider cache - instead of re-paying for the logs. An ML compressor in the loop would bust - the cache on every subtle variation; the regex passes never do. -- **Model attention** — LLMs lose things in the middle of walls of text. Feed - signal, not noise, and the first answer is the right one more often. - -## Quickstart - -```bash -pip install . # from a clone — installs the `lessismore` and `lm` commands -pip install '.[tokens]' # + tiktoken for exact token stats (else chars/4 estimate) -python3 test_lessismore.py # should print "ok" -``` - -```bash -lm dump.log -l 2 > small.log # file in, file out -docker logs myapp 2>&1 | lm -l 2 # pipe filter -lm big.txt -l 2 --budget 4000 # compress, then hard-cap at ~4k tokens -lm big.txt -l 2 --ml 0.5 # + LLMLingua last mile -lm --serve # paste-in demo page on localhost:7777 -``` - -No install needed either — `python3 lessismore.py` works the same from a bare -clone; it's one stdlib-only file. - -```python -from lessismore import compress, count_tokens, budget -small = budget(compress(big_log, level=2), 8000) -``` - -Token stats print to stderr, so pipes stay clean. - -## The dial - -Four levels, from byte-cautious to caveman. Pick by content, not by greed. - -| Level | What it eats | Point it at | Profile | -|---|---|---|---| -| **1** | whitespace runs, consecutive duplicate lines | code, scripts, anything | structure-safe: indentation and content untouched (whitespace *inside string literals* still collapses) | -| **2** | + JSON minify, `\r` redraw collapse, ANSI strip, ISO timestamps, base64/hex blobs, UUIDs, venv paths, scattered-duplicate aliasing, similar-line collapse | logs, CI dumps, traces, tool output | **the sweet spot** — destroys machine noise, keeps every distinct fact | -| **3** | + filler-phrase stripping ("could you please", "just") | prose, chat history | fine for text, never for strict logic | -| **4** | + `two_sticks` caveman mode: drops articles/copulas/auxiliaries, never negations or modals | gist-only prose, transcripts | lossy on style, protective of meaning — "do **not** delete" keeps its *not* | - -The crown jewel at level 2 is `alias_repeats`: ordinary dedupe only sees -*consecutive* repeats, so interleaved multi-service logs sail straight through -it. `alias_repeats` hunts scattered duplicates across the whole file and -dictionary-codes them (`@1 = ERROR [pool-3] psycopg2...` once in a legend, -2-token `@1` everywhere else). It's lossless — the legend keeps every line -verbatim — and on the interleaved benchmark it's the difference between -54,999 → 34,999 (dedupe alone, 1.6x) and 54,999 → 4,585 (**12x**). - -`--budget N` is the backstop, not the compressor: after the passes run, it -keeps head and tail lines and drops the middle with a `[~N tokens omitted]` -marker. It's the only pass with *guaranteed* bounded output — use it when the -context limit is a hard wall. - -## The receipts - -All reproducible: `python3 bench.py` (o200k counts via tiktoken). - -| Sample | Tokens | Why it wins | -|---|---|---| -| Mixed error log, 2000 lines | 86,914 → 7,895 (**11.0x**) | timestamp strip unmasks identical lines → dedupe; similar-line collapse catches the numbered stragglers | -| Interleaved 3-service log, zero consecutive repeats | 54,999 → 4,585 (**12.0x**) | `alias_repeats` — consecutive dedupe alone managed 1.6x on this input | -| ANSI-colored CI/docker build log | 21,198 → 2,387 (**8.9x**) | color codes make identical lines look different; strip them and the log collapses | -| Captured pip/tqdm output | 14,483 → 291 (**98%**) | every overwritten `\r` progress frame is invisible on screen but real tokens in a capture | -| pytest failure dump | 2,223 → 1,673 (25%) | one site-packages path prefix = 25 tokens → 7 | -| Pretty-printed JSON API response | 11,741 → 6,833 (42%) | minify (round-trip verified lossless) + UUID→8-hex squash | -| Chatty prose, level 4 | 427 → 264 (38%) | every dropped function word is a whole token; level 3 got 2% on the same text | - -Throughput: measured **~6 MB/s single-core** at level 2 (28 MB log in 4.4s). -No model, no network — break-even input size is effectively zero. - -## What we refused to build (measured so it stays dead) - -The graveyard is a feature. Each of these looks clever on paper and loses -against a real tokenizer: - -- **Shorthand codebooks** — `[fmt:md_tbl+hdr]` costs **8 tokens**; "Format - the output as a markdown table with headers." costs **10**. BPE already has - English baked in; bracket syntax shreds into off-distribution fragments. -- **Word→code recoding** ("database" → `qx`) — common words are already 1 - token, random codes cost 2, plus ~4 tokens/entry of codebook tax. You - cannot beat a 200k-entry codebook from inside its own encoding. Zipping a - zip. -- **Known abbreviations** (db, fn, env, auth) — measured 1 → 1 tokens. Zero. -- **Personalized shorthand skills** — scanned 2,917 real typed prompts across - 654 transcripts: 2,762 unique, and the repeats were already grunts ("yes", - "go"). A model-side decode skill costs ~1k tokens/turn to save ~10. - `grunts.py` keeps the half that works: mine your own history, emit - *client-side* slash-command stubs — expansion before the model sees it is - free. -- **Digit-masked dedupe** (0.0% — progress bars differ in glyphs, not - digits), **separator shortening** (a 78-char `----` is already 1 token), - **prefix hoisting** (one stray line kills it), **JSON→TSV / float - truncation / pointer squash** (real savings, worse trade). - -## Battle-tested - -An adversarial review agent was told to break it and confirmed 14 real -failure modes — negation-inverting filler stripping ("was **not just** the -db" → "was **not** the db"), URLs eaten as base64, crashes on empty and -non-UTF-8 stdin, distinct SHA-256s falsely merging as "repeated", -`two_sticks` eating "IT" and "US" as function words. Nine fixed with -regression tests, four documented below, one wontfix (adversarial in-band -marker collision). `python3 test_lessismore.py` — 30 asserts, no framework. - -## Where it does nothing (on purpose) - -Savings are proportional to **redundancy, not size**. This removes repetition -and machine noise; it cannot compress information, and doesn't pretend to. - -| Content | Expect | Verdict | -|---|---|---| -| Repetitive machine output | 5–12x, up to 50x on pathological repeats | the reason this exists | -| Structured data (JSON, tracebacks) | 25–42% | worthwhile, lossless where it fires | -| Varied prose | ~2% (level 3) / 38% lossy (level 4) | gist mode only | -| Clean code, unique dense text | ~0% by design | that's what `--ml` or truncation is for | - -`--ml RATE` bolts on Microsoft's LLMLingua-2 for perplexity pruning as a -last mile (`pip install llmlingua`) — runs *after* the deterministic passes so -you're not paying a classifier model to delete duplicate log lines. Only -worth it on multi-KB inputs, and it forfeits the cache-stability guarantee. - -## Try it in a browser - -`lm --serve` runs a paste-in demo page on `http://localhost:7777` — paste your -ugliest log, pick a level, watch the token count drop. Stdlib only, binds -localhost only. - -## License - -MIT — see [LICENSE](LICENSE). - -## Known tradeoffs - -- Levels 2+ assume line *order* matters but wall-clock timing doesn't. When - gaps and deltas are the signal (hang hunting), stay on level 1. -- Output is for LLM consumption, not round-tripping: markdown hard breaks - (trailing double-space) and diff context lines don't survive even level 1. -- At levels 3–4, dedupe counts describe the post-stripped text — five - differently-phrased "retry the job" lines can legitimately merge. -- In-band markers can collide with input that already contains them; - `alias_repeats` bails out if its own `@N` markers already appear as lines. diff --git a/lessismore.egg-info/SOURCES.txt b/lessismore.egg-info/SOURCES.txt deleted file mode 100644 index e36cee7..0000000 --- a/lessismore.egg-info/SOURCES.txt +++ /dev/null @@ -1,10 +0,0 @@ -LICENSE -README.md -lessismore.py -pyproject.toml -lessismore.egg-info/PKG-INFO -lessismore.egg-info/SOURCES.txt -lessismore.egg-info/dependency_links.txt -lessismore.egg-info/entry_points.txt -lessismore.egg-info/requires.txt -lessismore.egg-info/top_level.txt \ No newline at end of file diff --git a/lessismore.egg-info/dependency_links.txt b/lessismore.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/lessismore.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/lessismore.egg-info/entry_points.txt b/lessismore.egg-info/entry_points.txt deleted file mode 100644 index 09c13a3..0000000 --- a/lessismore.egg-info/entry_points.txt +++ /dev/null @@ -1,3 +0,0 @@ -[console_scripts] -lessismore = lessismore:main -lm = lessismore:main diff --git a/lessismore.egg-info/requires.txt b/lessismore.egg-info/requires.txt deleted file mode 100644 index b45aa97..0000000 --- a/lessismore.egg-info/requires.txt +++ /dev/null @@ -1,6 +0,0 @@ - -[ml] -llmlingua - -[tokens] -tiktoken diff --git a/lessismore.egg-info/top_level.txt b/lessismore.egg-info/top_level.txt deleted file mode 100644 index 218ecf5..0000000 --- a/lessismore.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -lessismore