From 160d0cc484567b94913fc13256b07231c5250f75 Mon Sep 17 00:00:00 2001 From: jing Date: Tue, 7 Jul 2026 17:20:35 +1000 Subject: [PATCH] bench_real.py: three public benchmarks with deterministic metrics; 0.3.0 - LogHub samples (real system logs): 3.0x over 1.15M tokens - LogChunks (797 Travis failures, human-labeled failure chunks): 1.9x over 66M tokens, 78.5% of labeled failure-explaining lines survive verbatim - LogDx-CI (35 real GitHub Actions failures, arXiv 2605.28876): 2.2x over 15M tokens, 95.2% critical diagnostic signals retained; with --budget 8000, 6,334 tok/case at 77.4% retention; haiku diagnosed 12/12 from compressed vs 12/12 raw on the 12-case subset - README: Real logs, real benchmarks section; updated synthetic receipts (templates lift ANSI CI to 15.9x, synthetic eval total to 6.0x, 8/8 re-verified); honest 2-4x expectation row for production logs Co-Authored-By: Claude Fable 5 --- README.md | 110 +++++++++++++++++++++++++++++++---------- bench_real.py | 129 +++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 3 files changed, 214 insertions(+), 27 deletions(-) create mode 100644 bench_real.py diff --git a/README.md b/README.md index ecee9e5..9c61bc4 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,16 @@ getting. tail -5000 app.log | python3 lessismore.py | llm "why did this crash?" ``` -**11x** on mixed error logs · **12x** on interleaved service logs · -**11.8x** on ANSI CI logs · **97%** on captured pip/tqdm output · +**11x** on mixed error logs · **16x** on ANSI CI logs · **97%** on captured +pip/tqdm output · **3.0x** on real LogHub system logs · **95% critical-signal +retention** on the LogDx-CI benchmark's 35 real GitHub Actions failures · **~6–12 MB/s** single-core · **0** dependencies And the part a compression ratio can't prove: in the root-cause eval ([eval.py](eval.py)), the model found the planted root cause in **8/8** -compressed logs vs **6/8** raw — smaller *and* more often right. +compressed logs vs **6/8** raw — smaller *and* more often right. On 12 real +LogDx-CI failures it scored 12/12 on both raw and compressed, at less than +half the tokens. ## Why this exists @@ -88,7 +91,7 @@ Four levels, from byte-cautious to caveman. Pick by content, not by greed. | Level | What it eats | Point it at | Profile | |---|---|---|---| | **1** | whitespace runs, consecutive duplicate lines | code, scripts, anything | structure-safe: indentation and content untouched (whitespace *inside string literals* still collapses) | -| **2** | + JSON minify (whole-input *and* embedded pretty blocks), `\r` redraw collapse, ANSI strip, ISO/syslog/nginx timestamps, base64/base64url/hex blobs, UUIDs, venv paths, scattered-duplicate aliasing, similar-line collapse with value summaries | logs, CI dumps, traces, tool output | **the sweet spot and the default** — destroys machine noise, keeps every distinct fact | +| **2** | + JSON minify (whole-input *and* embedded pretty blocks), `\r` redraw collapse, ANSI strip, timestamps (ISO/syslog/nginx/HDFS/BGL/HealthApp), base64/base64url/hex blobs, UUIDs, venv paths, scattered-duplicate aliasing, template mining, similar-line collapse with value summaries | logs, CI dumps, traces, tool output | **the sweet spot and the default** — destroys machine noise, keeps every distinct fact | | **3** | + filler-phrase stripping ("could you please", "just") | prose, chat history | fine for text, never for strict logic | | **4** | + `two_sticks` caveman mode: drops articles/copulas/auxiliaries, never negations or modals | gist-only prose, transcripts | lossy on style, protective of meaning — "do **not** delete" keeps its *not* | @@ -98,13 +101,23 @@ collapse, the digits that varied are summarized in the marker — digits (status codes, ports, exit codes) *are* the diagnosis. An earlier version silently ate them; the adversarial eval below is what caught it. -The crown jewel at level 2 is `alias_repeats`: ordinary dedupe only sees -*consecutive* repeats, so interleaved multi-service logs sail straight through -it. `alias_repeats` hunts scattered duplicates across the whole file and -dictionary-codes them (`@1 = ERROR [pool-3] psycopg2...` once in a legend, -2-token `@1` everywhere else). It's lossless — the legend keeps every line -verbatim — and on the interleaved benchmark it's the difference between -54,999 → 34,999 (dedupe alone, 1.6x) and 54,999 → 4,585 (**12x**). +The crown jewels at level 2 are the two dictionary passes: + +- `alias_repeats` — ordinary dedupe only sees *consecutive* repeats, so + interleaved multi-service logs sail straight through it. This pass hunts + scattered exact duplicates across the whole file and dictionary-codes them + (`@1 = ERROR [pool-3] psycopg2...` once in a legend, 2-token `@1` + everywhere else). Lossless — the legend keeps every line verbatim. On the + interleaved benchmark: 54,999 → 34,999 with dedupe alone (1.6x) vs + 54,999 → 4,585 with aliasing (**12x**). +- `alias_templates` — real logs rarely repeat *exact* lines; they repeat + templates: `Failed password for root from 10.2.3.4 port 2201`. Lines that + differ only in their digit-bearing tokens get one legend entry + (`@t1 = Failed password for root from <*> port <*> ssh2`) and per-line + values (`@t1 10.2.3.4 2201`). Digit-bearing tokens that never vary (ssh2, + v1.2.3) are inlined into the template. This is a deterministic, single-pass + Drain-lite, and it's what benchmarking on real logs demanded: it took the + LogHub corpus from 1.8x to 3.0x. `--budget N` is the backstop, not the compressor: after the passes run, it keeps head and tail lines and drops the middle with a `[~N tokens omitted]` @@ -119,16 +132,54 @@ this table claims). | Sample | Tokens | Why it wins | |---|---|---| -| Mixed error log, 2000 lines | 86,914 → 7,895 (**11.0x**) | timestamp strip unmasks identical lines → dedupe; similar-line collapse catches the numbered stragglers | +| Mixed error log, 2000 lines | 86,914 → 7,619 (**11.4x**) | timestamp strip unmasks identical lines → dedupe; templates catch the numbered stragglers | | Interleaved 3-service log, zero consecutive repeats | 54,999 → 4,585 (**12.0x**) | `alias_repeats` — consecutive dedupe alone managed 1.6x on this input | -| ANSI-colored CI log | 10,440 → 888 (**11.8x**) | color codes make identical lines look different; strip them and the log collapses | +| ANSI-colored CI log | 10,440 → 657 (**15.9x**) | color codes make identical lines look different; strip them and the log collapses | | Captured pip/tqdm output | 10,465 → 265 (**97%**) | every overwritten `\r` progress frame is invisible on screen but real tokens in a capture | | pytest failure dump | 732 → 515 (30%) | one site-packages path prefix = 25 tokens → 7 | | Pretty-printed JSON API response | 8,932 → 3,234 (**64%**) | minify (round-trip verified lossless) + UUID→8-hex squash | | Chatty prose, level 4 | 427 → 264 (38%) | every dropped function word is a whole token; level 3 got 2% on the same text | Throughput: measured **6–12 MB/s single-core** at level 2 (20 MB log in -1.7s). No model, no network — break-even input size is effectively zero. +2.6s). No model, no network — break-even input size is effectively zero. + +## Real logs, real benchmarks + +Synthetic logs flatter a compressor — they repeat exactly the way its passes +expect. So [bench_real.py](bench_real.py) runs three **public** corpora with +deterministic metrics (no model in the loop, no cherry-picking): + +**[LogHub](https://github.com/logpai/loghub) samples** — 10 real system logs +(HDFS, BGL, OpenSSH, OpenStack...), 1.15M tokens: **3.0x overall**, ranging +from 1.5x (HealthApp's pipe-format) to 6.5x (Apache). Fair warning made +explicit: real diverse logs compress 2–4x, not the 11x of repetitive error +walls. It was this benchmark that forced `alias_templates` and four new +timestamp formats into existence — the corpus started at 1.5x. + +**[LogChunks](https://zenodo.org/records/3632351)** — 797 real Travis CI +failure logs where *humans* marked the chunk that explains each failure +(externally validated with the developers who caused them). 66.2M tokens → +34.2M (**1.9x**), and **78.5% of labeled failure-explaining lines survive +byte-for-byte**; 73% of chunks survive fully intact. The rest is dominated by +similar-line collapse where the message survives once with a value summary — +represented, not deleted. + +**[LogDx-CI](https://arxiv.org/abs/2605.28876)** — the benchmark built for +exactly this question: 35 real GitHub Actions failures with per-case +ground-truth diagnostic signals. All 35 cases, 15.0M tokens → 6.9M +(**2.2x**), with **95.2% of critical diagnostic signals retained** (139/146; +of the 7 misses, most are layout artifacts where templating separates a value +from its line). With `--budget 8000` stacked on top: **6,334 tokens/case at +77.4% critical-signal retention** — the same token cost as the benchmark's +tail-200 baseline, with the whole log's worth of scattered signals instead of +just the tail. And the answer-quality check: on 12 cases, claude-haiku-4.5 +diagnosed **12/12 from raw and 12/12 from compressed** — parity at ~2.4x +fewer tokens. (Ground-truth caveat inherited from the benchmark: its labels +are AI-drafted with single-author verification, n=35.) + +One case in this run earned its keep twice: the benchmark caught a real +lessismore bug (a camelCase test name eaten as a base64 blob), which is now a +regression test. That's what public benchmarks are for. ## Does the model still get the answer? @@ -146,15 +197,15 @@ default runner is a logged-in `claude -p`. Rerun it yourself): | Scenario | Raw tokens | Compressed | Raw | Compressed | |---|---|---|---|---| -| OOM kill buried mid-log, connection-error red herring | 38,777 | 157 (**247x**) | ✗ blamed the red herring | ✓ found the OOM | -| 429 rate-limit burst hidden in a wall of status codes | 46,899 | 6,056 | ✗ "service overwhelmed" | ✓ named the 429s | -| Disk-full scattered through 3 interleaved services | 38,616 | 12,487 | ✓ | ✓ | +| OOM kill buried mid-log, connection-error red herring | 38,777 | 174 (**223x**) | ✗ blamed the red herring | ✓ found the OOM | +| 429 rate-limit burst hidden in a wall of status codes | 46,899 | 3,713 | ✗ "service overwhelmed" | ✓ named the 429s | +| Disk-full scattered through 3 interleaved services | 38,616 | 8,103 | ✓ | ✓ | | One failed assertion in 1,400 ANSI test lines | 46,812 | 21,563 | ✓ | ✓ | -| Dependency conflict under progress-bar walls | 16,304 | 446 | ✓ | ✓ | -| Expired TLS cert in an nginx access-log wall | 56,696 | 34,474 | ✓ | ✓ | +| Dependency conflict under progress-bar walls | 16,304 | 355 | ✓ | ✓ | +| Expired TLS cert in an nginx access-log wall | 56,696 | 189 (**300x**) | ✓ | ✓ | | Missing env var in a crash-loop traceback | 7,680 | 1,610 | ✓ | ✓ | -| Java deadlock in a 40-thread dump | 8,157 | 8,041 (1.0x) | ✓ | ✓ | -| **Total** | **259,941** | **84,834 (3.1x)** | **6/8** | **8/8** | +| Java deadlock in a 40-thread dump | 8,157 | 7,907 (1.0x) | ✓ | ✓ | +| **Total** | **259,941** | **43,614 (6.0x)** | **6/8** | **8/8** | The two raw failures are the lost-in-the-middle effect this tool exists to counter: the model latched onto the loud symptom cascade and never surfaced @@ -247,10 +298,11 @@ and machine noise; it cannot compress information, and doesn't pretend to. | Content | Expect | Verdict | |---|---|---| -| Repetitive machine output | 5–12x, up to 50x on pathological repeats | the reason this exists | +| Repetitive machine output | 5–16x, up to 300x on pathological repeats | the reason this exists | +| Real-world CI / system logs | 2–4x (measured on LogHub, LogChunks, LogDx-CI) | the honest production number | | Structured data (JSON, tracebacks) | 30–64% | worthwhile, lossless where it fires | | Varied prose | ~2% (level 3) / 38% lossy (level 4) | gist mode only | -| Clean code, unique dense text | ~0% by design | that's what `--ml` or truncation is for | +| Clean code, unique dense text, thread dumps | ~0% by design | that's what `--ml` or truncation is for | `--ml RATE` bolts on Microsoft's LLMLingua-2 for perplexity pruning as a last mile (`pip install llmlingua`) — runs *after* the deterministic passes so @@ -271,9 +323,10 @@ Honest placement, so you can pick the right tool: so it needs a cache-alignment component to patch the non-determinism lessismore doesn't have. - **[Drain3](https://github.com/logpai/Drain3)** — industrial log template - mining. `alias_repeats` + `dedupe_similar` are a single-pass, zero-dep - approximation of it; if you need streaming template state across files, - use Drain3. + mining. `alias_templates` is a deterministic, single-pass, zero-dep + Drain-lite (digit-token masking instead of similarity clustering); if you + need streaming template state across files or letter-varying parameters + (usernames, hostnames), use Drain3. - **[LLMLingua-2](https://github.com/microsoft/LLMLingua)** — ML token pruning; wrapped here as the optional `--ml` last mile, *after* the deterministic passes, so you never pay a classifier to delete duplicate @@ -313,3 +366,8 @@ MIT — see [LICENSE](LICENSE). - Similar-line collapse summarizes varying digits per column (`values 404/429/503`, or a min–max range past 4 distinct); if the digit *runs* per line differ in count, it falls back to a plain count marker. +- `alias_templates` reconstructs lines by filling `<*>` left to right with + the per-line values — lossless modulo whitespace runs (a templated line's + original spacing/alignment is not preserved). Parameters that vary in + *letters* (usernames, hostnames) split into separate templates rather than + merging — less compression, never confusion. diff --git a/bench_real.py b/bench_real.py new file mode 100644 index 0000000..09c9488 --- /dev/null +++ b/bench_real.py @@ -0,0 +1,129 @@ +"""bench_real.py — measure lessismore on REAL public log corpora, not synthetic ones. + +Three benchmarks, all free, all with a deterministic metric (no model calls): + + loghub 10 LogHub sample logs (HDFS, BGL, OpenSSH, ...) — token ratios on + real system logs. Downloads ~2.8 MB from GitHub on first run. + https://github.com/logpai/loghub (CC-BY-4.0) + + logchunks 797 Travis CI failure logs with HUMAN-labeled failure-explaining + chunks (MSR 2020). Metric: do the labeled chunk lines survive? + curl -L -o LogChunks.zip \\ + "https://zenodo.org/api/records/3632351/files/LogChunks.zip/content" + unzip LogChunks.zip + python3 bench_real.py logchunks ./LogChunks + + logdx LogDx-CI: 35 real GitHub Actions failures with per-case + required-signal ground truth (arXiv 2605.28876, CC-BY-4.0). + Metric: are the critical diagnostic signals still present? + pip install huggingface_hub + hf download eyuansu71/logdx-ci --repo-type dataset --local-dir ./logdx-ci + python3 bench_real.py logdx ./logdx-ci + +Chunk/signal matching normalizes whitespace and ANSI on both sides (the labels +themselves contain terminal color remnants), nothing else. +""" +import json +import re +import sys +import urllib.request +import xml.etree.ElementTree as ET +from pathlib import Path + +from lessismore import compress, count_tokens + +_CSI = re.compile(r"\x1b?\[[0-9;]*[A-Za-z]") # ANSI with or without the ESC byte + + +def norm(s): + return re.sub(r"\s+", " ", _CSI.sub("", s)).strip() + + +LOGHUB = ["HDFS", "BGL", "OpenStack", "Zookeeper", "Apache", + "OpenSSH", "Spark", "Thunderbird", "Linux", "HealthApp"] + + +def bench_loghub(cache=Path("loghub_samples")): + cache.mkdir(exist_ok=True) + tot_b = tot_a = 0 + print(f"{'dataset':14} {'raw tok':>8} {'small':>8} {'ratio':>6}") + for name in LOGHUB: + f = cache / f"{name}_2k.log" + if not f.exists(): + url = f"https://raw.githubusercontent.com/logpai/loghub/master/{name}/{name}_2k.log" + f.write_bytes(urllib.request.urlopen(url, timeout=60).read()) + text = f.read_text(encoding="utf-8", errors="replace") + b, a = count_tokens(text), count_tokens(compress(text, 2)) + tot_b += b; tot_a += a + print(f"{name:14} {b:>8,} {a:>8,} {b/a:>5.1f}x") + print(f"{'TOTAL':14} {tot_b:>8,} {tot_a:>8,} {tot_b/tot_a:>5.1f}x") + + +def bench_logchunks(root): + root = Path(root) + tot_b = tot_a = n = lv = la = 0 + full_v = 0 + for xf in sorted(root.glob("build-failure-reason/*/*.xml")): + try: + tree = ET.parse(xf) + except ET.ParseError: + continue + for ex in tree.getroot().iter("Example"): + p = root / "logs" / ex.findtext("Log", "").strip() + chunk = ex.findtext("Chunk", "") + if not (p.exists() and chunk.strip()): + continue + out = compress(p.read_text(encoding="utf-8", errors="replace"), 2) + tot_b += count_tokens(p.read_text(encoding="utf-8", errors="replace")) + tot_a += count_tokens(out) + n += 1 + nout = norm(out) + clines = [norm(l) for l in chunk.split("\n") if norm(l)] + v = sum(1 for l in clines if l in nout) + lv += v; la += len(clines); full_v += v == len(clines) + print(f"cases: {n} tokens {tot_b:,} -> {tot_a:,} ({tot_b/tot_a:.1f}x, {1-tot_a/tot_b:.0%} saved)") + print(f"labeled chunk lines retained verbatim: {lv}/{la} ({lv/la:.1%})") + print(f"chunks fully verbatim: {full_v}/{n} ({full_v/n:.1%})") + print("(the rest is mostly similar-line collapse: the message survives once " + "with a value summary — see README)") + + +def bench_logdx(root): + root = Path(root) + gts = sorted(root.glob("cases/**/ground_truth.json")) + tot_b = tot_a = ck = ca = 0 + for gt_path in gts: + raw_p = gt_path.parent / "raw.log" + if not raw_p.exists(): + continue + raw = raw_p.read_text(encoding="utf-8", errors="replace") + out = compress(raw, 2) + tot_b += count_tokens(raw); tot_a += count_tokens(out) + nout = norm(out) + for sig in json.loads(gt_path.read_text()).get("required_signals", []): + if sig.get("importance") != "critical": + continue + texts = [t for t in [sig.get("value"), sig.get("file")] + sig.get("aliases", []) if t] + ca += 1 + ck += any(norm(t) in nout for t in texts) + print(f"cases: {len(gts)} tokens {tot_b:,} -> {tot_a:,} ({tot_b/tot_a:.1f}x, {1-tot_a/tot_b:.0%} saved)") + print(f"critical diagnostic signals retained: {ck}/{ca} ({ck/ca:.1%})") + + +def main(): + if len(sys.argv) < 2 or sys.argv[1] not in ("loghub", "logchunks", "logdx"): + sys.exit(__doc__) + if sys.argv[1] == "loghub": + bench_loghub() + elif sys.argv[1] == "logchunks": + if len(sys.argv) < 3: + sys.exit("usage: bench_real.py logchunks ") + bench_logchunks(sys.argv[2]) + else: + if len(sys.argv) < 3: + sys.exit("usage: bench_real.py logdx ") + bench_logdx(sys.argv[2]) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 974f4ff..a543db2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "lessismore" -version = "0.2.0" +version = "0.3.0" description = "Squeeze text before it hits an LLM — deterministic prompt compression, measured in real tokens" readme = "README.md" license = { text = "MIT" }