From 0d4c5464b90c6c927e7110d1ae5861de9231cf26 Mon Sep 17 00:00:00 2001 From: jing Date: Tue, 7 Jul 2026 18:27:39 +1000 Subject: [PATCH] Two-tier template mining + faithful collapse + equal-budget receipt; 0.4.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit alias_templates is now two tiers with exact char accounting: - tier A masks hex/digit runs INSIDE tokens (run-level constants — dates, IP prefixes, the 2 in ssh2 — inline into the template); groups are re-costed under whole-token value encoding and take the cheaper - tier B masks whole digit-bearing tokens on what tier A left, so paths/ids varying in letters still group (compile/build logs) - dedupe_similar skeleton is hex-aware but keeps a leading @tN verbatim: refs from different templates can no longer merge as "similar" — an earlier build scored 8.9x on LogHub from exactly that silent value destruction, and v0.3.0's HDFS number had the same phantom; both gone Measured (bench_real.py): LogHub 3.0x -> 3.9x, LogDx-CI 2.2x -> 2.3x at unchanged 95.2% critical-signal retention, LogChunks 1.9x -> 2.1x. Equal-budget receipt added to bench_real logdx: at a 2k cap, compress-then-truncate retains 46.6% of critical signals vs 24.7% for plain truncation. Synthetic eval re-run on shipped code: 8/8 at 6.4x. Graveyard: Drain-style letter-merging measured flat (3.0x -> 3.0x, Linux regressed) and documented dead. Co-Authored-By: Claude Fable 5 --- README.md | 104 +++++++++++++++++---------- bench_real.py | 21 ++++-- lessismore.py | 171 ++++++++++++++++++++++++++++++++------------- pyproject.toml | 2 +- test_lessismore.py | 14 ++-- 5 files changed, 215 insertions(+), 97 deletions(-) diff --git a/README.md b/README.md index 9c61bc4..2f331f9 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,12 @@ getting. tail -5000 app.log | python3 lessismore.py | llm "why did this crash?" ``` -**11x** on mixed error logs · **16x** on ANSI CI logs · **97%** on captured -pip/tqdm output · **3.0x** on real LogHub system logs · **95% critical-signal -retention** on the LogDx-CI benchmark's 35 real GitHub Actions failures · -**~6–12 MB/s** single-core · **0** dependencies +**13x** on mixed error logs · **16x** on ANSI CI logs · **98%** on captured +pip/tqdm output · **3.9x** on real LogHub system logs · **95% critical-signal +retention** on the LogDx-CI benchmark's 35 real GitHub Actions failures — and +at an equal 2k-token budget, compress-then-truncate keeps **1.9x more** +diagnostic signals than plain truncation · **~6–12 MB/s** single-core · +**0** dependencies And the part a compression ratio can't prove: in the root-cause eval ([eval.py](eval.py)), the model found the planted root cause in **8/8** @@ -111,13 +113,17 @@ The crown jewels at level 2 are the two dictionary passes: interleaved benchmark: 54,999 → 34,999 with dedupe alone (1.6x) vs 54,999 → 4,585 with aliasing (**12x**). - `alias_templates` — real logs rarely repeat *exact* lines; they repeat - templates: `Failed password for root from 10.2.3.4 port 2201`. Lines that - differ only in their digit-bearing tokens get one legend entry - (`@t1 = Failed password for root from <*> port <*> ssh2`) and per-line - values (`@t1 10.2.3.4 2201`). Digit-bearing tokens that never vary (ssh2, - v1.2.3) are inlined into the template. This is a deterministic, single-pass - Drain-lite, and it's what benchmarking on real logs demanded: it took the - LogHub corpus from 1.8x to 3.0x. + templates: `Failed password for root from 10.2.3.4 port 2201`. Two tiers of + deterministic, single-pass Drain-lite mining. Tier A masks hex/digit runs + *inside* tokens, so run-level constants inline into the template + (`@t1 = Failed password for root from 10.2.3.<*> port <*> ssh2`, per-line + refs `@t1 44 2201`). Tier B masks whole digit-bearing tokens on whatever + tier A left behind, so paths and ids that vary in letters still group — + which is what crushes compile/build logs. Each group is costed with exact + character accounting under both value encodings and takes the cheaper; a + template only fires when it provably pays. Every tier exists because a + public benchmark regressed without it: this pass took the LogHub corpus + from 1.8x to 3.9x. `--budget N` is the backstop, not the compressor: after the passes run, it keeps head and tail lines and drops the middle with a `[~N tokens omitted]` @@ -132,10 +138,10 @@ this table claims). | Sample | Tokens | Why it wins | |---|---|---| -| Mixed error log, 2000 lines | 86,914 → 7,619 (**11.4x**) | timestamp strip unmasks identical lines → dedupe; templates catch the numbered stragglers | +| Mixed error log, 2000 lines | 86,914 → 6,730 (**12.9x**) | timestamp strip unmasks identical lines → dedupe; templates catch the numbered stragglers | | Interleaved 3-service log, zero consecutive repeats | 54,999 → 4,585 (**12.0x**) | `alias_repeats` — consecutive dedupe alone managed 1.6x on this input | -| ANSI-colored CI log | 10,440 → 657 (**15.9x**) | color codes make identical lines look different; strip them and the log collapses | -| Captured pip/tqdm output | 10,465 → 265 (**97%**) | every overwritten `\r` progress frame is invisible on screen but real tokens in a capture | +| ANSI-colored CI log | 10,440 → 668 (**15.6x**) | color codes make identical lines look different; strip them and the log collapses | +| Captured pip/tqdm output | 10,465 → 216 (**98%**) | every overwritten `\r` progress frame is invisible on screen but real tokens in a capture | | pytest failure dump | 732 → 515 (30%) | one site-packages path prefix = 25 tokens → 7 | | Pretty-printed JSON API response | 8,932 → 3,234 (**64%**) | minify (round-trip verified lossless) + UUID→8-hex squash | | Chatty prose, level 4 | 427 → 264 (38%) | every dropped function word is a whole token; level 3 got 2% on the same text | @@ -150,37 +156,53 @@ expect. So [bench_real.py](bench_real.py) runs three **public** corpora with deterministic metrics (no model in the loop, no cherry-picking): **[LogHub](https://github.com/logpai/loghub) samples** — 10 real system logs -(HDFS, BGL, OpenSSH, OpenStack...), 1.15M tokens: **3.0x overall**, ranging -from 1.5x (HealthApp's pipe-format) to 6.5x (Apache). Fair warning made -explicit: real diverse logs compress 2–4x, not the 11x of repetitive error -walls. It was this benchmark that forced `alias_templates` and four new -timestamp formats into existence — the corpus started at 1.5x. +(HDFS, BGL, OpenSSH, OpenStack...), 1.15M tokens: **3.9x overall**, ranging +from 2.0x (HDFS's unique block ids) to 9.8x (OpenStack). Fair warning made +explicit: real diverse logs compress 2–10x by structure, not the flat 13x of +repetitive error walls. It was this benchmark that forced `alias_templates` +and four new timestamp formats into existence — the corpus started at 1.5x. **[LogChunks](https://zenodo.org/records/3632351)** — 797 real Travis CI failure logs where *humans* marked the chunk that explains each failure (externally validated with the developers who caused them). 66.2M tokens → -34.2M (**1.9x**), and **78.5% of labeled failure-explaining lines survive -byte-for-byte**; 73% of chunks survive fully intact. The rest is dominated by -similar-line collapse where the message survives once with a value summary — -represented, not deleted. +31.6M (**2.1x**), and **76.9% of labeled failure-explaining lines survive +byte-for-byte**; 67.5% of chunks survive fully intact. The rest is dominated +by similar-line collapse and templating, where the message survives once with +per-line values — represented, not deleted. **[LogDx-CI](https://arxiv.org/abs/2605.28876)** — the benchmark built for exactly this question: 35 real GitHub Actions failures with per-case -ground-truth diagnostic signals. All 35 cases, 15.0M tokens → 6.9M -(**2.2x**), with **95.2% of critical diagnostic signals retained** (139/146; +ground-truth diagnostic signals. All 35 cases, 15.0M tokens → 6.6M +(**2.3x**), with **95.2% of critical diagnostic signals retained** (139/146; of the 7 misses, most are layout artifacts where templating separates a value -from its line). With `--budget 8000` stacked on top: **6,334 tokens/case at -77.4% critical-signal retention** — the same token cost as the benchmark's -tail-200 baseline, with the whole log's worth of scattered signals instead of -just the tail. And the answer-quality check: on 12 cases, claude-haiku-4.5 +from its line). And the answer-quality check: on 12 cases, claude-haiku-4.5 diagnosed **12/12 from raw and 12/12 from compressed** — parity at ~2.4x fewer tokens. (Ground-truth caveat inherited from the benchmark: its labels are AI-drafted with single-author verification, n=35.) +**The equal-budget receipt** — the question every harness actually faces is +not "compress or not" but "we cap tool output anyway; is compressing before +the cap worth it?" Measured on all 35 LogDx-CI cases (critical-signal +retention at the same token budget): + +| Budget | truncate only | compress, then truncate | +|---|---|---| +| 2,000 tokens | 24.7% | **46.6%** | +| 8,000 tokens | 65.8% | **78.8%** | +| 32,000 tokens | 80.8% | **87.7%** | + One case in this run earned its keep twice: the benchmark caught a real lessismore bug (a camelCase test name eaten as a base64 blob), which is now a regression test. That's what public benchmarks are for. +A faithfulness note that cost real ratio: an earlier build scored "8.9x" on +LogHub because refs from *different* templates were collapsing into one +"similar lines" blob, silently dropping the middle lines' values. That +over-merge is now prevented (a ref's template name is part of its identity), +and the honest number is 3.9x. The same audit exposed that v0.3.0's HDFS +"win" was powered by the same phantom — this codebase prefers the smaller +true number. + ## Does the model still get the answer? A compression ratio proves the log got smaller, not that the signal @@ -197,15 +219,15 @@ default runner is a logged-in `claude -p`. Rerun it yourself): | Scenario | Raw tokens | Compressed | Raw | Compressed | |---|---|---|---|---| -| OOM kill buried mid-log, connection-error red herring | 38,777 | 174 (**223x**) | ✗ blamed the red herring | ✓ found the OOM | -| 429 rate-limit burst hidden in a wall of status codes | 46,899 | 3,713 | ✗ "service overwhelmed" | ✓ named the 429s | -| Disk-full scattered through 3 interleaved services | 38,616 | 8,103 | ✓ | ✓ | -| One failed assertion in 1,400 ANSI test lines | 46,812 | 21,563 | ✓ | ✓ | +| OOM kill buried mid-log, connection-error red herring | 38,777 | 169 (**229x**) | ✗ blamed the red herring | ✓ found the OOM | +| 429 rate-limit burst hidden in a wall of status codes | 46,899 | 3,497 | ✗ "service overwhelmed" | ✓ named the 429s | +| Disk-full scattered through 3 interleaved services | 38,616 | 8,099 | ✓ | ✓ | +| One failed assertion in 1,400 ANSI test lines | 46,812 | 11,408 | ✓ | ✓ | | Dependency conflict under progress-bar walls | 16,304 | 355 | ✓ | ✓ | -| Expired TLS cert in an nginx access-log wall | 56,696 | 189 (**300x**) | ✓ | ✓ | +| Expired TLS cert in an nginx access-log wall | 56,696 | 11,855 | ✓ | ✓ | | Missing env var in a crash-loop traceback | 7,680 | 1,610 | ✓ | ✓ | -| Java deadlock in a 40-thread dump | 8,157 | 7,907 (1.0x) | ✓ | ✓ | -| **Total** | **259,941** | **43,614 (6.0x)** | **6/8** | **8/8** | +| Java deadlock in a 40-thread dump | 8,157 | 3,742 (2.2x) | ✓ | ✓ | +| **Total** | **259,941** | **40,735 (6.4x)** | **6/8** | **8/8** | The two raw failures are the lost-in-the-middle effect this tool exists to counter: the model latched onto the loud symptom cascade and never surfaced @@ -277,6 +299,14 @@ against a real tokenizer: digits), **separator shortening** (a 78-char `----` is already 1 token), **prefix hoisting** (one stray line kills it), **JSON→TSV / float truncation / pointer squash** (real savings, worse trade). +- **Drain-style letter-varying template merging** (clustering templates that + differ in ≤N word positions, so usernames/hostnames join one template) — + measured on LogHub: 3.0x → 3.0x, with Linux *regressing* 6.4x → 5.3x. The + extra per-line wildcard values eat exactly what the merged legends save. +- **Cross-template ref collapse** — letting `dedupe_similar` merge refs from + different templates scored 8.9x on LogHub, but the "compression" was + silently deleting the middle lines' values. Removed on faithfulness + grounds; the ratio was a lie. ## Battle-tested @@ -299,7 +329,7 @@ and machine noise; it cannot compress information, and doesn't pretend to. | Content | Expect | Verdict | |---|---|---| | Repetitive machine output | 5–16x, up to 300x on pathological repeats | the reason this exists | -| Real-world CI / system logs | 2–4x (measured on LogHub, LogChunks, LogDx-CI) | the honest production number | +| Real-world CI / system logs | 2–4x typical, up to 10x on template-heavy logs (measured on LogHub, LogChunks, LogDx-CI) | the honest production number | | Structured data (JSON, tracebacks) | 30–64% | worthwhile, lossless where it fires | | Varied prose | ~2% (level 3) / 38% lossy (level 4) | gist mode only | | Clean code, unique dense text, thread dumps | ~0% by design | that's what `--ml` or truncation is for | diff --git a/bench_real.py b/bench_real.py index 09c9488..2c90696 100644 --- a/bench_real.py +++ b/bench_real.py @@ -30,7 +30,7 @@ import urllib.request import xml.etree.ElementTree as ET from pathlib import Path -from lessismore import compress, count_tokens +from lessismore import budget, compress, count_tokens _CSI = re.compile(r"\x1b?\[[0-9;]*[A-Za-z]") # ANSI with or without the ESC byte @@ -90,8 +90,11 @@ def bench_logchunks(root): def bench_logdx(root): root = Path(root) - gts = sorted(root.glob("cases/**/ground_truth.json")) + gts = sorted(set(root.glob("cases/**/ground_truth.json"))) + caps = (2000, 8000, 32000) tot_b = tot_a = ck = ca = 0 + cap_raw = {c: 0 for c in caps} # truncate-only retention + cap_cmp = {c: 0 for c in caps} # compress-then-truncate retention for gt_path in gts: raw_p = gt_path.parent / "raw.log" if not raw_p.exists(): @@ -100,14 +103,24 @@ def bench_logdx(root): out = compress(raw, 2) tot_b += count_tokens(raw); tot_a += count_tokens(out) nout = norm(out) + ntr = {c: norm(budget(raw, c)) for c in caps} + ncm = {c: norm(budget(out, c)) for c in caps} for sig in json.loads(gt_path.read_text()).get("required_signals", []): if sig.get("importance") != "critical": continue - texts = [t for t in [sig.get("value"), sig.get("file")] + sig.get("aliases", []) if t] + texts = [norm(t) for t in [sig.get("value"), sig.get("file")] + + sig.get("aliases", []) if t] ca += 1 - ck += any(norm(t) in nout for t in texts) + ck += any(t in nout for t in texts) + for c in caps: + cap_raw[c] += any(t in ntr[c] for t in texts) + cap_cmp[c] += any(t in ncm[c] for t in texts) print(f"cases: {len(gts)} tokens {tot_b:,} -> {tot_a:,} ({tot_b/tot_a:.1f}x, {1-tot_a/tot_b:.0%} saved)") print(f"critical diagnostic signals retained: {ck}/{ca} ({ck/ca:.1%})") + print("equal-budget (the question a harness actually faces):") + for c in caps: + print(f" {c:>6,} tokens: truncate-only {cap_raw[c]/ca:.1%} " + f"compress-then-truncate {cap_cmp[c]/ca:.1%}") def main(): diff --git a/lessismore.py b/lessismore.py index 5ad47d2..cc8fa4d 100644 --- a/lessismore.py +++ b/lessismore.py @@ -222,16 +222,25 @@ def _variant_summary(lines: list) -> str: return "; values " + ", ".join(parts) if parts else "" +def _skel(l: str) -> str: + # hex runs masked first so goroutine addresses / template-ref hex values + # don't make otherwise-identical lines look distinct; a leading @tN ref + # name stays verbatim so different templates never merge as "similar" + m = _TREF.match(l) + head, tail = (l[:m.end()].strip() + " ", l[m.end():]) if m else ("", l) + return (head + _SKEL.sub(" ", _HEXRUN.sub(" ", tail)).strip()).strip() + + def dedupe_similar(text: str) -> str: - """Collapse runs of lines identical after masking non-letters — 'same words, - different numbers' (progress lines, per-item CI steps). Keeps first and last - so progression endpoints survive, and summarizes the varying values in the - marker so distinct facts (status codes, ports) aren't lost.""" + """Collapse runs of lines identical after masking hex runs and non-letters — + 'same words, different numbers' (progress lines, per-item CI steps). Keeps + first and last so progression endpoints survive, and summarizes the varying + values in the marker so distinct facts (status codes, ports) aren't lost.""" lines = text.split("\n") out, i = [], 0 while i < len(lines): - k, j = _SKEL.sub(" ", lines[i]).strip(), i - while j < len(lines) and _SKEL.sub(" ", lines[j]).strip() == k: + k, j = _skel(lines[i]), i + while j < len(lines) and _skel(lines[j]) == k: j += 1 if j - i >= 4 and k: out += [lines[i], @@ -244,60 +253,124 @@ def dedupe_similar(text: str) -> str: _TREF = re.compile(r"@t\d+(?:\s|$)") +# a maximal hex-char run containing at least one digit: catches decimal runs, +# hex ids, and addresses, but never plain words ("added" is hex chars, no digit) +_HEXRUN = re.compile(r"[0-9a-fA-F]*[0-9][0-9a-fA-F]*") +_W = "\x00" + + +def _mine(lines, live, key_fn, val_fn, min_count): + """One template-mining tier: group `live` line indexes by key_fn, keep + groups whose EXACT char accounting (original lines vs refs+legend) pays.""" + groups = {} + for idx in live: + toks = lines[idx].split() + k, v = key_fn(toks), val_fn(toks) + if v: + g = groups.setdefault(k, [[], [], 0]) + g[0].append(idx) + g[1].append(v) + g[2] += len(lines[idx]) + worth = {} # insertion order = first appearance: numbering is deterministic + for k, (idxs, vs, orig_chars) in groups.items(): + if len(vs) < min_count or len({len(v) for v in vs}) != 1: + continue + if sum(1 for t in k for c in t if c != _W) < 12: # junk guard + continue + # a value column that never varies (ssh2, a constant IP prefix) + # belongs in the template, not repeated in every ref line + const = [all(v[i] == vs[0][i] for v in vs) for i in range(len(vs[0]))] + it, tpl_toks = iter(range(len(vs[0]))), [] + for t in k: + if _W not in t: + tpl_toks.append(t) + continue + parts = t.split(_W) + filled = parts[0] + for p in parts[1:]: + i = next(it) + filled += (vs[0][i] if const[i] else "<*>") + p + tpl_toks.append(filled) + tpl = " ".join(tpl_toks) + if "<*>" not in tpl: # all-constant = identical lines = alias_repeats territory + continue + ref_chars = sum(5 + sum(len(x) + 1 for x, c in zip(v, const) if not c) + for v in vs) + if orig_chars - ref_chars - (len(tpl) + 8) > 100: # exact payoff, not a guess + worth[k] = (tpl, const, idxs) + return worth + + +def _cheaper_whole(lines, tpl_c_idxs, key): + """A mostly-varying token is cheaper carried whole (`10.251.91.84:52063`) + than as separate runs (`10 251 91 84 52063`). Re-cost the group with + whole-token values and switch encodings if that wins.""" + tpl, const, idxs = tpl_c_idxs + wild_tok = [_W in t for t in key] + wvs = [tuple(t for t, w in zip(lines[i].split(), wild_tok) if w) for i in idxs] + wconst = [all(v[i] == wvs[0][i] for v in wvs) for i in range(len(wvs[0]))] + it = iter(range(len(wvs[0]))) + wtpl = " ".join((wvs[0][i] if wconst[(i := next(it))] else "<*>") if w else t + for t, w in zip(key, wild_tok)) + sub_cost = sum(sum(len(x) + 1 for x, c in zip(_subtok_vals(lines[i].split()), const) + if not c) for i in idxs) + len(tpl) + whole_cost = sum(sum(len(x) + 1 for x, c in zip(v, wconst) if not c) + for v in wvs) + len(wtpl) + if whole_cost < sub_cost and "<*>" in wtpl: + return (wtpl, wconst, idxs), "b" + return tpl_c_idxs, "a" + + +def _subtok_vals(toks): + return tuple(m for t in toks for m in _HEXRUN.findall(t)) def alias_templates(text: str, min_count: int = 4) -> str: - """Drain-lite template mining: lines that differ ONLY in their digit-bearing - tokens (IDs, IPs, ports, sizes) get dictionary-coded as one template plus - per-line values — `@t1 = Failed password for <*> from <*> port <*>` in a - legend, `@t1 root 10.2.3.4 22` per line. + """Two-tier Drain-lite template mining. Lossless modulo whitespace runs — + values keep their order, so a line reconstructs by filling <*> left to right. - This is what real logs need that synthetic ones hide: they repeat - *templates*, not exact lines (measured on LogHub: this pass took the - corpus from 1.8x to 2.5x). Lossless modulo whitespace runs — values keep - their order, so the line reconstructs by filling <*> left to right. + Tier A masks hex/digit runs INSIDE tokens: `worker-3` and `10.2.3.44` + group as `worker-<*>` and `10.2.3.<*>`, and run-level constants (dates, + IP prefixes, the 2 in ssh2) inline into the template. + Tier B masks WHOLE digit-bearing tokens on whatever tier A left behind: + paths and ids that vary in letters (`.../hashtable.o` vs `.../sampler.o`) + still group, which is what crushes compile/build logs. + + Both tiers were forced by public benchmarks (LogHub, LogDx-CI), not + imagination — each exists because a real corpus regressed without it. """ lines = text.split("\n") if any(_TREF.match(l) for l in lines): # already templated, or real @tN content — bail return text - - keys, vals, groups = [], [], {} - for l in lines: - toks = l.split() - k = tuple("\x00" if any(c.isdigit() for c in t) else t for t in toks) - v = tuple(t for t in toks if any(c.isdigit() for c in t)) - keys.append(k) - vals.append(v) - # ≥3 constant tokens so junk like all-numeric lines never templates - if v and sum(t != "\x00" for t in k) >= 3: - groups.setdefault(k, []).append(v) - - worth = {} # insertion order = first appearance: numbering is deterministic - for k, vs in groups.items(): - if len(vs) < min_count: - continue - # a digit-bearing token that never varies (ssh2, v1.2.3) belongs in the - # template, not repeated in every value line - const = [all(v[i] == vs[0][i] for v in vs) for i in range(len(vs[0]))] - it, tpl_toks = iter(range(len(vs[0]))), [] - for t in k: - tpl_toks.append((vs[0][i] if const[(i := next(it))] else "<*>") - if t == "\x00" else t) - tpl = " ".join(tpl_toks) - # all-constant means identical lines — that's alias_repeats territory - if "<*>" in tpl and len(vs) * len(tpl) > 300: # same payoff bar as alias_repeats - worth[k] = (tpl, const) - if not worth: + live = range(len(lines)) + worth_a = _mine(lines, live, lambda toks: tuple(_HEXRUN.sub(_W, t) for t in toks), + _subtok_vals, min_count) + taken = {i for _, _, idxs in worth_a.values() for i in idxs} + worth_b = _mine(lines, [i for i in live if i not in taken], + lambda toks: tuple(_W if any(c.isdigit() for c in t) else t + for t in toks), + lambda toks: tuple(t for t in toks if any(c.isdigit() for c in t)), + min_count) + if not (worth_a or worth_b): return text - names = {k: f"@t{i}" for i, k in enumerate(worth, 1)} - legend = [f"{names[k]} = {worth[k][0]}" for k in worth] + entries = [_cheaper_whole(lines, w, k) for k, w in worth_a.items()] + entries += [(w, "b") for w in worth_b.values()] + legend, ref_of = [], {} + for n, ((tpl, const, idxs), tier) in enumerate(entries, 1): + legend.append(f"@t{n} = {tpl}") + for i in idxs: + ref_of[i] = (f"@t{n}", const, tier) out = [] - for l, k, v in zip(lines, keys, vals): - if k in worth: - keep = [x for x, c in zip(v, worth[k][1]) if not c] - out.append(" ".join([names[k]] + keep)) - else: + for i, l in enumerate(lines): + r = ref_of.get(i) + if not r: out.append(l) + continue + name, const, tier = r + toks = l.split() + v = _subtok_vals(toks) if tier == "a" else \ + tuple(t for t in toks if any(c.isdigit() for c in t)) + out.append(" ".join([name] + [x for x, c in zip(v, const) if not c])) return "\n".join(["[templated lines, <*> = per-line values:]"] + legend + [""] + out) diff --git a/pyproject.toml b/pyproject.toml index a543db2..117fdb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "lessismore" -version = "0.3.0" +version = "0.4.0" description = "Squeeze text before it hits an LLM — deterministic prompt compression, measured in real tokens" readme = "README.md" license = { text = "MIT" } diff --git a/test_lessismore.py b/test_lessismore.py index aa153ee..c3b2929 100644 --- a/test_lessismore.py +++ b/test_lessismore.py @@ -68,21 +68,23 @@ def test(): # same-words-different-numbers: template mined, endpoints kept, ranges summarized sim = "\n".join(f"Downloading chunk {i} of 50 at {i * 3}kbps" for i in range(50)) out = compress(sim, 2) - assert "@t1 = Downloading chunk <*> of 50 at <*>" in out # constant 50 inlined + # "of 50", "kbps", and "ssh2"-style run-level constants inline into the template + assert "@t1 = Downloading chunk <*> of 50 at <*>kbps" in out assert "48 similar lines omitted; values 0–49, 0–147" in out - assert "@t1 0 0kbps" in out and "@t1 49 147kbps" in out + assert "@t1 0 0" in out and "@t1 49 147" in out # scattered template repeats (the real-log shape): same message, different # IPs/ports, interleaved with a second message — dictionary-coded as - # template + per-line values, and every value survives + # template + per-line values, and every varying value survives ssh = "\n".join(f"Failed password for {['root', 'admin', 'guest'][i % 3]} " f"from 10.2.3.{i} port {2200 + i} ssh2\n" f"pam_unix(sshd:auth): authentication failure; rhost=10.2.3.{i}" for i in range(24)) out = compress(ssh, 2) - assert "@t1 = Failed password for root from <*> port <*> ssh2" in out - assert "\n@t1 10.2.3.0 2200\n" in out # constant ssh2 inlined, not repeated per line - for v in ("10.2.3.17", "2217"): + # constant IP prefix and ssh2 inline; interleaved refs never merge as "similar" + assert "@t1 = Failed password for root from 10.2.3.<*> port <*> ssh2" in out + assert "\n@t1 0 2200\n" in out + for v in ("@t4 17 2217", "@t2 23"): assert v in out, v assert count_tokens(out) < count_tokens(ssh) / 1.5