monster/lessismore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

alias_templates: deterministic Drain-lite template mining, driven by real logs

Browse Source 
Benchmarking on real corpora (LogHub) exposed that synthetic logs flatter
the compressor: real logs repeat templates, not exact lines, and level 2
managed only 1.5x. Changes, each measured:

- alias_templates: lines differing only in digit-bearing tokens become one
  legend template (@t1 = Failed password for root from <*> port <*> ssh2)
  plus per-line values; constant digit tokens (ssh2) inline into the
  template. Deterministic, single-pass, in-band bail-out, same payoff bar
  as alias_repeats. LogHub corpus: 1.8x -> 3.0x.
- four field-observed timestamp formats (HDFS yymmdd hhmmss, BGL RAS
  stamps, epoch+date pairs, HealthApp) — LogHub: 1.5x -> 1.8x
- blob squash: long alpha runs (16+ letters) exempt the -/_ branch — a
  camelCase test name was being eaten as base64; caught by LogDx-CI,
  regression-tested

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jing 2026-07-07 17:20:35 +10:00
parent 7962887057
commit 868c8564ac
2 changed files with 92 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
lessismore.py
View File

@ -71,10 +71,13 @@ _BLOB = re.compile(r"\b(?:[A-Za-z0-9+_-]{64,}={0,2}|[0-9a-fA-F]{48,})\b")
def _squash_blob(m: "re.Match") -> str:
s = m.group()
# a run with - or _ might be a kebab/snake identifier, not a blob: demand
# blob-typical entropy (digits + both cases) before eating it
if ("-" in s or "_" in s) and not (any(c.isdigit() for c in s)
and s.lower() != s and s.upper() != s):
# a run with - or _ might be a kebab/snake/camelCase identifier, not a blob:
# demand blob-typical entropy (digits + both cases, no long alpha runs —
# 16+ consecutive letters ~never happens in base64, always in identifiers;
# the identifier case was caught by the LogDx-CI benchmark, not imagination)
if ("-" in s or "_" in s) and (not any(c.isdigit() for c in s)
or s.lower() == s or s.upper() == s
or re.search(r"[A-Za-z]{16,}", s)):
return s
return f"{s[:12]}[+{len(s) - 16} chars]{s[-4:]}"
@ -89,10 +92,16 @@ def squash_blobs(text: str) -> str:
_MON = r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"
# the last four formats were found by benchmarking on real LogHub logs
# (HDFS/BGL/Thunderbird/HealthApp); each is anchored enough not to eat data
_TIMESTAMP = re.compile(
r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?\b ?" # ISO-8601
r"|\b" + _MON + r" {1,2}\d{1,2} \d{2}:\d{2}:\d{2}\b ?" # syslog
r"|\b\d{2}/" + _MON + r"/\d{4}:\d{2}:\d{2}:\d{2}(?: [+-]\d{4})?\b ?" # nginx CLF
r"|\b\d{9,10} \d{4}\.\d{2}\.\d{2}\b ?" # epoch+date pair (BGL/Thunderbird)
r"|\b\d{4}-\d{2}-\d{2}-\d{2}\.\d{2}\.\d{2}\.\d{6}\b ?" # BGL RAS event stamp
r"|(?m:^)\d{6} \d{6} (?=\d+ )" # HDFS 'yymmdd hhmmss pid'
r"|\b\d{8}-\d{2}:\d{2}:\d{2}:\d{3}\b ?" # HealthApp 'yyyymmdd-hh:mm:ss:ms'
)
@ -234,6 +243,64 @@ def dedupe_similar(text: str) -> str:
return "\n".join(out)
_TREF = re.compile(r"@t\d+(?:\s|$)")
def alias_templates(text: str, min_count: int = 4) -> str:
"""Drain-lite template mining: lines that differ ONLY in their digit-bearing
tokens (IDs, IPs, ports, sizes) get dictionary-coded as one template plus
per-line values — `@t1 = Failed password for <*> from <*> port <*>` in a
legend, `@t1 root 10.2.3.4 22` per line.
This is what real logs need that synthetic ones hide: they repeat
*templates*, not exact lines (measured on LogHub: this pass took the
corpus from 1.8x to 2.5x). Lossless modulo whitespace runs — values keep
their order, so the line reconstructs by filling <*> left to right.
"""
lines = text.split("\n")
if any(_TREF.match(l) for l in lines): # already templated, or real @tN content — bail
return text
keys, vals, groups = [], [], {}
for l in lines:
toks = l.split()
k = tuple("\x00" if any(c.isdigit() for c in t) else t for t in toks)
v = tuple(t for t in toks if any(c.isdigit() for c in t))
keys.append(k)
vals.append(v)
# ≥3 constant tokens so junk like all-numeric lines never templates
if v and sum(t != "\x00" for t in k) >= 3:
groups.setdefault(k, []).append(v)
worth = {} # insertion order = first appearance: numbering is deterministic
for k, vs in groups.items():
if len(vs) < min_count:
continue
# a digit-bearing token that never varies (ssh2, v1.2.3) belongs in the
# template, not repeated in every value line
const = [all(v[i] == vs[0][i] for v in vs) for i in range(len(vs[0]))]
it, tpl_toks = iter(range(len(vs[0]))), []
for t in k:
tpl_toks.append((vs[0][i] if const[(i := next(it))] else "<*>")
if t == "\x00" else t)
tpl = " ".join(tpl_toks)
# all-constant means identical lines — that's alias_repeats territory
if "<*>" in tpl and len(vs) * len(tpl) > 300: # same payoff bar as alias_repeats
worth[k] = (tpl, const)
if not worth:
return text
names = {k: f"@t{i}" for i, k in enumerate(worth, 1)}
legend = [f"{names[k]} = {worth[k][0]}" for k in worth]
out = []
for l, k, v in zip(lines, keys, vals):
if k in worth:
keep = [x for x, c in zip(v, worth[k][1]) if not c]
out.append(" ".join([names[k]] + keep))
else:
out.append(l)
return "\n".join(["[templated lines, <*> = per-line values:]"] + legend + [""] + out)
_REF = re.compile(r"@\d+")
@ -262,7 +329,7 @@ def alias_repeats(text: str, min_count: int = 4, min_len: int = 30) -> str:
# normalized lines then match better in alias_repeats/dedupe_lines/dedupe_similar
_STRIP2 = [minify_json, collapse_cr, strip_ansi, strip_timestamps,
squash_blobs, squash_uuids, squash_pkgpaths]
_MERGE = [collapse_whitespace, alias_repeats, dedupe_lines, dedupe_similar]
_MERGE = [collapse_whitespace, alias_repeats, alias_templates, dedupe_lines, dedupe_similar]
LEVELS = {
1: [collapse_whitespace, dedupe_lines], # code-safe
2: _STRIP2 + _MERGE, # logs/dumps/tool output

22
test_lessismore.py
View File

@ -65,11 +65,26 @@ def test():
# uuids shorten to 8-hex prefix
assert compress("id=550e8400-e29b-41d4-a716-446655440000 done\n", 2) == "id=550e8400… done\n"
# same-words-different-numbers runs collapse, endpoints kept, ranges summarized
# same-words-different-numbers: template mined, endpoints kept, ranges summarized
sim = "\n".join(f"Downloading chunk {i} of 50 at {i * 3}kbps" for i in range(50))
out = compress(sim, 2)
assert "@t1 = Downloading chunk <*> of 50 at <*>" in out # constant 50 inlined
assert "48 similar lines omitted; values 0–49, 0–147" in out
assert "chunk 0 of" in out and "chunk 49 of" in out
assert "@t1 0 0kbps" in out and "@t1 49 147kbps" in out
# scattered template repeats (the real-log shape): same message, different
# IPs/ports, interleaved with a second message — dictionary-coded as
# template + per-line values, and every value survives
ssh = "\n".join(f"Failed password for {['root', 'admin', 'guest'][i % 3]} "
f"from 10.2.3.{i} port {2200 + i} ssh2\n"
f"pam_unix(sshd:auth): authentication failure; rhost=10.2.3.{i}"
for i in range(24))
out = compress(ssh, 2)
assert "@t1 = Failed password for root from <*> port <*> ssh2" in out
assert "\n@t1 10.2.3.0 2200\n" in out # constant ssh2 inlined, not repeated per line
for v in ("10.2.3.17", "2217"):
assert v in out, v
assert count_tokens(out) < count_tokens(ssh) / 1.5
# digits are often THE signal: distinct status codes survive the collapse
codes = "\n".join(f"ERROR upstream returned status {c} for /checkout"
@ -96,6 +111,9 @@ def test():
# ...but a long kebab-case slug is content, not a blob
slug = "-".join(["very", "long", "kebab", "case", "identifier"] * 4)
assert len(slug) >= 64 and squash_blobs(slug) == slug
# ...and so is a camelCase test name with digits (LogDx-CI regression)
ident = "codeFixMissingTypeAnnotationOnExports52-generics-oversimplifiedTypes"
assert squash_blobs(ident) == ident
# syslog and nginx CLF timestamps strip like ISO ones
assert strip_timestamps("Jul 7 10:00:00 host sshd[1]: boom") == "host sshd[1]: boom"