493c3e912a
- LICENSE (MIT) and pyproject.toml: pip install . gives lessismore + lm commands; [tokens] and [ml] extras - budget(): middle-out hard cap, the only guaranteed-bounded pass; --budget N runs after compression as the backstop; char-slicing fallback for single giant lines - serve(): stdlib paste-in demo page, localhost only; lm --serve - Scrubbed internal hostnames/IPs from samples and README; benchmarks re-measured after scrub (interleaved now 54,999 -> 4,585, 12.0x; dedupe alone 1.6x on same input) - 30-assert suite, green on system python and tiktoken venv Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
43 lines
2.1 KiB
Python
43 lines
2.1 KiB
Python
"""Reproduce the README benchmark numbers. pip install tiktoken for exact counts."""
|
|
import random
|
|
|
|
from lessismore import compress, count_tokens
|
|
|
|
|
|
def bench(name, text, level):
|
|
b, a = count_tokens(text), count_tokens(compress(text, level))
|
|
print(f"{name:22} level {level} {b:>7,} -> {a:>6,} ({b / a:4.1f}x, {1 - a / b:.0%} saved)")
|
|
|
|
|
|
# mixed log: 85% repeated ERROR line, 15% INFO lines differing only in numbers, one JWT
|
|
random.seed(1)
|
|
lines = []
|
|
for i in range(2000):
|
|
ts = f"2026-07-07T10:{i // 60 % 60:02d}:{i % 60:02d}.{random.randint(100, 999)}Z"
|
|
if random.random() < 0.85:
|
|
lines.append(f"{ts} ERROR [pool-3] psycopg2.OperationalError: connection to "
|
|
f"server at 10.0.0.5 failed: Connection refused")
|
|
else:
|
|
lines.append(f"{ts} INFO [worker-{random.randint(1, 4)}] processed batch {i} ok")
|
|
lines.insert(500, "2026-07-07T10:08:20.123Z DEBUG auth token=" + "eyJhbGciOiJIUzI1NiJ9" * 8)
|
|
bench("mixed error log", "\n".join(lines), 2)
|
|
|
|
# interleaved log: three services alternating — zero consecutive runs, dedupe-proof
|
|
random.seed(2)
|
|
msgs = ["ERROR [pool-3] psycopg2.OperationalError: connection to server at "
|
|
"10.0.0.5 failed: Connection refused",
|
|
"WARN [redis-1] retry queue depth exceeded threshold, backing off 500ms",
|
|
"ERROR [worker-2] TimeoutError: upstream api.example.com did not respond within 30s"]
|
|
bench("interleaved log", "\n".join(
|
|
f"2026-07-07T10:{i // 60 % 60:02d}:{i % 60:02d}Z " + msgs[i % 3] for i in range(1500)), 2)
|
|
|
|
# chatty prose: levels 3 vs 4 show where filler stripping ends and caveman starts
|
|
prose = ("So basically what happened was that the team had been trying to get the "
|
|
"deployment pipeline working for about three weeks, and it turned out that the "
|
|
"main problem was that the configuration file was pointing to the wrong database "
|
|
"server. I think we should also mention that the staging environment has been "
|
|
"flaky and there are a couple of monitoring alerts that have not been "
|
|
"investigated yet. ") * 6
|
|
bench("chatty prose", prose, 3)
|
|
bench("chatty prose", prose, 4)
|