493c3e912a
- LICENSE (MIT) and pyproject.toml: pip install . gives lessismore + lm commands; [tokens] and [ml] extras - budget(): middle-out hard cap, the only guaranteed-bounded pass; --budget N runs after compression as the backstop; char-slicing fallback for single giant lines - serve(): stdlib paste-in demo page, localhost only; lm --serve - Scrubbed internal hostnames/IPs from samples and README; benchmarks re-measured after scrub (interleaved now 54,999 -> 4,585, 12.0x; dedupe alone 1.6x on same input) - 30-assert suite, green on system python and tiktoken venv Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
101 lines
4.2 KiB
Python
101 lines
4.2 KiB
Python
import subprocess
|
|
import sys
|
|
|
|
from lessismore import (budget, compress, count_tokens, dedupe_lines,
|
|
squash_blobs, strip_filler, strip_timestamps, two_sticks)
|
|
|
|
|
|
def test():
|
|
# dedupe: long-line runs collapse, short runs where the marker wouldn't pay stay
|
|
line = "ERROR connection refused by upstream database"
|
|
assert dedupe_lines((line + "\n") * 5 + "b") == \
|
|
line + "\n[previous line repeated 4 more times]\nb"
|
|
assert dedupe_lines("a\n" * 5 + "b") == "a\n" * 5 + "b"
|
|
assert dedupe_lines(line + "\n" + line + "\nb") == line + "\n" + line + "\nb"
|
|
|
|
# blobs squash with head+tail stub; URLs are not blobs
|
|
assert "[+84 chars]" in squash_blobs("token=" + "A" * 100)
|
|
url = ("GET https://api.example.com/api/v1/users/12345/orders/67890/"
|
|
"items/status/pending/verbose HTTP/1.1")
|
|
assert squash_blobs(url) == url
|
|
|
|
# distinct blobs keep distinct stubs — no false "repeated" merge at level 2
|
|
blobs = "\n".join("stored blob deadbeefcafe" + f"{i:052x}" for i in range(4))
|
|
assert "repeated" not in compress(blobs, 2)
|
|
|
|
# timestamps
|
|
assert strip_timestamps("2026-07-07T10:00:00.123Z boom") == "boom"
|
|
|
|
# filler stripping never inverts negation
|
|
assert strip_filler("This is not just a warning.") == "This is not just a warning."
|
|
assert strip_filler("just do it") == "do it"
|
|
|
|
# level 1 is code-safe: indentation and content untouched
|
|
code = "def f(x):\n return x + 1\n"
|
|
assert compress(code, 1) == code
|
|
|
|
# two_sticks drops function words, never negations, and spares acronyms
|
|
assert two_sticks("The server is down") == "server down"
|
|
assert "not" in two_sticks("do not delete the backup")
|
|
assert two_sticks("The IT team said it is a US issue") == "IT team said US issue"
|
|
|
|
# interleaved repeats defeat consecutive dedupe but get dictionary-aliased
|
|
inter = ("db connection refused on host alpha-primary\n"
|
|
"redis queue depth exceeded on host beta-cache\n") * 30
|
|
out = compress(inter, 2)
|
|
assert "[repeated lines aliased:]" in out and "@1" in out
|
|
assert count_tokens(out) < count_tokens(inter) / 2
|
|
|
|
# a repetitive log shrinks hard, and stripping leaves no doubled spaces
|
|
log = "2026-07-07T10:00:01Z ERROR conn refused db=orders retry\n" * 500
|
|
out = compress(log, 2)
|
|
assert count_tokens(out) < count_tokens(log) / 50
|
|
assert " " not in out
|
|
|
|
# \r progress redraws keep only the final state
|
|
assert compress("downloading 1%\rdownloading 99%\rdone\n", 2) == "done\n"
|
|
|
|
# ANSI escapes stripped
|
|
assert compress("\x1b[32mok\x1b[0m\n", 2) == "ok\n"
|
|
|
|
# venv path spam squashed, file:line still unique
|
|
tb = 'File "/Users/j/venv/lib/python3.14/site-packages/httpx/_client.py", line 5\n'
|
|
assert compress(tb, 2) == 'File "…/httpx/_client.py", line 5\n'
|
|
|
|
# uuids shorten to 8-hex prefix
|
|
assert compress("id=550e8400-e29b-41d4-a716-446655440000 done\n", 2) == "id=550e8400… done\n"
|
|
|
|
# same-words-different-numbers runs collapse, endpoints kept
|
|
sim = "\n".join(f"Downloading chunk {i} of 50 at {i * 3}kbps" for i in range(50))
|
|
out = compress(sim, 2)
|
|
assert "[48 similar lines omitted]" in out
|
|
assert "chunk 0 of" in out and "chunk 49 of" in out
|
|
|
|
# whole-input pretty JSON minifies losslessly
|
|
import json
|
|
pretty = json.dumps({"items": [{"id": i, "name": f"x{i}"} for i in range(10)]}, indent=2)
|
|
out = compress(pretty, 2)
|
|
assert json.loads(out) == json.loads(pretty)
|
|
assert len(out) < len(pretty) / 1.5
|
|
|
|
# budget: guaranteed cap, head and tail kept, middle dropped
|
|
text = "\n".join(f"line {i} of the log with some words" for i in range(200))
|
|
out = budget(text, 100)
|
|
assert count_tokens(out) <= 110
|
|
assert "line 0 " in out and "line 199" in out and "tokens omitted]" in out
|
|
assert budget("tiny", 100) == "tiny"
|
|
giant = ("word " * 3000).strip() # single huge line falls back to char slicing
|
|
assert count_tokens(budget(giant, 50)) <= 80
|
|
|
|
# CLI survives empty and non-UTF-8 stdin
|
|
for stdin in (b"", b"ok\n\xff\xfe\n"):
|
|
r = subprocess.run([sys.executable, "lessismore.py"],
|
|
input=stdin, capture_output=True)
|
|
assert r.returncode == 0, r.stderr
|
|
|
|
print("ok")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
test()
|