diff --git a/.gitignore b/.gitignore index e285bb9..7427ab7 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,9 @@ bin/brush-app-aarch64-apple-darwin/ # GPU worker-pool config — primary only (a leaf worker must NOT have this) nodes.json + +# perfcheck runtime state — per-runner, not shared. The authoritative copy lives on the +# m4mini watchdog; baselines.json is derived from history.jsonl and regenerated each run. +scripts/perfcheck/history.jsonl +scripts/perfcheck/baselines.json +scripts/perfcheck/pending.json diff --git a/BENCHMARKS.md b/BENCHMARKS.md index b2ef3a2..14fbebb 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -114,7 +114,36 @@ Raised the config from the laptop-tuned defaults to `octree_resolution 384` + `r 2. Either `huggingface-cli login` on the machine, or paste an HF token into Settings → "HuggingFace token" (injected as `HF_TOKEN` for the operators). +## Fleet hardware baseline (perfcheck, torch 2.13.0/MPS, 2026-07-17) + +What each node's GPU is *capable* of, independent of any model — the denominator for every +number above, and the reference for "is this machine still healthy". Measured nightly by +`scripts/perfcheck/`; see **[PERFCHECK.md](PERFCHECK.md)** for the full story. + +| node | chip | RAM | matmul fp16 | bandwidth | sdpa 56/64 | +|---|---|---|---|---|---| +| m3ultra | M3 Ultra | 256GB | 25202 | 627 GB/s | 4.35x | +| m1 | M1 Ultra | 128GB | ~17100 | 639 GB/s | 3.89x | +| studio | M1 Max | 32GB | 6996 | 348 GB/s | 3.09x | +| m4 | M4 Pro | 24GB | 5678 | 241 GB/s | 1.96x | +| m4mini | M4 | 16GB | 3774 | 105 GB/s | 5.10x | +| mini | M1 (2020) | 8GB | 2310 | 61 GB/s | 5.26x | + +GFLOP/s. The M3 Ultra is **~2.2x** the M1 Ultra on fp16 matmul but they share the same ~625 +GB/s memory bandwidth — so bandwidth-bound stages (VAE decode, big texture bakes) run at +similar speed on both, while compute-bound ones (denoise) scale with the newer silicon. Both +Ultras reach only ~78% of their ~800 GB/s spec on a single kernel; the smaller Macs hit ~88%. + +`sdpa 56/64` is the fused-attention cliff at CorridorKey's awkward `head_dim=56` — it's why +padding to 64 wins (`CORRIDORKEY.md`), reproduced independently on all 6 nodes, and it is +watched nightly so we learn if a future torch ever closes it. + ## Method Timings are wall-clock from the job runner (`started_at`→`finished_at`), single job at a time (gpu lane = 1). Re-run `tests/smoke.sh` for the framework regression suite (12 checks, ~30s). + +**Benchmarking on this fleet is contended.** These are working machines; a neighbouring job +can make a node look 5–11x slower and it is invisible to load average (the M3 Ultra reads load +2.45 with its GPU pinned). Anything measured here should use min-of-runs and check +`ioreg … IOAccelerator` "Device Utilization %" *before* touching the GPU. See PERFCHECK.md. diff --git a/PERFCHECK.md b/PERFCHECK.md new file mode 100644 index 0000000..658db7e --- /dev/null +++ b/PERFCHECK.md @@ -0,0 +1,120 @@ +# PERFCHECK — nightly GPU performance canary across the fleet + +`BENCHMARKS.md` is a *snapshot*: what each model cost on the day it was measured. Perfcheck is +the *tripwire*. A macOS point update, a torch/MLX bump, or a thermal fault can halve a node's +throughput, and nothing else in the fleet would notice — you'd just quietly wait twice as long +for every render until someone got suspicious months later. This catches it the next morning. + +It runs as part of godcheck on the m4mini watchdog (`~/godcheck/nightly.sh`, cron 03:30), +and appends a table + drift lines to `~/Documents/GODCHECK_LATEST.md` on ultra. Whole fleet, +6 nodes, **~35s**. + +## The numbers (torch 2.13.0 / MPS, 2026-07-17) + +Every node runs `MODELBEAST/venvs/rmbg/bin/python` — the same torch is already installed +fleet-wide, so nothing new gets installed (notably nothing lands on the disk-tight m1max). + +| node | chip | RAM | matmul fp16 | matmul fp32 | bandwidth | sdpa 64 | sdpa 56 | **56/64** | +|---|---|---|---|---|---|---|---|---| +| **m3ultra** | M3 Ultra | 256GB | 25202 | 22995 | 627 GB/s | 0.98ms | 4.27ms | **4.35x** | +| **m1** | M1 Ultra | 128GB | ~17100 | ~16000 | 639 GB/s | 1.50ms | 5.83ms | **3.89x** | +| **studio** | M1 Max | 32GB | 6996 | 6538 | 348 GB/s | 3.48ms | 10.73ms | **3.09x** | +| **m4** | M4 Pro | 24GB | 5678 | 3869 | 241 GB/s | 6.59ms | 12.88ms | **1.96x** | +| **m4mini** | M4 | 16GB | 3774 | 3436 | 105 GB/s | 5.46ms | 27.83ms | **5.10x** | +| **mini** | M1 (2020) | 8GB | 2310 | 1781 | 61 GB/s | 9.88ms | 51.84ms | **5.26x** | + +GFLOP/s, higher better. m1/m3ultra are *best observed* rather than medians — both were serving +real work all session (see "busy nodes" below), so their baselines are still warming up. + +Bandwidth tracks spec closely on the small machines — mini 61/68 GB/s (89%), m4mini 105/120 +(88%), m4 241/273 (88%), studio 348/400 (87%) — but **both Ultras land near 625 GB/s against a +spec of ~800**, i.e. ~78%. A single `add_` kernel doesn't saturate the Ultra fabric; that's a +property of the interconnect, not a fault, and it is the number to beat if a workload ever +needs to. + +### The 56/64 column is the point + +CorridorKey's Hiera backbone uses `head_dim=56`, which misses the fused-attention fast path on +Apple Silicon — padding to 64 is why our fork is 1.3x–4.9x faster (see `CORRIDORKEY.md`). This +bench reproduces that cliff directly and independently, on every machine, every night: + +- It confirms the padding win is real fleet-wide (**1.96x–5.26x**, never below 1). +- **If a future torch closes the gap, this column collapses toward 1.0 and we find out** — + instead of carrying a workaround forever without noticing it became unnecessary. + +The M4 Pro benefits least (1.96x); the small M4/M1 benefit most (>5x). Note these are pure +microbenchmark ratios — a full CorridorKey run dilutes them with non-attention work, so they +run higher than the end-to-end gains in `CORRIDORKEY.md`. + +## Usage + +```sh +scripts/perfcheck/run_fleet.py # bench fleet, compare to history, report +scripts/perfcheck/run_fleet.py --only m1,m4 # subset +scripts/perfcheck/run_fleet.py --show # print current baselines, don't bench +scripts/perfcheck/deploy_to_watchdog.sh # push to m4mini — RE-RUN AFTER EDITING bench.py +``` + +Exit 1 on a confirmed regression. State (`history.jsonl`, `baselines.json`, `pending.json`) is +gitignored and per-runner; the authoritative copy lives on the m4mini. + +**Editing `bench.py` changes what the numbers mean.** Bump `SCHEMA` in it (and in +`run_fleet.py`) whenever a probe's shape or size changes — older history is then ignored rather +than being median'd against a different benchmark and mistaken for a regression. Then re-run +`deploy_to_watchdog.sh`, or the fleet keeps being measured by the old bench. + +## Why it's built the way it is + +Everything below is a scar. The fleet runs real work while being measured, and that fact +dominates the whole design. + +**Naive benchmarking here is off by 11x.** The first version reported the M3 Ultra at 2154 +GFLOP/s and 84 GB/s. The true figures are ~24900 and ~620. Two causes: `n=2048` matmul is +dispatch-bound rather than compute-bound (it swung 11700 vs 17361 across runs — 48%), and +`median` lets a noisy neighbour through. `n=4096` reproduced to **0.1%** across runs *while the +GPU was contended*. Sizes here are chosen for stability, not speed — shrinking them to make the +bench faster turns it back into a random number generator. + +**min, not mean/median.** A concurrent `trellis_mac` job dragged a median-of-5 matmul from +~24500 to ~2150 — a 5x error that looks exactly like a catastrophic regression. The minimum +time is the least-interfered sample and the only estimate that survives a busy neighbour. + +**Sub-millisecond probes measure nothing.** The obvious sdpa size (8 heads x 1024) is ~0.4ms +and pure dispatch noise: three consecutive runs on one machine gave ratios of 0.79, 3.95 and +2.35 — the first would have "proven" head_dim=56 is *faster*, contradicting the real 8-machine +result. At 16 x 2048 it lands ~1-4ms and reproduces to 0.6%. + +**Busy nodes are excluded, not blamed.** GPU contention is invisible to load average — the M3 +Ultra sat at **load 2.45 with its GPU fully pinned** by trellis_mac. So `bench.py` samples GPU +utilization from `ioreg` *before it touches the GPU* (our own matmul pins the device at ~100%, +so sampling during the run cannot separate us from a neighbour — the ordering is the whole +trick). Above 50% the node is marked `[~] BUSY`, and its numbers raise no alert and never enter +a baseline. Two quirks: the first `ioreg` read is garbage (it is since-last-read, and an idle +m4mini reported 97% cold), and a GPU job alternates phases (100% CPU / 0% GPU one moment, 97% +GPU the next) — hence discard-first and max-of-several. + +**Baselines are the median of recent history, not a saved "best".** These machines have +neighbours: the M4 Pro also serves Ollama and its fp32 is bimodal (~3200 vs ~5500 depending on +whether a request is in flight); the M1 Ultra runs `mrpgi` on its GPU and swings 8301–15891 +(1.9x). A best-observed baseline pins to a lucky outlier the node rarely reaches again — which +means alerting every night forever. A median is what the machine actually does. + +**A regression must repeat before it's believed.** Transient contention is far more common than +real regression and doesn't recur on schedule. First sighting is `[~] watching`; only a second +consecutive one is `[!] CONFIRMED` and fails the run. Getting *faster* is reported immediately +— it's never an alarm. + +## Fleet gotcha found while building this + +**The m4mini is the only node with Tailscale SSH enabled (`RunSSH: true`), and it does not +propagate remote exit codes.** `ssh m4mini "exit 7"` returns **0**; so does `ssh m4mini +"false"`. Every other node propagates correctly. This means **any `if ssh m4mini ...; then` +test silently always passes** — a reach-check written that way will report every node healthy +no matter what. Test on *output* instead (`x=$(ssh … && echo YES)`), which is what godcheck +already does throughout, and why godcheck is unaffected. + +It also means the watchdog cannot ssh to *itself* ("Host key verification failed"), so +`run_fleet.py` detects its own tailnet IP and benches the local node through the shell instead. + +Worth deciding: `tailscale set --ssh=false` on the m4mini would make it behave like the rest of +the fleet, but it's a live config change on the always-on watchdog and hasn't been made. diff --git a/scripts/perfcheck/bench.py b/scripts/perfcheck/bench.py new file mode 100644 index 0000000..b4e25fa --- /dev/null +++ b/scripts/perfcheck/bench.py @@ -0,0 +1,194 @@ +"""PERFCHECK — per-node GPU performance canary. Prints one JSON object to stdout. + +Runs on ``MODELBEAST/venvs/rmbg/bin/python``, which carries an identical torch on every node +in the fleet — so numbers are comparable machine-to-machine without installing anything new +(nothing lands on the disk-critical m1max). + +Probes (~20s, sized to survive the 8GB M1 mini): + + matmul_fp16 / matmul_fp32 GFLOP/s — raw compute + bandwidth GB/s — unified-memory read/write + sdpa_64 / sdpa_56 ms — attention at a fast-path head_dim (64) vs the + awkward one CorridorKey's Hiera actually uses (56) + +The sdpa pair is the point. CorridorKey is slow on Apple Silicon because head_dim=56 misses +the fused attention path; padding to 64 buys 1.3x-4.9x depending on the machine. ``sdpa_ratio`` +(56/64) tracks that cliff per node, so if a future torch closes it we find out — instead of +carrying the padding workaround forever without noticing it became unnecessary. + +MEASUREMENT NOTES — these are load-bearing, do not "simplify" them: + +* **min, not mean/median.** These nodes serve jobs. A concurrent trellis_mac run on the M3 Ultra + dragged a median-of-5 matmul from ~24500 down to ~2150 GFLOP/s — a 5x error that looks + exactly like a catastrophic regression. The minimum time is the least-interfered sample and + the only estimate of the machine's actual capability that survives a noisy neighbour. +* **Sizes are chosen for stability, not for speed.** n=4096 reproduced to within 0.1% across + runs *while the GPU was contended*; n=2048 swung 48% (11700 vs 17361) because it is + dispatch-bound rather than compute-bound. Shrinking these to make the bench faster silently + turns it back into a random number generator. +* **Every probe syncs MPS before stopping the clock.** MPS dispatch is async; an unsynced timer + measures queue-submission, not work. +* ``load1`` is reported so a consumer can distrust a sample taken on a busy node. Under + contention the mins mostly hold up, but a hard-loaded node can still read low. +""" + +from __future__ import annotations + +import json +import os +import platform +import subprocess +import sys +import time + +os.environ.setdefault("PYTORCH_ENABLE_MPS_FALLBACK", "1") +os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE") + +import torch # noqa: E402 + +# 2: matmul 2048->4096, sdpa 8x1024 -> 16x2048, median->min, added gpu_busy_pre. +SCHEMA = 2 + + +def _sync() -> None: + torch.mps.synchronize() + + +def _best(fn, runs: int = 7, warmup: int = 3) -> float: + """Fastest of ``runs`` seconds-per-call, MPS-synced. See MEASUREMENT NOTES on why min.""" + for _ in range(warmup): + fn() + _sync() + best = float("inf") + for _ in range(runs): + t0 = time.perf_counter() + fn() + _sync() + best = min(best, time.perf_counter() - t0) + return best + + +def bench_matmul(dtype: torch.dtype, n: int = 4096) -> float: + """GFLOP/s for an n^3 matmul. n=4096 is the smallest size that is compute- not + dispatch-bound here (~67MB/tensor fp32 — fine on the 8GB mini).""" + a = torch.randn(n, n, device="mps", dtype=dtype) + b = torch.randn(n, n, device="mps", dtype=dtype) + secs = _best(lambda: a @ b) + return (2.0 * n**3) / secs / 1e9 + + +def bench_bandwidth(mb: int = 512) -> float: + """GB/s. x.add_(1) reads and writes the whole buffer -> 2x bytes moved. + + runs=25 rather than the default: each call is only ~1.6ms, so extra samples are nearly + free and they reject neighbour interference. At runs=9 this spread 16% on a contended + node (479 vs 570) — enough to trip a 20% drift alarm on its own. At runs=25: 0.3%. + """ + n = (mb * 1024 * 1024) // 4 + x = torch.randn(n, device="mps", dtype=torch.float32) + secs = _best(lambda: x.add_(1.0), runs=25) + return (2.0 * n * 4) / secs / 1e9 + + +def bench_sdpa(head_dim: int, heads: int = 16, seq: int = 2048) -> float: + """Milliseconds for one fused-attention call at the given head_dim. + + heads=16/seq=2048 is deliberate: it lands ~1-4ms, where the ratio reproduces to 0.6% + (4.28/4.30/4.33 across runs on a contended M3 Ultra). The obvious smaller 8x1024 is + sub-millisecond and pure dispatch noise — it produced ratios of 0.79, 3.95 and 2.35 on + three consecutive runs of the same machine, i.e. it would have "proven" head_dim=56 is + *faster*, contradicting the real 8-machine result. + """ + shape = (1, heads, seq, head_dim) + q = torch.randn(*shape, device="mps", dtype=torch.float16) + k = torch.randn(*shape, device="mps", dtype=torch.float16) + v = torch.randn(*shape, device="mps", dtype=torch.float16) + secs = _best(lambda: torch.nn.functional.scaled_dot_product_attention(q, k, v), runs=9) + return secs * 1000.0 + + +def gpu_busy_pre() -> int: + """Peak GPU utilization % observed over ~1.2s, sampled BEFORE any torch work starts. + + Must run before we touch the GPU: our own matmul pins the device at ~100%, so this can + only tell us about a *neighbour* while we are still idle. That ordering is the whole trick + — sampling during the bench is worthless. + + ``load1`` cannot substitute: the M3 Ultra sat at load 2.45 with its GPU fully held by a + trellis_mac job. Conversely a GPU job alternates phases (100% CPU / 0% GPU one moment, + 97% GPU the next), so a single reading misses it — hence max-of-several. The first ioreg + read is discarded because the counter is since-last-read and reports garbage when cold (an + idle m4mini reported 97% on its first sample, 0% thereafter). + """ + def sample() -> int: + try: + out = subprocess.run( + ["ioreg", "-r", "-d", "1", "-w", "0", "-c", "IOAccelerator"], + capture_output=True, text=True, timeout=8, + ).stdout + except Exception: + return 0 + vals = [int(s.split("=")[1]) for s in out.split(",") + if s.strip().startswith('"Device Utilization %"')] + return max(vals) if vals else 0 + + sample() # discard the cold read + peak = 0 + for _ in range(4): + time.sleep(0.3) + peak = max(peak, sample()) + return peak + + +def _sysctl(key: str) -> str: + try: + return subprocess.run( + ["sysctl", "-n", key], capture_output=True, text=True, timeout=5 + ).stdout.strip() + except Exception: + return "" + + +def host_facts() -> dict: + mem = _sysctl("hw.memsize") + return { + "host": platform.node().split(".")[0], + "chip": _sysctl("machdep.cpu.brand_string"), + "cores": _sysctl("hw.ncpu"), + "ram_gb": round(int(mem) / 1024**3) if mem.isdigit() else None, + "macos": platform.mac_ver()[0], + "torch": torch.__version__, + "load1": round(os.getloadavg()[0], 2), + } + + +def main() -> int: + if not torch.backends.mps.is_available(): + print(json.dumps({"error": "MPS unavailable", **host_facts()})) + return 1 + + # Before any torch work — see gpu_busy_pre(). + busy = gpu_busy_pre() + + # BUMP `schema` whenever a probe's shape/size changes. Numbers are only comparable within + # a schema, and run_fleet drops older samples from the median rather than averaging a + # 2048-matmul against a 4096-matmul and calling the difference a regression. + out = {"schema": SCHEMA, **host_facts(), "gpu_busy_pre": busy, "metrics": {}} + m = out["metrics"] + m["matmul_fp16_gflops"] = round(bench_matmul(torch.float16), 1) + m["matmul_fp32_gflops"] = round(bench_matmul(torch.float32), 1) + m["bandwidth_gbs"] = round(bench_bandwidth(), 1) + sdpa64 = bench_sdpa(64) + sdpa56 = bench_sdpa(56) + m["sdpa_64_ms"] = round(sdpa64, 3) + m["sdpa_56_ms"] = round(sdpa56, 3) + # >1 means the awkward head_dim is slower, i.e. CorridorKey's padding win still stands. + m["sdpa_ratio"] = round(sdpa56 / sdpa64, 2) if sdpa64 else None + out["load1_end"] = round(os.getloadavg()[0], 2) + + print(json.dumps(out)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/perfcheck/deploy_to_watchdog.sh b/scripts/perfcheck/deploy_to_watchdog.sh new file mode 100755 index 0000000..32ec23b --- /dev/null +++ b/scripts/perfcheck/deploy_to_watchdog.sh @@ -0,0 +1,38 @@ +#!/bin/zsh +# Push perfcheck to the m4mini watchdog, where the nightly godcheck cron runs it. +# +# Idempotent — re-run it whenever bench.py or run_fleet.py changes, otherwise the fleet keeps +# being measured by the old benchmark. Accumulated state (history/baselines/pending) is NEVER +# overwritten: on the first deploy the local history is seeded across so the baselines carry +# over, and after that the m4mini's copy is authoritative. +# +# Lives beside godcheck (~/godcheck/perfcheck) rather than in ~/MODELBEAST, to match how +# godcheck itself is a standalone script and to keep the cron independent of the repo state. +set -u +W=m4mini@100.124.220.31 +DEST=godcheck/perfcheck +HERE=${0:A:h} + +echo "==> deploying perfcheck to $W:~/$DEST" +ssh -o ConnectTimeout=10 -o BatchMode=yes "$W" "mkdir -p ~/$DEST" +scp -q "$HERE/bench.py" "$HERE/run_fleet.py" "$W:$DEST/" +ssh -o ConnectTimeout=10 -o BatchMode=yes "$W" "chmod +x ~/$DEST/run_fleet.py" + +# Seed history only if the watchdog has none, so a redeploy never clobbers real history. +# +# Tested on OUTPUT, never on ssh's exit status: the m4mini is the one node in the fleet with +# Tailscale SSH enabled (`RunSSH: true`), and tailscaled handles the session itself rather than +# sshd — it does not propagate the remote exit code. `ssh m4mini "exit 7"` returns 0, so every +# `if ssh m4mini test ...` silently succeeds. Every other node propagates correctly. (godcheck +# is output-based throughout and is unaffected.) +have=$(ssh -o ConnectTimeout=10 -o BatchMode=yes "$W" "test -s ~/$DEST/history.jsonl && echo YES") +if [ "$have" = "YES" ]; then + echo "==> watchdog already has history — left untouched" +elif [ -s "$HERE/history.jsonl" ]; then + echo "==> seeding history.jsonl (first deploy)" + scp -q "$HERE/history.jsonl" "$W:$DEST/" +fi + +echo "==> smoke test (one node)" +ssh -o ConnectTimeout=10 -o BatchMode=yes "$W" "cd ~/$DEST && /usr/bin/python3 run_fleet.py --only m4mini --quiet" +echo "==> done" diff --git a/scripts/perfcheck/run_fleet.py b/scripts/perfcheck/run_fleet.py new file mode 100755 index 0000000..c19a099 --- /dev/null +++ b/scripts/perfcheck/run_fleet.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""PERFCHECK fleet runner — benchmark every MODELBEAST node, detect drift, write a report. + +Stdlib only, so it runs on any node's system python3 (notably the m4mini, which hosts the +godcheck cron). ``bench.py`` is copied to each node on every run, so a node can never drift +onto a stale copy of the benchmark itself. + + ./run_fleet.py # bench the fleet, compare against recent history, report + ./run_fleet.py --only m1,m4 # subset + ./run_fleet.py --show # print the current baselines and exit (no benching) + +Exit 1 if any node shows a CONFIRMED regression, so cron/CI can react. + +WHY IT LOOKS LIKE THIS — the fleet serves real work, and that dominates every design choice: + +* **Baselines are the median of recent history, not a saved "best".** These machines have + neighbours. The M4 Pro also serves Ollama and its fp32 is bimodal (~3200 vs ~5500 GFLOP/s) + depending on whether a request is in flight; the M1 Ultra runs `mrpgi` on its GPU and swings + 8301-15891 (1.9x). A best-observed baseline pins to a lucky outlier the node rarely reaches + again, which means alerting every night forever. A median is what the machine actually does. +* **A busy node is excluded, not blamed.** GPU contention is invisible to load average (the M3 + Ultra read load 2.45 with its GPU pinned by trellis_mac), so bench.py samples GPU% before it + touches the GPU. Above BUSY_PCT the sample is noise: no alert, and kept out of the baseline. +* **A regression must repeat before it is believed.** Transient contention is far more common + than real regression and does not recur on schedule. First sighting is [~] watching; only a + second consecutive one is [!] CONFIRMED and fails the run. +* **Suspicious nodes are retried and the best attempt kept.** Contention only ever makes a node + look slower, so best-of-attempts is the least-interfered estimate. Only nodes that look off + pay for this, so the common case stays one pass. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +HERE = Path(__file__).resolve().parent +BENCH = HERE / "bench.py" +HISTORY = HERE / "history.jsonl" # every sample ever taken, append-only +BASELINES = HERE / "baselines.json" # derived view of HISTORY, rewritten each run +PENDING = HERE / "pending.json" # breaches seen last run — see confirm() + +# name -> (ssh target, MODELBEAST root). Mirrors nodes.json + the local primary. Embedded +# rather than read from nodes.json because that file is gitignored and primary-only, so the +# m4mini cron would not see it; godcheck hardcodes its hosts the same way. +NODES = [ + ("m3ultra", "m3ultra@100.89.131.57", "/Users/m3ultra/Documents/MODELBEAST"), + ("m1", "johnking@100.91.239.7", "/Users/johnking/MODELBEAST"), + ("m4", "m4pro@100.69.21.128", "/Users/m4pro/MODELBEAST"), + ("studio", "studio@100.92.78.24", "/Users/studio/MODELBEAST"), + ("m4mini", "m4mini@100.124.220.31", "/Users/m4mini/MODELBEAST"), + ("mini", "mini@100.79.229.50", "/Users/mini/MODELBEAST"), +] + +HIGHER_BETTER = {"matmul_fp16_gflops", "matmul_fp32_gflops", "bandwidth_gbs"} +# sdpa_ratio is diagnostic: a change means the fused-attention landscape moved (e.g. a torch +# release finally handles head_dim=56), which is a headline to read, not a regression to fail. +DIAGNOSTIC = {"sdpa_ratio"} + +THRESHOLD = 0.20 # flag a >20% move vs baseline +BUSY_PCT = 50 # GPU% (pre-bench) above which a node is unmeasurable +HISTORY_WINDOW = 9 # samples per node the median is drawn from +MIN_SAMPLES = 3 # below this a node has no baseline yet and cannot alert +# Numbers are only comparable within a bench schema. bench.py bumps SCHEMA when a probe's +# shape changes; older history is then ignored rather than being median'd against the new +# shape and mistaken for a regression. Keep in step with bench.py's SCHEMA. +SCHEMA = 2 +SSH = ["ssh", "-o", "ConnectTimeout=10", "-o", "BatchMode=yes"] + + +# --------------------------------------------------------------------------- +# Running +# --------------------------------------------------------------------------- +def local_ip() -> str: + """This machine's tailnet IPv4, or "" — used to spot our own entry in NODES.""" + for cmd in (["tailscale", "ip", "-4"], + ["/Applications/Tailscale.app/Contents/MacOS/Tailscale", "ip", "-4"]): + try: + out = subprocess.run(cmd, capture_output=True, text=True, timeout=8) + if out.returncode == 0 and out.stdout.strip(): + return out.stdout.strip().splitlines()[0].strip() + except Exception: + continue + return "" + + +def run_node(name: str, host: str, root: str, self_ip: str = "") -> dict: + """Run bench.py on a node (locally if that node is us) and return parsed JSON. + + Benching ourselves goes through the shell, not ssh: a host looping back to its own tailnet + address needs its own host key trusted and its own pubkey authorized, which fails out of the + box ("Host key verification failed" on the m4mini watchdog, which must bench itself as part + of the fleet). Running local also skips a pointless copy and round-trip. + """ + remote = "/tmp/perfcheck_bench.py" + is_self = bool(self_ip) and host.split("@")[-1] == self_ip + try: + if is_self: + proc = subprocess.run( + [f"{root}/venvs/rmbg/bin/python", str(BENCH)], + capture_output=True, text=True, timeout=600, + ) + else: + subprocess.run( + ["scp", "-q", "-o", "ConnectTimeout=10", "-o", "BatchMode=yes", + str(BENCH), f"{host}:{remote}"], + check=True, capture_output=True, timeout=60, + ) + proc = subprocess.run( + SSH + [host, f"{root}/venvs/rmbg/bin/python {remote}"], + capture_output=True, text=True, timeout=600, + ) + except subprocess.TimeoutExpired: + return {"node": name, "error": "timeout"} + except subprocess.CalledProcessError as e: + return {"node": name, "error": f"scp failed: {e.stderr.decode()[:120].strip()}"} + + if proc.returncode != 0: + tail = (proc.stderr or proc.stdout).strip().splitlines() + return {"node": name, "error": (tail[-1][:160] if tail else f"exit {proc.returncode}")} + try: + # The bench prints one JSON object; take the last line in case a lib warns first. + return {"node": name, **json.loads(proc.stdout.strip().splitlines()[-1])} + except (ValueError, IndexError): + return {"node": name, "error": f"unparseable: {proc.stdout[:120]!r}"} + + +def usable(r: dict) -> bool: + """True if a sample is trustworthy enough to compare or to feed the baseline.""" + return ( + "error" not in r + and r.get("schema") == SCHEMA + and r.get("gpu_busy_pre", 0) < BUSY_PCT + and bool(r.get("metrics")) + ) + + +def merge_best(runs: list[dict]) -> dict: + """Fold repeat attempts at one node into its least-contended sample, per metric. + + Contention can only make a node look slower, never faster, so the best value across + attempts is the closest estimate of what the machine can do right now. Same argument as + min-of-runs inside bench.py, lifted one level. + """ + ok = [r for r in runs if r and "error" not in r and r.get("metrics")] + if not ok: + return runs[0] + out = dict(ok[-1]) + m = {} + for key in ok[0]["metrics"]: + vals = [r["metrics"][key] for r in ok if r["metrics"].get(key) is not None] + if vals: + m[key] = max(vals) if key in HIGHER_BETTER else min(vals) + # Recompute so the ratio stays consistent with the sdpa numbers reported beside it. + if m.get("sdpa_64_ms"): + m["sdpa_ratio"] = round(m["sdpa_56_ms"] / m["sdpa_64_ms"], 2) + out["metrics"] = m + out["attempts"] = len(ok) + # Least-contended attempt wins, so carry the quietest GPU reading with it. + out["gpu_busy_pre"] = min(r.get("gpu_busy_pre", 0) for r in ok) + return out + + +# --------------------------------------------------------------------------- +# Baselines (derived from history) +# --------------------------------------------------------------------------- +def _median(xs: list[float]) -> float: + xs = sorted(xs) + n = len(xs) + return xs[n // 2] if n % 2 else (xs[n // 2 - 1] + xs[n // 2]) / 2.0 + + +def baselines_from_history() -> dict: + """node -> {metrics: {k: median-of-recent}, n: samples}. Only clean samples count.""" + per_node: dict[str, list[dict]] = {} + if HISTORY.exists(): + for line in HISTORY.read_text().splitlines(): + try: + r = json.loads(line) + except ValueError: + continue + if usable(r): + per_node.setdefault(r["node"], []).append(r["metrics"]) + + out = {} + for node, samples in per_node.items(): + recent = samples[-HISTORY_WINDOW:] + if len(recent) < MIN_SAMPLES: + out[node] = {"metrics": {}, "n": len(recent)} + continue + keys = {k for s in recent for k in s} + med = {} + for k in keys: + vals = [s[k] for s in recent if s.get(k) is not None] + if vals: + med[k] = round(_median(vals), 3) + out[node] = {"metrics": med, "n": len(recent)} + return out + + +def compare(results: list[dict], baselines: dict) -> dict[str, str]: + """Breaches keyed ``node/metric`` -> readable line. ``{}`` when all within threshold.""" + out = {} + for r in results: + if not usable(r): + continue + base = baselines.get(r["node"], {}).get("metrics") + if not base: + continue # not enough history yet + for key, now in r["metrics"].items(): + was = base.get(key) + if was in (None, 0) or now is None or key in DIAGNOSTIC: + continue + change = (now - was) / was + slower = -change if key in HIGHER_BETTER else change + if slower > THRESHOLD: + out[f"{r['node']}/{key}"] = f"{r['node']} {key}: {was} -> {now} ({slower:+.0%} SLOWER)" + elif -slower > THRESHOLD: + out[f"+{r['node']}/{key}"] = f"{r['node']} {key}: {was} -> {now} ({-slower:+.0%} faster)" + return out + + +def confirm(breaches: dict[str, str]) -> tuple[list[str], list[str]]: + """Split breaches into (confirmed, provisional) by whether they also fired last run. + + A regression is believed only once it repeats: transient contention is the overwhelmingly + common cause of one slow sample and it does not recur on schedule, while a real regression + does. A node getting *faster* is reported immediately — it is never an alarm. + """ + try: + prev = set(json.loads(PENDING.read_text())) + except Exception: + prev = set() + + confirmed, provisional = [], [] + for key, line in sorted(breaches.items()): + if key.startswith("+"): + confirmed.append(f"[+] {line}") + elif key in prev: + confirmed.append(f"[!] {line} — CONFIRMED (2 runs)") + else: + provisional.append(f"[~] {line} — unconfirmed, watching") + PENDING.write_text(json.dumps([k for k in breaches if not k.startswith("+")]) + "\n") + return confirmed, provisional + + +# --------------------------------------------------------------------------- +# Reporting +# --------------------------------------------------------------------------- +def report(results: list[dict], confirmed: list[str], provisional: list[str], + baselines: dict, stamp: str) -> str: + out = [f"# PERFCHECK {stamp}", ""] + out.append("| node | chip | RAM | matmul fp16 | matmul fp32 | bandwidth | sdpa 64 | sdpa 56 | 56/64 |") + out.append("|---|---|---|---|---|---|---|---|---|") + for r in sorted(results, key=lambda x: x["node"]): + if "error" in r: + out.append(f"| **{r['node']}** | [X] {r['error']} | | | | | | | |") + continue + m, chip = r["metrics"], (r.get("chip") or "").replace("Apple ", "") + busy = r.get("gpu_busy_pre", 0) + if busy >= BUSY_PCT: + chip += f" [~] BUSY {busy}%" + out.append( + f"| **{r['node']}** | {chip} | {r.get('ram_gb')}GB | " + f"{m['matmul_fp16_gflops']:.0f} | {m['matmul_fp32_gflops']:.0f} | " + f"{m['bandwidth_gbs']:.0f} GB/s | {m['sdpa_64_ms']:.2f}ms | " + f"{m['sdpa_56_ms']:.2f}ms | **{m['sdpa_ratio']}x** |" + ) + thin = [n for n, b in baselines.items() if b["n"] < MIN_SAMPLES] + out.append("") + out.append("## drift vs baseline (median of last %d clean runs)" % HISTORY_WINDOW) + out.extend(confirmed or ["- no confirmed regressions"]) + if provisional: + out.append("") + out.extend(provisional) + if thin: + out.append(f"- (warming up, <{MIN_SAMPLES} clean samples: {', '.join(sorted(thin))})") + return "\n".join(out) + "\n" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--only", help="comma-separated node names") + ap.add_argument("--quiet", action="store_true", help="print only the report") + ap.add_argument("--show", action="store_true", help="print current baselines and exit") + ap.add_argument("--attempts", type=int, default=2, + help="extra attempts for a node that looks regressed (default 2)") + a = ap.parse_args() + + if a.show: + print(json.dumps(baselines_from_history(), indent=2, sort_keys=True)) + return 0 + + nodes = NODES + if a.only: + want = {s.strip() for s in a.only.split(",")} + nodes = [n for n in NODES if n[0] in want] + + baselines = baselines_from_history() + self_ip = local_ip() + stamp = datetime.now(timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M %Z") + + results = [] + for name, host, root in nodes: + if not a.quiet: + print(f" benching {name} ...", file=sys.stderr, flush=True) + results.append(run_node(name, host, root, self_ip)) + + # Adaptive retry: only a node that looks regressed pays for extra attempts. A busy + # neighbour is far more likely than a real regression, and best-of-attempts costs nothing + # when the node is idle — so the common case stays a single pass over the fleet. + for i, r in enumerate(results): + if not usable(r) or not compare([r], baselines): + continue + name, host, root = next(n for n in nodes if n[0] == r["node"]) + if not a.quiet: + print(f" {name} looks off — {a.attempts} more attempt(s) ...", file=sys.stderr, flush=True) + results[i] = merge_best( + [r] + [run_node(name, host, root, self_ip) for _ in range(a.attempts)] + ) + + confirmed, provisional = confirm(compare(results, baselines)) + + with HISTORY.open("a") as fh: + for r in results: + fh.write(json.dumps({"ts": stamp, **r}) + "\n") + # Rewrite the derived view *after* appending, so it reflects this run too. + BASELINES.write_text(json.dumps(baselines_from_history(), indent=2, sort_keys=True) + "\n") + + print(report(results, confirmed, provisional, baselines, stamp)) + return 1 if any(x.startswith("[!]") for x in confirmed) else 0 + + +if __name__ == "__main__": + os.chdir(HERE) + sys.exit(main())