6fcf677cef
Fleet benchmarking exposed the problem: throughput was ANTI-correlated with GPU core count. The 80-core M3 Ultra came in slowest at 128^3/128ch (81.6ms) behind a 38-core M2 Max (58.3ms), M1 Ultra (66.7ms) and even a 32-core M1 Max (69.2ms). That ordering only makes sense if the op is bound by dispatch latency rather than compute - the per-offset loop issued 2*K^3 = 54 tiny GPU ops per layer, none big enough to occupy the machine, and the Ultra's fused-die design penalises exactly that. Concatenating the K^3 neighbour taps along the channel axis collapses it to a single [N, K^3*Cin] x [K^3*Cin, Cout] matmul. Chunked over rows so peak memory stays ~256MB (the unchunked buffer is ~2.9GB at 128^3/128ch - fine on a Studio, not fine on an 8GB mini). m3ultra 128^3/128ch: 81.6ms -> 39.3ms (2.08x), 2.57 -> 5.34 Mvox/s m3ultra 64^3/128ch: 21.9ms -> 6.4ms (3.4x) 7/7 tests still pass against the naive reference.
59 lines
2.3 KiB
Bash
Executable File
59 lines
2.3 KiB
Bash
Executable File
#!/bin/zsh
|
|
# Run the SubMConv3d benchmark on every Apple Silicon box in the fleet.
|
|
# Ships code only (no weights); each box gets a small mlx+numpy venv under ~/lato-bench.
|
|
set -u
|
|
SRC=/Users/m3ultra/Documents/lato.2_mrp_mlx
|
|
OUT=$SRC/bench
|
|
SSH_OPTS=(-o ConnectTimeout=10 -o BatchMode=yes -o StrictHostKeyChecking=accept-new)
|
|
SSH_STR="ssh -o ConnectTimeout=10 -o BatchMode=yes -o StrictHostKeyChecking=accept-new"
|
|
|
|
# host:label (m3ultra runs locally, already done)
|
|
HOSTS=(
|
|
"m1max@100.92.78.24:m1max"
|
|
"m2max@100.120.83.110:m2max"
|
|
"100.69.21.128:m4pro"
|
|
"johnking@100.91.239.7:m1ultra"
|
|
)
|
|
|
|
hb(){ mkdir -p ~/.jobs; echo "$(date '+%F %T') | $1 | $2" > ~/.jobs/lato-fleet-bench.status; }
|
|
hb "starting" "${#HOSTS[@]} hosts"
|
|
|
|
for entry in $HOSTS; do
|
|
host=${entry%%:*}; label=${entry##*:}
|
|
echo "===== $label ($host)"
|
|
hb "$label" "connecting"
|
|
|
|
if ! ssh "${SSH_OPTS[@]}" "$host" 'true' 2>/dev/null; then
|
|
echo " SKIP unreachable"; continue
|
|
fi
|
|
|
|
# m1max is disk-critical (~20GB free per fleet notes) - refuse rather than fill it
|
|
free_mb=$(ssh "${SSH_OPTS[@]}" "$host" "df -m / | tail -1 | awk '{print \$4}'" 2>/dev/null)
|
|
if [ -n "$free_mb" ] && [ "$free_mb" -lt 3000 ]; then
|
|
echo " SKIP only ${free_mb}MB free - not installing a venv here"; continue
|
|
fi
|
|
|
|
ssh "${SSH_OPTS[@]}" "$host" 'mkdir -p ~/lato-bench' 2>/dev/null
|
|
rsync -a -e "$SSH_STR" --delete \
|
|
"$SRC/lato_mlx" "$SRC/bench" "$host:~/lato-bench/" 2>&1 | tail -1
|
|
|
|
hb "$label" "installing venv"
|
|
ssh "${SSH_OPTS[@]}" "$host" '
|
|
cd ~/lato-bench || exit 1
|
|
UV=$(command -v uv || echo /opt/homebrew/bin/uv)
|
|
if [ ! -x "$UV" ]; then echo " no uv on this host"; exit 3; fi
|
|
[ -d .venv ] || "$UV" venv --python 3.12 .venv >/dev/null 2>&1
|
|
VIRTUAL_ENV=.venv "$UV" pip install -q mlx numpy >/dev/null 2>&1
|
|
.venv/bin/python -c "import mlx.core" 2>/dev/null || { echo " mlx install failed"; exit 4; }
|
|
' || { echo " SKIP setup failed (rc=$?)"; continue; }
|
|
|
|
hb "$label" "benchmarking"
|
|
ssh "${SSH_OPTS[@]}" "$host" \
|
|
'cd ~/lato-bench && .venv/bin/python bench/bench_subm.py --out bench/result.json' 2>&1 | tail -8
|
|
rsync -a -e "$SSH_STR" "$host:~/lato-bench/bench/result.json" "$OUT/$label.json" 2>/dev/null \
|
|
&& echo " -> $OUT/$label.json" || echo " (no result pulled)"
|
|
done
|
|
|
|
hb "DONE" "results in $OUT"
|
|
echo "===== done"
|