diff --git a/depot/README.md b/depot/README.md index 6147307..f48333e 100644 --- a/depot/README.md +++ b/depot/README.md @@ -78,3 +78,27 @@ Verify freshness the same way as the other flows: ⚠️ Shares the whole-fleet risk noted in the backup-verify skill: the `gdrive:` remote uses rclone's shared Google client_id, which Google retires during 2026. When it dies every Drive flow breaks at once, this one included. + +### Schedule +`games.monsterrobot.depot-archive` — launchd on ultra, daily **05:30** (after meshgod at 05:00), +logging to `~/depot-backups/launchd.log`. Plist kept here alongside the script. + +### Pruning (`prune.py`) +Reclaims VPS disk **without** the 404 risk, because age alone is not a safe criterion — a +three-year-old asset a game still names is load-bearing, not stale. A GLB is deleted only when +**all three** hold: +1. older than `--days` +2. its name appears in **no** consumer's source (js/html/json/ts across thriftgod, procity, 90sDJsim) +3. an **identical-size copy is confirmed in the Drive archive** + +Dry-run by default (`plan`); `apply` re-verifies every rule before deleting. Thumbnails and +`meta.json` are never touched, so the depot keeps listing pruned assets — same behaviour as the +MeshGod archive. + +```bash +python3 ~/depot-backups/prune.py plan --days 180 # on ultra, where rclone lives +``` + +As of 2026-07-24 this deletes **nothing**: of 774 assets, 35 are game-referenced and 724 are +newer than the cutoff. Every asset in the depot is under 30 days old. The mechanism is armed for +when that changes. diff --git a/depot/games.monsterrobot.depot-archive.plist b/depot/games.monsterrobot.depot-archive.plist new file mode 100644 index 0000000..22bd669 --- /dev/null +++ b/depot/games.monsterrobot.depot-archive.plist @@ -0,0 +1,13 @@ + + + + Labelgames.monsterrobot.depot-archive + ProgramArguments + /bin/sh/Users/johnking/depot-backups/archive.sh + EnvironmentVariables + PATH/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin + StartCalendarIntervalHour5Minute30 + StandardOutPath/Users/johnking/depot-backups/launchd.log + StandardErrorPath/Users/johnking/depot-backups/launchd.log + WorkingDirectory/Users/johnking + diff --git a/depot/prune.py b/depot/prune.py new file mode 100644 index 0000000..f3ff96c --- /dev/null +++ b/depot/prune.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Prune old depot GLBs from the live VPS — safely. + +Age alone is NOT a safe criterion here. Two shipping games resolve depot assets BY FILENAME at +runtime (thriftgod web/index.html:1176, procity web/js/core/loaders.js), so deleting a mesh some +game still names is a 404 in a released build. A three-year-old asset that a game references is +not stale, it is load-bearing. + +So a file is only ever deleted when ALL of these hold: + 1. it is older than --days + 2. its name appears in NO consumer's source (js/html/json/ts across every game repo) + 3. an identical-size copy is confirmed present in the Drive archive + +Dry-run by default. Thumbnails and meta.json are never touched, so the depot keeps listing the +asset — matching how the MeshGod archive behaves after a prune. + + prune.py plan [--days 180] + prune.py apply [--days 180] # actually deletes, after re-verifying every rule +""" +import json, os, re, subprocess, sys, time, urllib.parse, urllib.request + +DEPOT = os.environ.get('WG_GOD3', 'http://100.94.195.115:8788') +DRIVE = os.environ.get('WG_DEPOT_DRIVE', 'gdrive:DB-BACKUP/3god-depot') +RCLONE = os.environ.get('WG_RCLONE', '/opt/homebrew/bin/rclone') +ULTRA = 'johnking@100.91.239.7' + +# Every tree that can name a depot asset. Local paths are on JING5; ultra ones are fetched by ssh. +LOCAL_SOURCES = [os.path.expanduser('~/Documents/thriftgod/web'), + os.path.expanduser('~/Documents/PROCITY/web')] +ULTRA_SOURCES = ['Documents/90sDJsim/web', 'Documents/thriftgod/web'] +GLB_RE = r'[A-Za-z0-9._-]+\.glb' + + +def referenced(): + """Every .glb filename mentioned anywhere in any consumer's source.""" + names = set() + for d in LOCAL_SOURCES: + if not os.path.isdir(d): + continue + r = subprocess.run(['grep', '-rhoaE', GLB_RE, d], capture_output=True, text=True) + names |= {x.strip() for x in r.stdout.splitlines() if x.strip()} + for d in ULTRA_SOURCES: + r = subprocess.run(['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', ULTRA, + f'grep -rhoaE "{GLB_RE}" ~/{d} 2>/dev/null'], + capture_output=True, text=True) + names |= {x.strip() for x in r.stdout.splitlines() if x.strip()} + return names + + +def depot_list(): + with urllib.request.urlopen(DEPOT + '/api/list', timeout=60) as r: + return json.load(r).get('assets', []) + + +def drive_index(): + """{name: size} from the Drive archive — the proof a delete is recoverable.""" + r = subprocess.run([RCLONE, 'lsjson', DRIVE], capture_output=True, text=True, timeout=600) + if r.returncode != 0: + raise SystemExit(f'cannot read the Drive archive ({DRIVE}): {r.stderr.strip()[:200]}\n' + 'Refusing to plan a prune without a verified backup.') + return {e['Name']: e['Size'] for e in json.loads(r.stdout or '[]')} + + +def plan(days): + refs, assets, drive = referenced(), depot_list(), drive_index() + cutoff = time.time() - days * 86400 + out = {'days': days, 'depot_total': len(assets), 'referenced': len(refs), + 'drive_objects': len(drive), 'delete': [], 'kept': {}} + keep_ref = keep_young = keep_unbacked = 0 + for a in assets: + f = a.get('file', '') + if not f.lower().endswith('.glb'): + continue + if f in refs: + keep_ref += 1; continue + if a.get('added', 0) > cutoff: + keep_young += 1; continue + if drive.get(f) != a.get('size'): + # present-but-different-size counts as unbacked; never delete on a maybe + keep_unbacked += 1; continue + out['delete'].append({'file': f, 'size': a.get('size'), + 'age_days': int((time.time() - a.get('added', 0)) / 86400)}) + out['kept'] = {'referenced_by_a_game': keep_ref, 'newer_than_cutoff': keep_young, + 'NOT_in_drive_backup': keep_unbacked} + out['reclaim_bytes'] = sum(d['size'] for d in out['delete']) + return out + + +def apply(days): + p = plan(days) + ok, fail = [], [] + for d in p['delete']: + try: + req = urllib.request.Request( + DEPOT + '/api/del?name=' + urllib.parse.quote(d['file']), method='POST') + with urllib.request.urlopen(req, timeout=60) as r: + r.read() + ok.append(d['file']) + except Exception as e: + fail.append({'file': d['file'], 'error': str(e)[:120]}) + p['deleted'], p['failed'] = ok, fail + return p + + +if __name__ == '__main__': + cmd = sys.argv[1] if len(sys.argv) > 1 else 'plan' + days = int(sys.argv[sys.argv.index('--days') + 1]) if '--days' in sys.argv else 180 + res = apply(days) if cmd == 'apply' else plan(days) + res['reclaim_gb'] = round(res.get('reclaim_bytes', 0) / 1073741824, 2) + sample = res['delete'][:8] + res['delete_count'] = len(res['delete']) + res['delete'] = sample if cmd != 'apply' else res['delete'][:8] + print(json.dumps(res, indent=1))