wardrobegod/depot/prune.py

#!/usr/bin/env python3
"""Prune old depot GLBs from the live VPS — safely.

Age alone is NOT a safe criterion here. Two shipping games resolve depot assets BY FILENAME at
runtime (thriftgod web/index.html:1176, procity web/js/core/loaders.js), so deleting a mesh some
game still names is a 404 in a released build. A three-year-old asset that a game references is
not stale, it is load-bearing.

So a file is only ever deleted when ALL of these hold:
  1. it is older than --days
  2. its name appears in NO consumer's source (js/html/json/ts across every game repo)
  3. an identical-size copy is confirmed present in the Drive archive

Dry-run by default. Thumbnails and meta.json are never touched, so the depot keeps listing the
asset — matching how the MeshGod archive behaves after a prune.

    prune.py plan  [--days 180]
    prune.py apply [--days 180]        # actually deletes, after re-verifying every rule
"""
import json, os, re, subprocess, sys, time, urllib.parse, urllib.request

DEPOT = os.environ.get('WG_GOD3', 'http://100.94.195.115:8788')
DRIVE = os.environ.get('WG_DEPOT_DRIVE', 'gdrive:DB-BACKUP/3god-depot')
RCLONE = os.environ.get('WG_RCLONE', '/opt/homebrew/bin/rclone')
ULTRA = 'johnking@100.91.239.7'

# Every tree that can name a depot asset. Local paths are on JING5; ultra ones are fetched by ssh.
LOCAL_SOURCES = [os.path.expanduser('~/Documents/thriftgod/web'),
                 os.path.expanduser('~/Documents/PROCITY/web')]
ULTRA_SOURCES = ['Documents/90sDJsim/web', 'Documents/thriftgod/web']
GLB_RE = r'[A-Za-z0-9._-]+\.glb'


def referenced():
    """Every .glb filename mentioned anywhere in any consumer's source."""
    names = set()
    for d in LOCAL_SOURCES:
        if not os.path.isdir(d):
            continue
        r = subprocess.run(['grep', '-rhoaE', GLB_RE, d], capture_output=True, text=True)
        names |= {x.strip() for x in r.stdout.splitlines() if x.strip()}
    for d in ULTRA_SOURCES:
        r = subprocess.run(['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', ULTRA,
                            f'grep -rhoaE "{GLB_RE}" ~/{d} 2>/dev/null'],
                           capture_output=True, text=True)
        names |= {x.strip() for x in r.stdout.splitlines() if x.strip()}
    return names


def depot_list():
    with urllib.request.urlopen(DEPOT + '/api/list', timeout=60) as r:
        return json.load(r).get('assets', [])


def drive_index():
    """{name: size} from the Drive archive — the proof a delete is recoverable."""
    r = subprocess.run([RCLONE, 'lsjson', DRIVE], capture_output=True, text=True, timeout=600)
    if r.returncode != 0:
        raise SystemExit(f'cannot read the Drive archive ({DRIVE}): {r.stderr.strip()[:200]}\n'
                         'Refusing to plan a prune without a verified backup.')
    return {e['Name']: e['Size'] for e in json.loads(r.stdout or '[]')}


def plan(days):
    refs, assets, drive = referenced(), depot_list(), drive_index()
    cutoff = time.time() - days * 86400
    out = {'days': days, 'depot_total': len(assets), 'referenced': len(refs),
           'drive_objects': len(drive), 'delete': [], 'kept': {}}
    keep_ref = keep_young = keep_unbacked = 0
    for a in assets:
        f = a.get('file', '')
        if not f.lower().endswith('.glb'):
            continue
        if f in refs:
            keep_ref += 1; continue
        if a.get('added', 0) > cutoff:
            keep_young += 1; continue
        if drive.get(f) != a.get('size'):
            # present-but-different-size counts as unbacked; never delete on a maybe
            keep_unbacked += 1; continue
        out['delete'].append({'file': f, 'size': a.get('size'),
                              'age_days': int((time.time() - a.get('added', 0)) / 86400)})
    out['kept'] = {'referenced_by_a_game': keep_ref, 'newer_than_cutoff': keep_young,
                   'NOT_in_drive_backup': keep_unbacked}
    out['reclaim_bytes'] = sum(d['size'] for d in out['delete'])
    return out


def apply(days):
    p = plan(days)
    ok, fail = [], []
    for d in p['delete']:
        try:
            req = urllib.request.Request(
                DEPOT + '/api/del?name=' + urllib.parse.quote(d['file']), method='POST')
            with urllib.request.urlopen(req, timeout=60) as r:
                r.read()
            ok.append(d['file'])
        except Exception as e:
            fail.append({'file': d['file'], 'error': str(e)[:120]})
    p['deleted'], p['failed'] = ok, fail
    return p


if __name__ == '__main__':
    cmd = sys.argv[1] if len(sys.argv) > 1 else 'plan'
    days = int(sys.argv[sys.argv.index('--days') + 1]) if '--days' in sys.argv else 180
    res = apply(days) if cmd == 'apply' else plan(days)
    res['reclaim_gb'] = round(res.get('reclaim_bytes', 0) / 1073741824, 2)
    sample = res['delete'][:8]
    res['delete_count'] = len(res['delete'])
    res['delete'] = sample if cmd != 'apply' else res['delete'][:8]
    print(json.dumps(res, indent=1))