PROCITY/pipeline/godverse_town.py

#!/usr/bin/env python3
"""Lane G — GODVERSE town adapter: thriftgod's REAL shop census -> a procity-town-cache/2.

Marries the two halves: roads[] come from an existing OSM cache (Lane E's build_towns.py),
shops[] come from thriftgod's Postgres (the real-business census DealGod scraped). Result:
walk the town's actual streets past its actual secondhand shops.

    python3 pipeline/godverse_town.py newtown_real newtown_godverse   # roads-donor key, out key
    python3 pipeline/godverse_town.py redhill_real redhill_godverse   # THE shop's real street

Reads thriftgod via the `psql` CLI (R24): psycopg2 isn't installed on the m3, and one less dep
between the census and the cache. Works unchanged wherever thriftgod lives.

`godverseShopId` (R24 ledger #2, with Lane A) — the OPAQUE key that resolves
`stock_godverse/<godverseShopId>/`. **It is not `shop.id` and not "the census id"**: census shops
carry their thriftgod id (max 2992), while Monster Robot Party isn't in the census at all and
carries its dealgod store id (3962749). Two disjoint id spaces, one opaque field — G emits it,
everyone else reads it, nobody derives it. A's `validateTownCache` wants a positive integer,
unique per cache; **missing is a WARN, not an error** (measured — A's rule is deliberate: absent
stock identity must never fail a town's boot, per the charter's determinism boundary). So the
GODVERSE layer is keyed and the inherited texture layer is not — see Ruling 2 below for why that
is the honest shape rather than a gap.
"""
import json, math, os, subprocess, sys

DONOR, OUT = sys.argv[1], sys.argv[2]
HERE = os.path.dirname(os.path.abspath(__file__))
TOWNS = os.path.join(HERE, '..', 'web', 'assets', 'towns')

# thriftgod shop_type -> procity SHOP_TYPES (registry.js is the vocabulary of record)
TYPE_MAP = {'music': 'record', 'charity': 'opshop', 'second_hand': 'opshop', 'antiques': 'opshop',
            'books': 'book', 'toys': 'toy', 'video_games': 'video', 'games': 'toy',
            'pawnbroker': 'pawn', 'market': 'dept'}

# REAL shops that exist in neither OSM nor thriftgod's census, added by explicit, sourced ruling.
# This is not a mapping table (the R12 gigKey law) — it's a hand-verified roster of the shops the
# epoch is actually about. Every entry states where its identity and its coordinates came from.
#
# Monster Robot Party: the v5 alpha's shop — its POS *is* `recordgod`, its own site hosts the
# covers, and dealgod scrapes it as store 3962749 (which is therefore its godverseShopId). OSM
# doesn't know it (no name match, no 147 Musgrave Rd address node) and the census doesn't either,
# so the coordinate cannot be derived from any dataset we hold: John supplied the shop's Google
# Maps place (place `/g/11g6rlbbc0`), whose point is -27.4553791, 153.0076244 — inside
# `redhill_real`'s bbox and on Musgrave Road's own geometry. Verified, not estimated.
EXTRA_SHOPS = {
    'redhill_godverse': [
        {'id': 3962749, 'godverseShopId': 3962749, 'name': 'Monster Robot Party', 'type': 'record',
         'lat': -27.4553791, 'lon': 153.0076244, 'suburb': 'Red Hill',
         'source': 'GODVERSE/dealgod store 3962749 · POS recordgod · coords: Google Maps place '
                   '/g/11g6rlbbc0 (147 Musgrave Road, Red Hill QLD), John-supplied R24'},
    ],
}


def psql(db, sql):
    out = subprocess.run(['psql', '-d', db, '-tAF\t', '-c', sql],
                         capture_output=True, text=True, check=True).stdout
    return [ln.split('\t') for ln in out.splitlines() if ln.strip()]


donor = json.load(open(os.path.join(TOWNS, f'{DONOR}.json')))
bb = donor['bbox']
rows = psql(os.environ.get('THRIFTGOD_DB', 'thriftgod'), f"""
    SELECT id, COALESCE(name,''), COALESCE(shop_type,''), lat, lng, COALESCE(suburb,'')
    FROM shop
    WHERE lat BETWEEN {bb['minLat']} AND {bb['maxLat']}
      AND lng BETWEEN {bb['minLon']} AND {bb['maxLon']}
    ORDER BY id""")

shops = [{'id': int(r[0]),
          'godverseShopId': int(r[0]),      # census shops: the thriftgod id IS the key (not derived
                                            # from `id` by law — they coincide, see the docstring)
          'name': r[1],
          'type': TYPE_MAP.get(r[2], 'opshop'),
          'lat': float(r[3]), 'lon': float(r[4]),
          'suburb': r[5].split('|')[0]} for r in rows]
census_n = len(shops)

extras = EXTRA_SHOPS.get(OUT, [])
known = {s['id'] for s in shops}
for e in sorted(extras, key=lambda s: s['id']):
    if e['id'] in known:                    # census wins a collision, and we count the swap
        continue
    shops.append({k: e[k] for k in ('id', 'godverseShopId', 'name', 'type', 'lat', 'lon', 'suburb')})
extra_n = len(shops) - census_n


# ── Fable's Ruling 2 (R22, implemented R24): INHERIT THE DONOR'S TEXTURE LAYER ────────────────
# "Merge census heroes WITH the donor's texture shops; census heroes always win a collision
# (dedupe by proximity + name), drop the donor hero, keep yours, count the swap. Result: real
# census shops on a street that also feels alive."
#
# Not cosmetic — MEASURED (R24): the census layer alone is 9 shops across Red Hill's 2.4 km bbox →
# A's spacing warn at 261 m ("scattered, not a high street"), *worse than the 255 m that retired
# ballarat this round*, plus a frontage poster seated inside edge 1037's kerb. With the donor's
# texture layer merged (37 shops) both go away and E's donor's own density is preserved.
#
# THE MERGED SHOPS CARRY NO `godverseShopId`, DELIBERATELY. The field is the GODVERSE stock
# identity; an OSM cafe has none, and minting one from its OSM id would push a third id space into
# a field that already carries two (thriftgod census ≤2992 · dealgod store ids). OSM ids are the
# same order of magnitude as dealgod store ids, so a collision is *possible* — and A's validator
# calls a duplicate godverseShopId mis-stocking (charter risk #3: two shops keyed to one atlas,
# one of them Monster Robot's crate). Unkeyed is both true and safe: they fall soft to tier 0,
# which is exactly what a milkbar with no stock should do. A's coverage warn states this honestly.
def norm(n):
    return ' '.join((n or '').lower().split())


def near(a, b, m=150.0):                    # metres, flat-earth at suburb scale — plenty at 150 m
    dy = (a['lat'] - b['lat']) * 111_320.0
    dx = (a['lon'] - b['lon']) * 111_320.0 * math.cos(math.radians(a['lat']))
    return math.hypot(dx, dy) <= m


swaps = 0
for d in sorted(donor.get('shops', []), key=lambda s: s['id']):
    hit = next((s for s in shops if norm(s['name']) == norm(d['name']) and near(s, d)), None)
    if hit:                                 # the donor's own entry for a shop we already hold
        swaps += 1
        continue
    if d['id'] in {s['id'] for s in shops}:
        continue
    shops.append({'id': d['id'], 'name': d['name'], 'type': d['type'],
                  'lat': d['lat'], 'lon': d['lon'], 'suburb': d.get('suburb', '')})
texture_n = len(shops) - census_n - extra_n
shops.sort(key=lambda s: s['id'])           # deterministic order, extras + texture included

out = dict(donor)
out.update({
    'key': OUT,
    'town': donor['town'],
    'source': 'godverse+osm',                       # shops = thriftgod census, roads = OSM (donor)
    'generator': 'pipeline/godverse_town.py',
    'attribution': donor['attribution'] + ' · shop census: GODVERSE/thriftgod',
    'shops': shops,
    # The donor's shop-side counts (raw/heroes/texture/textureDropped) describe the shop layer we
    # REPLACE, so carrying them would state a census of shops this cache doesn't contain. Keep the
    # donor's road count; recount our own shops.
    'counts': {**{k: v for k, v in donor.get('counts', {}).items() if k == 'roads'},
               'shops': len(shops), 'census': census_n, 'extra': extra_n,
               'texture': texture_n, 'donorSwaps': swaps},
})
dst = os.path.join(TOWNS, f'{OUT}.json')
with open(dst, 'w') as fh:
    json.dump(out, fh, indent=1)
types = {}
for s in shops:
    types[s['type']] = types.get(s['type'], 0) + 1
keyed = sum(1 for s in shops if 'godverseShopId' in s)
print(f"{OUT}: {len(shops)} shops = {census_n} census + {extra_n} extra + {texture_n} donor-texture "
      f"({swaps} donor heroes swapped for census) · {keyed} keyed w/ godverseShopId")
print(f"  {types} + {len(out.get('roads', []))} OSM roads -> {dst}")