#!/usr/bin/env python3
"""build_towns.py — the real-map scout (ROUND17 ledger #6). A port of thriftgod `build_city()`'s
Overpass route to PER-TOWN real Australian caches in the Lane-A `procity-town-cache/1` contract.

    python3 pipeline/build_towns.py                 # build every town (cached raw reused, no re-fetch)
    python3 pipeline/build_towns.py katoomba_real    # one town
    python3 pipeline/build_towns.py --refetch        # ignore cached raw, re-hit Overpass (deliberate)

Pipeline per town: bounded Overpass query → RAW response cached in-repo (`web/assets/towns/_raw/`, so
re-runs NEVER re-fetch) → dedupe (OSM maps a shop as node AND way) → suburb fill (untagged adopt the
nearest tagged neighbour) → parody the big trademarked charity/franchise names (indie shops keep their
real OSM name) → map OSM `shop=*` to a registry SHOP_TYPE → `web/assets/towns/<key>.json`.

DATA DISCIPLINE (ledger #6): deterministic (same raw → byte-identical cache; shops sorted by OSM id,
no RNG); bounded queries (a cache is ONE compact town, span < 5 km — the mega-strip risk); ODbL
attribution ships IN every cache (`license`/`attribution`) + `SOURCES.md`. Hours are NOT cached — the
registry supplies them at plan time (contract shop = {id,name,type,lat,lon,suburb}). Overpass is an
outward-facing fetch (John-authorized scout, once per town, raw cached).
"""
import os, sys, json, re, math, time, datetime, hashlib, urllib.request, urllib.parse, urllib.error

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TOWNS_DIR = os.path.join(ROOT, "web", "assets", "towns")
RAW_DIR = os.path.join(TOWNS_DIR, "_raw")
OVERPASS = "https://overpass-api.de/api/interpreter"
UA = "PROCITY-townscout/0.1 (real-map scout; OpenStreetMap ODbL data)"
COURTESY_SLEEP = 4        # pause between Overpass calls — the standing auth covers the fetches,
                          # courtesy covers the cadence (R21 pack scale: ~2 calls × ~20 towns)

# Compact real AU towns: center (lat,lon) + query box span (km). `katoomba_real` is the fixture
# comparison (vs the hand-made `katoomba` in osm_fixture.js). Spans kept tight so each is ONE town.
TOWNS = {
    # ── the beta five (v4.0-alpha/beta — A has these absorbed + pinned) ─────────────────────────
    "katoomba_real":    {"town": "Katoomba",    "state": "NSW", "center": (-33.7145, 150.3120), "span_km": 3.0},
    "newtown_real":     {"town": "Newtown",     "state": "NSW", "center": (-33.8985, 151.1790), "span_km": 2.6},
    "fremantle_real":   {"town": "Fremantle",   "state": "WA",  "center": (-32.0555, 115.7480), "span_km": 2.6},
    "bendigo_real":     {"town": "Bendigo",     "state": "VIC", "center": (-36.7580, 144.2810), "span_km": 3.4},
    "castlemaine_real": {"town": "Castlemaine", "state": "VIC", "center": (-37.0640, 144.2170), "span_km": 3.0},
    # ── the v4.0 curated pack (ROUND21 ledger #5 — DATA-ONLY; A absorbs + pins in R22) ──────────
    # Curated on the instincts the first five earned: pick for real SECONDHAND density (the heroes
    # are the game — a town with no op shops isn't a PROCITY town), a walkable centre that fits a
    # ~2–3 km bbox (the mega-strip law), and a national spread. Any town that misses the ≥6-shop
    # floor or the 5 km span is dropped by the builder and counted — the pack is what survives.
    "glebe_real":        {"town": "Glebe",        "state": "NSW", "center": (-33.8790, 151.1870), "span_km": 2.2},
    "marrickville_real": {"town": "Marrickville", "state": "NSW", "center": (-33.9110, 151.1550), "span_km": 2.4},
    # Newcastle: centred on the Hamilton / Beaumont St strip, NOT the CBD — the CBD bbox held only 2
    # heroes; Hamilton is where Newcastle's actual secondhand shops are (my first centre missed it).
    "newcastle_real":    {"town": "Newcastle",    "state": "NSW", "center": (-32.9235, 151.7440), "span_km": 2.4},
    "wollongong_real":   {"town": "Wollongong",   "state": "NSW", "center": (-34.4250, 150.8940), "span_km": 2.8},
    "bowral_real":       {"town": "Bowral",       "state": "NSW", "center": (-34.4780, 150.4180), "span_km": 2.2},
    "fitzroy_real":      {"town": "Fitzroy",      "state": "VIC", "center": (-37.7990, 144.9790), "span_km": 2.2},
    "brunswick_real":    {"town": "Brunswick",    "state": "VIC", "center": (-37.7700, 144.9600), "span_km": 2.6},
    # R24 — RETIRED (counted), and for a DIFFERENT illness than Toowoomba. A's spacing warn caught
    # it at median NN 255 m. Ballarat is NOT dead: its raw bbox holds 103 POIs and its densest 400 m
    # disc is 47 of them (Toowoomba's densest was 3). But its retail core is alive exactly where the
    # GAME isn't — that disc is cafes/clothes/bakeries, while the op-shops sit 854 m, 1042 m and
    # 1709 m out. The heroes and the density don't coincide, so no bbox holds both. Measured, every
    # span (real builds, not simulated):
    #     span    shops heroes  medNN  hub160     op-shops
    #     1.0 km      8      2   184 m  4/8  (50%)   0   <- the live core: a thrift town with NO thrift shop
    #     2.0 km     12      3   186 m  3/12 (25%)   1
    #     3.0 km     20      5   255 m  4/20 (20%)   2   <- as shipped
    # Every row is worse than any shipped town (darwin 68 m/58%, glebe 60 m, redhill 73 m). The
    # binding constraint is mine, not OSM's: with 2 heroes the 3x cap keeps 6 of 47 texture POIs and
    # the subsample is spatially blind, so the survivors scatter no matter how tight the box. Fixing
    # THAT (seeding texture near heroes) is a pipeline change that would move every town's golden —
    # a v5.x proposal, not a curation move. Cache + raw snapshots removed. See LANE_E_NOTES R24.
    # "ballarat_real":     {"town": "Ballarat",     "state": "VIC", "center": (-37.5620, 143.8500), "span_km": 3.0},
    "daylesford_real":   {"town": "Daylesford",   "state": "VIC", "center": (-37.3430, 144.1430), "span_km": 2.0},
    "geelong_real":      {"town": "Geelong",      "state": "VIC", "center": (-38.1490, 144.3600), "span_km": 3.0},
    "westend_real":      {"town": "West End",     "state": "QLD", "center": (-27.4830, 153.0100), "span_km": 2.2},
    # R24 ledger #3 — the v5 crate's real street: **Monster Robot Party, 147 Musgrave Rd, Red Hill,
    # Brisbane** (John's ruling: the crate ships as the shop it really is, not Newtown). Centre/span
    # per G's ask (they geocoded the address); westend_real is the only other Brisbane cache and stops
    # ~2.3 km south. This is the DONOR G runs godverse_town.py against → redhill_godverse.
    "redhill_real":      {"town": "Red Hill",     "state": "QLD", "center": (-27.4553, 153.0064), "span_km": 2.4},
    # ── RETIRED R23 (E's curation call, counted) — toowoomba_real ────────────────────────────────
    # D's R22 finding: DEAD — median NN 388 m, 3/12 shops near the hub, **0 patronage finds in 1417
    # checks**. I tried the re-bbox first (3.0 km CBD-wide → 1.6 km on the Ruthven St heart, centre
    # -27.5610,151.9535): it FIXED the named metric (median NN 395.7 → 89.4 m raw — normal for the
    # pack) but made the real thing WORSE — hub density 3/12 → **2/12 within 160 m, the worst in the
    # pack** (D's alive-darwin is 7/12). Toowoomba's shops are PAIRS strung along a road, not a high
    # street; its densest 300 m cluster is only 3 shops, below MIN_TOWN_SHOPS, so no tighter box
    # exists. Shipping it would have gamed the spacing metric while leaving a town nobody shops in.
    # Retired rather than padded. (QLD keeps westend_real, 11 heroes / 44 shops.)
    # "toowoomba_real":  {"town": "Toowoomba", "state": "QLD", "center": (-27.5600, 151.9540), "span_km": 3.0},
    "northbridge_real":  {"town": "Northbridge",  "state": "WA",  "center": (-31.9490, 115.8570), "span_km": 2.4},
    "adelaide_real":     {"town": "Adelaide",     "state": "SA",  "center": (-34.9240, 138.6050), "span_km": 2.8},
    "hobart_real":       {"town": "Hobart",       "state": "TAS", "center": (-42.8820, 147.3270), "span_km": 2.8},
    "launceston_real":   {"town": "Launceston",   "state": "TAS", "center": (-41.4380, 147.1370), "span_km": 2.6},
    "braddon_real":      {"town": "Braddon",      "state": "ACT", "center": (-35.2740, 149.1330), "span_km": 2.4},
    "darwin_real":       {"town": "Darwin",       "state": "NT",  "center": (-12.4630, 130.8420), "span_km": 2.6},
}
# the beta five are already pinned by A; `--pack` builds only the new curated candidates
BETA_FIVE = ("katoomba_real", "newtown_real", "fremantle_real", "bendigo_real", "castlemaine_real")

# ── the class list (v4.0-beta, ROUND20 ledger #2 — C's LANE_C_PUB §6 table, John's SUBTLE directive) ──
# HEROES = the secondhand shops (they ARE the game) — never subsampled. TEXTURE = the everyday
# main-street staples a real AU town has (bakery/cafe/chemist/newsagent/clothes/…). Tasteful staples
# ONLY per C's guardrail: no vehicle/tyres/fuel, funeral, storage, industrial, and no pure services
# the player never browses (hairdresser/beauty/travel_agency/bank/laundry/estate_agent).
HERO_CLASSES = {"charity", "second_hand", "antiques", "vintage", "books", "bookshop",
                "music", "video", "games", "video_games", "pawnbroker", "toys"}
TEXTURE_CLASSES = {
    "bakery", "cafe", "confectionery", "pastry", "chocolate", "deli", "delicatessen", "butcher",
    "cheese", "coffee", "tea", "convenience", "kiosk", "general", "dairy", "newsagent", "stationery",
    "chemist", "pharmacy", "cosmetics", "perfumery", "clothes", "fashion", "boutique", "shoes", "bag",
    "jewelry", "jewellery", "department_store", "variety_store", "supermarket", "hardware",
    "doityourself", "trade", "electronics", "mobile_phone", "furniture", "houseware", "homeware",
    "greengrocer", "florist", "farm", "garden_centre",
}
AMENITY_CLASSES = {"cafe"}                                   # C's †: cafe is amenity=*, not shop=*
SHOP_CLASSES = (HERO_CLASSES | TEXTURE_CLASSES) - AMENITY_CLASSES

# C's copy-paste map (LANE_C_PUB §6). Every target is an EXISTING registry type — no new archetypes.
# (+ `video_games`→video: C listed video/games→video; video_games is the same family and was already
#    shipping as `video` pre-widening — kept so those shops don't regress to opshop. Flagged to C.)
TYPE_MAP = {
    "charity": "opshop", "second_hand": "opshop", "antiques": "opshop", "vintage": "opshop",
    "books": "book", "bookshop": "book", "music": "record", "video": "video", "games": "video",
    "video_games": "video", "pawnbroker": "pawn", "toys": "toy",
    "bakery": "milkbar", "cafe": "milkbar", "confectionery": "milkbar", "pastry": "milkbar",
    "chocolate": "milkbar", "deli": "milkbar", "delicatessen": "milkbar", "butcher": "milkbar",
    "cheese": "milkbar", "coffee": "milkbar", "tea": "milkbar", "convenience": "milkbar",
    "kiosk": "milkbar", "general": "milkbar", "dairy": "milkbar", "newsagent": "milkbar",
    "stationery": "milkbar", "chemist": "milkbar", "pharmacy": "milkbar", "cosmetics": "milkbar",
    "perfumery": "milkbar",
    "clothes": "dept", "fashion": "dept", "boutique": "dept", "shoes": "dept", "bag": "dept",
    "jewelry": "dept", "jewellery": "dept", "department_store": "dept", "variety_store": "dept",
    "supermarket": "dept", "hardware": "dept", "doityourself": "dept", "trade": "dept",
    "electronics": "dept", "mobile_phone": "dept", "furniture": "dept", "houseware": "dept",
    "homeware": "dept",
    "greengrocer": "stall", "florist": "stall", "farm": "stall", "garden_centre": "stall",
}
# John's directive: texture, not takeover. If a town's general retail would drown the secondhand
# strip, seed-subsample the TEXTURE classes toward C's ~2–3× band; heroes are never touched and
# every drop is counted (counts.textureDropped).
TEXTURE_CAP = 3.0

# ── roads[] (schema v2, ROUND18 REAL ROADS) ─────────────────────────────────────────────────────
# We FETCH every OSM highway tier A's ROAD_KIND maps (→ raw cache, full provenance / beta-ready) but
# EMIT only the town STREET graph. Fidelity cut (charter risk #4, flagged to A who owns the knob):
# the service/track/footway/path/steps/cycleway tier is EXCLUDED — in Katoomba that's 1461 of 2506
# ways (Blue-Mountains bushwalking tracks, sidewalks, stairs, driveways, parking aisles), noise for a
# town street graph and a way-count A's lift would build 2500+ edges from. The full set stays in the
# raw response if A wants a tier back. `kind` ships as the raw OSM highway class; A maps it (ROAD_KIND).
ROAD_HW = ("motorway|trunk|primary|secondary|tertiary|unclassified|residential|living_street|road"
           "|service|track|pedestrian|footway|path|steps|cycleway")
STREET_KINDS = {"motorway", "trunk", "primary", "secondary", "tertiary", "unclassified",
                "residential", "living_street", "road", "pedestrian"}

# The big trademarked charity/franchise chains get loving parodies (ported from thriftgod, John's call
# for the live game); independent shops keep their real OSM names (factual ODbL data).
PARODY = [
    (r"salvos( stores?)?|(the )?salvation army", "Slave-ohs"),
    (r"lifeline", "Tightrope"),
    (r"rspca", "RSYMCA"),
    (r"vinnies|st\.? ?vincent('?s| de paul)?( society)?|svdp", "Sinnies"),
    (r"(australian )?red cross", "Red Floss"),
    (r"save the children", "Shave the Children"),
    (r"anglicare", "Tanglecare"),
    (r"uniting ?care|unitingcare|uniting", "Unknitting"),
    (r"good sammys?|good samaritans?", "Okay Sammys"),
    (r"mission australia", "Mission Improbable"),
    (r"brotherhood of st\.? ?laurence", "Brotherhood of St Lozenge"),
    (r"cash ?converters?", "Cashtheists"),
]


def parodize(name):
    for pat, rep in PARODY:
        name = re.sub(pat, rep, name, flags=re.I)
    return name


def punct_blind(s):
    return re.sub(r"[^a-z0-9]", "", s.lower())


def bbox(center, span_km):
    lat, lon = center
    dlat = (span_km / 2.0) / 111.0
    dlon = (span_km / 2.0) / (111.0 * math.cos(math.radians(lat)))
    return {"minLat": lat - dlat, "minLon": lon - dlon, "maxLat": lat + dlat, "maxLon": lon + dlon}


def haversine_m(a, b):
    R = 6371000.0
    p1, p2 = math.radians(a[0]), math.radians(b[0])
    dp = math.radians(b[0] - a[0]); dl = math.radians(b[1] - a[1])
    h = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2
    return 2 * R * math.asin(min(1.0, math.sqrt(h)))


def overpass_post(q):
    """POST an Overpass query with retry/backoff — the public API 504s / times out under load."""
    req = urllib.request.Request(OVERPASS, data=urllib.parse.urlencode({"data": q}).encode(),
                                 headers={"User-Agent": UA})
    for attempt in range(4):
        try:
            with urllib.request.urlopen(req, timeout=180) as r:
                return json.load(r).get("elements", [])
        except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as e:
            if attempt == 3:
                raise
            wait = 15 * (attempt + 1)
            print(f"    Overpass {getattr(e, 'code', type(e).__name__)} — retry {attempt + 1}/3 in {wait}s")
            time.sleep(wait)


def shops_query(bb):
    box = f'{bb["minLat"]:.5f},{bb["minLon"]:.5f},{bb["maxLat"]:.5f},{bb["maxLon"]:.5f}'
    return (f'[out:json][timeout:180];('
            f'nwr["shop"~"^({"|".join(sorted(SHOP_CLASSES))})$"]["name"]({box});'
            f'nwr["amenity"~"^({"|".join(sorted(AMENITY_CLASSES))})$"]["name"]({box});'
            f');out center;')


def fetch_raw(key, bb, refetch):
    """Raw Overpass elements, cached in-repo so re-runs never re-fetch. Auto-refetches when the CLASS
    LIST changes (each snapshot records its query) — that's how the R20 widening lands as a NEW raw
    snapshot without anyone remembering --refetch."""
    os.makedirs(RAW_DIR, exist_ok=True)
    path = os.path.join(RAW_DIR, key + ".overpass.json")
    q = shops_query(bb)
    if os.path.exists(path) and not refetch:
        w = json.load(open(path))
        if w.get("query") == q:
            return w["elements"], w.get("fetchedAt")
        print(f"  [{key}] class list changed → new raw snapshot")
    print(f"  [{key}] fetching Overpass shops (widened, bounded)")
    els = overpass_post(q)
    fetched = datetime.date.today().isoformat()
    json.dump({"fetchedAt": fetched, "endpoint": OVERPASS, "query": q, "bbox": bb, "elements": els},
              open(path, "w"), indent=1)
    time.sleep(COURTESY_SLEEP)                  # be a good Overpass citizen between towns
    return els, fetched


def fetch_roads_raw(key, bb, refetch):
    """Raw OSM highway ways (with geometry) for the town bbox, cached in-repo like the shops — and,
    like the shops, auto-refetched when the QUERY changes. That symmetry matters: the bbox lives in
    the query, so re-centring a town must invalidate BOTH snapshots. (R21: it didn't. Re-centring
    Newcastle refetched the shops but silently reused the old CBD roads — Hamilton shops seated
    against a road graph 2.5 km away, and the town marched out with ZERO shops.)"""
    os.makedirs(RAW_DIR, exist_ok=True)
    path = os.path.join(RAW_DIR, key + ".roads.overpass.json")
    q = (f'[out:json][timeout:120];'
         f'way["highway"~"^({ROAD_HW})$"]'
         f'({bb["minLat"]:.5f},{bb["minLon"]:.5f},{bb["maxLat"]:.5f},{bb["maxLon"]:.5f});'
         f'out geom;')
    if os.path.exists(path) and not refetch:
        w = json.load(open(path))
        if w.get("query") == q:
            return w["elements"]
        print(f"  [{key}] roads bbox/query changed → new raw snapshot")
    print(f"  [{key}] fetching Overpass roads (highway ways, bounded)")
    ways = overpass_post(q)
    json.dump({"fetchedAt": datetime.date.today().isoformat(), "endpoint": OVERPASS, "query": q,
               "bbox": bb, "elements": ways}, open(path, "w"), indent=1)
    time.sleep(COURTESY_SLEEP)
    return ways


def roads_from_ways(ways):
    """OSM highway ways → roads[] (contract v2): {id, kind, name?, pts:[[lat,lon],…]}. Emit only the
    STREET tier; ship full geometry (A simplifies with Douglas–Peucker); deterministic (id-sorted)."""
    roads = []
    for w in ways:
        hwy = w.get("tags", {}).get("highway")
        geom = w.get("geometry")
        if hwy not in STREET_KINDS or not isinstance(geom, list) or len(geom) < 2:
            continue
        pts = [[round(g["lat"], 6), round(g["lon"], 6)] for g in geom
               if isinstance(g.get("lat"), (int, float)) and isinstance(g.get("lon"), (int, float))]
        if len(pts) < 2:
            continue
        rd = {"id": w["id"], "kind": hwy, "pts": pts}
        name = w.get("tags", {}).get("name")
        if name:
            rd["name"] = name[:60]
        roads.append(rd)
    return sorted(roads, key=lambda r: r["id"])


def process(key, cfg, els, fetched, roads=None):
    center = cfg["center"]
    rows, seen = [], set()
    for el in els:
        t = el.get("tags", {})
        name, osm_shop = t.get("name"), (t.get("shop") or t.get("amenity"))   # cafe rides amenity=*
        lat = el.get("lat", el.get("center", {}).get("lat"))
        lon = el.get("lon", el.get("center", {}).get("lon"))
        if not (name and osm_shop and isinstance(lat, (int, float)) and isinstance(lon, (int, float))):
            continue
        dkey = (punct_blind(name), round(lat, 3), round(lon, 3))   # node/way dup fuzz (~110 m)
        if dkey in seen:
            continue
        seen.add(dkey)
        suburb = next((t[k] for k in ("addr:suburb", "addr:city", "addr:town", "addr:place") if t.get(k)), "")
        rows.append({"id": el["id"], "name": name[:60], "osm": osm_shop, "lat": lat, "lon": lon, "suburb": suburb})
    # suburb fill: untagged shops adopt the nearest tagged neighbour within ~3 km (that's a suburb)
    tagged = [r for r in rows if r["suburb"]]
    for r in rows:
        if r["suburb"] or not tagged:
            continue
        near = min(tagged, key=lambda s: haversine_m((r["lat"], r["lon"]), (s["lat"], s["lon"])))
        if haversine_m((r["lat"], r["lon"]), (near["lat"], near["lon"])) <= 3000:
            r["suburb"] = near["suburb"]
    # parody + type-map + second dedupe (parodized name in same suburb = same shop); id-sorted = deterministic
    shops, seen2 = [], set()
    for r in sorted(rows, key=lambda r: r["id"]):
        pname = parodize(r["name"])[:60]
        k2 = (punct_blind(pname), r["suburb"])
        if k2 in seen2:
            continue
        seen2.add(k2)
        shops.append({"id": r["id"], "name": pname, "type": TYPE_MAP.get(r["osm"], "opshop"),
                      "lat": round(r["lat"], 6), "lon": round(r["lon"], 6), "suburb": r["suburb"],
                      "_osm": r["osm"]})
    # ── SUBTLE (John's directive): texture, not takeover ────────────────────────────────────────
    # The secondhand HEROES are never subsampled — they're the game. If a town's general retail would
    # drown the strip, seed-subsample the TEXTURE toward C's ~2–3× band (deterministic: rank by
    # sha256(townKey:osmId), keep the top `cap`) and COUNT the drops like every other drop.
    heroes = [s for s in shops if s["_osm"] in HERO_CLASSES]
    texture = [s for s in shops if s["_osm"] not in HERO_CLASSES]
    cap, dropped = int(round(TEXTURE_CAP * len(heroes))), 0
    if len(texture) > cap:
        texture.sort(key=lambda s: hashlib.sha256(f"{key}:{s['id']}".encode()).hexdigest())
        dropped, texture = len(texture) - cap, texture[:cap]
    shops = sorted(heroes + texture, key=lambda s: s["id"])       # deterministic emit order
    for s in shops:
        s.pop("_osm", None)
    bb = bbox(center, cfg["span_km"])
    counts = {"raw": len(els), "shops": len(shops), "heroes": len(heroes),
              "texture": len(texture), "textureDropped": dropped}
    cache = {
        "schema": "procity-town-cache/2" if roads else "procity-town-cache/1",
        "key": key, "town": cfg["town"], "source": "osm",
        "license": "ODbL 1.0", "attribution": "© OpenStreetMap contributors",
        "generator": "pipeline/build_towns.py", "fetchedAt": fetched,
        "center": {"lat": center[0], "lon": center[1]},
        "bbox": bb, "counts": counts, "shops": shops,
    }
    if roads:
        counts["roads"] = len(roads)
        cache["roads"] = roads                  # schema v2 — A's plan_osm builds the real street graph
    return cache


def span_km(shops):
    if not shops:
        return 0.0
    lats = [s["lat"] for s in shops]; lons = [s["lon"] for s in shops]
    latm = haversine_m((min(lats), lons[0]), (max(lats), lons[0]))
    lonm = haversine_m((lats[0], min(lons)), (lats[0], max(lons)))
    return max(latm, lonm) / 1000.0


def main():
    args = [a for a in sys.argv[1:] if not a.startswith("-")]
    refetch = "--refetch" in sys.argv
    no_roads = "--no-roads" in sys.argv          # build a v1 cache (marched fallback), no road fetch
    pack = "--pack" in sys.argv                  # only the NEW curated candidates (the beta five are
    keys = args or ([k for k in TOWNS if k not in BETA_FIVE] if pack else list(TOWNS))  # pinned by A)
    os.makedirs(TOWNS_DIR, exist_ok=True)
    kept, dropped = [], []
    for key in keys:
        cfg = TOWNS[key]
        bb = bbox(cfg["center"], cfg["span_km"])
        els, fetched = fetch_raw(key, bb, refetch)
        roads = None
        if not no_roads:
            roads = roads_from_ways(fetch_roads_raw(key, bb, refetch))
        cache = process(key, cfg, els, fetched, roads)
        n, sp = len(cache["shops"]), span_km(cache["shops"])
        types = {}
        for s in cache["shops"]:
            types[s["type"]] = types.get(s["type"], 0) + 1
        ok = n >= 6 and sp < 5.0
        if ok:
            json.dump(cache, open(os.path.join(TOWNS_DIR, key + ".json"), "w"), indent=1)
            kept.append(key)
        else:
            dropped.append((key, n, sp))
        c = cache["counts"]
        sub = f" −{c['textureDropped']} subsampled" if c.get("textureDropped") else ""
        print(f"  {'OK ' if ok else 'LOW'} {key:16} shops={n:>3} (heroes {c['heroes']:>2} + texture "
              f"{c['texture']:>3}{sub}) roads={len(roads or [])!s:>4}  {dict(sorted(types.items()))}")
    print(f"\nBUILT {len(kept)} town cache(s) → web/assets/towns/: {', '.join(kept)}")
    if dropped:
        print(f"DROPPED (< 6 shops or span ≥ 5 km): {dropped}")
    write_index()


def write_index():
    """web/assets/towns/index.json (ROUND20 ledger #2) — B's selector derives the town list from this
    instead of hardcoding REAL_TOWNS. Built from every cache on disk, not just this run."""
    towns = []
    for f in sorted(x for x in os.listdir(TOWNS_DIR) if x.endswith(".json") and x != "index.json"):
        c = json.load(open(os.path.join(TOWNS_DIR, f)))
        k = c.get("key", f[:-5])
        # state: the cache's own field first (so ANY lane's town self-describes — e.g. Lane G's
        # godverse caches), then my TOWNS config, then blank. The index rosters every cache in the
        # dir, not just mine.
        state = c.get("state") or TOWNS.get(k, {}).get("state", "")
        towns.append({"key": k, "town": c.get("town", k), "state": state,
                      "shops": len(c.get("shops", [])), "roads": bool(c.get("roads")),
                      "source": c.get("source", "osm")})
    idx = {"schema": "procity-towns-index/1", "generated": datetime.date.today().isoformat(),
           "license": "ODbL 1.0", "attribution": "© OpenStreetMap contributors", "towns": towns}
    json.dump(idx, open(os.path.join(TOWNS_DIR, "index.json"), "w"), indent=1)
    print(f"INDEX  → web/assets/towns/index.json ({len(towns)} towns: "
          f"{', '.join(t['key'] for t in towns)})")


if __name__ == "__main__":
    main()