PROCITY/pipeline/build_towns.py

#!/usr/bin/env python3
"""build_towns.py — the real-map scout (ROUND17 ledger #6). A port of thriftgod `build_city()`'s
Overpass route to PER-TOWN real Australian caches in the Lane-A `procity-town-cache/1` contract.

    python3 pipeline/build_towns.py                 # build every town (cached raw reused, no re-fetch)
    python3 pipeline/build_towns.py katoomba_real    # one town
    python3 pipeline/build_towns.py --refetch        # ignore cached raw, re-hit Overpass (deliberate)

Pipeline per town: bounded Overpass query → RAW response cached in-repo (`web/assets/towns/_raw/`, so
re-runs NEVER re-fetch) → dedupe (OSM maps a shop as node AND way) → suburb fill (untagged adopt the
nearest tagged neighbour) → parody the big trademarked charity/franchise names (indie shops keep their
real OSM name) → map OSM `shop=*` to a registry SHOP_TYPE → `web/assets/towns/<key>.json`.

DATA DISCIPLINE (ledger #6): deterministic (same raw → byte-identical cache; shops sorted by OSM id,
no RNG); bounded queries (a cache is ONE compact town, span < 5 km — the mega-strip risk); ODbL
attribution ships IN every cache (`license`/`attribution`) + `SOURCES.md`. Hours are NOT cached — the
registry supplies them at plan time (contract shop = {id,name,type,lat,lon,suburb}). Overpass is an
outward-facing fetch (John-authorized scout, once per town, raw cached).
"""
import os, sys, json, re, math, time, datetime, hashlib, urllib.request, urllib.parse, urllib.error

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TOWNS_DIR = os.path.join(ROOT, "web", "assets", "towns")
RAW_DIR = os.path.join(TOWNS_DIR, "_raw")
OVERPASS = "https://overpass-api.de/api/interpreter"
UA = "PROCITY-townscout/0.1 (real-map scout; OpenStreetMap ODbL data)"

# Compact real AU towns: center (lat,lon) + query box span (km). `katoomba_real` is the fixture
# comparison (vs the hand-made `katoomba` in osm_fixture.js). Spans kept tight so each is ONE town.
TOWNS = {
    "katoomba_real":    {"town": "Katoomba",    "state": "NSW", "center": (-33.7145, 150.3120), "span_km": 3.0},
    "newtown_real":     {"town": "Newtown",     "state": "NSW", "center": (-33.8985, 151.1790), "span_km": 2.6},
    "fremantle_real":   {"town": "Fremantle",   "state": "WA",  "center": (-32.0555, 115.7480), "span_km": 2.6},
    "bendigo_real":     {"town": "Bendigo",     "state": "VIC", "center": (-36.7580, 144.2810), "span_km": 3.4},
    "castlemaine_real": {"town": "Castlemaine", "state": "VIC", "center": (-37.0640, 144.2170), "span_km": 3.0},
}

# ── the class list (v4.0-beta, ROUND20 ledger #2 — C's LANE_C_PUB §6 table, John's SUBTLE directive) ──
# HEROES = the secondhand shops (they ARE the game) — never subsampled. TEXTURE = the everyday
# main-street staples a real AU town has (bakery/cafe/chemist/newsagent/clothes/…). Tasteful staples
# ONLY per C's guardrail: no vehicle/tyres/fuel, funeral, storage, industrial, and no pure services
# the player never browses (hairdresser/beauty/travel_agency/bank/laundry/estate_agent).
HERO_CLASSES = {"charity", "second_hand", "antiques", "vintage", "books", "bookshop",
                "music", "video", "games", "video_games", "pawnbroker", "toys"}
TEXTURE_CLASSES = {
    "bakery", "cafe", "confectionery", "pastry", "chocolate", "deli", "delicatessen", "butcher",
    "cheese", "coffee", "tea", "convenience", "kiosk", "general", "dairy", "newsagent", "stationery",
    "chemist", "pharmacy", "cosmetics", "perfumery", "clothes", "fashion", "boutique", "shoes", "bag",
    "jewelry", "jewellery", "department_store", "variety_store", "supermarket", "hardware",
    "doityourself", "trade", "electronics", "mobile_phone", "furniture", "houseware", "homeware",
    "greengrocer", "florist", "farm", "garden_centre",
}
AMENITY_CLASSES = {"cafe"}                                   # C's †: cafe is amenity=*, not shop=*
SHOP_CLASSES = (HERO_CLASSES | TEXTURE_CLASSES) - AMENITY_CLASSES

# C's copy-paste map (LANE_C_PUB §6). Every target is an EXISTING registry type — no new archetypes.
# (+ `video_games`→video: C listed video/games→video; video_games is the same family and was already
#    shipping as `video` pre-widening — kept so those shops don't regress to opshop. Flagged to C.)
TYPE_MAP = {
    "charity": "opshop", "second_hand": "opshop", "antiques": "opshop", "vintage": "opshop",
    "books": "book", "bookshop": "book", "music": "record", "video": "video", "games": "video",
    "video_games": "video", "pawnbroker": "pawn", "toys": "toy",
    "bakery": "milkbar", "cafe": "milkbar", "confectionery": "milkbar", "pastry": "milkbar",
    "chocolate": "milkbar", "deli": "milkbar", "delicatessen": "milkbar", "butcher": "milkbar",
    "cheese": "milkbar", "coffee": "milkbar", "tea": "milkbar", "convenience": "milkbar",
    "kiosk": "milkbar", "general": "milkbar", "dairy": "milkbar", "newsagent": "milkbar",
    "stationery": "milkbar", "chemist": "milkbar", "pharmacy": "milkbar", "cosmetics": "milkbar",
    "perfumery": "milkbar",
    "clothes": "dept", "fashion": "dept", "boutique": "dept", "shoes": "dept", "bag": "dept",
    "jewelry": "dept", "jewellery": "dept", "department_store": "dept", "variety_store": "dept",
    "supermarket": "dept", "hardware": "dept", "doityourself": "dept", "trade": "dept",
    "electronics": "dept", "mobile_phone": "dept", "furniture": "dept", "houseware": "dept",
    "homeware": "dept",
    "greengrocer": "stall", "florist": "stall", "farm": "stall", "garden_centre": "stall",
}
# John's directive: texture, not takeover. If a town's general retail would drown the secondhand
# strip, seed-subsample the TEXTURE classes toward C's ~2–3× band; heroes are never touched and
# every drop is counted (counts.textureDropped).
TEXTURE_CAP = 3.0

# ── roads[] (schema v2, ROUND18 REAL ROADS) ─────────────────────────────────────────────────────
# We FETCH every OSM highway tier A's ROAD_KIND maps (→ raw cache, full provenance / beta-ready) but
# EMIT only the town STREET graph. Fidelity cut (charter risk #4, flagged to A who owns the knob):
# the service/track/footway/path/steps/cycleway tier is EXCLUDED — in Katoomba that's 1461 of 2506
# ways (Blue-Mountains bushwalking tracks, sidewalks, stairs, driveways, parking aisles), noise for a
# town street graph and a way-count A's lift would build 2500+ edges from. The full set stays in the
# raw response if A wants a tier back. `kind` ships as the raw OSM highway class; A maps it (ROAD_KIND).
ROAD_HW = ("motorway|trunk|primary|secondary|tertiary|unclassified|residential|living_street|road"
           "|service|track|pedestrian|footway|path|steps|cycleway")
STREET_KINDS = {"motorway", "trunk", "primary", "secondary", "tertiary", "unclassified",
                "residential", "living_street", "road", "pedestrian"}

# The big trademarked charity/franchise chains get loving parodies (ported from thriftgod, John's call
# for the live game); independent shops keep their real OSM names (factual ODbL data).
PARODY = [
    (r"salvos( stores?)?|(the )?salvation army", "Slave-ohs"),
    (r"lifeline", "Tightrope"),
    (r"rspca", "RSYMCA"),
    (r"vinnies|st\.? ?vincent('?s| de paul)?( society)?|svdp", "Sinnies"),
    (r"(australian )?red cross", "Red Floss"),
    (r"save the children", "Shave the Children"),
    (r"anglicare", "Tanglecare"),
    (r"uniting ?care|unitingcare|uniting", "Unknitting"),
    (r"good sammys?|good samaritans?", "Okay Sammys"),
    (r"mission australia", "Mission Improbable"),
    (r"brotherhood of st\.? ?laurence", "Brotherhood of St Lozenge"),
    (r"cash ?converters?", "Cashtheists"),
]


def parodize(name):
    for pat, rep in PARODY:
        name = re.sub(pat, rep, name, flags=re.I)
    return name


def punct_blind(s):
    return re.sub(r"[^a-z0-9]", "", s.lower())


def bbox(center, span_km):
    lat, lon = center
    dlat = (span_km / 2.0) / 111.0
    dlon = (span_km / 2.0) / (111.0 * math.cos(math.radians(lat)))
    return {"minLat": lat - dlat, "minLon": lon - dlon, "maxLat": lat + dlat, "maxLon": lon + dlon}


def haversine_m(a, b):
    R = 6371000.0
    p1, p2 = math.radians(a[0]), math.radians(b[0])
    dp = math.radians(b[0] - a[0]); dl = math.radians(b[1] - a[1])
    h = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2
    return 2 * R * math.asin(min(1.0, math.sqrt(h)))


def overpass_post(q):
    """POST an Overpass query with retry/backoff — the public API 504s / times out under load."""
    req = urllib.request.Request(OVERPASS, data=urllib.parse.urlencode({"data": q}).encode(),
                                 headers={"User-Agent": UA})
    for attempt in range(4):
        try:
            with urllib.request.urlopen(req, timeout=180) as r:
                return json.load(r).get("elements", [])
        except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as e:
            if attempt == 3:
                raise
            wait = 15 * (attempt + 1)
            print(f"    Overpass {getattr(e, 'code', type(e).__name__)} — retry {attempt + 1}/3 in {wait}s")
            time.sleep(wait)


def shops_query(bb):
    box = f'{bb["minLat"]:.5f},{bb["minLon"]:.5f},{bb["maxLat"]:.5f},{bb["maxLon"]:.5f}'
    return (f'[out:json][timeout:180];('
            f'nwr["shop"~"^({"|".join(sorted(SHOP_CLASSES))})$"]["name"]({box});'
            f'nwr["amenity"~"^({"|".join(sorted(AMENITY_CLASSES))})$"]["name"]({box});'
            f');out center;')


def fetch_raw(key, bb, refetch):
    """Raw Overpass elements, cached in-repo so re-runs never re-fetch. Auto-refetches when the CLASS
    LIST changes (each snapshot records its query) — that's how the R20 widening lands as a NEW raw
    snapshot without anyone remembering --refetch."""
    os.makedirs(RAW_DIR, exist_ok=True)
    path = os.path.join(RAW_DIR, key + ".overpass.json")
    q = shops_query(bb)
    if os.path.exists(path) and not refetch:
        w = json.load(open(path))
        if w.get("query") == q:
            return w["elements"], w.get("fetchedAt")
        print(f"  [{key}] class list changed → new raw snapshot")
    print(f"  [{key}] fetching Overpass shops (widened, bounded)")
    els = overpass_post(q)
    fetched = datetime.date.today().isoformat()
    json.dump({"fetchedAt": fetched, "endpoint": OVERPASS, "query": q, "bbox": bb, "elements": els},
              open(path, "w"), indent=1)
    time.sleep(2)                               # be a good Overpass citizen between towns
    return els, fetched


def fetch_roads_raw(key, bb, refetch):
    """Return the raw OSM highway ways (with geometry) for the town bbox, cached in-repo like the shops."""
    os.makedirs(RAW_DIR, exist_ok=True)
    path = os.path.join(RAW_DIR, key + ".roads.overpass.json")
    if os.path.exists(path) and not refetch:
        return json.load(open(path))["elements"]
    q = (f'[out:json][timeout:120];'
         f'way["highway"~"^({ROAD_HW})$"]'
         f'({bb["minLat"]:.5f},{bb["minLon"]:.5f},{bb["maxLat"]:.5f},{bb["maxLon"]:.5f});'
         f'out geom;')
    print(f"  [{key}] fetching Overpass roads (highway ways, bounded)")
    ways = overpass_post(q)
    json.dump({"fetchedAt": datetime.date.today().isoformat(), "endpoint": OVERPASS, "query": q,
               "bbox": bb, "elements": ways}, open(path, "w"), indent=1)
    time.sleep(2)
    return ways


def roads_from_ways(ways):
    """OSM highway ways → roads[] (contract v2): {id, kind, name?, pts:[[lat,lon],…]}. Emit only the
    STREET tier; ship full geometry (A simplifies with Douglas–Peucker); deterministic (id-sorted)."""
    roads = []
    for w in ways:
        hwy = w.get("tags", {}).get("highway")
        geom = w.get("geometry")
        if hwy not in STREET_KINDS or not isinstance(geom, list) or len(geom) < 2:
            continue
        pts = [[round(g["lat"], 6), round(g["lon"], 6)] for g in geom
               if isinstance(g.get("lat"), (int, float)) and isinstance(g.get("lon"), (int, float))]
        if len(pts) < 2:
            continue
        rd = {"id": w["id"], "kind": hwy, "pts": pts}
        name = w.get("tags", {}).get("name")
        if name:
            rd["name"] = name[:60]
        roads.append(rd)
    return sorted(roads, key=lambda r: r["id"])


def process(key, cfg, els, fetched, roads=None):
    center = cfg["center"]
    rows, seen = [], set()
    for el in els:
        t = el.get("tags", {})
        name, osm_shop = t.get("name"), (t.get("shop") or t.get("amenity"))   # cafe rides amenity=*
        lat = el.get("lat", el.get("center", {}).get("lat"))
        lon = el.get("lon", el.get("center", {}).get("lon"))
        if not (name and osm_shop and isinstance(lat, (int, float)) and isinstance(lon, (int, float))):
            continue
        dkey = (punct_blind(name), round(lat, 3), round(lon, 3))   # node/way dup fuzz (~110 m)
        if dkey in seen:
            continue
        seen.add(dkey)
        suburb = next((t[k] for k in ("addr:suburb", "addr:city", "addr:town", "addr:place") if t.get(k)), "")
        rows.append({"id": el["id"], "name": name[:60], "osm": osm_shop, "lat": lat, "lon": lon, "suburb": suburb})
    # suburb fill: untagged shops adopt the nearest tagged neighbour within ~3 km (that's a suburb)
    tagged = [r for r in rows if r["suburb"]]
    for r in rows:
        if r["suburb"] or not tagged:
            continue
        near = min(tagged, key=lambda s: haversine_m((r["lat"], r["lon"]), (s["lat"], s["lon"])))
        if haversine_m((r["lat"], r["lon"]), (near["lat"], near["lon"])) <= 3000:
            r["suburb"] = near["suburb"]
    # parody + type-map + second dedupe (parodized name in same suburb = same shop); id-sorted = deterministic
    shops, seen2 = [], set()
    for r in sorted(rows, key=lambda r: r["id"]):
        pname = parodize(r["name"])[:60]
        k2 = (punct_blind(pname), r["suburb"])
        if k2 in seen2:
            continue
        seen2.add(k2)
        shops.append({"id": r["id"], "name": pname, "type": TYPE_MAP.get(r["osm"], "opshop"),
                      "lat": round(r["lat"], 6), "lon": round(r["lon"], 6), "suburb": r["suburb"],
                      "_osm": r["osm"]})
    # ── SUBTLE (John's directive): texture, not takeover ────────────────────────────────────────
    # The secondhand HEROES are never subsampled — they're the game. If a town's general retail would
    # drown the strip, seed-subsample the TEXTURE toward C's ~2–3× band (deterministic: rank by
    # sha256(townKey:osmId), keep the top `cap`) and COUNT the drops like every other drop.
    heroes = [s for s in shops if s["_osm"] in HERO_CLASSES]
    texture = [s for s in shops if s["_osm"] not in HERO_CLASSES]
    cap, dropped = int(round(TEXTURE_CAP * len(heroes))), 0
    if len(texture) > cap:
        texture.sort(key=lambda s: hashlib.sha256(f"{key}:{s['id']}".encode()).hexdigest())
        dropped, texture = len(texture) - cap, texture[:cap]
    shops = sorted(heroes + texture, key=lambda s: s["id"])       # deterministic emit order
    for s in shops:
        s.pop("_osm", None)
    bb = bbox(center, cfg["span_km"])
    counts = {"raw": len(els), "shops": len(shops), "heroes": len(heroes),
              "texture": len(texture), "textureDropped": dropped}
    cache = {
        "schema": "procity-town-cache/2" if roads else "procity-town-cache/1",
        "key": key, "town": cfg["town"], "source": "osm",
        "license": "ODbL 1.0", "attribution": "© OpenStreetMap contributors",
        "generator": "pipeline/build_towns.py", "fetchedAt": fetched,
        "center": {"lat": center[0], "lon": center[1]},
        "bbox": bb, "counts": counts, "shops": shops,
    }
    if roads:
        counts["roads"] = len(roads)
        cache["roads"] = roads                  # schema v2 — A's plan_osm builds the real street graph
    return cache


def span_km(shops):
    if not shops:
        return 0.0
    lats = [s["lat"] for s in shops]; lons = [s["lon"] for s in shops]
    latm = haversine_m((min(lats), lons[0]), (max(lats), lons[0]))
    lonm = haversine_m((lats[0], min(lons)), (lats[0], max(lons)))
    return max(latm, lonm) / 1000.0


def main():
    args = [a for a in sys.argv[1:] if not a.startswith("-")]
    refetch = "--refetch" in sys.argv
    no_roads = "--no-roads" in sys.argv          # build a v1 cache (marched fallback), no road fetch
    keys = args or list(TOWNS)
    os.makedirs(TOWNS_DIR, exist_ok=True)
    kept, dropped = [], []
    for key in keys:
        cfg = TOWNS[key]
        bb = bbox(cfg["center"], cfg["span_km"])
        els, fetched = fetch_raw(key, bb, refetch)
        roads = None
        if not no_roads:
            roads = roads_from_ways(fetch_roads_raw(key, bb, refetch))
        cache = process(key, cfg, els, fetched, roads)
        n, sp = len(cache["shops"]), span_km(cache["shops"])
        types = {}
        for s in cache["shops"]:
            types[s["type"]] = types.get(s["type"], 0) + 1
        ok = n >= 6 and sp < 5.0
        if ok:
            json.dump(cache, open(os.path.join(TOWNS_DIR, key + ".json"), "w"), indent=1)
            kept.append(key)
        else:
            dropped.append((key, n, sp))
        c = cache["counts"]
        sub = f" −{c['textureDropped']} subsampled" if c.get("textureDropped") else ""
        print(f"  {'OK ' if ok else 'LOW'} {key:16} shops={n:>3} (heroes {c['heroes']:>2} + texture "
              f"{c['texture']:>3}{sub}) roads={len(roads or [])!s:>4}  {dict(sorted(types.items()))}")
    print(f"\nBUILT {len(kept)} town cache(s) → web/assets/towns/: {', '.join(kept)}")
    if dropped:
        print(f"DROPPED (< 6 shops or span ≥ 5 km): {dropped}")
    write_index()


def write_index():
    """web/assets/towns/index.json (ROUND20 ledger #2) — B's selector derives the town list from this
    instead of hardcoding REAL_TOWNS. Built from every cache on disk, not just this run."""
    towns = []
    for f in sorted(x for x in os.listdir(TOWNS_DIR) if x.endswith(".json") and x != "index.json"):
        c = json.load(open(os.path.join(TOWNS_DIR, f)))
        k = c.get("key", f[:-5])
        towns.append({"key": k, "town": c.get("town", k), "state": TOWNS.get(k, {}).get("state", ""),
                      "shops": len(c.get("shops", [])), "roads": bool(c.get("roads"))})
    idx = {"schema": "procity-towns-index/1", "generated": datetime.date.today().isoformat(),
           "license": "ODbL 1.0", "attribution": "© OpenStreetMap contributors", "towns": towns}
    json.dump(idx, open(os.path.join(TOWNS_DIR, "index.json"), "w"), indent=1)
    print(f"INDEX  → web/assets/towns/index.json ({len(towns)} towns: "
          f"{', '.join(t['key'] for t in towns)})")


if __name__ == "__main__":
    main()