PROCITY/pipeline/build_towns.py

#!/usr/bin/env python3
"""build_towns.py — the real-map scout (ROUND17 ledger #6). A port of thriftgod `build_city()`'s
Overpass route to PER-TOWN real Australian caches in the Lane-A `procity-town-cache/1` contract.

    python3 pipeline/build_towns.py                 # build every town (cached raw reused, no re-fetch)
    python3 pipeline/build_towns.py katoomba_real    # one town
    python3 pipeline/build_towns.py --refetch        # ignore cached raw, re-hit Overpass (deliberate)

Pipeline per town: bounded Overpass query → RAW response cached in-repo (`web/assets/towns/_raw/`, so
re-runs NEVER re-fetch) → dedupe (OSM maps a shop as node AND way) → suburb fill (untagged adopt the
nearest tagged neighbour) → parody the big trademarked charity/franchise names (indie shops keep their
real OSM name) → map OSM `shop=*` to a registry SHOP_TYPE → `web/assets/towns/<key>.json`.

DATA DISCIPLINE (ledger #6): deterministic (same raw → byte-identical cache; shops sorted by OSM id,
no RNG); bounded queries (a cache is ONE compact town, span < 5 km — the mega-strip risk); ODbL
attribution ships IN every cache (`license`/`attribution`) + `SOURCES.md`. Hours are NOT cached — the
registry supplies them at plan time (contract shop = {id,name,type,lat,lon,suburb}). Overpass is an
outward-facing fetch (John-authorized scout, once per town, raw cached).
"""
import os, sys, json, re, math, time, datetime, urllib.request, urllib.parse, urllib.error

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TOWNS_DIR = os.path.join(ROOT, "web", "assets", "towns")
RAW_DIR = os.path.join(TOWNS_DIR, "_raw")
OVERPASS = "https://overpass-api.de/api/interpreter"
UA = "PROCITY-townscout/0.1 (real-map scout; OpenStreetMap ODbL data)"

# Compact real AU towns: center (lat,lon) + query box span (km). `katoomba_real` is the fixture
# comparison (vs the hand-made `katoomba` in osm_fixture.js). Spans kept tight so each is ONE town.
TOWNS = {
    "katoomba_real":    {"town": "Katoomba",    "center": (-33.7145, 150.3120), "span_km": 3.0},
    "newtown_real":     {"town": "Newtown",     "center": (-33.8985, 151.1790), "span_km": 2.6},
    "fremantle_real":   {"town": "Fremantle",   "center": (-32.0555, 115.7480), "span_km": 2.6},
    "bendigo_real":     {"town": "Bendigo",     "center": (-36.7580, 144.2810), "span_km": 3.4},
    "castlemaine_real": {"town": "Castlemaine", "center": (-37.0640, 144.2170), "span_km": 3.0},
}

# OSM shop tags we scout (on-theme: the game's secondhand/media/hobby world) → registry SHOP_TYPE.
SHOP_RE = "charity|second_hand|music|antiques|books|toys|games|video_games|pawnbroker"
TYPE_MAP = {
    "books": "book", "music": "record", "toys": "toy", "games": "toy",
    "video_games": "video", "video": "video", "pawnbroker": "pawn",
    "charity": "opshop", "second_hand": "opshop", "antiques": "opshop",
}

# ── roads[] (schema v2, ROUND18 REAL ROADS) ─────────────────────────────────────────────────────
# We FETCH every OSM highway tier A's ROAD_KIND maps (→ raw cache, full provenance / beta-ready) but
# EMIT only the town STREET graph. Fidelity cut (charter risk #4, flagged to A who owns the knob):
# the service/track/footway/path/steps/cycleway tier is EXCLUDED — in Katoomba that's 1461 of 2506
# ways (Blue-Mountains bushwalking tracks, sidewalks, stairs, driveways, parking aisles), noise for a
# town street graph and a way-count A's lift would build 2500+ edges from. The full set stays in the
# raw response if A wants a tier back. `kind` ships as the raw OSM highway class; A maps it (ROAD_KIND).
ROAD_HW = ("motorway|trunk|primary|secondary|tertiary|unclassified|residential|living_street|road"
           "|service|track|pedestrian|footway|path|steps|cycleway")
STREET_KINDS = {"motorway", "trunk", "primary", "secondary", "tertiary", "unclassified",
                "residential", "living_street", "road", "pedestrian"}

# The big trademarked charity/franchise chains get loving parodies (ported from thriftgod, John's call
# for the live game); independent shops keep their real OSM names (factual ODbL data).
PARODY = [
    (r"salvos( stores?)?|(the )?salvation army", "Slave-ohs"),
    (r"lifeline", "Tightrope"),
    (r"rspca", "RSYMCA"),
    (r"vinnies|st\.? ?vincent('?s| de paul)?( society)?|svdp", "Sinnies"),
    (r"(australian )?red cross", "Red Floss"),
    (r"save the children", "Shave the Children"),
    (r"anglicare", "Tanglecare"),
    (r"uniting ?care|unitingcare|uniting", "Unknitting"),
    (r"good sammys?|good samaritans?", "Okay Sammys"),
    (r"mission australia", "Mission Improbable"),
    (r"brotherhood of st\.? ?laurence", "Brotherhood of St Lozenge"),
    (r"cash ?converters?", "Cashtheists"),
]


def parodize(name):
    for pat, rep in PARODY:
        name = re.sub(pat, rep, name, flags=re.I)
    return name


def punct_blind(s):
    return re.sub(r"[^a-z0-9]", "", s.lower())


def bbox(center, span_km):
    lat, lon = center
    dlat = (span_km / 2.0) / 111.0
    dlon = (span_km / 2.0) / (111.0 * math.cos(math.radians(lat)))
    return {"minLat": lat - dlat, "minLon": lon - dlon, "maxLat": lat + dlat, "maxLon": lon + dlon}


def haversine_m(a, b):
    R = 6371000.0
    p1, p2 = math.radians(a[0]), math.radians(b[0])
    dp = math.radians(b[0] - a[0]); dl = math.radians(b[1] - a[1])
    h = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2
    return 2 * R * math.asin(min(1.0, math.sqrt(h)))


def overpass_post(q):
    """POST an Overpass query with retry/backoff — the public API 504s / times out under load."""
    req = urllib.request.Request(OVERPASS, data=urllib.parse.urlencode({"data": q}).encode(),
                                 headers={"User-Agent": UA})
    for attempt in range(4):
        try:
            with urllib.request.urlopen(req, timeout=180) as r:
                return json.load(r).get("elements", [])
        except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as e:
            if attempt == 3:
                raise
            wait = 15 * (attempt + 1)
            print(f"    Overpass {getattr(e, 'code', type(e).__name__)} — retry {attempt + 1}/3 in {wait}s")
            time.sleep(wait)


def fetch_raw(key, bb, refetch):
    """Return the raw Overpass elements list, cached in-repo so re-runs never re-fetch."""
    os.makedirs(RAW_DIR, exist_ok=True)
    path = os.path.join(RAW_DIR, key + ".overpass.json")
    if os.path.exists(path) and not refetch:
        w = json.load(open(path))
        return w["elements"], w.get("fetchedAt")
    q = (f'[out:json][timeout:120];'
         f'nwr["shop"~"^({SHOP_RE})$"]["name"]'
         f'({bb["minLat"]:.5f},{bb["minLon"]:.5f},{bb["maxLat"]:.5f},{bb["maxLon"]:.5f});'
         f'out center;')
    print(f"  [{key}] fetching Overpass (bounded {bb['minLat']:.3f},{bb['minLon']:.3f} … {bb['maxLat']:.3f},{bb['maxLon']:.3f})")
    els = overpass_post(q)
    fetched = datetime.date.today().isoformat()
    json.dump({"fetchedAt": fetched, "endpoint": OVERPASS, "query": q, "bbox": bb, "elements": els},
              open(path, "w"), indent=1)
    time.sleep(2)                               # be a good Overpass citizen between towns
    return els, fetched


def fetch_roads_raw(key, bb, refetch):
    """Return the raw OSM highway ways (with geometry) for the town bbox, cached in-repo like the shops."""
    os.makedirs(RAW_DIR, exist_ok=True)
    path = os.path.join(RAW_DIR, key + ".roads.overpass.json")
    if os.path.exists(path) and not refetch:
        return json.load(open(path))["elements"]
    q = (f'[out:json][timeout:120];'
         f'way["highway"~"^({ROAD_HW})$"]'
         f'({bb["minLat"]:.5f},{bb["minLon"]:.5f},{bb["maxLat"]:.5f},{bb["maxLon"]:.5f});'
         f'out geom;')
    print(f"  [{key}] fetching Overpass roads (highway ways, bounded)")
    ways = overpass_post(q)
    json.dump({"fetchedAt": datetime.date.today().isoformat(), "endpoint": OVERPASS, "query": q,
               "bbox": bb, "elements": ways}, open(path, "w"), indent=1)
    time.sleep(2)
    return ways


def roads_from_ways(ways):
    """OSM highway ways → roads[] (contract v2): {id, kind, name?, pts:[[lat,lon],…]}. Emit only the
    STREET tier; ship full geometry (A simplifies with Douglas–Peucker); deterministic (id-sorted)."""
    roads = []
    for w in ways:
        hwy = w.get("tags", {}).get("highway")
        geom = w.get("geometry")
        if hwy not in STREET_KINDS or not isinstance(geom, list) or len(geom) < 2:
            continue
        pts = [[round(g["lat"], 6), round(g["lon"], 6)] for g in geom
               if isinstance(g.get("lat"), (int, float)) and isinstance(g.get("lon"), (int, float))]
        if len(pts) < 2:
            continue
        rd = {"id": w["id"], "kind": hwy, "pts": pts}
        name = w.get("tags", {}).get("name")
        if name:
            rd["name"] = name[:60]
        roads.append(rd)
    return sorted(roads, key=lambda r: r["id"])


def process(key, cfg, els, fetched, roads=None):
    center = cfg["center"]
    rows, seen = [], set()
    for el in els:
        t = el.get("tags", {})
        name, osm_shop = t.get("name"), t.get("shop")
        lat = el.get("lat", el.get("center", {}).get("lat"))
        lon = el.get("lon", el.get("center", {}).get("lon"))
        if not (name and osm_shop and isinstance(lat, (int, float)) and isinstance(lon, (int, float))):
            continue
        dkey = (punct_blind(name), round(lat, 3), round(lon, 3))   # node/way dup fuzz (~110 m)
        if dkey in seen:
            continue
        seen.add(dkey)
        suburb = next((t[k] for k in ("addr:suburb", "addr:city", "addr:town", "addr:place") if t.get(k)), "")
        rows.append({"id": el["id"], "name": name[:60], "osm": osm_shop, "lat": lat, "lon": lon, "suburb": suburb})
    # suburb fill: untagged shops adopt the nearest tagged neighbour within ~3 km (that's a suburb)
    tagged = [r for r in rows if r["suburb"]]
    for r in rows:
        if r["suburb"] or not tagged:
            continue
        near = min(tagged, key=lambda s: haversine_m((r["lat"], r["lon"]), (s["lat"], s["lon"])))
        if haversine_m((r["lat"], r["lon"]), (near["lat"], near["lon"])) <= 3000:
            r["suburb"] = near["suburb"]
    # parody + type-map + second dedupe (parodized name in same suburb = same shop); id-sorted = deterministic
    shops, seen2 = [], set()
    for r in sorted(rows, key=lambda r: r["id"]):
        pname = parodize(r["name"])[:60]
        k2 = (punct_blind(pname), r["suburb"])
        if k2 in seen2:
            continue
        seen2.add(k2)
        shops.append({"id": r["id"], "name": pname, "type": TYPE_MAP.get(r["osm"], "opshop"),
                      "lat": round(r["lat"], 6), "lon": round(r["lon"], 6), "suburb": r["suburb"]})
    bb = bbox(center, cfg["span_km"])
    counts = {"raw": len(els), "shops": len(shops)}
    cache = {
        "schema": "procity-town-cache/2" if roads else "procity-town-cache/1",
        "key": key, "town": cfg["town"], "source": "osm",
        "license": "ODbL 1.0", "attribution": "© OpenStreetMap contributors",
        "generator": "pipeline/build_towns.py", "fetchedAt": fetched,
        "center": {"lat": center[0], "lon": center[1]},
        "bbox": bb, "counts": counts, "shops": shops,
    }
    if roads:
        counts["roads"] = len(roads)
        cache["roads"] = roads                  # schema v2 — A's plan_osm builds the real street graph
    return cache


def span_km(shops):
    if not shops:
        return 0.0
    lats = [s["lat"] for s in shops]; lons = [s["lon"] for s in shops]
    latm = haversine_m((min(lats), lons[0]), (max(lats), lons[0]))
    lonm = haversine_m((lats[0], min(lons)), (lats[0], max(lons)))
    return max(latm, lonm) / 1000.0


def main():
    args = [a for a in sys.argv[1:] if not a.startswith("-")]
    refetch = "--refetch" in sys.argv
    no_roads = "--no-roads" in sys.argv          # build a v1 cache (marched fallback), no road fetch
    keys = args or list(TOWNS)
    os.makedirs(TOWNS_DIR, exist_ok=True)
    kept, dropped = [], []
    for key in keys:
        cfg = TOWNS[key]
        bb = bbox(cfg["center"], cfg["span_km"])
        els, fetched = fetch_raw(key, bb, refetch)
        roads = None
        if not no_roads:
            roads = roads_from_ways(fetch_roads_raw(key, bb, refetch))
        cache = process(key, cfg, els, fetched, roads)
        n, sp = len(cache["shops"]), span_km(cache["shops"])
        types = {}
        for s in cache["shops"]:
            types[s["type"]] = types.get(s["type"], 0) + 1
        ok = n >= 6 and sp < 5.0
        if ok:
            json.dump(cache, open(os.path.join(TOWNS_DIR, key + ".json"), "w"), indent=1)
            kept.append(key)
        else:
            dropped.append((key, n, sp))
        rstr = ""
        if roads is not None:
            rk = {}
            for r in roads:
                rk[r["kind"]] = rk.get(r["kind"], 0) + 1
            rstr = f"  roads={len(roads)} {dict(sorted(rk.items(), key=lambda x: -x[1]))}"
        print(f"  {'OK ' if ok else 'LOW'} {key:16} shops={n:>3} span={sp:.2f}km {dict(sorted(types.items()))}{rstr}")
    print(f"\nBUILT {len(kept)} town cache(s) → web/assets/towns/: {', '.join(kept)}")
    if dropped:
        print(f"DROPPED (< 6 shops or span ≥ 5 km): {dropped}")


if __name__ == "__main__":
    main()