From e4af0fb01d38899965a68633cbb7188d4be0d0d3 Mon Sep 17 00:00:00 2001 From: type-two Date: Thu, 30 Jul 2026 11:29:00 +1000 Subject: [PATCH] =?UTF-8?q?feat:=20Wave=207=20=E2=80=94=20REAL=20live=20gl?= =?UTF-8?q?obal=20shipping=20via=20aisstream.io?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ais_bridge.py: a small daemon holding the aisstream.io websocket (free key, server-side only), tracking every vessel heard (15-min freshness window) and atomically writing ais_snapshot.json into the web root every 20 s. The js/layers/ais.js layer polls that static file — no request-path server code, identical under dev serve.py and prod nginx; rolling query bucket defeats CDN caching. Verified locally: 15,844 live vessels within minutes of connect — the Dover Strait separation lanes are visibly full of real traffic; click any ship for name/MMSI/type/speed/course/status/destination (colored by type: cargo green, tanker amber, passenger blue, fishing purple, anchored grey). Live-only (hidden when scrubbed). Demo ships demoted to default-off; the dormant browser-side AIS scaffold in ships.js stays inert (key would be browser-exposed — the bridge supersedes it). Co-Authored-By: Claude Fable 5 --- .gitignore | 2 + ais_bridge.py | 152 +++++++++++++++++++++++++++++++++++++++++++++ js/layers/ais.js | 125 +++++++++++++++++++++++++++++++++++++ js/layers/ships.js | 3 +- js/main.js | 1 + js/ui.js | 2 +- 6 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 ais_bridge.py create mode 100644 js/layers/ais.js diff --git a/.gitignore b/.gitignore index d4918c6..52e1b1d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ openskycredentials.json *credentials*.json *.secret .env +# Generated by ais_bridge.py — live data, never committed. +ais_snapshot.json diff --git a/ais_bridge.py b/ais_bridge.py new file mode 100644 index 0000000..c4fb781 --- /dev/null +++ b/ais_bridge.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""AIS bridge (Wave 7) — real live global shipping for GODSIGH. + +Holds a websocket to aisstream.io (free API key), keeps an in-memory table of +every ship seen recently, and atomically writes a snapshot JSON into the web +root every WRITE_SEC. The browser layer polls that static file — no server +code in the request path, works identically under dev serve.py and prod nginx +(the layer cache-busts with a rolling query param, so CDN caching is moot). + +Usage: python3 ais_bridge.py + e.g. dev : ~/.venvs/godsigh/bin/python ais_bridge.py aiscredentials.json ais_snapshot.json + prod: /home/humanjing/.venvs/godsigh/bin/python ais_bridge.py \ + /home/humanjing/godsigh-secret/aiscredentials.json \ + /home/humanjing/godsigh/ais_snapshot.json + +The key file may be full JSON, a brace-less "key": "…" fragment, or a raw +token. Reconnects with backoff forever. Heartbeat: ~/.jobs/godsigh-ais.status +(fleet jobs convention). +""" +import json +import os +import re +import sys +import tempfile +import threading +import time + +import websocket # websocket-client + +STALE_SEC = 900 # drop ships not heard from in 15 min +WRITE_SEC = 20 # snapshot cadence +MAX_SHIPS = 20000 # snapshot cap (newest first) — keeps the JSON sane +URL = "wss://stream.aisstream.io/v0/stream" + +ships = {} # mmsi -> dict +lock = threading.Lock() +stats = {"msgs": 0, "connected_at": None, "last_msg": None} + + +def read_key(path): + raw = open(path).read().strip() + for attempt in (raw, "{" + raw + "}"): + try: + d = json.loads(attempt) + if isinstance(d, dict): + for v in d.values(): + if isinstance(v, str) and len(v) >= 16: + return v + except Exception: + pass + m = re.findall(r"[A-Za-z0-9_-]{16,}", raw) + if not m: + raise SystemExit("no key found in " + path) + return m[0] + + +def on_message(_ws, raw): + try: + m = json.loads(raw) + except Exception: + return + meta = m.get("MetaData") or {} + mmsi = meta.get("MMSI") + if not mmsi: + return + now = time.time() + stats["msgs"] += 1 + stats["last_msg"] = now + t = m.get("MessageType") + with lock: + rec = ships.get(mmsi) or {"mmsi": mmsi} + if t == "PositionReport": + p = (m.get("Message") or {}).get("PositionReport") or {} + rec.update( + lat=p.get("Latitude"), lon=p.get("Longitude"), + sog=p.get("Sog"), cog=p.get("Cog"), + hdg=p.get("TrueHeading"), nav=p.get("NavigationalStatus"), + ts=now, + ) + elif t == "ShipStaticData": + s = (m.get("Message") or {}).get("ShipStaticData") or {} + rec["type"] = s.get("Type") + dest = (s.get("Destination") or "").strip() + if dest: + rec["dest"] = dest[:24] + name = (meta.get("ShipName") or "").strip() + if name: + rec["name"] = name[:32] + # MetaData carries a position too — fallback for static-only sightings + if rec.get("lat") is None and meta.get("latitude") is not None: + rec["lat"], rec["lon"], rec["ts"] = meta.get("latitude"), meta.get("longitude"), now + ships[mmsi] = rec + + +def writer(out_path): + jobs = os.path.expanduser("~/.jobs") + os.makedirs(jobs, exist_ok=True) + while True: + time.sleep(WRITE_SEC) + now = time.time() + with lock: + for k in [k for k, v in ships.items() if now - (v.get("ts") or 0) > STALE_SEC]: + del ships[k] + rows = [v for v in ships.values() if v.get("lat") is not None] + rows.sort(key=lambda r: r.get("ts", 0), reverse=True) + rows = rows[:MAX_SHIPS] + snap = {"ts": int(now), "count": len(rows), "ships": [ + {k: r.get(k) for k in ("mmsi", "name", "lat", "lon", "sog", "cog", "hdg", "nav", "type", "dest") if r.get(k) is not None} + for r in rows + ]} + body = json.dumps(snap, separators=(",", ":")) + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(os.path.abspath(out_path))) + with os.fdopen(fd, "w") as f: + f.write(body) + os.replace(tmp, out_path) # atomic — readers never see a torn file + with open(os.path.join(jobs, "godsigh-ais.status"), "w") as f: + f.write(json.dumps({"ts": int(now), "ships": len(rows), "msgs": stats["msgs"]})) + print(f"[ais] {len(rows)} ships · {stats['msgs']} msgs", flush=True) + + +def main(): + if len(sys.argv) != 3: + raise SystemExit(__doc__) + key = read_key(sys.argv[1]) + threading.Thread(target=writer, args=(sys.argv[2],), daemon=True).start() + backoff = 2 + while True: + try: + ws = websocket.WebSocketApp( + URL, + on_open=lambda w: ( + stats.__setitem__("connected_at", time.time()), + w.send(json.dumps({ + "APIKey": key, + "BoundingBoxes": [[[-90, -180], [90, 180]]], + "FilterMessageTypes": ["PositionReport", "ShipStaticData"], + })), + print("[ais] connected + subscribed (global)", flush=True), + ), + on_message=on_message, + on_error=lambda w, e: print(f"[ais] error: {e}", flush=True), + ) + ws.run_forever(ping_interval=30, ping_timeout=10) + except Exception as e: + print(f"[ais] crashed: {e}", flush=True) + print(f"[ais] reconnecting in {backoff}s", flush=True) + time.sleep(backoff) + backoff = min(backoff * 2, 60) + + +if __name__ == "__main__": + main() diff --git a/js/layers/ais.js b/js/layers/ais.js new file mode 100644 index 0000000..1e77269 --- /dev/null +++ b/js/layers/ais.js @@ -0,0 +1,125 @@ +// Live AIS shipping (Wave 7) — every real vessel aisstream.io can hear, +// worldwide. ais_bridge.py (a tiny daemon next to the web root) holds the +// websocket and drops ais_snapshot.json every ~20 s; this layer just polls +// that static file. Live-only: hidden when the clock is scrubbed off now +// (there's no ship history recorder). + +const POLL_MS = 30000; +const NAV = { + 0: 'under way', 1: 'at anchor', 2: 'not under command', 3: 'restricted manoeuvre', + 4: 'constrained by draught', 5: 'moored', 6: 'aground', 7: 'fishing', 8: 'under sail', +}; +function typeLabel(t) { + if (t == null) return null; + if (t === 30) return 'fishing'; + if (t === 35) return 'military'; + if (t === 36 || t === 37) return 'sailing / pleasure'; + if (t === 52) return 'tug'; + if (t >= 60 && t <= 69) return 'passenger'; + if (t >= 70 && t <= 79) return 'cargo'; + if (t >= 80 && t <= 89) return 'tanker'; + return `type ${t}`; +} + +export default function create(ctx) { + const { viewer, Cesium, ui } = ctx; + const pts = viewer.scene.primitives.add(new Cesium.PointPrimitiveCollection()); + pts.show = false; + + const C_CARGO = Cesium.Color.fromCssColorString('#39d98a'); + const C_TANKER = Cesium.Color.fromCssColorString('#ff9f43'); + const C_PASS = Cesium.Color.fromCssColorString('#54a0ff'); + const C_FISH = Cesium.Color.fromCssColorString('#c58fff'); + const C_OTHER = Cesium.Color.fromCssColorString('#9fd8cb'); + const C_STILL = Cesium.Color.fromCssColorString('#7c8b97'); + const OUTLINE = Cesium.Color.BLACK; + const colorFor = (s) => { + if ((s.sog ?? 0) < 0.5 || s.nav === 1 || s.nav === 5) return C_STILL; + const t = s.type; + if (t >= 70 && t <= 79) return C_CARGO; + if (t >= 80 && t <= 89) return C_TANKER; + if (t >= 60 && t <= 69) return C_PASS; + if (t === 30) return C_FISH; + return C_OTHER; + }; + + let enabled = false; + let live = true; + let timer = null; + let lastCount = 0; + let lastTs = null; + + ui.addLayer('ais', 'Ships (live AIS)', true, (on) => { + enabled = on; + pts.show = on && live; + if (on && !timer) { poll(); timer = setInterval(poll, POLL_MS); } + if (!on && timer) { clearInterval(timer); timer = null; } + if (!on) { ui.hidePickOverlay(); ui.setStatus('ais', '', 'ok'); } + else status(); + }, 'Sea'); + enabled = true; + pts.show = true; // defaultOn — addLayer doesn't fire onToggle for the initial state + poll(); + timer = setInterval(poll, POLL_MS); + + function status() { + if (!enabled) return; + if (!live) { ui.setStatus('ais', 'live only — return clock to now', 'warn'); return; } + const age = lastTs ? Math.round(Date.now() / 1000 - lastTs) : null; + ui.setStatus('ais', lastCount ? `${lastCount} live vessels · ${age}s ago · aisstream.io` : 'waiting for bridge…', lastCount ? 'ok' : 'warn'); + } + + async function poll() { + if (!enabled || document.hidden) return; + let res; + try { + // rolling bucket busts any CDN cache; no-store busts the browser's + res = await fetch(`ais_snapshot.json?t=${Math.floor(Date.now() / POLL_MS)}`, { cache: 'no-store' }); + } catch { ui.setStatus('ais', 'network error', 'warn'); return; } + if (res.status === 404) { ui.setStatus('ais', 'bridge offline (no snapshot)', 'err'); return; } + if (!res.ok) { ui.setStatus('ais', `HTTP ${res.status}`, 'warn'); return; } + let snap; + try { snap = await res.json(); } catch { ui.setStatus('ais', 'bad snapshot', 'err'); return; } + const ships = snap.ships || []; + pts.removeAll(); + for (const s of ships) { + if (!isFinite(s.lat) || !isFinite(s.lon)) continue; + pts.add({ + position: Cesium.Cartesian3.fromDegrees(s.lon, s.lat), + pixelSize: (s.sog ?? 0) > 0.5 ? 4 : 3, + color: colorFor(s), + outlineColor: OUTLINE, + outlineWidth: 1, + disableDepthTestDistance: 50000, + id: { layer: 'ais', ...s }, + }); + } + lastCount = ships.length; + lastTs = snap.ts; + status(); + } + + function onClockTick(_t, isLive) { + if (live === isLive) return; + live = isLive; + pts.show = enabled && live; + status(); + } + + function handlePick(picked) { + if (!enabled || picked?.id?.layer !== 'ais') return false; + const d = picked.id; + ui.showPickOverlay('🚢 ' + (d.name || `MMSI ${d.mmsi}`), [ + ['MMSI', String(d.mmsi)], + ['Type', typeLabel(d.type) || '—'], + ['Speed', d.sog != null ? `${d.sog.toFixed(1)} kn` : '—'], + ['Course', d.cog != null ? `${Math.round(d.cog)}°` : '—'], + ['Status', NAV[d.nav] || '—'], + ['Destination', d.dest || '—'], + ]); + return true; + } + function clearPick() { ui.hidePickOverlay(); } + + return { id: 'ais', handlePick, clearPick, onClockTick }; +} diff --git a/js/layers/ships.js b/js/layers/ships.js index 7d2d7a7..a46c08f 100644 --- a/js/layers/ships.js +++ b/js/layers/ships.js @@ -9,6 +9,7 @@ export default function create(ctx) { const shipsDS = new Cesium.CustomDataSource('ships'); viewer.dataSources.add(shipsDS); + shipsDS.show = false; // default-off: the real AIS layer (ais.js) is the ships layer now // ---- helpers ---- const at = (h) => Cesium.JulianDate.addHours(now, h, new Cesium.JulianDate()); @@ -84,7 +85,7 @@ export default function create(ctx) { // resurrects the hidden demo fleet on top of live vessels. let aisDS = null; let liveActive = false; - ui.addLayer('ships', 'Ships [DEMO]', true, (on) => { + ui.addLayer('ships', 'Ships [DEMO]', false, (on) => { if (liveActive) { if (aisDS) aisDS.show = on; } else { shipsDS.show = on; } }); diff --git a/js/main.js b/js/main.js index 627f54a..a2bbe37 100644 --- a/js/main.js +++ b/js/main.js @@ -179,6 +179,7 @@ const LAYER_MODULES = [ './layers/infra.js', './layers/events.js', './layers/ships.js', + './layers/ais.js', './layers/aircraft.js', './layers/military.js', './layers/radius.js', diff --git a/js/ui.js b/js/ui.js index 82e10f1..f1e8ae1 100644 --- a/js/ui.js +++ b/js/ui.js @@ -11,7 +11,7 @@ const CATEGORY_BY_ID = { satellites: 'Space', 'sat-paths': 'Space', launches: 'Space', celestial: 'Space', aircraft: 'Air', military: 'Air', 'aircraft-mil': 'Air', emergency: 'Air', 'rare-types': 'Air', shadow: 'Air', radius: 'Air', - ships: 'Sea', + ships: 'Sea', ais: 'Sea', quakes: 'Earth & hazards', fires: 'Earth & hazards', gdacs: 'Earth & hazards', nws: 'Earth & hazards', infra: 'Context', events: 'Context', radio: 'Context', roadcams: 'Context', rfs: 'Regional — Australia',