monster/GODSIGH
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
d0d3da116c
GODSIGH/serve.py
type-two d0d3da116c polish: emoji favicon (kills the /favicon.ico 404) + adsb.lol 420/429 → clean empty result 
- inline SVG 🌍 favicon so the browser never requests /favicon.ico
- proxy/adsb: on adsb.lol rate-limit (420 'Enhance Your Calm' or 429) with no
  cached fallback, return 200 {"ac":[]} instead of the error — layer shows
  nothing until next poll, no scary browser console error; widened upstream
  spacing 1.0→1.3s to hit the limit less often

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 10:19:21 +10:00

440 lines
20 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""GODSIGH dev server: static files + same-origin proxy for feeds that block CORS.
Also implements the **OpenSky shared cache v1** contract (see SPEC2.md §4), which
lets GODSIGH and its sibling project godstrument poll OpenSky from the same IP
without racing each other into the anonymous daily quota. The contract is
mirrored verbatim in godstrument's brief — keep the two in sync.
"""
import base64
import json
import os
import re
import sqlite3
import sys
import tempfile
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import zlib
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
UPSTREAMS = {
# OpenSky pins Access-Control-Allow-Origin to its own domain, so the browser
# can't call it directly — we forward it here instead.
"opensky": "https://opensky-network.org/api/states/all",
# Celestrak sends ACAO:* today; proxied too so the app keeps working if that changes.
"celestrak": "https://celestrak.org/NORAD/elements/gp.php",
}
# ---- OpenSky shared cache v1 ------------------------------------------------
# Path: ~/.cache/godverse/opensky-states.json (overridable via OPENSKY_CACHE_FILE).
# Content: the raw, unmodified /states/all JSON body (global, no bbox). Freshness
# = file mtime; readers treat <120 s as fresh. On upstream 429/failure a reader
# may serve a stale cache rather than nothing.
OPENSKY_CACHE_FRESH_SEC = 120
# OpenSky OAuth2 (client-credentials) → 4000 credits/day vs 400 anonymous. The
# clientId/clientSecret live in gitignored openskycredentials.json; a 30-min
# bearer token is fetched on demand and cached. If the file is absent or auth
# fails, we simply fetch anonymously (graceful fallback).
OPENSKY_CREDS = Path(__file__).resolve().parent / "openskycredentials.json"
OPENSKY_TOKEN_URL = ("https://auth.opensky-network.org/auth/realms/"
"opensky-network/protocol/openid-connect/token")
_opensky_token = {"value": None, "exp": 0.0}
# ---- Historical replay (record.py writes this db; dev-only) -----------------
# GET history/aircraft?t=<epoch_s> returns the nearest recorded snapshot within
# ±10 min, or 404. Prod has no recorder → the file is absent → graceful 404.
HISTORY_DB = Path(__file__).resolve().parent / "data" / "history.db"
HISTORY_TOLERANCE_SEC = 600
# ---- Military ADS-B via adsb.lol (FREE, no key, community feed) --------------
# proxy/mil forwards api.adsb.lol/v2/mil (no CORS header, so it needs a same-origin
# proxy) with a short in-memory cache. Free and unmetered — the cache is just
# politeness/coalescing, not a quota guard. Same {ac:[...]} shape as ADS-B
# Exchange, so js/layers/military.js is source-agnostic.
ADSBLOL_MIL_URL = "https://api.adsb.lol/v2/mil"
ADSBLOL_CACHE_SEC = 60
_adsblol_cache = {"body": None, "ts": 0.0}
# ---- Generic adsb.lol query proxy (proxy/adsb/<v2-path>) ---------------------
# Powers the OSINT layers (emergency squawks, rare-type hunter, LADD/PIA shadow
# aircraft, radius focus). The regex is the security allowlist — only these v2
# shapes are forwardable, so it can't be turned into an open proxy/SSRF. Each
# distinct path is cached 60s, and upstream calls are spaced (adsb.lol 429s on
# rapid bursts — the same guard the prod nginx regex-location relies on). The
# prod nginx block mirrors this allowlist; keep the two in sync.
ADSB_PATH_RE = re.compile(
r'^(mil|ladd|pia'
r'|sqk/\d{3,4}'
r'|type/[A-Za-z0-9]{2,5}'
r'|hex/[0-9a-fA-F]{6}'
r'|callsign/[A-Za-z0-9]{1,8}'
r'|reg/[A-Za-z0-9\-]{1,10}'
r'|lat/-?\d{1,3}(\.\d+)?/lon/-?\d{1,3}(\.\d+)?/dist/\d{1,4}(\.\d+)?'
r'|closest/-?\d{1,3}(\.\d+)?/-?\d{1,3}(\.\d+)?/\d{1,4}(\.\d+)?)$')
ADSB_CACHE_SEC = 120
ADSB_MIN_INTERVAL = 1.3 # min seconds between upstream adsb.lol calls (anti-burst; it 429s fast)
_adsb_cache = {} # path -> (body, ts)
_adsb_lock = threading.Lock()
_adsb_next = [0.0] # earliest wall-time the next upstream call may start
# ---- ADS-B Exchange military feed (paid RapidAPI, ~10k req/month) ------------
# proxy/adsbx-mil injects the RapidAPI key server-side (never in the browser) and
# HARD-caches the /v2/mil/ response — the quota guard. The $10 BASIC tier bills
# $0.0015/request OVER 10k/month with NO hard block, so the cache TTL must keep
# us safely under: 10 min → at most ~144 calls/day (~4.4k/month, <half the cap).
ADSBX_CREDS = Path(__file__).resolve().parent / "adsbxcredentials.json"
ADSBX_CACHE_SEC = 600
_adsbx_cache = {"body": None, "ts": 0.0}
def load_adsbx_creds():
try:
d = json.loads(ADSBX_CREDS.read_text())
return d.get("host"), d.get("key")
except (OSError, ValueError):
return None, None
def opensky_cache_file():
override = os.environ.get("OPENSKY_CACHE_FILE")
return Path(override if override else "~/.cache/godverse/opensky-states.json").expanduser()
def opensky_bearer():
# A valid access token (refreshed ~1 min before expiry), or None if creds are
# missing/auth fails → caller fetches anonymously.
try:
d = json.loads(OPENSKY_CREDS.read_text())
cid, csec = d.get("clientId"), d.get("clientSecret")
except (OSError, ValueError):
return None
if not cid or not csec:
return None
now = time.time()
if _opensky_token["value"] and now < _opensky_token["exp"]:
return _opensky_token["value"]
try:
data = urllib.parse.urlencode({
"grant_type": "client_credentials", "client_id": cid, "client_secret": csec,
}).encode()
req = urllib.request.Request(
OPENSKY_TOKEN_URL, data=data,
headers={"Content-Type": "application/x-www-form-urlencoded"})
with urllib.request.urlopen(req, timeout=15) as r:
tok = json.loads(r.read())
_opensky_token["value"] = tok["access_token"]
_opensky_token["exp"] = now + max(60, int(tok.get("expires_in", 1800)) - 60)
return _opensky_token["value"]
except Exception:
return None
def has_bbox(query):
# A global states/all request has no bounding-box params; only those bypass the cache.
q = query.lower()
return any(k in q for k in ("lamin", "lomin", "lamax", "lomax"))
def read_cache_if_fresh(path, max_age):
try:
st = path.stat()
except OSError:
return None
if time.time() - st.st_mtime >= max_age:
return None
try:
return path.read_bytes()
except OSError:
return None
def read_cache_any(path):
try:
return path.read_bytes()
except OSError:
return None
def write_cache(path, body):
# Atomic write: a unique temp file *beside* the target (same filesystem, so
# os.replace is atomic) then os.replace over the target. A unique temp name
# means concurrent writers (GODSIGH + godstrument, or two threads here) never
# corrupt each other's cache or expose a half-written file. Best-effort —
# caching must never break the actual response.
try:
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=path.name + ".", suffix=".tmp")
try:
with os.fdopen(fd, "wb") as f:
f.write(body)
os.replace(tmp, path)
except BaseException:
try:
os.unlink(tmp)
except OSError:
pass
raise
except OSError:
pass
class Handler(SimpleHTTPRequestHandler):
def end_headers(self):
# Dev server: never let the browser cache our static assets, so edits to
# js/css always take effect on reload (module graphs cache aggressively).
self.send_header("Cache-Control", "no-store")
super().end_headers()
def do_GET(self):
if self.path.startswith("/history/"):
return self._serve_history()
if not self.path.startswith("/proxy/"):
return super().do_GET()
name, _, query = self.path[len("/proxy/"):].partition("?")
if name == "mil":
return self._serve_mil()
if name.startswith("adsb/"):
return self._serve_adsb(name[len("adsb/"):])
if name == "adsbx-mil":
return self._serve_adsbx_mil()
base = UPSTREAMS.get(name)
if not base:
return self.send_error(404, f"unknown upstream {name!r}")
# Shared cache applies only to the OpenSky global states feed.
opensky_global = name == "opensky" and not has_bbox(query)
cache = opensky_cache_file()
# 1) Serve-from-cache: fresh (<120 s) global request costs zero quota.
if opensky_global:
fresh = read_cache_if_fresh(cache, OPENSKY_CACHE_FRESH_SEC)
if fresh is not None:
return self._send(200, "application/json", fresh, {"X-Godsigh-Cache": "hit"})
url = base + ("?" + query if query else "")
passthrough = {}
def grab(hdrs):
for k in ("X-Rate-Limit-Remaining", "X-Rate-Limit-Limit", "X-Expires-After"):
if hdrs.get(k) is not None:
passthrough[k] = hdrs.get(k)
req_headers = {"User-Agent": "godsigh-dev/0.1"}
if name == "opensky":
tok = opensky_bearer() # authenticated → 4000/day; None → anonymous 400/day
if tok:
req_headers["Authorization"] = "Bearer " + tok
try:
req = urllib.request.Request(url, headers=req_headers)
with urllib.request.urlopen(req, timeout=45) as r:
body = r.read()
status, ctype = r.status, r.headers.get("Content-Type", "text/plain")
grab(r.headers)
# 2) Write-through: a healthy global fetch refreshes the shared cache.
if opensky_global and status == 200:
write_cache(cache, body)
passthrough["X-Godsigh-Cache"] = "miss"
except urllib.error.HTTPError as e:
# 3) Stale-on-error: on 429 (quota) or any 5xx (upstream failure), keep
# the client working from the last-known cache. 4xx client errors are
# NOT masked with stale data.
if opensky_global and (e.code == 429 or e.code >= 500):
stale = read_cache_any(cache)
if stale is not None:
return self._send(200, "application/json", stale, {"X-Godsigh-Cache": "stale"})
body = e.read() or str(e).encode()
status, ctype = e.code, "text/plain"
grab(e.headers)
except Exception as e:
# Same stale fallback for connection errors/timeouts.
if opensky_global:
stale = read_cache_any(cache)
if stale is not None:
return self._send(200, "application/json", stale, {"X-Godsigh-Cache": "stale"})
body, status, ctype = str(e).encode(), 502, "text/plain"
self._send(status, ctype, body, passthrough)
def _send(self, status, ctype, body, extra=None):
self.send_response(status)
self.send_header("Content-Type", ctype)
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header(
"Access-Control-Expose-Headers",
"X-Rate-Limit-Remaining, X-Rate-Limit-Limit, X-Expires-After, X-Godsigh-Cache",
)
for k, v in (extra or {}).items():
self.send_header(k, v)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def do_POST(self):
# Dev-only screenshot sink: POST snap?name=<name> with a base64 PNG (or a
# full data:image/png;base64,... URL) body → writes docs/<name>.png. The
# server binds 127.0.0.1 only, and the name is sanitized to [a-z0-9-] so
# the write can never escape docs/. serve.py is never deployed.
if not (self.path == "/snap" or self.path.startswith("/snap?")):
return self.send_error(404, "not found")
_, _, query = self.path.partition("?")
raw_name = (urllib.parse.parse_qs(query).get("name") or ["screenshot"])[0]
name = re.sub(r"[^a-z0-9-]", "", raw_name.lower()) or "screenshot"
length = int(self.headers.get("Content-Length") or 0)
text = self.rfile.read(length).decode("utf-8", "replace") if length else ""
if text.strip().startswith("data:") and "," in text:
text = text.split(",", 1)[1]
try:
png = base64.b64decode(text, validate=False)
except Exception:
return self._send_json(400, {"error": "body must be base64 PNG (optionally a data: URL)"})
if not png:
return self._send_json(400, {"error": "empty image"})
docs = (Path(__file__).resolve().parent / "docs")
docs.mkdir(parents=True, exist_ok=True)
out = (docs / f"{name}.png").resolve()
if out.parent != docs.resolve():
return self._send_json(400, {"error": "invalid name"})
out.write_bytes(png)
return self._send_json(200, {"ok": True, "path": f"docs/{name}.png", "bytes": len(png)})
def _serve_mil(self):
# Free adsb.lol military feed; short in-memory cache (politeness, not quota).
now = time.time()
if _adsblol_cache["body"] is not None and now - _adsblol_cache["ts"] < ADSBLOL_CACHE_SEC:
return self._send(200, "application/json", _adsblol_cache["body"], {"X-Godsigh-Cache": "hit"})
try:
req = urllib.request.Request(ADSBLOL_MIL_URL, headers={"User-Agent": "godsigh/1.0 (partly.party)"})
with urllib.request.urlopen(req, timeout=30) as r:
body = r.read()
_adsblol_cache["body"] = body
_adsblol_cache["ts"] = now
return self._send(200, "application/json", body, {"X-Godsigh-Cache": "miss"})
except urllib.error.HTTPError as e:
if _adsblol_cache["body"] is not None:
return self._send(200, "application/json", _adsblol_cache["body"], {"X-Godsigh-Cache": "stale"})
return self._send_json(e.code, {"error": f"adsb.lol upstream {e.code}"})
except Exception as e:
if _adsblol_cache["body"] is not None:
return self._send(200, "application/json", _adsblol_cache["body"], {"X-Godsigh-Cache": "stale"})
return self._send_json(502, {"error": str(e)})
def _serve_adsb(self, path):
# Generic adsb.lol v2 query proxy, allowlisted by ADSB_PATH_RE.
if not ADSB_PATH_RE.match(path):
return self._send_json(400, {"error": "unsupported adsb path"})
hit = _adsb_cache.get(path)
if hit and time.time() - hit[1] < ADSB_CACHE_SEC:
return self._send(200, "application/json", hit[0], {"X-Godsigh-Cache": "hit"})
# Reserve a rate-limit slot so call STARTS are spaced ≥ADSB_MIN_INTERVAL,
# but do NOT hold the lock during the fetch — otherwise a big response
# (ladd, ~300 aircraft) would stall every other layer's poll.
with _adsb_lock:
hit = _adsb_cache.get(path) # re-check under lock
if hit and time.time() - hit[1] < ADSB_CACHE_SEC:
return self._send(200, "application/json", hit[0], {"X-Godsigh-Cache": "hit"})
start_at = max(time.time(), _adsb_next[0])
_adsb_next[0] = start_at + ADSB_MIN_INTERVAL
wait = start_at - time.time()
if wait > 0:
time.sleep(wait)
try:
req = urllib.request.Request(f"https://api.adsb.lol/v2/{path}",
headers={"User-Agent": "godsigh/1.0 (partly.party)"})
with urllib.request.urlopen(req, timeout=12) as r: # fail fast; adsb.lol can hang
body = r.read()
_adsb_cache[path] = (body, time.time())
return self._send(200, "application/json", body, {"X-Godsigh-Cache": "miss"})
except urllib.error.HTTPError as e:
if hit is not None:
return self._send(200, "application/json", hit[0], {"X-Godsigh-Cache": "stale"})
# adsb.lol rate-limits with 429 or 420 ("Enhance Your Calm"). With no
# cache to fall back on, return a clean empty result (200) rather than
# surfacing the error — the layer just shows nothing until the next
# poll, and the browser console stays quiet.
if e.code in (420, 429):
return self._send(200, "application/json", b'{"ac":[]}', {"X-Godsigh-Cache": "ratelimited"})
return self._send_json(e.code, {"error": f"adsb.lol upstream {e.code}"})
except Exception as e:
if hit is not None:
return self._send(200, "application/json", hit[0], {"X-Godsigh-Cache": "stale"})
return self._send_json(502, {"error": str(e)})
def _serve_adsbx_mil(self):
host, key = load_adsbx_creds()
if not host or not key:
# Not configured (e.g. no key file) — the layer degrades gracefully.
return self._send_json(503, {"error": "ADS-B Exchange not configured"})
now = time.time()
if _adsbx_cache["body"] is not None and now - _adsbx_cache["ts"] < ADSBX_CACHE_SEC:
return self._send(200, "application/json", _adsbx_cache["body"], {"X-Godsigh-Cache": "hit"})
try:
req = urllib.request.Request(
f"https://{host}/v2/mil/",
headers={"x-rapidapi-host": host, "x-rapidapi-key": key,
"User-Agent": "godsigh-dev/0.1"},
)
with urllib.request.urlopen(req, timeout=30) as r:
body = r.read()
_adsbx_cache["body"] = body
_adsbx_cache["ts"] = now
return self._send(200, "application/json", body, {"X-Godsigh-Cache": "miss"})
except urllib.error.HTTPError as e:
if _adsbx_cache["body"] is not None:
return self._send(200, "application/json", _adsbx_cache["body"], {"X-Godsigh-Cache": "stale"})
return self._send_json(e.code, {"error": f"adsbx upstream {e.code}"})
except Exception as e:
if _adsbx_cache["body"] is not None:
return self._send(200, "application/json", _adsbx_cache["body"], {"X-Godsigh-Cache": "stale"})
return self._send_json(502, {"error": str(e)})
def _send_json(self, status, obj, extra=None):
self._send(status, "application/json", json.dumps(obj).encode(), extra)
def _serve_history(self):
# /history/aircraft?t=<epoch_s> → nearest snapshot within ±10 min, or 404.
kind, _, query = self.path[len("/history/"):].partition("?")
params = urllib.parse.parse_qs(query)
t_raw = (params.get("t") or [None])[0]
if kind != "aircraft" or t_raw is None:
return self._send_json(400, {"error": "usage: history/aircraft?t=<epoch_s>"})
try:
t = int(float(t_raw))
except (ValueError, OverflowError): # OverflowError: t=inf / 1e999
return self._send_json(400, {"error": "t must be epoch seconds"})
if not HISTORY_DB.exists():
return self._send_json(404, {"error": "no history for this time"})
try:
# Read-only, short-lived connection per request — WAL lets us read
# without ever locking out the recorder's writes.
con = sqlite3.connect(f"file:{HISTORY_DB}?mode=ro", uri=True, timeout=2)
try:
row = con.execute(
"SELECT ts, body FROM snapshots WHERE kind='aircraft' "
"ORDER BY ABS(ts - ?) ASC LIMIT 1", (t,)).fetchone()
finally:
con.close()
except sqlite3.Error as e:
return self._send_json(500, {"error": f"history db error: {e}"})
if not row or abs(row[0] - t) > HISTORY_TOLERANCE_SEC:
return self._send_json(404, {"error": "no history for this time"})
try:
snap = zlib.decompress(row[1]) # already-JSON bytes: {"t":..,"planes":[...]}
except zlib.error:
return self._send_json(500, {"error": "corrupt snapshot"})
return self._send(200, "application/json", snap, {"X-Godsigh-History": "hit"})
def log_message(self, fmt, *args):
sys.stderr.write("%s - %s\n" % (self.address_string(), fmt % args))
if __name__ == "__main__":
port = int(sys.argv[1]) if len(sys.argv) > 1 else 8137
print(f"GODSIGH serving on http://127.0.0.1:{port}")
ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever()
Reference in New Issue View Git Blame Copy Permalink