GODSIGH/record.py

#!/usr/bin/env python3
"""GODSIGH history recorder — dev-side daemon (NOT shipped to prod).

Snapshots live aircraft every 180 s into a local SQLite db so the timeline can
replay ACTUAL past traffic instead of just hiding aircraft when scrubbed off
now. Fetches through the local dev proxy (http://127.0.0.1:8137/proxy/opensky)
so the shared OpenSky cache dedupes quota with the live app — it never calls
upstream directly. If the dev server is down it simply skips the cycle.

Prod has no recorder; the app's history endpoint 404s gracefully there, so the
aircraft layer just falls back to hide-when-scrubbed. Wire this into John's
heartbeat convention: status at ~/.jobs/godsigh-recorder.status.

Run it alongside the dev server:  python3 record.py
"""
import json
import sqlite3
import sys
import time
import urllib.request
import zlib
from datetime import datetime
from pathlib import Path

PROXY_URL = "http://127.0.0.1:8137/proxy/opensky"
DB_PATH = Path(__file__).resolve().parent / "data" / "history.db"
SNAPSHOT_SEC = 180          # one snapshot every 3 min
RETENTION_SEC = 72 * 3600   # keep 72 h (~70 MB steady-state)
PRUNE_EVERY_SEC = 3600      # prune + checkpoint hourly
JOB = "godsigh-recorder"


def heartbeat(count, note=""):
    # John's convention: overwrite ~/.jobs/<job>.status every iteration so
    # "is the recorder alive?" is one `cat` away.
    try:
        d = Path.home() / ".jobs"
        d.mkdir(exist_ok=True)
        (d / f"{JOB}.status").write_text(
            f"{datetime.now():%Y-%m-%d %H:%M:%S} | {count} snapshots | {note}\n")
    except OSError:
        pass


def connect():
    DB_PATH.parent.mkdir(parents=True, exist_ok=True)
    con = sqlite3.connect(str(DB_PATH))
    con.execute("PRAGMA journal_mode=WAL")
    con.execute("CREATE TABLE IF NOT EXISTS snapshots (ts INTEGER PRIMARY KEY, kind TEXT, body BLOB)")
    con.commit()
    return con


def reduce_states(data):
    # Reduce OpenSky /states/all to airborne essentials:
    # [icao24, callsign, lon, lat, baro_alt, track, vel]
    planes = []
    for s in data.get("states") or []:
        if not s:
            continue
        lon, lat, on_ground = s[5], s[6], s[8]
        if lon is None or lat is None or on_ground:
            continue
        planes.append([s[0], (s[1] or "").strip(), lon, lat, s[7], s[10], s[9]])
    return planes


def record_once(con):
    req = urllib.request.Request(PROXY_URL, headers={"User-Agent": "godsigh-recorder/0.1"})
    with urllib.request.urlopen(req, timeout=45) as r:
        data = json.loads(r.read())
    t = int(data.get("time") or time.time())
    snap = {"t": t, "planes": reduce_states(data)}
    body = zlib.compress(json.dumps(snap, separators=(",", ":")).encode())
    # INSERT OR REPLACE: if a stale cache hands us the same upstream time twice
    # (quota exhausted), we keep one row per timestamp instead of erroring.
    con.execute("INSERT OR REPLACE INTO snapshots (ts, kind, body) VALUES (?, ?, ?)",
                (t, "aircraft", body))
    con.commit()
    return t, len(snap["planes"])


def prune(con):
    cutoff = int(time.time()) - RETENTION_SEC
    con.execute("DELETE FROM snapshots WHERE ts < ?", (cutoff,))
    con.commit()
    con.execute("PRAGMA wal_checkpoint(TRUNCATE)")


def count_snaps(con):
    return con.execute("SELECT COUNT(*) FROM snapshots").fetchone()[0]


def main():
    con = connect()
    try:
        prune(con)
    except sqlite3.Error as e:
        print(f"[recorder] initial prune failed: {e}", file=sys.stderr)
    last_prune = time.time()
    n = count_snaps(con)
    heartbeat(n, "starting")
    print(f"[recorder] db={DB_PATH} snapshots={n} — every {SNAPSHOT_SEC}s via {PROXY_URL}")
    while True:
        try:
            t, planes = record_once(con)
            n = count_snaps(con)
            note = f"planes={planes} t={t}"
            print(f"[recorder] snapshot ok — {note} (retained {n})")
            heartbeat(n, note)
        except Exception as e:
            # Dev server down or upstream hiccup — skip this cycle, retry next
            # tick. Never fetch OpenSky directly (that would defeat the cache).
            print(f"[recorder] snapshot skipped: {e}", file=sys.stderr)
            heartbeat(n, f"skip: {e}")
        now = time.time()
        if now - last_prune > PRUNE_EVERY_SEC:
            try:
                prune(con)
                last_prune = now
            except sqlite3.Error as e:
                print(f"[recorder] prune failed: {e}", file=sys.stderr)
        time.sleep(SNAPSHOT_SEC)


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        print("\n[recorder] stopped")