// Voice control (Wave 4.1) — hands-free GODSIGH via the OpenAI Realtime API // over WebSocket, in the spirit of the open-sourced God's Eye View voice agent // (28 tools there; the ponytail cut here is 7 that cover fly/layers/styles/ // basemap/time/report). // // BYOK, strictly: the key is asked for once in a browser prompt and lives ONLY // in this browser's localStorage ('godsigh_openai_key'). It is never in the // repo, never on the server, never in assets/keys.json, and is sent only to // api.openai.com. No key → the button explains itself and nothing else changes. // // Costs are the user's own: Realtime audio is metered (~$0.10–0.30/min). const MODEL = 'gpt-realtime'; const KEY_STORE = 'godsigh_openai_key'; // host: { viewer, ui, Cesium, CONFIG, styles, setMode, getMode } export function initVoice(host) { const btn = document.getElementById('voice-btn'); const statusEl = document.getElementById('voice-status'); if (!btn) return; let session = null; const setStatus = (text, state) => { if (!statusEl) return; statusEl.hidden = !text; statusEl.textContent = text || ''; statusEl.dataset.state = state || 'ok'; }; btn.addEventListener('click', async () => { if (session) { session.stop('voice off'); return; } let key = localStorage.getItem(KEY_STORE); if (!key) { key = window.prompt( 'OpenAI API key for voice control.\nStored ONLY in this browser (localStorage); sent only to api.openai.com. Realtime audio is metered — set a usage cap on your OpenAI account.'); if (!key || !key.trim()) return; key = key.trim(); localStorage.setItem(KEY_STORE, key); } btn.disabled = true; session = await startSession(host, key, setStatus, () => { session = null; btn.classList.remove('active'); }); btn.disabled = false; if (session) btn.classList.add('active'); }); } async function startSession(host, key, setStatus, onEnd) { const { Cesium, viewer } = host; // ---- microphone ---- let stream; try { stream = await navigator.mediaDevices.getUserMedia({ audio: true }); } catch { setStatus('microphone denied — voice needs mic access', 'err'); onEnd(); return null; } const ac = new AudioContext({ sampleRate: 24000 }); let ws = null; let opened = false; let gotSession = false; let stopped = false; let playTime = 0; // next scheduled playback time let playing = []; // scheduled AudioBufferSourceNodes (for barge-in) function stop(msg, bad) { if (stopped) return; stopped = true; try { ws && ws.close(); } catch { /* already closed */ } stream.getTracks().forEach((t) => t.stop()); try { ac.close(); } catch { /* already closed */ } setStatus(msg || '', bad ? 'err' : 'ok'); onEnd(); } // ---- websocket (browser auth via subprotocol) ---- const startedAt = Date.now(); try { ws = new WebSocket( `wss://api.openai.com/v1/realtime?model=${MODEL}`, ['realtime', `openai-insecure-api-key.${key}`, 'openai-beta.realtime-v1'], ); } catch { stop('connection failed', true); return null; } ws.onopen = () => { opened = true; ws.send(JSON.stringify({ type: 'session.update', session: { type: 'realtime', instructions: 'You are the voice of GODSIGH, a live OSINT world globe. Speak like a terse mission operator: confirmations under 8 words. ' + 'Use the tools for every action and for any question about what is on screen (situation_report). ' + 'Never invent data — if a tool result is empty, say so.', tools: toolDefs(host), tool_choice: 'auto', }, })); setStatus('listening…', 'ok'); }; ws.onmessage = async (m) => { let ev; try { ev = JSON.parse(m.data); } catch { return; } if (ev.type === 'session.created' || ev.type === 'session.updated') gotSession = true; else if (ev.type === 'response.output_audio.delta' || ev.type === 'response.audio.delta') play(ev.delta); else if (ev.type === 'input_audio_buffer.speech_started') bargeIn(); else if (ev.type === 'response.function_call_arguments.done') { let args = {}; try { args = JSON.parse(ev.arguments || '{}'); } catch { /* leave {} */ } const output = await dispatch(host, ev.name, args); ws.send(JSON.stringify({ type: 'conversation.item.create', item: { type: 'function_call_output', call_id: ev.call_id, output } })); ws.send(JSON.stringify({ type: 'response.create' })); } else if (ev.type === 'error') { // Session-fatal errors surface here (bad session config, bad key scope…) setStatus(`voice error: ${(ev.error && ev.error.message) || 'unknown'}`, 'err'); } }; ws.onclose = () => { // Rejected key: closes almost immediately, before any session event. if (!gotSession && Date.now() - startedAt < 5000) { localStorage.removeItem(KEY_STORE); stop('key rejected — stored key cleared, click to retry', true); } else { stop(stopped ? '' : 'voice disconnected', !stopped); } }; ws.onerror = () => { /* onclose always follows and handles it */ }; // ---- mic → pcm16 base64 → input_audio_buffer.append ---- const src = ac.createMediaStreamSource(stream); // ponytail: ScriptProcessor is deprecated but universal; AudioWorklet when it breaks const proc = ac.createScriptProcessor(4096, 1, 1); proc.onaudioprocess = (e) => { if (!ws || ws.readyState !== 1) return; ws.send(JSON.stringify({ type: 'input_audio_buffer.append', audio: pcm16b64(e.inputBuffer.getChannelData(0)) })); }; src.connect(proc); proc.connect(ac.destination); // required sink; we never write output = silence // ---- model audio out (pcm16 24k base64 deltas) ---- function play(b64) { const bin = atob(b64); const n = bin.length / 2; if (!n) return; const buf = ac.createBuffer(1, n, 24000); const ch = buf.getChannelData(0); for (let i = 0; i < n; i++) { const v = (bin.charCodeAt(2 * i) | (bin.charCodeAt(2 * i + 1) << 8)) << 16 >> 16; ch[i] = v / 32768; } const node = ac.createBufferSource(); node.buffer = buf; node.connect(ac.destination); playTime = Math.max(playTime, ac.currentTime); node.start(playTime); playTime += buf.duration; playing.push(node); node.onended = () => { playing = playing.filter((x) => x !== node); }; } function bargeIn() { // user spoke over the model — cut playback for (const node of playing) { try { node.stop(); } catch { /* raced */ } } playing = []; playTime = 0; } return { stop }; } function pcm16b64(f32) { const out = new Uint8Array(f32.length * 2); for (let i = 0; i < f32.length; i++) { const s = Math.max(-1, Math.min(1, f32[i])); const v = s < 0 ? s * 0x8000 : s * 0x7fff; out[2 * i] = v & 0xff; out[2 * i + 1] = (v >> 8) & 0xff; } let bin = ''; for (let i = 0; i < out.length; i += 8192) bin += String.fromCharCode.apply(null, out.subarray(i, i + 8192)); return btoa(bin); } // ---- tools ------------------------------------------------------------------ function toolDefs(host) { const layerNames = host.ui.layerReport().map((l) => l.name).join(' · '); const obj = (props, required) => ({ type: 'object', properties: props, required }); return [ { type: 'function', name: 'fly_to', description: 'Fly the camera to a named place on Earth (city, landmark, strait, airport…).', parameters: obj({ place: { type: 'string' }, height_km: { type: 'number', description: 'eye height in km, default 40' } }, ['place']) }, { type: 'function', name: 'set_layer', description: `Turn a data layer on or off. Layers: ${layerNames}`, parameters: obj({ name: { type: 'string' }, on: { type: 'boolean' } }, ['name', 'on']) }, { type: 'function', name: 'set_style', description: 'Set the full-screen sensor style.', parameters: obj({ style: { type: 'string', enum: ['none', 'nvg', 'flir', 'crt'] } }, ['style']) }, { type: 'function', name: 'set_basemap', description: 'Switch basemap: data (dark), photo (Sentinel-2), bing (aerial, needs key), 3d (Google photoreal, needs key).', parameters: obj({ mode: { type: 'string', enum: ['data', 'photo', 'bing', '3d'] } }, ['mode']) }, { type: 'function', name: 'set_time', description: 'Scrub the timeline. Offset in minutes from now: negative = past, 0 = back to live. Window is ±6 hours.', parameters: obj({ minutes: { type: 'number' } }, ['minutes']) }, { type: 'function', name: 'reset_view', description: 'Reset to the default Persian Gulf overview, dark basemap.', parameters: obj({}, []) }, { type: 'function', name: 'situation_report', description: 'Read the current console state: every layer with its live status, basemap, style, camera position.', parameters: obj({}, []) }, ]; } async function dispatch(host, name, args) { const { viewer, ui, Cesium, CONFIG, styles, setMode, getMode } = host; try { if (name === 'fly_to') { const r = await fetch(`https://nominatim.openstreetmap.org/search?format=jsonv2&limit=1&q=${encodeURIComponent(args.place)}`); const j = await r.json(); if (!j[0]) return `No result for "${args.place}"`; viewer.camera.flyTo({ destination: Cesium.Cartesian3.fromDegrees(+j[0].lon, +j[0].lat, (args.height_km || 40) * 1000), duration: 3, }); return `Flying to ${j[0].display_name}`; } if (name === 'set_layer') { const q = String(args.name || '').toLowerCase(); const hit = ui.layerReport().find((l) => l.id === q || l.name.toLowerCase().includes(q)); if (!hit) return `No layer matching "${args.name}"`; ui.setLayerChecked(hit.id, !!args.on); return `${hit.name} ${args.on ? 'on' : 'off'}`; } if (name === 'set_style') { styles.setStyle(args.style); return `Style: ${args.style}`; } if (name === 'set_basemap') { setMode(args.mode); const got = getMode(); return got === args.mode ? `Basemap: ${got}` : `"${args.mode}" needs a key that isn't configured — staying on ${got}`; } if (name === 'set_time') { let jd = Cesium.JulianDate.addSeconds(Cesium.JulianDate.now(), (args.minutes || 0) * 60, new Cesium.JulianDate()); if (Cesium.JulianDate.lessThan(jd, viewer.clock.startTime)) jd = viewer.clock.startTime.clone(); if (Cesium.JulianDate.greaterThan(jd, viewer.clock.stopTime)) jd = viewer.clock.stopTime.clone(); viewer.clock.currentTime = jd; return args.minutes === 0 ? 'Back to live' : `Clock at now ${args.minutes > 0 ? '+' : ''}${args.minutes} min`; } if (name === 'reset_view') { setMode('data'); viewer.camera.flyTo({ destination: Cesium.Cartesian3.fromDegrees(CONFIG.camera.lon, CONFIG.camera.lat, CONFIG.camera.height), duration: 2 }); return 'Reset to Gulf overview'; } if (name === 'situation_report') { const carto = viewer.camera.positionCartographic; return JSON.stringify({ basemap: getMode(), style: styles.getStyle(), camera: { lat: +Cesium.Math.toDegrees(carto.latitude).toFixed(2), lon: +Cesium.Math.toDegrees(carto.longitude).toFixed(2), height_km: Math.round(carto.height / 1000) }, layers: ui.layerReport().filter((l) => l.on).map((l) => `${l.name}: ${l.status || 'on'}`), }); } return `Unknown tool ${name}`; } catch (err) { return `Tool failed: ${err && err.message}`; } }