266 lines
11 KiB
JavaScript
266 lines
11 KiB
JavaScript
// Voice control (Wave 4.1) — hands-free GODSIGH via the OpenAI Realtime API
|
||
// over WebSocket, in the spirit of the open-sourced God's Eye View voice agent
|
||
// (28 tools there; the ponytail cut here is 7 that cover fly/layers/styles/
|
||
// basemap/time/report).
|
||
//
|
||
// BYOK, strictly: the key is asked for once in a browser prompt and lives ONLY
|
||
// in this browser's localStorage ('godsigh_openai_key'). It is never in the
|
||
// repo, never on the server, never in assets/keys.json, and is sent only to
|
||
// api.openai.com. No key → the button explains itself and nothing else changes.
|
||
//
|
||
// Costs are the user's own: Realtime audio is metered (~$0.10–0.30/min).
|
||
|
||
const MODEL = 'gpt-realtime';
|
||
const KEY_STORE = 'godsigh_openai_key';
|
||
|
||
// host: { viewer, ui, Cesium, CONFIG, styles, setMode, getMode }
|
||
export function initVoice(host) {
|
||
const btn = document.getElementById('voice-btn');
|
||
const statusEl = document.getElementById('voice-status');
|
||
if (!btn) return;
|
||
|
||
let session = null;
|
||
const setStatus = (text, state) => {
|
||
if (!statusEl) return;
|
||
statusEl.hidden = !text;
|
||
statusEl.textContent = text || '';
|
||
statusEl.dataset.state = state || 'ok';
|
||
};
|
||
|
||
btn.addEventListener('click', async () => {
|
||
if (session) { session.stop('voice off'); return; }
|
||
let key = localStorage.getItem(KEY_STORE);
|
||
if (!key) {
|
||
key = window.prompt(
|
||
'OpenAI API key for voice control.\nStored ONLY in this browser (localStorage); sent only to api.openai.com. Realtime audio is metered — set a usage cap on your OpenAI account.');
|
||
if (!key || !key.trim()) return;
|
||
key = key.trim();
|
||
localStorage.setItem(KEY_STORE, key);
|
||
}
|
||
btn.disabled = true;
|
||
session = await startSession(host, key, setStatus, () => {
|
||
session = null;
|
||
btn.classList.remove('active');
|
||
});
|
||
btn.disabled = false;
|
||
if (session) btn.classList.add('active');
|
||
});
|
||
}
|
||
|
||
async function startSession(host, key, setStatus, onEnd) {
|
||
const { Cesium, viewer } = host;
|
||
|
||
// ---- microphone ----
|
||
let stream;
|
||
try {
|
||
stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||
} catch {
|
||
setStatus('microphone denied — voice needs mic access', 'err');
|
||
onEnd();
|
||
return null;
|
||
}
|
||
|
||
const ac = new AudioContext({ sampleRate: 24000 });
|
||
let ws = null;
|
||
let opened = false;
|
||
let gotSession = false;
|
||
let stopped = false;
|
||
let playTime = 0; // next scheduled playback time
|
||
let playing = []; // scheduled AudioBufferSourceNodes (for barge-in)
|
||
|
||
function stop(msg, bad) {
|
||
if (stopped) return;
|
||
stopped = true;
|
||
try { ws && ws.close(); } catch { /* already closed */ }
|
||
stream.getTracks().forEach((t) => t.stop());
|
||
try { ac.close(); } catch { /* already closed */ }
|
||
setStatus(msg || '', bad ? 'err' : 'ok');
|
||
onEnd();
|
||
}
|
||
|
||
// ---- websocket (browser auth via subprotocol) ----
|
||
const startedAt = Date.now();
|
||
try {
|
||
ws = new WebSocket(
|
||
`wss://api.openai.com/v1/realtime?model=${MODEL}`,
|
||
['realtime', `openai-insecure-api-key.${key}`, 'openai-beta.realtime-v1'],
|
||
);
|
||
} catch {
|
||
stop('connection failed', true);
|
||
return null;
|
||
}
|
||
|
||
ws.onopen = () => {
|
||
opened = true;
|
||
ws.send(JSON.stringify({
|
||
type: 'session.update',
|
||
session: {
|
||
type: 'realtime',
|
||
instructions:
|
||
'You are the voice of GODSIGH, a live OSINT world globe. Speak like a terse mission operator: confirmations under 8 words. ' +
|
||
'Use the tools for every action and for any question about what is on screen (situation_report). ' +
|
||
'Never invent data — if a tool result is empty, say so.',
|
||
tools: toolDefs(host),
|
||
tool_choice: 'auto',
|
||
},
|
||
}));
|
||
setStatus('listening…', 'ok');
|
||
};
|
||
|
||
ws.onmessage = async (m) => {
|
||
let ev;
|
||
try { ev = JSON.parse(m.data); } catch { return; }
|
||
if (ev.type === 'session.created' || ev.type === 'session.updated') gotSession = true;
|
||
else if (ev.type === 'response.output_audio.delta' || ev.type === 'response.audio.delta') play(ev.delta);
|
||
else if (ev.type === 'input_audio_buffer.speech_started') bargeIn();
|
||
else if (ev.type === 'response.function_call_arguments.done') {
|
||
let args = {};
|
||
try { args = JSON.parse(ev.arguments || '{}'); } catch { /* leave {} */ }
|
||
const output = await dispatch(host, ev.name, args);
|
||
ws.send(JSON.stringify({ type: 'conversation.item.create', item: { type: 'function_call_output', call_id: ev.call_id, output } }));
|
||
ws.send(JSON.stringify({ type: 'response.create' }));
|
||
} else if (ev.type === 'error') {
|
||
// Session-fatal errors surface here (bad session config, bad key scope…)
|
||
setStatus(`voice error: ${(ev.error && ev.error.message) || 'unknown'}`, 'err');
|
||
}
|
||
};
|
||
|
||
ws.onclose = () => {
|
||
// Rejected key: closes almost immediately, before any session event.
|
||
if (!gotSession && Date.now() - startedAt < 5000) {
|
||
localStorage.removeItem(KEY_STORE);
|
||
stop('key rejected — stored key cleared, click to retry', true);
|
||
} else {
|
||
stop(stopped ? '' : 'voice disconnected', !stopped);
|
||
}
|
||
};
|
||
ws.onerror = () => { /* onclose always follows and handles it */ };
|
||
|
||
// ---- mic → pcm16 base64 → input_audio_buffer.append ----
|
||
const src = ac.createMediaStreamSource(stream);
|
||
// ponytail: ScriptProcessor is deprecated but universal; AudioWorklet when it breaks
|
||
const proc = ac.createScriptProcessor(4096, 1, 1);
|
||
proc.onaudioprocess = (e) => {
|
||
if (!ws || ws.readyState !== 1) return;
|
||
ws.send(JSON.stringify({ type: 'input_audio_buffer.append', audio: pcm16b64(e.inputBuffer.getChannelData(0)) }));
|
||
};
|
||
src.connect(proc);
|
||
proc.connect(ac.destination); // required sink; we never write output = silence
|
||
|
||
// ---- model audio out (pcm16 24k base64 deltas) ----
|
||
function play(b64) {
|
||
const bin = atob(b64);
|
||
const n = bin.length / 2;
|
||
if (!n) return;
|
||
const buf = ac.createBuffer(1, n, 24000);
|
||
const ch = buf.getChannelData(0);
|
||
for (let i = 0; i < n; i++) {
|
||
const v = (bin.charCodeAt(2 * i) | (bin.charCodeAt(2 * i + 1) << 8)) << 16 >> 16;
|
||
ch[i] = v / 32768;
|
||
}
|
||
const node = ac.createBufferSource();
|
||
node.buffer = buf;
|
||
node.connect(ac.destination);
|
||
playTime = Math.max(playTime, ac.currentTime);
|
||
node.start(playTime);
|
||
playTime += buf.duration;
|
||
playing.push(node);
|
||
node.onended = () => { playing = playing.filter((x) => x !== node); };
|
||
}
|
||
|
||
function bargeIn() { // user spoke over the model — cut playback
|
||
for (const node of playing) { try { node.stop(); } catch { /* raced */ } }
|
||
playing = [];
|
||
playTime = 0;
|
||
}
|
||
|
||
return { stop };
|
||
}
|
||
|
||
function pcm16b64(f32) {
|
||
const out = new Uint8Array(f32.length * 2);
|
||
for (let i = 0; i < f32.length; i++) {
|
||
const s = Math.max(-1, Math.min(1, f32[i]));
|
||
const v = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||
out[2 * i] = v & 0xff;
|
||
out[2 * i + 1] = (v >> 8) & 0xff;
|
||
}
|
||
let bin = '';
|
||
for (let i = 0; i < out.length; i += 8192) bin += String.fromCharCode.apply(null, out.subarray(i, i + 8192));
|
||
return btoa(bin);
|
||
}
|
||
|
||
// ---- tools ------------------------------------------------------------------
|
||
|
||
function toolDefs(host) {
|
||
const layerNames = host.ui.layerReport().map((l) => l.name).join(' · ');
|
||
const obj = (props, required) => ({ type: 'object', properties: props, required });
|
||
return [
|
||
{ type: 'function', name: 'fly_to', description: 'Fly the camera to a named place on Earth (city, landmark, strait, airport…).',
|
||
parameters: obj({ place: { type: 'string' }, height_km: { type: 'number', description: 'eye height in km, default 40' } }, ['place']) },
|
||
{ type: 'function', name: 'set_layer', description: `Turn a data layer on or off. Layers: ${layerNames}`,
|
||
parameters: obj({ name: { type: 'string' }, on: { type: 'boolean' } }, ['name', 'on']) },
|
||
{ type: 'function', name: 'set_style', description: 'Set the full-screen sensor style.',
|
||
parameters: obj({ style: { type: 'string', enum: ['none', 'nvg', 'flir', 'crt'] } }, ['style']) },
|
||
{ type: 'function', name: 'set_basemap', description: 'Switch basemap: data (dark), photo (Sentinel-2), bing (aerial, needs key), 3d (Google photoreal, needs key).',
|
||
parameters: obj({ mode: { type: 'string', enum: ['data', 'photo', 'bing', '3d'] } }, ['mode']) },
|
||
{ type: 'function', name: 'set_time', description: 'Scrub the timeline. Offset in minutes from now: negative = past, 0 = back to live. Window is ±6 hours.',
|
||
parameters: obj({ minutes: { type: 'number' } }, ['minutes']) },
|
||
{ type: 'function', name: 'reset_view', description: 'Reset to the default Persian Gulf overview, dark basemap.', parameters: obj({}, []) },
|
||
{ type: 'function', name: 'situation_report', description: 'Read the current console state: every layer with its live status, basemap, style, camera position.', parameters: obj({}, []) },
|
||
];
|
||
}
|
||
|
||
async function dispatch(host, name, args) {
|
||
const { viewer, ui, Cesium, CONFIG, styles, setMode, getMode } = host;
|
||
try {
|
||
if (name === 'fly_to') {
|
||
const r = await fetch(`https://nominatim.openstreetmap.org/search?format=jsonv2&limit=1&q=${encodeURIComponent(args.place)}`);
|
||
const j = await r.json();
|
||
if (!j[0]) return `No result for "${args.place}"`;
|
||
viewer.camera.flyTo({
|
||
destination: Cesium.Cartesian3.fromDegrees(+j[0].lon, +j[0].lat, (args.height_km || 40) * 1000),
|
||
duration: 3,
|
||
});
|
||
return `Flying to ${j[0].display_name}`;
|
||
}
|
||
if (name === 'set_layer') {
|
||
const q = String(args.name || '').toLowerCase();
|
||
const hit = ui.layerReport().find((l) => l.id === q || l.name.toLowerCase().includes(q));
|
||
if (!hit) return `No layer matching "${args.name}"`;
|
||
ui.setLayerChecked(hit.id, !!args.on);
|
||
return `${hit.name} ${args.on ? 'on' : 'off'}`;
|
||
}
|
||
if (name === 'set_style') { styles.setStyle(args.style); return `Style: ${args.style}`; }
|
||
if (name === 'set_basemap') {
|
||
setMode(args.mode);
|
||
const got = getMode();
|
||
return got === args.mode ? `Basemap: ${got}` : `"${args.mode}" needs a key that isn't configured — staying on ${got}`;
|
||
}
|
||
if (name === 'set_time') {
|
||
let jd = Cesium.JulianDate.addSeconds(Cesium.JulianDate.now(), (args.minutes || 0) * 60, new Cesium.JulianDate());
|
||
if (Cesium.JulianDate.lessThan(jd, viewer.clock.startTime)) jd = viewer.clock.startTime.clone();
|
||
if (Cesium.JulianDate.greaterThan(jd, viewer.clock.stopTime)) jd = viewer.clock.stopTime.clone();
|
||
viewer.clock.currentTime = jd;
|
||
return args.minutes === 0 ? 'Back to live' : `Clock at now ${args.minutes > 0 ? '+' : ''}${args.minutes} min`;
|
||
}
|
||
if (name === 'reset_view') {
|
||
setMode('data');
|
||
viewer.camera.flyTo({ destination: Cesium.Cartesian3.fromDegrees(CONFIG.camera.lon, CONFIG.camera.lat, CONFIG.camera.height), duration: 2 });
|
||
return 'Reset to Gulf overview';
|
||
}
|
||
if (name === 'situation_report') {
|
||
const carto = viewer.camera.positionCartographic;
|
||
return JSON.stringify({
|
||
basemap: getMode(),
|
||
style: styles.getStyle(),
|
||
camera: { lat: +Cesium.Math.toDegrees(carto.latitude).toFixed(2), lon: +Cesium.Math.toDegrees(carto.longitude).toFixed(2), height_km: Math.round(carto.height / 1000) },
|
||
layers: ui.layerReport().filter((l) => l.on).map((l) => `${l.name}: ${l.status || 'on'}`),
|
||
});
|
||
}
|
||
return `Unknown tool ${name}`;
|
||
} catch (err) {
|
||
return `Tool failed: ${err && err.message}`;
|
||
}
|
||
}
|