From 1945c19fb3ac86357acc48fe2c33bd8053dfc56a Mon Sep 17 00:00:00 2001 From: type-two Date: Sat, 13 Jun 2026 19:54:33 +1000 Subject: [PATCH] discgod: API-token Discogs seller-inventory tracker Lean MV3 extension + standalone single-file ingest server that pulls watched sellers' full inventories from the Discogs API into discogs_full (mp_* schema), tracking price changes and sales over time. Successor to pliceclogs/marketwatcher; runs alongside the legacy plice server on a separate port (5057). Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 7 + README.md | 160 ++++++++ deploy.sh | 42 ++ extension/background.js | 312 ++++++++++++++ extension/content.js | 41 ++ extension/icons/icon128.png | Bin 0 -> 1688 bytes extension/icons/icon16.png | Bin 0 -> 613 bytes extension/icons/icon48.png | Bin 0 -> 3062 bytes extension/manifest.json | 46 +++ extension/popup.css | 114 ++++++ extension/popup.html | 82 ++++ extension/popup.js | 182 +++++++++ seeds/aus_sellers.txt | 104 +++++ server/discgod_crawl.py | 211 ++++++++++ server/discgod_server.py | 785 ++++++++++++++++++++++++++++++++++++ server/requirements.txt | 3 + server/schema.sql | 115 ++++++ 17 files changed, 2204 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 deploy.sh create mode 100644 extension/background.js create mode 100644 extension/content.js create mode 100644 extension/icons/icon128.png create mode 100644 extension/icons/icon16.png create mode 100644 extension/icons/icon48.png create mode 100644 extension/manifest.json create mode 100644 extension/popup.css create mode 100644 extension/popup.html create mode 100644 extension/popup.js create mode 100644 seeds/aus_sellers.txt create mode 100644 server/discgod_crawl.py create mode 100644 server/discgod_server.py create mode 100644 server/requirements.txt create mode 100644 server/schema.sql diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d278c3a --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.log +*.pyc +__pycache__/ +.DS_Store +.discgod_token +server/discgod.env +*_backup_*.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..84da26e --- /dev/null +++ b/README.md @@ -0,0 +1,160 @@ +# discgod + +A lean, all-in-one Discogs **seller inventory tracker**. Paste a Discogs API +token, pick the sellers you care about (Aussie or anyone), and discgod pulls +their **full inventories from the official Discogs API** on a schedule into the +`discogs_full` Postgres DB — building seller profiles and tracking price +changes and sales over time. No buttons, no DOM scraping. + +It's the token-driven successor to **pliceclogs / marketwatcher**: same database, +same proven `mp_*` schema and upsert/diff/sold-detection engine, but the data +now comes from the API (robust, no fragile selectors, no Cloudflare) instead of +scraping rendered pages. + +``` +┌─────────────────────────┐ Discogs API (your token, 60/min) +│ extension (background) │ ───────────────────────────────────► api.discogs.com +│ • holds token │ ◄─────────────────────────────────── /users/{u}/inventory +│ • schedules + paginates│ +│ • rate-limits + queue │ raw listing pages (HTTP POST) +└───────────┬─────────────┘ ─────────────► ┌────────────────────┐ + │ │ discgod_server.py │ ──► discogs_full + popup = control panel │ (maps + upserts) │ (mp_* tables) + └────────────────────┘ + optional, for 24/7 even with the browser closed: + discgod_crawl.py ──(same engine, server-side)──► discogs_full +``` + +## Layout + +``` +discgod/ + extension/ Chrome MV3 extension (load unpacked) + manifest.json + background.js engine: token, scheduler, API pagination, rate-limit, retry queue + popup.html/.css/.js control panel: token, watchlist, scrape-now, live stats + content.js "▶ Track in discgod" button on Discogs seller pages + icons/ + server/ + discgod_server.py STANDALONE single-file ingest server (port 5057) — + schema + API→mp_* mapping + upsert/diff/gone + watchlist + + HTTP, all in one. Coexists with the legacy :5002 server; + paste its handlers into pliceclogs_server.py to merge later. + discgod_crawl.py headless crawler for cron / 24-7 (imports the engine + from discgod_server; no browser needed) + schema.sql reference (server auto-applies it on startup) + requirements.txt just psycopg2 + seeds/ + aus_sellers.txt top 100 Australian sellers, ready to seed the watchlist + deploy.sh backup + deploy + restart the server on the tailscale box +``` + +## Quick start + +### 1. Server (writes Postgres) +The Ultra (`ultra.local` = tailscale `100.91.239.7`) hosts `discogs_full` +locally, so there's nothing to deploy — just run the standalone server *on the +Ultra itself*. It listens on **:5057** and coexists with the legacy plice server +on :5002 (separate process, separate port, same DB, untouched legacy endpoints): + +```bash +cd server +nohup /opt/homebrew/bin/python3 discgod_server.py > discgod_server.log 2>&1 & +curl -s localhost:5057/discgod/health # {"ok":true,"server":"discgod",...} +``` +It creates the `mp_*` tables and `discgod_seller_watch` in `discogs_full` on +startup (additive — touches nothing existing). `deploy.sh` is only needed if you +ever run the server on a *different* box. + +To keep it running across reboots, either add that `nohup` line to your +MONSTERPANEL startup alongside the plice server, or wrap it in a launchd plist. + +### 2. Extension (the engine + control panel) +1. `chrome://extensions` → enable Developer mode → **Load unpacked** → pick `discgod/extension`. +2. Open the popup, paste your **Discogs API token** + ([Settings → Developers](https://www.discogs.com/settings/developers) → *Generate token*), + click **Test** (should say "signed in as …"). +3. Set the **Ingest server URL** (default `http://100.91.239.7:5057`) and an + interval, tick **Auto-crawl**, **Save**. +4. Add sellers — type a username (or paste a `/seller/NAME` URL) and **Track**, + or hit the **▶ Track in discgod** button while on any Discogs seller page. + Add an optional filter like `format=Vinyl` or `ships_from=Australia`. +5. **Scrape all now**, or wait for the schedule. Watch the live stats fill in. + +That's the whole "add token and go" loop. + +### 3. (Optional) 24/7 headless crawling on the box +The extension only runs while Chrome is open. For always-on tracking, run the +crawler on the box — it shares the same watchlist: + +```bash +ssh johnking@100.91.239.7 +echo YOUR_TOKEN > ~/.discgod_token && chmod 600 ~/.discgod_token +cd /Users/johnking/Documents/MONSTERPANEL/discgod + +python3 discgod_crawl.py --seed /tmp/aus_sellers.txt # seed the 100 Aussie sellers +python3 discgod_crawl.py # crawl everything once +python3 discgod_crawl.py Dennis-brisbane-qld # just one seller +python3 discgod_crawl.py --loop 43200 # crawl all, repeat every 12 h +``` + +Or cron it (daily 3am): +``` +0 3 * * * cd /Users/johnking/Documents/MONSTERPANEL/discgod && /opt/homebrew/bin/python3 discgod_crawl.py >> crawl.log 2>&1 +``` + +## What gets captured + +| Table | Contents | +|---|---| +| `mp_listing` | One row per listing (`item_id` PK, `release_id` joins to your XML `release` table). Condition, sleeve, notes, price/currency, shipping, converted total, have/want, ships-from, seller. `first_seen` / `last_seen` / `times_seen` / `status` (`active`/`gone`). | +| `mp_listing_event` | Append-only history: `listed`, `price_change` (old/new), `condition_change`, `gone` (presumed sold/delisted, with last price), `relisted`. | +| `mp_seller` | Each seller seen — id, username, rating, ships-from. | +| `mp_seller_snapshot` | Inventory-size time series (`source='api_inventory'`), from the API's authoritative pagination total. Deduped within 6 h per (seller, filter). | +| `mp_page_log` | Provenance: every captured inventory page + the item_ids it held. | +| `discgod_seller_watch` | The watchlist: who to crawl, optional per-seller filter, last-crawled status. | + +**Sold detection:** every full crawl stamps each listing with a `crawl_id`. When +the crawl finishes, listings the crawl *didn't* see (`last_crawl_id` differs) are +flipped to `gone` with a `gone` event — your sold/delisted feed. Filtered crawls +(e.g. `format=Vinyl`) skip gone-marking unless they cover the seller's whole +inventory. + +## Useful queries + +```sql +-- A seller's inventory size over time +SELECT captured_at::date, max(total_listings) +FROM mp_seller_snapshot +WHERE username = 'Dennis-brisbane-qld' AND source = 'api_inventory' +GROUP BY 1 ORDER BY 1; + +-- Presumed sales last 30 days, with full release info from the XML dump +SELECT e.recorded_at::date, e.seller_username, e.old_price, r.title, ra.artist_name +FROM mp_listing_event e +JOIN release r ON r.id = e.release_id +LEFT JOIN release_artist ra ON ra.release_id = r.id AND ra.position = 1 +WHERE e.event = 'gone' AND e.recorded_at > now() - interval '30 days' +ORDER BY e.recorded_at DESC; + +-- Price drops +SELECT e.recorded_at, e.seller_username, e.old_price, e.new_price, r.title +FROM mp_listing_event e JOIN release r ON r.id = e.release_id +WHERE e.event = 'price_change' AND e.new_price < e.old_price +ORDER BY e.recorded_at DESC LIMIT 50; +``` + +## Notes & limits + +- **Rate limit:** the Discogs API allows ~60 authenticated requests/min. discgod + paces itself (~55/min) and backs off on 429. A 26k-listing seller is ~270 API + pages ≈ 5 min — fine periodically; the headless crawler is better for big + bulk sweeps. +- **Public inventories only** — the API returns another user's listings that are + "For Sale". That's every real seller. +- The extension's background worker can be suspended by Chrome on very long + crawls; discgod persists the per-seller queue and a 1-minute keepalive so a + sweep resumes where it left off. For unattended bulk work, prefer the headless + crawler. +- Off-box server? Set `PGHOST`/`PGUSER`/`PGPASSWORD` (and `DISCGOD_DB`) before + launching `discgod_server.py`. On the box it uses local peer auth, no creds. diff --git a/deploy.sh b/deploy.sh new file mode 100644 index 0000000..f01586d --- /dev/null +++ b/deploy.sh @@ -0,0 +1,42 @@ +#!/bin/zsh +# Deploy the discgod ingest server (+ headless crawler) to the tailscale box +# and restart it. Backs up the existing server first. Run from anywhere: +# zsh /Users/johnking/discgod/deploy.sh +set -e + +HOST=johnking@100.91.239.7 +DIR=/Users/johnking/Documents/MONSTERPANEL/discgod +PORT=5057 + +# Resolve this script's directory so the local paths work from any machine. +HERE=${0:A:h} +LOCAL_SERVER=$HERE/server + +scp "$LOCAL_SERVER/discgod_server.py" \ + "$LOCAL_SERVER/discgod_crawl.py" \ + "$LOCAL_SERVER/schema.sql" \ + "$HOST:/tmp/" + +ssh "$HOST" " +set -e +mkdir -p $DIR +cd $DIR +# Back up any existing server before overwriting. +[ -f discgod_server.py ] && cp discgod_server.py discgod_server_backup_\$(date +%Y%m%d_%H%M).py || true +cp /tmp/discgod_server.py /tmp/discgod_crawl.py /tmp/schema.sql . +chmod +x discgod_crawl.py + +# Restart the ingest server. +OLDPID=\$(lsof -tiTCP:$PORT -sTCP:LISTEN || true) +[ -n \"\$OLDPID\" ] && kill \$OLDPID && sleep 2 || true +nohup /opt/homebrew/bin/python3 discgod_server.py > discgod_server.log 2>&1 & +sleep 3 +echo '--- health ---'; curl -s http://localhost:$PORT/discgod/health; echo +echo '--- stats ---'; curl -s http://localhost:$PORT/discgod/stats; echo +echo '--- log ---'; head -5 discgod_server.log +" +echo +echo "Done. Tables (mp_* + discgod_seller_watch) are created in discogs_full on startup." +echo "Next: put your token in ~/.discgod_token on the box for the headless crawler:" +echo " ssh $HOST 'echo YOUR_TOKEN > ~/.discgod_token && chmod 600 ~/.discgod_token'" +echo "Seed the Aussie sellers: ssh $HOST 'cd $DIR && python3 discgod_crawl.py --seed /tmp/aus_sellers.txt'" diff --git a/extension/background.js b/extension/background.js new file mode 100644 index 0000000..837b919 --- /dev/null +++ b/extension/background.js @@ -0,0 +1,312 @@ +/* + * discgod — background service worker (the engine). + * + * Holds the Discogs API token, runs the schedule, paginates each watched + * seller's full inventory from api.discogs.com (rate-limited, client-side so it + * uses *your* token's 60/min quota), and POSTs the raw listing pages to the + * ingest server, which maps + upserts them into discogs_full and detects price + * changes / sales. + * + * The watchlist itself lives server-side (discgod_seller_watch) so the popup, + * this worker, and the headless cron crawler all agree on who to track. + */ + +const DEFAULTS = { + token: "", + serverUrl: "http://100.91.239.7:5057", + intervalHours: 12, + status: "For Sale", + perPage: 100, + baseDelayMs: 1100, // ~55 Discogs calls/min, under the 60/min ceiling + autoCrawl: true, +}; + +const ALARM_CRAWL = "discgod-crawl"; // periodic full sweep +const ALARM_KEEPALIVE = "discgod-tick"; // resurrects the worker mid-crawl / drains queue +const QUEUE_KEY = "sendQueue"; // failed server POSTs, retried later + +// --------------------------------------------------------------------------- +// Config +// --------------------------------------------------------------------------- +async function getConfig() { + const stored = await chrome.storage.local.get(Object.keys(DEFAULTS)); + return { ...DEFAULTS, ...stored }; +} +async function setConfig(patch) { + await chrome.storage.local.set(patch); + if ("intervalHours" in patch) await scheduleCrawl(); + return getConfig(); +} + +// --------------------------------------------------------------------------- +// Lifecycle +// --------------------------------------------------------------------------- +chrome.runtime.onInstalled.addListener(scheduleCrawl); +chrome.runtime.onStartup.addListener(scheduleCrawl); + +async function scheduleCrawl() { + const { intervalHours, autoCrawl } = await getConfig(); + await chrome.alarms.clear(ALARM_CRAWL); + if (autoCrawl) { + const period = Math.max(15, Math.round(intervalHours * 60)); // minutes; min 15 + chrome.alarms.create(ALARM_CRAWL, { periodInMinutes: period, delayInMinutes: 1 }); + } +} + +chrome.alarms.onAlarm.addListener(async (alarm) => { + if (alarm.name === ALARM_CRAWL) { + crawlAll().catch((e) => log("crawlAll failed", e.message)); + } else if (alarm.name === ALARM_KEEPALIVE) { + // Worker was revived (or never died) mid-crawl: keep draining. + drainQueue(); + const { crawling } = await chrome.storage.local.get("crawling"); + if (crawling) processQueue().catch(() => {}); + } +}); + +function keepaliveOn() { + chrome.alarms.create(ALARM_KEEPALIVE, { periodInMinutes: 1 }); +} +function keepaliveOff() { + chrome.alarms.clear(ALARM_KEEPALIVE); +} + +// --------------------------------------------------------------------------- +// Discogs API (client-side, authenticated, rate-limited) +// --------------------------------------------------------------------------- +async function discogsGet(path, params) { + const cfg = await getConfig(); + if (!cfg.token) throw new Error("No Discogs token set"); + const url = new URL("https://api.discogs.com" + path); + if (params) for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); + + for (let attempt = 0; attempt < 5; attempt++) { + const resp = await fetch(url, { + headers: { Authorization: `Discogs token=${cfg.token}`, Accept: "application/json" }, + }); + if (resp.status === 429) { + const wait = Math.min(60000, 2000 * 2 ** attempt); + await sleep(wait); + continue; + } + if (resp.status === 401) throw new Error("401 — bad or missing token"); + if (resp.status === 404) throw new Error("404 — no such user/inventory"); + if (!resp.ok) { + await sleep(1500 * (attempt + 1)); + continue; + } + const remaining = parseInt(resp.headers.get("X-Discogs-Ratelimit-Remaining") || "60", 10); + await sleep(remaining <= 2 ? 5000 : cfg.baseDelayMs); + return resp.json(); + } + throw new Error(`Discogs API gave up: ${path}`); +} + +// Quick token check — returns the authenticated identity. +async function testToken() { + const me = await discogsGet("/oauth/identity"); + return { ok: true, username: me.username }; +} + +// --------------------------------------------------------------------------- +// Server I/O (the thing that writes Postgres). Failed posts queue + retry. +// --------------------------------------------------------------------------- +async function serverFetch(endpoint, { method = "GET", body } = {}) { + const { serverUrl } = await getConfig(); + const resp = await fetch(serverUrl + endpoint, { + method, + headers: { "Content-Type": "application/json" }, + body: body ? JSON.stringify(body) : undefined, + }); + if (!resp.ok) throw new Error(`server ${endpoint} -> ${resp.status}`); + return resp.json(); +} + +// Post to the ingest server; on failure, queue for later retry so a momentary +// server/network blip never loses a captured page. +async function postOrQueue(endpoint, body) { + try { + return await serverFetch(endpoint, { method: "POST", body }); + } catch (e) { + const { [QUEUE_KEY]: q = [] } = await chrome.storage.local.get(QUEUE_KEY); + q.push({ endpoint, body, attempts: 0, ts: Date.now() }); + await chrome.storage.local.set({ [QUEUE_KEY]: q }); + setBadge(String(q.length), "#c0392b"); + throw e; + } +} + +async function drainQueue() { + const { [QUEUE_KEY]: q = [] } = await chrome.storage.local.get(QUEUE_KEY); + if (!q.length) return; + const keep = []; + for (const item of q) { + try { + await serverFetch(item.endpoint, { method: "POST", body: item.body }); + } catch { + item.attempts++; + if (item.attempts < 10) keep.push(item); + } + } + await chrome.storage.local.set({ [QUEUE_KEY]: keep }); + setBadge(keep.length ? String(keep.length) : "", keep.length ? "#c0392b" : "#2e7d32"); +} + +// --------------------------------------------------------------------------- +// Crawl one seller: paginate their whole inventory, post each page, finish. +// --------------------------------------------------------------------------- +async function crawlSeller(username, filter) { + const cfg = await getConfig(); + filter = filter || {}; + const crawlId = `${username}-${Date.now()}`; + + // Profile gives seller_id + a reliable num_for_sale for the snapshot. + let profile = null, sellerId = null; + try { + profile = await discogsGet(`/users/${encodeURIComponent(username)}`); + sellerId = profile.id; + } catch (e) { + log(`profile ${username}`, e.message); + } + + let page = 1, pages = 1, total = null; + const baseParams = { + per_page: cfg.perPage, status: cfg.status, sort: "listed", sort_order: "desc", ...filter, + }; + + while (page <= pages) { + const data = await discogsGet(`/users/${encodeURIComponent(username)}/inventory`, + { ...baseParams, page }); + const pg = data.pagination || {}; + pages = pg.pages || 1; + total = pg.items ?? total; + + await postOrQueue("/discgod/ingest", { + username, seller_id: sellerId, + seller: page === 1 ? profile : null, + page, total, crawl_id: crawlId, filter, + listings: data.listings || [], + }); + setBadge("⟳", "#1565c0"); + page++; + } + + const filtered = Object.keys(filter).length > 0; + await postOrQueue("/discgod/crawl_complete", { + username, crawl_id: crawlId, filter, + treat_complete: !filtered, listing_count: total, + }); + return total; +} + +// --------------------------------------------------------------------------- +// Crawl every watched seller (queue-driven so it survives worker restarts). +// --------------------------------------------------------------------------- +async function crawlAll() { + let tracked; + try { + const res = await serverFetch("/discgod/tracked"); + tracked = (res.sellers || []).filter((s) => s.enabled); + } catch (e) { + log("cannot load watchlist", e.message); + return; + } + const queue = tracked.map((s) => ({ username: s.username, filter: s.filter_params || {} })); + await chrome.storage.local.set({ crawlQueue: queue, crawling: true }); + await processQueue(); +} + +async function crawlOne(username) { + await chrome.storage.local.set({ + crawlQueue: [{ username, filter: {} }], crawling: true, + }); + await processQueue(); +} + +// Drains crawlQueue one seller at a time, persisting progress between sellers. +let processing = false; +async function processQueue() { + if (processing) return; + processing = true; + keepaliveOn(); + try { + for (;;) { + const { crawlQueue = [] } = await chrome.storage.local.get("crawlQueue"); + if (!crawlQueue.length) break; + const job = crawlQueue[0]; + try { + const n = await crawlSeller(job.username, job.filter); + log(`crawled ${job.username}`, `${n} listings`); + } catch (e) { + log(`crawl ${job.username} failed`, e.message); + } + const { crawlQueue: cur = [] } = await chrome.storage.local.get("crawlQueue"); + cur.shift(); + await chrome.storage.local.set({ crawlQueue: cur, lastCrawlAt: Date.now() }); + } + await drainQueue(); + } finally { + await chrome.storage.local.set({ crawling: false }); + keepaliveOff(); + setBadge("", "#2e7d32"); + processing = false; + notify("discgod", "Inventory sweep complete"); + } +} + +// --------------------------------------------------------------------------- +// Messages from the popup / content script +// --------------------------------------------------------------------------- +chrome.runtime.onMessage.addListener((msg, _sender, sendResponse) => { + (async () => { + try { + switch (msg.type) { + case "getConfig": return sendResponse(await getConfig()); + case "setConfig": return sendResponse(await setConfig(msg.patch)); + case "testToken": return sendResponse(await testToken()); + case "getStats": return sendResponse(await serverFetch("/discgod/stats")); + case "getTracked": return sendResponse(await serverFetch("/discgod/tracked")); + case "track": return sendResponse(await serverFetch("/discgod/track", + { method: "POST", body: msg.body })); + case "untrack": return sendResponse(await serverFetch("/discgod/untrack", + { method: "POST", body: { username: msg.username } })); + case "scrapeAll": crawlAll(); return sendResponse({ ok: true, started: true }); + case "scrapeOne": crawlOne(msg.username); return sendResponse({ ok: true, started: true }); + case "status": return sendResponse(await statusSnapshot()); + default: return sendResponse({ ok: false, error: "unknown message" }); + } + } catch (e) { + sendResponse({ ok: false, error: e.message }); + } + })(); + return true; // async +}); + +async function statusSnapshot() { + const { crawling, crawlQueue = [], lastCrawlAt, [QUEUE_KEY]: q = [] } = + await chrome.storage.local.get(["crawling", "crawlQueue", "lastCrawlAt", QUEUE_KEY]); + return { ok: true, crawling: !!crawling, remaining: crawlQueue.length, + queued: q.length, lastCrawlAt: lastCrawlAt || null }; +} + +// --------------------------------------------------------------------------- +// Small utilities +// --------------------------------------------------------------------------- +function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); } + +function setBadge(text, color) { + chrome.action.setBadgeText({ text: text || "" }); + if (color) chrome.action.setBadgeBackgroundColor({ color }); +} + +function notify(title, message) { + try { + chrome.notifications.create({ + type: "basic", iconUrl: "icons/icon128.png", title, message, + }); + } catch { /* notifications optional */ } +} + +function log(label, detail) { + console.log(`[discgod] ${label}${detail ? ": " + detail : ""}`); +} diff --git a/extension/content.js b/extension/content.js new file mode 100644 index 0000000..e8b0545 --- /dev/null +++ b/extension/content.js @@ -0,0 +1,41 @@ +/* discgod content script — adds a floating "Track in discgod" button on + * Discogs seller pages. Reads the username from the URL and asks the background + * worker to add it to the watchlist. Purely a convenience; all the real work + * happens in the background worker + server. */ + +(function () { + const m = location.pathname.match(/\/seller\/([^/?#]+)/); + if (!m) return; + const username = decodeURIComponent(m[1]); + if (document.getElementById("discgod-track-btn")) return; + + const btn = document.createElement("button"); + btn.id = "discgod-track-btn"; + btn.textContent = "▶ Track in discgod"; + Object.assign(btn.style, { + position: "fixed", right: "16px", bottom: "16px", zIndex: 99999, + padding: "9px 14px", borderRadius: "8px", border: "none", + background: "#4f9dff", color: "#07111f", fontWeight: "700", + fontSize: "13px", cursor: "pointer", fontFamily: "system-ui, sans-serif", + boxShadow: "0 3px 12px rgba(0,0,0,.35)", + }); + + btn.addEventListener("click", () => { + btn.disabled = true; + btn.textContent = "…"; + chrome.runtime.sendMessage({ type: "track", body: { username, filter: {} } }, (r) => { + const ok = r && r.ok; + btn.textContent = ok ? `✓ Tracking ${username}` : "✕ Failed"; + btn.style.background = ok ? "#38b000" : "#e5484d"; + btn.style.color = "#fff"; + setTimeout(() => { + btn.textContent = "▶ Track in discgod"; + btn.style.background = "#4f9dff"; + btn.style.color = "#07111f"; + btn.disabled = false; + }, 2500); + }); + }); + + document.body.appendChild(btn); +})(); diff --git a/extension/icons/icon128.png b/extension/icons/icon128.png new file mode 100644 index 0000000000000000000000000000000000000000..cdb402492ae78b2d76ad253139ac63912f4399cb GIT binary patch literal 1688 zcmV;J250$+P) zOOD(&5Qf{0IfI>mF^~+h@ilri_i>I9-^Z)IMmB*Rz!ApwC1MDOY(fx47VAyczp^nB zrPlwkiVumrTtcBxC=?3Ks236!dHK`p|0J$6qP~9q#Oe`S;bFQD7)T2Y)BV9fQb-u@ z7lbSJ2Eu$@VV^$=fH2>NFg|pJk1&3mlmLzZVZLoa{Q&_G=KB!Dr)B`cd|zty3*8F{ z`BoDEQTu(W#kVQ|VZLwG{5u68%pXH%K1Lg0auorH+CP>`em-0#(a*m;g3o#W_rqm$ z4GBPx*{$V_2E~g}!hg;54}bp!F7xsCH>ZF12*ZA^z%>9R(|gRuLg595ApkfszGixQ z{Tui?%cqZE7_5U(1;DcT5aaiT*UZF};SGcw0v7=6jc+l%$MBk3%HP9a5d>EN{}ScAX~0B7cJ21uG-Qt4i-LSP;MFUETX!88Ch^DP>`*U^1ig1{^QCG)9j zJ)|qR5M3)T#|i`{0ocr^to4|#U?S*7)}nzH1m*ysG@q+ikPzisgTNF3N9Om0e96XP zBDy~<2+RObGT*B4R3K)jpyd)ZhSIc?dV#9DDYVeiB`8t~#02KZGGigi` zEkI)-Wo^$%qF12O4WtVF>oJ-$gv@7 z()g2%4!3t-ihsMmeT6Q+HUI>fGuY+{xCVyAMgp;uI9T(~OfMaWfuVO|2z#!}6u|vs zTg|6B&r_<%c z-xIJQID)1G;K=N!3C$m>8@P7Kpk2Wygy|!60jG`tAp-zd9wMD($eSF1?Zw6IGiOO7 zrNZ)E=sGzZT%U~(mjbxWxet>2T>vmb8c4=xHLoL@%vDS8#N}Fm<2q_hrZ(#?9Qs{< z7qD*#YQeLbhBfYvD1*vG&UJ&-Q-DN-=r!=pSnDl>f6xha%LP>#c--Gi`uev3Kxg-o1l2#qQykxdEVtAU`)3G*ol- z3N*KlHw(b3e4q%soxD~z@a&%R*1necrU5vEz|-3{>7>?cn{OU~76?{Z1R)jf&64>p z0PqGva$>wY0BR6$O#HN}c3rFHy8~d=eipGbe9R3)YAAm{tZU7DmjG-!S}3^%hF&02 zV;M(d`4)|D0>C;1P!Klt%o{8fXuKJ}H{UG)n@UTkb8fi!SeUu9Zv2n{0ki7LLjl%?z2PFOXd%?PGHTX9w?q2o877*0hrZH(l>&R!bQlhN`AWfJUm3N zQvf0gjG;3hs}Ud|@U5DEO(`HC@TnHxq75*)ngNI?;7hH3fg(Ua;6o6fz$qXguq~)R zsKFl3^97^nYv;fnsu38`z=kkBpnCxUfv&L6pLD!GQi7bQDjwT}PryJgj%X_~;}{Y@ literal 0 HcmV?d00001 diff --git a/extension/icons/icon16.png b/extension/icons/icon16.png new file mode 100644 index 0000000000000000000000000000000000000000..9ce5280298b3a44037188ab6175a56d8d0d42f88 GIT binary patch literal 613 zcmV-r0-F7aP)ogN;Xd*ErP7>I$n7$}+;U#2D^Vxa$kiK$@dWrB$qikN$X7?=v? z$|b8H;_e@EAdSbTu7KTD5qQ`V!pb7iSQ31iOuoO< z1JdJ@+dGem_z6Jvg9tL9z|Mn6#pQxAwzm7C z%Ee0pfpluG}d|{)pbJo;Yz`RiUY%?=EwDk<@WiSfMcg4PPDa9bZ}8kDyi&v-%4^Si{%Xj}ZP$P!t8e zyu1r0f-F0CIiL(t57pzcC@~My4D8o8e13Y(Ix_)&xNdB!(Hc@}mkNCM7tIBn!(cEp z+Dbl8nOAJHh2AkjQyVLh2yYw+NmYc#Art!{mcJ97$Spxe1G35r?G(CBG6e#`8kWEk3S~|d#dIyS8x<}qk%w%Neoek(wR;>J#Eir-`6tVzxHW6!%TZx$HA4H?tS)NXRZJ8 z{nu6C%EPNa@Agmt{Q7#?mzNtPyyDH=+>O_1ANOjF_ZnmDF2A{|!&+5%Ngw~fV?6WK979YPri+fevXpi)HE?Jph%%LMU-R*1pduy%P&FXh2*n&TIVB3K-F@0x6a#i}ENG!D0 zK~+`1SXo|tl(dgsR5`zJ^m(oH=dCqm{cPf4jUnmqJRd<2LVG^KFqUVIHHE4wP!u_; zvP4m2@?7h&Aos-#C)*;cj`ey=Z;{YyVPRpug8h^e{-nNNZ4n+fnuy{iwAQF917j_c zG;y;O=UI5ZhcIeF#Tr@zd6pqf2B@mCW@KXeE3G|04DEdYo+1Obf_J~Cy{jZWN=)Qw#^2Tg*-%IK@3I@Z~WB z&`RG@uT2%~$pAIgZcibMTUcMe0P@YvD`NyAlxppMw}#eAB<=YQotqnL!fdBAjp^BW zY|1(Yit5=NQo^Nyx|vs5&03p{ZER2CjluMx!zhXpt1IWAJx>VE^IQn>eP;)L5IBj} zvAZk^R8=L=0aCG-FpRLd*+URUn3-F^=GqGKA{Qv`H54p8<(9EdM!YrXPS2s=?~5Ul zz$_WTSc4#l5Jr(0EE%D}@;b&1847{`Pz_;*a7&V*7(2hOo?k|HY8rObN0Dc>UF@xi ziM@@J{1NmAo3a6krZg$7R;w#>@+=i%_>KhZ#58}yA(D?`v4K2Ggc08lL>n9HYnVNB z7{h)~1TcwBOu&GGsYoG&3V{A3u!+P?wy?T-P9Q>pqbNcUh5`c`t{;X% z05wHM%d&7DiEh-6V?=;PvnB7+B*ADjL=?yJEKP7TRG$b7@+hSq`+*jD%}RgqS0sxBZNd8w_F?;4g^}B=LzA~Rwzd6 z@~Mhcyp=FS!%q`rc_xxq$`eh{2B;<4k<7_5fh{$}WWUEoji(u*~`hJ9C(M|0kh+g=FOn21{>1`60ZpA*=KEp7sPn)Eu?OduoV+ z$jM-%DYoJ=1hroV82)`xf1akYD5EPq$XH5oqamA%ttk)IR4}H%`yO}#pLx#9aSvu(pQ}SK|rh+vE?)=vCxb49oA?hAQl~yQ*C6wRB!kzcyt{=XLsbe2S zIa(Ln0JWNDZY}GgIPYmM+=xLf*h&ib$$-4new`99o2(>Q58mT??parWsnaG1kE;XeeiMk%^9&NrL;(!(k1@Os+8+^4>y>8U)ct( zzC=f#N{0(F;1p#phRRJTF)8<|0%2z!NAG?FWnzRWFVv`t9HR^85Hy&92FOPh zWzZDzDH!d;>oI z=rh<@d>uD`_95gW$0XP6FTHlv`UHBnmlZt^gHLTpWmSkB7{f*Z6Ejcl>DvcOr?7VV z1$Ys0ZBVAb+^rwOEyo`Mi&O_Ui)^Eh^RJwM7c`tk>i8jh)jps0Mq7}$!s5kMiu+-J zRh0`|Bv{m5=lvcXyQM2F;4!EXw-(TTh||x02jwV7*b2bXx)|liHZxf9I12HMkAIx_ z3Hpn#!HZi$s210(WIge6h6Gzz{CH!Assp%DR#^4Mq>zl11^u%YNzq3$=x`o4(Ln?0 z+9FOq@c?>nzKpQzBkY8TIuXK-4^xit>XSdf;tRh*(4G~htTp0&xDSbDvX>W`OGlK` zfW(AasQa=3x<^Jr)lniP*C}y@3X`nSXdphE;Xg}L=|6c<6TLV8jFV4%5%ag-i$k{^ zhYl3_Z=b^XSAK{7;;RVShw3JutKK}%r6Nq`sCT+j=0%(#`wZ9XHp~IjX)@7nccs|y zD4{Hs@7$c3B=4)BLYYg2hoCtPTV+`K>$7{h{JR=z|?dsQe*T@OLiX;m!ph^q6ub|Ow7_Wff8f&5VWSZ z{k#a=z|=JYnU5PSnJ?*h+;A0MomOy87z9%I)Oa{}AaezKigtHen4o&4ePWEjhXy7a zDTQXMEw;jWB-SvB^HZ{lt38EEF30AYwM9SKxeJJT+tz8Lmv>Dz-th+1IZ zh7}SwBJTalAWcUiMb^XA0xK#qCxyL=T@oloJ&l~O(9YXbrC?qvy{O@;I)RZzquG$> zo$eI+8*5jmq#Td_`UT7!TEOb+a;*h}WHdq;hG?}rf>mmUg$N~0D7f^1zMhp=R(l(l z2u4v`v`A)X1C%)XLesNz((IzGu>jgDkWR#aoD)cM5xs~NA5PEWBB{P8k0(rAcY-c560c<$!95RC-9sqz6odZclE0|2 zrL1ey?L8b}3}(HxWlYb^BdWcU^VjVEL?Ufh$!(7yU|z4c43qt&APfXnG+4T3kICMo zt#*a&)?%T&=X_&+%9D_Ze(Q>0^e&u75XESBrUb`yi-d=*wXuC<=p;?4YflT&)I>7s zPO+W)oj{0{Vy0)*3CBl$Tr+><=u2Ab4_RhY`#B@1vCaI5Zjlj`DZ!=DQ!>gr z3CyyQXpCc$M(!oUD5Z2&;Lg?6Gk+f+C6!j;X{FVN+0L1q0oQp`k_KCe2cuzstHvD* zlnSE_5LWKg@tq7#I4FJ3TJ!4R)z0gy^%0Y##6gC%)GrT( z?XvMeqpGhxW$iu7%Zsmz0>_yOa4J_Qz6oQ_YOR^*J72fou?vC!4e?4TPiyTdYrkI; z@9#Wjm6Ms}<>m9RxX)PgN1n^p+NtV~c^5vVU|u@BcG;BW!^_KyPXV8igUQSJ|E&}7 z`Rm^Eb7j@nwbt=Fj!m!8P&MQ%aI&=c+M4^ni7nz~&SieL#l*tG!c7Ljaaeo-8po_# zOU5|7x>yTvl-Z0kn?1^=YAFLjV8(07*qoM6N<$ Eg5EyTHUIzs literal 0 HcmV?d00001 diff --git a/extension/manifest.json b/extension/manifest.json new file mode 100644 index 0000000..1f58805 --- /dev/null +++ b/extension/manifest.json @@ -0,0 +1,46 @@ +{ + "manifest_version": 3, + "name": "discgod — Discogs seller tracker", + "version": "1.0", + "description": "Paste a Discogs API token, pick sellers, and auto-pull their full inventories into the discogs_full Postgres DB over time. Tracks price changes and sales.", + "permissions": [ + "storage", + "alarms", + "activeTab", + "scripting", + "notifications" + ], + "host_permissions": [ + "https://api.discogs.com/*", + "*://*.discogs.com/*", + "http://100.91.239.7:5057/*", + "http://localhost:5057/*" + ], + "background": { + "service_worker": "background.js" + }, + "action": { + "default_popup": "popup.html", + "default_title": "discgod", + "default_icon": { + "16": "icons/icon16.png", + "48": "icons/icon48.png", + "128": "icons/icon128.png" + } + }, + "content_scripts": [ + { + "matches": [ + "*://www.discogs.com/seller/*", + "*://www.discogs.com/sell/*" + ], + "js": ["content.js"], + "run_at": "document_idle" + } + ], + "icons": { + "16": "icons/icon16.png", + "48": "icons/icon48.png", + "128": "icons/icon128.png" + } +} diff --git a/extension/popup.css b/extension/popup.css new file mode 100644 index 0000000..578e391 --- /dev/null +++ b/extension/popup.css @@ -0,0 +1,114 @@ +:root { + --bg: #16181d; + --panel: #1e2128; + --panel2: #262a33; + --line: #2f343f; + --text: #e6e8ec; + --muted: #9aa3b2; + --accent: #4f9dff; + --good: #38b000; + --bad: #e5484d; + --warn: #f5a623; +} + +* { box-sizing: border-box; } + +body { + width: 380px; + margin: 0; + font: 13px/1.4 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + background: var(--bg); + color: var(--text); +} + +header { + display: flex; + align-items: center; + gap: 8px; + padding: 12px 14px; + border-bottom: 1px solid var(--line); +} +header .logo { font-size: 18px; font-weight: 700; letter-spacing: 0.5px; } +header .logo span { color: var(--accent); } +header .dot { + width: 9px; height: 9px; border-radius: 50%; + background: var(--muted); margin-left: auto; +} +header .dot.ok { background: var(--good); } +header .dot.busy { background: var(--accent); animation: pulse 1s infinite; } +header .dot.err { background: var(--bad); } +@keyframes pulse { 50% { opacity: 0.35; } } + +.panel { padding: 12px 14px; border-bottom: 1px solid var(--line); } +.panel h2 { + margin: 0 0 8px; font-size: 11px; font-weight: 600; + text-transform: uppercase; letter-spacing: 0.6px; color: var(--muted); +} + +label { display: block; font-size: 11px; color: var(--muted); margin: 8px 0 3px; } + +input, select { + width: 100%; padding: 7px 9px; border-radius: 7px; + border: 1px solid var(--line); background: var(--panel2); + color: var(--text); font-size: 13px; +} +input:focus, select:focus { outline: none; border-color: var(--accent); } + +.row { display: flex; gap: 8px; } +.row > * { flex: 1; } +.row.tight { gap: 6px; align-items: flex-end; } + +button { + padding: 7px 11px; border: none; border-radius: 7px; + background: var(--panel2); color: var(--text); + font-size: 12px; font-weight: 600; cursor: pointer; +} +button:hover { background: #30353f; } +button.primary { background: var(--accent); color: #07111f; } +button.primary:hover { filter: brightness(1.08); } +button.ghost { background: transparent; border: 1px solid var(--line); } +button.danger { background: transparent; color: var(--bad); padding: 4px 7px; } +button:disabled { opacity: 0.5; cursor: default; } + +.hint { font-size: 11px; color: var(--muted); margin-top: 6px; } +.hint a { color: var(--accent); } + +.statusline { + font-size: 12px; color: var(--muted); margin-top: 10px; + display: flex; justify-content: space-between; +} +.statusline b { color: var(--text); font-weight: 600; } + +/* watchlist */ +#watchlist { max-height: 220px; overflow-y: auto; margin-top: 4px; } +.seller { + display: flex; align-items: center; gap: 8px; + padding: 7px 8px; border-radius: 7px; background: var(--panel2); + margin-bottom: 5px; +} +.seller .name { font-weight: 600; flex: 1; min-width: 0; + overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.seller .meta { font-size: 10px; color: var(--muted); } +.seller .count { + font-size: 11px; background: #11151c; border-radius: 10px; + padding: 2px 7px; color: var(--accent); +} +.seller .err { color: var(--bad); } +.seller button { padding: 3px 7px; font-size: 11px; } +.empty { color: var(--muted); font-size: 12px; padding: 8px 2px; } + +/* stats grid */ +.stats { display: grid; grid-template-columns: 1fr 1fr; gap: 7px; } +.stat { + background: var(--panel2); border-radius: 8px; padding: 9px 10px; +} +.stat .n { font-size: 18px; font-weight: 700; } +.stat .l { font-size: 10px; color: var(--muted); text-transform: uppercase; letter-spacing: 0.4px; } + +.toast { + position: sticky; bottom: 0; padding: 7px 14px; font-size: 12px; + background: var(--panel2); border-top: 1px solid var(--line); display: none; +} +.toast.show { display: block; } +.toast.ok { color: var(--good); } +.toast.err { color: var(--bad); } diff --git a/extension/popup.html b/extension/popup.html new file mode 100644 index 0000000..228c50a --- /dev/null +++ b/extension/popup.html @@ -0,0 +1,82 @@ + + + + + + + +
+ +
+
+ + +
+

Setup

+ +
+ + +
+
+ Get one at Settings → Developers → “Generate token”. +
+ +
+
+ + +
+
+ + +
+
+ +
+ + +
+
+ + +
+

Sellers tracked

+
+ + + +
+
No sellers yet.
+
+ + +
+
idle
+
+ + +
+

Database

+
+
–
Active listings
+
–
Sellers watched
+
–
Gone (7d)
+
–
Price changes (7d)
+
+
last capture–
+
+ +
+ + + + diff --git a/extension/popup.js b/extension/popup.js new file mode 100644 index 0000000..b4b484a --- /dev/null +++ b/extension/popup.js @@ -0,0 +1,182 @@ +/* discgod popup — talks to the background worker only (which owns the token, + * the Discogs API calls, and the server I/O). */ + +const $ = (id) => document.getElementById(id); +const send = (msg) => chrome.runtime.sendMessage(msg); + +function toast(text, kind = "ok") { + const t = $("toast"); + t.textContent = text; + t.className = `toast show ${kind}`; + setTimeout(() => (t.className = "toast"), 2600); +} + +function setDot(state) { + $("statusDot").className = "dot" + (state ? " " + state : ""); +} + +// --- load config ------------------------------------------------------------ +async function loadConfig() { + const cfg = await send({ type: "getConfig" }); + $("token").value = cfg.token || ""; + $("serverUrl").value = cfg.serverUrl || ""; + $("intervalHours").value = String(cfg.intervalHours || 12); + $("autoCrawl").checked = cfg.autoCrawl !== false; +} + +// --- watchlist -------------------------------------------------------------- +function renderSellers(sellers) { + const box = $("watchlist"); + if (!sellers || !sellers.length) { + box.innerHTML = '
No sellers yet — add one above.
'; + return; + } + box.innerHTML = ""; + for (const s of sellers) { + const el = document.createElement("div"); + el.className = "seller"; + const last = s.last_crawled_at + ? new Date(s.last_crawled_at).toLocaleDateString() + : "never"; + const statusErr = (s.last_status || "").startsWith("error"); + el.innerHTML = ` +
+
${escapeHtml(s.username)}
+
+ last ${last}${statusErr ? " · failed" : ""} +
+
+ ${s.active_listings ?? 0} + + `; + box.appendChild(el); + } + box.querySelectorAll("button").forEach((b) => + b.addEventListener("click", onSellerAction) + ); +} + +async function onSellerAction(e) { + const u = e.currentTarget.dataset.u; + const act = e.currentTarget.dataset.act; + if (act === "remove") { + await send({ type: "untrack", username: u }); + toast(`Stopped tracking ${u}`); + refresh(); + } else if (act === "scrape") { + await send({ type: "scrapeOne", username: u }); + toast(`Scraping ${u}…`); + setDot("busy"); + pollStatus(); + } +} + +async function refresh() { + const tracked = await send({ type: "getTracked" }).catch(() => null); + if (tracked && tracked.ok) renderSellers(tracked.sellers); + else if (tracked && !tracked.ok) toast(tracked.error || "server unreachable", "err"); + + const stats = await send({ type: "getStats" }).catch(() => null); + if (stats && stats.ok) { + $("s_active").textContent = fmt(stats.active_listings); + $("s_watched").textContent = fmt(stats.sellers_watched); + $("s_gone").textContent = fmt(stats.gone_7d); + $("s_price").textContent = fmt(stats.price_changes_7d); + $("lastCapture").lastElementChild.textContent = stats.last_capture + ? new Date(stats.last_capture).toLocaleString() + : "–"; + setDot("ok"); + } else { + setDot("err"); + } + pollStatus(); +} + +async function pollStatus() { + const st = await send({ type: "status" }).catch(() => null); + const line = $("statusLine"); + if (st && st.crawling) { + setDot("busy"); + line.firstElementChild.textContent = `crawling… ${st.remaining} seller(s) left`; + line.lastElementChild.textContent = st.queued ? `${st.queued} queued` : ""; + setTimeout(pollStatus, 2000); + } else if (st) { + line.firstElementChild.textContent = st.lastCrawlAt + ? "last sweep " + new Date(st.lastCrawlAt).toLocaleTimeString() + : "idle"; + line.lastElementChild.textContent = st.queued ? `${st.queued} queued (retrying)` : ""; + } +} + +// --- events ----------------------------------------------------------------- +$("save").addEventListener("click", async () => { + await send({ + type: "setConfig", + patch: { + token: $("token").value.trim(), + serverUrl: $("serverUrl").value.trim().replace(/\/$/, ""), + intervalHours: parseInt($("intervalHours").value, 10), + autoCrawl: $("autoCrawl").checked, + }, + }); + toast("Saved"); + refresh(); +}); + +$("testToken").addEventListener("click", async () => { + await send({ type: "setConfig", patch: { token: $("token").value.trim() } }); + const r = await send({ type: "testToken" }); + if (r.ok) toast(`Token OK — signed in as ${r.username}`); + else toast(r.error || "token failed", "err"); +}); + +$("track").addEventListener("click", async () => { + const username = $("newSeller").value.trim().replace(/^.*\/seller\//, "").split(/[/?]/)[0]; + if (!username) return; + const filter = parseFilter($("newFilter").value.trim()); + const r = await send({ type: "track", body: { username, filter } }); + if (r.ok) { + $("newSeller").value = ""; + $("newFilter").value = ""; + toast(`Tracking ${username}`); + refresh(); + } else toast(r.error || "could not track", "err"); +}); + +$("scrapeAll").addEventListener("click", async () => { + await send({ type: "scrapeAll" }); + toast("Scraping all watched sellers…"); + setDot("busy"); + pollStatus(); +}); + +$("refresh").addEventListener("click", refresh); + +// Enter-to-track +$("newSeller").addEventListener("keydown", (e) => { + if (e.key === "Enter") $("track").click(); +}); + +// --- helpers ---------------------------------------------------------------- +function parseFilter(str) { + // "format=Vinyl&ships_from=Australia" -> {format:'Vinyl', ships_from:'Australia'} + const out = {}; + if (!str) return out; + for (const part of str.split("&")) { + const [k, v] = part.split("="); + if (k && v) out[k.trim()] = v.trim(); + } + return out; +} +function fmt(n) { return n == null ? "–" : Number(n).toLocaleString(); } +function escapeHtml(s) { + return String(s).replace(/[&<>"']/g, (c) => + ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c])); +} +function escapeAttr(s) { return escapeHtml(s).replace(/"/g, """); } + +// --- init ------------------------------------------------------------------- +(async function init() { + await loadConfig(); + await refresh(); +})(); diff --git a/seeds/aus_sellers.txt b/seeds/aus_sellers.txt new file mode 100644 index 0000000..e5c6685 --- /dev/null +++ b/seeds/aus_sellers.txt @@ -0,0 +1,104 @@ +# discgod watchlist seed — top Australian Discogs sellers (by listing count, 11 Jun 2026) +# Seed the server watchlist with: python discgod_crawl.py --seed seeds/aus_sellers.txt +# One username per line; blank lines and #comments ignored. + +Dennis-brisbane-qld +ozfelixtc +musikshak +plusonerecords +Mezabel +GlitterRecords +Timewarprecordss +revolverecordsaust +vicious_sloth +AZRecordsAU +scarcesounds +adelaidesoundworks +DADDYRICHRECORDS +eclectico_melbourne +tek-tonics +Mad_Vic_Records +Kaiserp1964 +Splendid-Spin +Jungleroom1957 +ozsoul +milanmircev +reeferrecords +muscleshoals +MartinJames +Binylio +Harboursiderecords +rubymusic808 +ozmusiconline +WaxBuildup +from99c +Revinylise118 +kool-mar-v +lostreasures +clockworkculture +Gregory22 +-mattbro +Vinyl.Galore +wolfa4 +vinylsol +TREVORDUNEN +SEMKASET +Rechoard +Plastic_Passion +Discopolis.AU +DamVinylRecords +famousa +gazjazz +FootscrayRecords +UrbanWorld +ReplayRecordsOZ +dynodynamic +Dynomite_Records +franklycollectible +thomasthelby +plugseven +Dada_Records +Rockabilly_Main_Man +fanatico_records +Bluerecord54 +tektonicsharddance +BebopRecords +ResolutionMusic +FairPlayed +zat +d-JCB +gordon_cole +VinyljunkieOz +Nicks_Discs +djstevemc +DiscogRecordsPreston +waynewafa +recordstoresydney +ErkoBlue +Audio_dude +WMR +Sammy55 +_florecords +RingosRecords +aussiefunk +jppictorial +hunter_and_robinson +mushroomjb +ozpoprock +Discrepancy-Records +jobbles81 +El-Roy +Legcyvinyl +stevieraymusic23 +Mr.S +RareCollect +raisemastervolume +Simplyrecord +neilmacmillan +My-Vinyl-Revolution +redundancy_records +amowlam +RockSaltRecords +RareRecordsOz +MysteryTram +jamie_df diff --git a/server/discgod_crawl.py b/server/discgod_crawl.py new file mode 100644 index 0000000..35d26ce --- /dev/null +++ b/server/discgod_crawl.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +""" +discgod headless crawler. + +Pulls full seller inventories from the Discogs API and writes straight into +`discogs_full` via discgod_ingest — no browser required. Use it for 24/7 +tracking on the box (cron or --loop). The extension and this script share the +same `discgod_seller_watch` watchlist, so a seller added in either shows up in +both. + +Token (a Discogs personal access token) is read from, in order: + 1. $DISCGOD_TOKEN + 2. the file pointed to by $DISCGOD_TOKEN_FILE + 3. ~/.discgod_token + +Usage + python discgod_crawl.py # crawl every enabled watched seller + python discgod_crawl.py NAME [NAME...] # crawl just these sellers + python discgod_crawl.py --add NAME... # add sellers to the watchlist, exit + python discgod_crawl.py --seed FILE # add every username in FILE, exit + python discgod_crawl.py --loop 21600 # crawl all, then repeat every 6 h +""" + +import os +import sys +import time +import json +import urllib.parse +import urllib.request +from pathlib import Path + +# The standalone server doubles as the importable engine (schema, mapping, +# upserts, watchlist) — one source of truth, shared with the HTTP server. +import discgod_server as ing + +API = "https://api.discogs.com" +USER_AGENT = "discgod/1.0 +https://discogs.com" +PER_PAGE = 100 +BASE_DELAY = float(os.environ.get("DISCGOD_DELAY", "1.1")) # seconds between API calls (~55/min) +MAX_RETRIES = 5 + + +def get_token(): + tok = os.environ.get("DISCGOD_TOKEN") + if tok: + return tok.strip() + path = os.environ.get("DISCGOD_TOKEN_FILE") or str(Path.home() / ".discgod_token") + try: + return Path(path).read_text().strip() + except OSError: + return None + + +def api_get(token, path, params=None): + """GET a Discogs API resource with auth, rate-limit awareness and backoff.""" + url = API + path + if params: + url += "?" + urllib.parse.urlencode(params) + req = urllib.request.Request(url, headers={ + "Authorization": f"Discogs token={token}", + "User-Agent": USER_AGENT, + "Accept": "application/json", + }) + for attempt in range(MAX_RETRIES): + try: + with urllib.request.urlopen(req, timeout=45) as resp: + remaining = resp.headers.get("X-Discogs-Ratelimit-Remaining") + data = json.loads(resp.read().decode()) + # Ease off as we approach the per-minute ceiling. + if remaining is not None and remaining.isdigit() and int(remaining) <= 2: + time.sleep(5) + else: + time.sleep(BASE_DELAY) + return data + except urllib.error.HTTPError as e: + if e.code == 429: + wait = min(60, 2 ** (attempt + 1)) + print(f" 429 rate-limited; backing off {wait}s") + time.sleep(wait) + continue + if e.code in (404, 401, 403): + raise RuntimeError(f"HTTP {e.code} for {path}") from e + time.sleep(2 ** attempt) + except (urllib.error.URLError, TimeoutError) as e: + print(f" network error ({e}); retrying") + time.sleep(2 ** attempt) + raise RuntimeError(f"giving up on {path} after {MAX_RETRIES} attempts") + + +def crawl_seller(token, username, filter_params=None, conn=None): + """Pull a seller's whole inventory and ingest it. Returns total listing count.""" + filter_params = filter_params or {} + crawl_id = f"{username}-{int(time.time())}" + owns = conn is None + conn = conn or ing.get_conn() + try: + # Profile first: gives seller_id and a reliable num_for_sale. + profile = None + seller_id = None + try: + profile = api_get(token, f"/users/{urllib.parse.quote(username)}") + seller_id = profile.get("id") + except RuntimeError as e: + print(f" {username}: profile fetch failed ({e}); continuing") + + page, pages, total = 1, 1, None + params = {"per_page": PER_PAGE, "status": "For Sale", + "sort": "listed", "sort_order": "desc"} + params.update(filter_params) + + while page <= pages: + params["page"] = page + data = api_get(token, f"/users/{urllib.parse.quote(username)}/inventory", params) + pg = data.get("pagination", {}) + pages = pg.get("pages", 1) or 1 + total = pg.get("items", total) + listings = data.get("listings", []) + + res = ing.ingest_inventory_page({ + "username": username, + "seller_id": seller_id, + "seller": profile if page == 1 else None, + "page": page, + "total": total, + "crawl_id": crawl_id, + "filter": filter_params, + "listings": listings, + }, conn=conn) + print(f" {username} p{page}/{pages}: {res['items']} items " + f"({res['new']} new, {res['price_changes']} price changes)") + page += 1 + + filtered = bool(ing.filter_key_from_params(filter_params)) + done = ing.finish_crawl({ + "username": username, "crawl_id": crawl_id, + "filter": filter_params, "treat_complete": not filtered, + }, conn=conn) + ing.mark_watch_crawled(username, total, "ok", conn=conn) + print(f" {username}: done — {total} listed, {done.get('marked_gone', 0)} marked gone") + return total + except Exception as e: + ing.mark_watch_crawled(username, None, f"error:{e}", conn=conn) + print(f" {username}: ERROR {e}") + return None + finally: + if owns: + conn.close() + + +def crawl_all(token, only=None): + conn = ing.get_conn() + try: + ing.ensure_schema(conn) + watched = ing.list_tracked(conn=conn, enabled_only=True) + if only: + wanted = {u.lower() for u in only} + watched = [w for w in watched if w["username"].lower() in wanted] + # Allow crawling sellers passed on the CLI even if not yet watched. + known = {w["username"].lower() for w in watched} + for u in only: + if u.lower() not in known: + watched.append({"username": u, "filter_params": None}) + print(f"discgod crawl: {len(watched)} seller(s)") + for w in watched: + crawl_seller(token, w["username"], w.get("filter_params"), conn=conn) + finally: + conn.close() + + +def main(argv): + token = get_token() + + if argv and argv[0] == "--add": + for u in argv[1:]: + ing.track_seller(u) + print(f"tracking {u}") + return + if argv and argv[0] == "--seed": + path = argv[1] + added = 0 + for line in Path(path).read_text().splitlines(): + name = line.strip() + if name and not name.startswith(("#", "http")): + ing.track_seller(name) + added += 1 + print(f"seeded {added} sellers from {path}") + return + + if not token: + print("No Discogs token. Set $DISCGOD_TOKEN or write ~/.discgod_token", file=sys.stderr) + sys.exit(1) + + loop_secs = None + only = [] + i = 0 + while i < len(argv): + if argv[i] == "--loop": + loop_secs = int(argv[i + 1]); i += 2 + else: + only.append(argv[i]); i += 1 + + while True: + crawl_all(token, only or None) + if not loop_secs: + break + print(f"sleeping {loop_secs}s…") + time.sleep(loop_secs) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/server/discgod_server.py b/server/discgod_server.py new file mode 100644 index 0000000..b484a1f --- /dev/null +++ b/server/discgod_server.py @@ -0,0 +1,785 @@ +#!/usr/bin/env python3 +""" +discgod ingest server — STANDALONE, single file. + +Everything (schema, Discogs-API→DB mapping, upsert/price-diff/sold-detection, +watchlist, stats, HTTP server) lives here so it's drop-in: copy it to the box, +run it, done. It coexists with the legacy pliceclogs server on :5002 — discgod +listens on :5057, writes the same `discogs_full` DB, and touches none of the +legacy endpoints. If you ever want one server, the handlers below paste straight +into pliceclogs_server.py. + +The extension's background worker pulls each watched seller's full inventory +from api.discogs.com (client-side, with your token) and POSTs the raw listing +pages here; this server maps + upserts them into the proven `mp_*` tables and +detects price changes / sales. + +Stdlib + psycopg2 only. DB auth: plain psycopg2.connect(dbname=...) — local peer +auth on the Ultra (run as johnking). Off-box: set PGHOST/PGUSER/PGPASSWORD. + +Run: + python3 discgod_server.py + # or, like the legacy server: + nohup /opt/homebrew/bin/python3 discgod_server.py > discgod_server.log 2>&1 & + +Endpoints + GET /discgod/health + GET /discgod/stats + GET /discgod/tracked + POST /discgod/track {username, note?, filter?, enabled?, seller_id?} + POST /discgod/untrack {username} + POST /discgod/ingest {username, seller_id?, seller?, page, total?, crawl_id, filter?, listings:[...]} + POST /discgod/crawl_complete {username, crawl_id, filter?, treat_complete?} +""" + +import json +import os +import sys +from datetime import datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import psycopg2 +import psycopg2.extras + +try: + sys.stdout.reconfigure(line_buffering=True) +except Exception: + pass + +DB_NAME = os.environ.get("DISCGOD_DB", os.environ.get("PLICE_DB", "discogs_full")) +PORT = int(os.environ.get("DISCGOD_PORT", "5057")) + +# Params that describe the search *filter*, not a position within it. +NON_FILTER_PARAMS = {"page", "pages", "per_page", "limit", "sort", "sort_order", "status", "token"} + + +def get_conn(): + return psycopg2.connect(dbname=DB_NAME) + + +# =========================================================================== +# Schema (idempotent). mp_* mirror the marketwatcher schema exactly so existing +# queries and the join to the XML `release` table keep working. +# =========================================================================== + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS mp_seller ( + seller_id BIGINT PRIMARY KEY, + username TEXT NOT NULL, + rating NUMERIC(3,1), + rating_percent NUMERIC(5,2), + ratings_count INTEGER, + ships_from TEXT, + first_seen TIMESTAMPTZ DEFAULT now(), + last_seen TIMESTAMPTZ DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_mp_seller_username ON mp_seller (username); + +CREATE TABLE IF NOT EXISTS mp_listing ( + item_id BIGINT PRIMARY KEY, + release_id BIGINT NOT NULL, + seller_id BIGINT, + seller_username TEXT, + title TEXT, + label TEXT, + cat_no TEXT, + media_condition TEXT, + sleeve_condition TEXT, + notes TEXT, + price_value NUMERIC(12,2), + price_currency TEXT, + shipping_value NUMERIC(12,2), + converted_value NUMERIC(12,2), + have_count INTEGER, + want_count INTEGER, + ships_from TEXT, + status TEXT NOT NULL DEFAULT 'active', + gone_at TIMESTAMPTZ, + first_seen TIMESTAMPTZ DEFAULT now(), + last_seen TIMESTAMPTZ DEFAULT now(), + times_seen INTEGER DEFAULT 1, + last_crawl_id TEXT +); +CREATE INDEX IF NOT EXISTS idx_mp_listing_release ON mp_listing (release_id); +CREATE INDEX IF NOT EXISTS idx_mp_listing_seller ON mp_listing (seller_username, status); +CREATE INDEX IF NOT EXISTS idx_mp_listing_seen ON mp_listing (last_seen DESC); + +CREATE TABLE IF NOT EXISTS mp_listing_event ( + id BIGSERIAL PRIMARY KEY, + item_id BIGINT NOT NULL, + release_id BIGINT, + seller_username TEXT, + event TEXT NOT NULL, + old_price NUMERIC(12,2), + new_price NUMERIC(12,2), + currency TEXT, + details JSONB, + recorded_at TIMESTAMPTZ DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_mp_event_item ON mp_listing_event (item_id, recorded_at DESC); +CREATE INDEX IF NOT EXISTS idx_mp_event_type ON mp_listing_event (event, recorded_at DESC); +CREATE INDEX IF NOT EXISTS idx_mp_event_release ON mp_listing_event (release_id); + +CREATE TABLE IF NOT EXISTS mp_seller_snapshot ( + id BIGSERIAL PRIMARY KEY, + seller_id BIGINT, + username TEXT NOT NULL, + total_listings BIGINT, + source TEXT NOT NULL, + filter_key TEXT, + filter_params JSONB, + captured_at TIMESTAMPTZ DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_mp_snapshot_series + ON mp_seller_snapshot (username, source, filter_key, captured_at DESC); + +CREATE TABLE IF NOT EXISTS mp_page_log ( + id BIGSERIAL PRIMARY KEY, + url TEXT, + page_type TEXT, + seller_page TEXT, + params JSONB, + page INTEGER, + total_results BIGINT, + item_count INTEGER, + item_ids BIGINT[], + new_items INTEGER, + price_changes INTEGER, + crawl_id TEXT, + scraped_at TIMESTAMPTZ DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_mp_page_log_time ON mp_page_log (scraped_at DESC); + +-- discgod watchlist: who to auto-crawl, shared by the extension + cron crawler. +CREATE TABLE IF NOT EXISTS discgod_seller_watch ( + username TEXT PRIMARY KEY, + seller_id BIGINT, + enabled BOOLEAN DEFAULT TRUE, + note TEXT, + filter_params JSONB, + added_at TIMESTAMPTZ DEFAULT now(), + last_crawled_at TIMESTAMPTZ, + last_listing_count INTEGER, + last_status TEXT +); +CREATE INDEX IF NOT EXISTS idx_discgod_watch_enabled ON discgod_seller_watch (enabled); +""" + + +def ensure_schema(conn): + with conn.cursor() as cur: + cur.execute(SCHEMA) + cur.execute("ALTER TABLE mp_listing ADD COLUMN IF NOT EXISTS last_crawl_id TEXT") + cur.execute("ALTER TABLE mp_page_log ADD COLUMN IF NOT EXISTS crawl_id TEXT") + conn.commit() + + +# =========================================================================== +# Coercion helpers +# =========================================================================== + +def _num(val): + if val is None or val == "": + return None + try: + return float(val) + except (TypeError, ValueError): + return None + + +def _int(val): + if val is None or val == "": + return None + try: + return int(val) + except (TypeError, ValueError): + return None + + +def _txt(val): + if val is None: + return None + s = str(val).strip() + return s if s else None + + +def _cents(val): + return None if val is None else int(round(val * 100)) + + +def filter_key_from_params(params): + if not params: + return "" + items = sorted((k, str(v)) for k, v in params.items() if k not in NON_FILTER_PARAMS) + return "&".join(f"{k}={v}" for k, v in items) + + +# =========================================================================== +# Map a raw Discogs API inventory listing -> our flat item dict +# =========================================================================== + +def map_listing(raw, seller_username=None, seller_id=None): + """ + Convert one Discogs API listing object (from /users/{u}/inventory) into the + flat shape our upsert expects. Defensive about missing/renamed fields; + returns None if it lacks the two required keys. + """ + item_id = _int(raw.get("id")) + release = raw.get("release") or {} + release_id = _int(release.get("id")) + if not item_id or not release_id: + return None + + price = raw.get("price") or {} + # original_price = price converted into the authed account's currency + # (Discogs' confusingly-named field). Good as "converted_value". + converted = raw.get("original_price") or {} + shipping = raw.get("shipping_price") or {} + + stats = release.get("stats") or {} + community = stats.get("community") or release.get("community") or {} + + seller = raw.get("seller") or {} + sstats = seller.get("stats") or {} + + return { + "item_id": item_id, + "release_id": release_id, + "seller_id": _int(seller.get("id")) or _int(seller_id), + "seller_username": _txt(seller.get("username")) or _txt(seller_username), + # Discogs: stats.stars = 0-5 star score (-> rating NUMERIC(3,1)); + # stats.rating = percent positive, e.g. 100.0 (-> rating_percent). + "seller_rating": _num(sstats.get("stars")), + "seller_percent": _num(sstats.get("rating")), + "seller_ratings_count": _int(sstats.get("total")), + "title": _txt(release.get("description")), + "label": _txt(release.get("label")), + "cat_no": _txt(release.get("catalog_number") or release.get("catno")), + "media_condition": _txt(raw.get("condition")), + "sleeve_condition": _txt(raw.get("sleeve_condition")), + "notes": _txt(raw.get("comments")), + "price_value": _num(price.get("value")), + "price_currency": _txt(price.get("currency")), + "shipping_value": _num(shipping.get("value")), + "converted_value": _num(converted.get("value")), + "have_count": _int(community.get("in_collection")), + "want_count": _int(community.get("in_wantlist")), + "ships_from": _txt(raw.get("ships_from")), + } + + +# =========================================================================== +# Upserts (adapted from the marketwatcher engine, mp_* compatible) +# =========================================================================== + +def upsert_seller_profile(cur, username, seller_id, profile): + """Record seller identity + location from the /users/{username} blob. + Reputation (stars/percent/count) comes from each listing's seller.stats, + not from this profile resource (whose `rating` field is ambiguous and can + overflow rating NUMERIC(3,1)); COALESCE preserves it.""" + username = _txt(username) + seller_id = _int(seller_id) or _int((profile or {}).get("id")) + if not seller_id or not username: + return 0 + profile = profile or {} + cur.execute(""" + INSERT INTO mp_seller (seller_id, username, ships_from) + VALUES (%s, %s, %s) + ON CONFLICT (seller_id) DO UPDATE SET + username = EXCLUDED.username, + ships_from = COALESCE(EXCLUDED.ships_from, mp_seller.ships_from), + last_seen = now() + """, (seller_id, username, _txt(profile.get("location")))) + return 1 + + +def upsert_sellers_from_items(cur, items): + """Upsert sellers gleaned from listing items (when listings carry seller).""" + sellers = {} + for it in items: + sid = _int(it.get("seller_id")) + if not sid or not _txt(it.get("seller_username")): + continue + sellers[sid] = ( + sid, + _txt(it.get("seller_username")), + _num(it.get("seller_rating")), + _num(it.get("seller_percent")), + _int(it.get("seller_ratings_count")), + _txt(it.get("ships_from")), + ) + if not sellers: + return 0 + psycopg2.extras.execute_values(cur, """ + INSERT INTO mp_seller (seller_id, username, rating, rating_percent, ratings_count, ships_from) + VALUES %s + ON CONFLICT (seller_id) DO UPDATE SET + username = EXCLUDED.username, + rating = COALESCE(EXCLUDED.rating, mp_seller.rating), + rating_percent = COALESCE(EXCLUDED.rating_percent, mp_seller.rating_percent), + ratings_count = COALESCE(EXCLUDED.ratings_count, mp_seller.ratings_count), + ships_from = COALESCE(EXCLUDED.ships_from, mp_seller.ships_from), + last_seen = now() + """, list(sellers.values())) + return len(sellers) + + +def upsert_listings(cur, items, crawl_id=None): + """Upsert listings, emitting events for new / price-changed / condition- + changed / relisted items. Returns (new, price_changes, relisted).""" + by_id = {} + for it in items: + if _int(it.get("item_id")) and _int(it.get("release_id")): + by_id[_int(it["item_id"])] = it + valid = list(by_id.values()) + if not valid: + return 0, 0, 0 + + ids = list(by_id.keys()) + cur.execute(""" + SELECT item_id, price_value, media_condition, sleeve_condition, status + FROM mp_listing WHERE item_id = ANY(%s) + FOR UPDATE + """, (ids,)) + existing = {row[0]: row for row in cur.fetchall()} + + rows, events = [], [] + new_count = price_changes = relists = 0 + + for it in valid: + item_id = _int(it["item_id"]) + release_id = _int(it["release_id"]) + price = _num(it.get("price_value")) + currency = _txt(it.get("price_currency")) + media = _txt(it.get("media_condition")) + sleeve = _txt(it.get("sleeve_condition")) + seller = _txt(it.get("seller_username")) + + prev = existing.get(item_id) + if prev is None: + new_count += 1 + events.append((item_id, release_id, seller, "listed", None, price, currency, + json.dumps({"media": media, "sleeve": sleeve}))) + else: + _, prev_price, prev_media, prev_sleeve, prev_status = prev + prev_price = float(prev_price) if prev_price is not None else None + if prev_status == "gone": + relists += 1 + events.append((item_id, release_id, seller, "relisted", prev_price, price, currency, None)) + if price is not None and prev_price is not None and _cents(price) != _cents(prev_price): + price_changes += 1 + events.append((item_id, release_id, seller, "price_change", prev_price, price, currency, None)) + if (media != prev_media and media and prev_media) or \ + (sleeve != prev_sleeve and sleeve and prev_sleeve): + events.append((item_id, release_id, seller, "condition_change", None, None, None, + json.dumps({"old_media": prev_media, "new_media": media, + "old_sleeve": prev_sleeve, "new_sleeve": sleeve}))) + + rows.append(( + item_id, release_id, _int(it.get("seller_id")), seller, + _txt(it.get("title")), _txt(it.get("label")), _txt(it.get("cat_no")), + media, sleeve, _txt(it.get("notes")), + price, currency, _num(it.get("shipping_value")), _num(it.get("converted_value")), + _int(it.get("have_count")), _int(it.get("want_count")), _txt(it.get("ships_from")), + crawl_id, + )) + + psycopg2.extras.execute_values(cur, """ + INSERT INTO mp_listing + (item_id, release_id, seller_id, seller_username, title, label, cat_no, + media_condition, sleeve_condition, notes, + price_value, price_currency, shipping_value, converted_value, + have_count, want_count, ships_from, last_crawl_id) + VALUES %s + ON CONFLICT (item_id) DO UPDATE SET + release_id = EXCLUDED.release_id, + seller_id = COALESCE(EXCLUDED.seller_id, mp_listing.seller_id), + seller_username = COALESCE(EXCLUDED.seller_username, mp_listing.seller_username), + title = COALESCE(EXCLUDED.title, mp_listing.title), + label = COALESCE(EXCLUDED.label, mp_listing.label), + cat_no = COALESCE(EXCLUDED.cat_no, mp_listing.cat_no), + media_condition = COALESCE(EXCLUDED.media_condition, mp_listing.media_condition), + sleeve_condition = COALESCE(EXCLUDED.sleeve_condition, mp_listing.sleeve_condition), + notes = COALESCE(EXCLUDED.notes, mp_listing.notes), + price_value = COALESCE(EXCLUDED.price_value, mp_listing.price_value), + price_currency = COALESCE(EXCLUDED.price_currency, mp_listing.price_currency), + shipping_value = COALESCE(EXCLUDED.shipping_value, mp_listing.shipping_value), + converted_value = COALESCE(EXCLUDED.converted_value, mp_listing.converted_value), + have_count = COALESCE(EXCLUDED.have_count, mp_listing.have_count), + want_count = COALESCE(EXCLUDED.want_count, mp_listing.want_count), + ships_from = COALESCE(EXCLUDED.ships_from, mp_listing.ships_from), + last_crawl_id = COALESCE(EXCLUDED.last_crawl_id, mp_listing.last_crawl_id), + status = 'active', + gone_at = NULL, + last_seen = now(), + times_seen = mp_listing.times_seen + 1 + """, rows) + + if events: + psycopg2.extras.execute_values(cur, """ + INSERT INTO mp_listing_event + (item_id, release_id, seller_username, event, old_price, new_price, currency, details) + VALUES %s + """, events) + + return new_count, price_changes, relists + + +SNAPSHOT_DEDUP_HOURS = 6 + + +def record_seller_snapshot(cur, username, seller_id, total, source, params): + """Insert an inventory-size snapshot unless an identical recent one exists.""" + if not username or total is None: + return False + fkey = filter_key_from_params(params) + cur.execute(""" + SELECT total_listings FROM mp_seller_snapshot + WHERE username = %s AND source = %s AND filter_key = %s + AND captured_at > now() - make_interval(hours => %s) + ORDER BY captured_at DESC LIMIT 1 + """, (username, source, fkey, SNAPSHOT_DEDUP_HOURS)) + row = cur.fetchone() + if row and row[0] == total: + return False + cur.execute(""" + INSERT INTO mp_seller_snapshot (seller_id, username, total_listings, source, filter_key, filter_params) + VALUES (%s, %s, %s, %s, %s, %s::jsonb) + """, (seller_id, username, total, source, fkey, json.dumps(params or {}))) + return True + + +def mark_gone(cur, username, crawl_id, filtered, treat_complete): + """After a full inventory crawl, mark this seller's active listings the crawl + did NOT touch (last_crawl_id != crawl_id) as gone — sold or delisted. Only + safe when the crawl was the complete inventory. Returns count, or None when + skipped.""" + username = _txt(username) + crawl_id = _txt(crawl_id) + if not username or not crawl_id: + return None + if filtered and not treat_complete: + return None + cur.execute(""" + UPDATE mp_listing + SET status = 'gone', gone_at = now() + WHERE seller_username = %s AND status = 'active' + AND last_crawl_id IS DISTINCT FROM %s + RETURNING item_id, release_id, price_value, price_currency + """, (username, crawl_id)) + gone = cur.fetchall() + if gone: + psycopg2.extras.execute_values(cur, """ + INSERT INTO mp_listing_event + (item_id, release_id, seller_username, event, old_price, new_price, currency, details) + VALUES %s + """, [ + (item_id, release_id, username, "gone", price, None, currency, + json.dumps({"presumed": "sold_or_delisted", "crawl_filtered": filtered})) + for item_id, release_id, price, currency in gone + ]) + return len(gone) + + +# =========================================================================== +# High-level operations (own their own transaction unless a conn is passed) +# =========================================================================== + +def ingest_inventory_page(payload, conn=None): + """Process one page of a seller's inventory. + payload = {username, seller_id?, seller?, page, total?, crawl_id, filter?, + listings:[raw API objects]}""" + owns = conn is None + conn = conn or get_conn() + try: + username = _txt(payload.get("username")) + seller_id = _int(payload.get("seller_id")) + profile = payload.get("seller") + page = _int(payload.get("page")) or 1 + total = _int(payload.get("total")) + crawl_id = _txt(payload.get("crawl_id")) + params = payload.get("filter") or {} + raw_listings = payload.get("listings") or [] + + items = [m for m in (map_listing(r, username, seller_id) for r in raw_listings) if m] + + with conn.cursor() as cur: + sellers = 0 + if profile or seller_id: + sellers += upsert_seller_profile(cur, username, seller_id, profile) + sellers += upsert_sellers_from_items(cur, items) + + new_count, price_changes, relists = upsert_listings(cur, items, crawl_id) + + snapshots = 0 + if page == 1 and total is not None: + if record_seller_snapshot(cur, username, seller_id, total, "api_inventory", params): + snapshots += 1 + + item_ids = [it["item_id"] for it in items] + cur.execute(""" + INSERT INTO mp_page_log + (url, page_type, seller_page, params, page, total_results, + item_count, item_ids, new_items, price_changes, crawl_id) + VALUES (%s, %s, %s, %s::jsonb, %s, %s, %s, %s, %s, %s, %s) + """, ( + f"api:/users/{username}/inventory?page={page}", + "api_inventory", username, json.dumps(params), page, total, + len(items), item_ids, new_count, price_changes, crawl_id, + )) + conn.commit() + return {"ok": True, "username": username, "page": page, + "items": len(items), "new": new_count, + "price_changes": price_changes, "relisted": relists, + "sellers": sellers, "snapshots": snapshots} + finally: + if owns: + conn.close() + + +def finish_crawl(payload, conn=None): + """Close out a full inventory crawl: mark untouched listings gone. + payload = {username, crawl_id, filter?, treat_complete?}""" + owns = conn is None + conn = conn or get_conn() + try: + username = _txt(payload.get("username")) + crawl_id = _txt(payload.get("crawl_id")) + params = payload.get("filter") or {} + filtered = bool(filter_key_from_params(params)) + treat_complete = bool(payload.get("treat_complete")) + if not username or not crawl_id: + return {"ok": False, "error": "username and crawl_id required"} + with conn.cursor() as cur: + marked = mark_gone(cur, username, crawl_id, filtered, treat_complete) + conn.commit() + if marked is None: + return {"ok": True, "marked_gone": 0, + "skipped": "filtered crawl not treated as complete inventory"} + return {"ok": True, "marked_gone": marked} + finally: + if owns: + conn.close() + + +# ---- watchlist ------------------------------------------------------------ + +def list_tracked(conn=None, enabled_only=False): + owns = conn is None + conn = conn or get_conn() + try: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute(f""" + SELECT w.username, w.seller_id, w.enabled, w.note, w.filter_params, + w.added_at, w.last_crawled_at, w.last_listing_count, w.last_status, + (SELECT count(*) FROM mp_listing l + WHERE l.seller_username = w.username AND l.status = 'active') AS active_listings + FROM discgod_seller_watch w + {"WHERE w.enabled" if enabled_only else ""} + ORDER BY w.last_crawled_at DESC NULLS LAST, w.username + """) + rows = cur.fetchall() + for r in rows: + for k in ("added_at", "last_crawled_at"): + if r.get(k): + r[k] = r[k].isoformat() + return rows + finally: + if owns: + conn.close() + + +def track_seller(username, note=None, filter_params=None, enabled=True, seller_id=None, conn=None): + owns = conn is None + conn = conn or get_conn() + try: + username = _txt(username) + if not username: + return {"ok": False, "error": "username required"} + with conn.cursor() as cur: + cur.execute(""" + INSERT INTO discgod_seller_watch (username, seller_id, enabled, note, filter_params) + VALUES (%s, %s, %s, %s, %s::jsonb) + ON CONFLICT (username) DO UPDATE SET + enabled = EXCLUDED.enabled, + note = COALESCE(EXCLUDED.note, discgod_seller_watch.note), + filter_params = COALESCE(EXCLUDED.filter_params, discgod_seller_watch.filter_params), + seller_id = COALESCE(EXCLUDED.seller_id, discgod_seller_watch.seller_id) + """, (username, _int(seller_id), bool(enabled), _txt(note), + json.dumps(filter_params) if filter_params else None)) + conn.commit() + return {"ok": True, "username": username} + finally: + if owns: + conn.close() + + +def untrack_seller(username, conn=None): + owns = conn is None + conn = conn or get_conn() + try: + with conn.cursor() as cur: + cur.execute("DELETE FROM discgod_seller_watch WHERE username = %s", (_txt(username),)) + n = cur.rowcount + conn.commit() + return {"ok": True, "removed": n} + finally: + if owns: + conn.close() + + +def mark_watch_crawled(username, listing_count, status="ok", conn=None): + owns = conn is None + conn = conn or get_conn() + try: + with conn.cursor() as cur: + cur.execute(""" + UPDATE discgod_seller_watch + SET last_crawled_at = now(), last_listing_count = %s, last_status = %s + WHERE username = %s + """, (_int(listing_count), _txt(status), _txt(username))) + conn.commit() + finally: + if owns: + conn.close() + + +# ---- dashboard stats ------------------------------------------------------ + +def stats(conn=None): + owns = conn is None + conn = conn or get_conn() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT + (SELECT count(*) FROM mp_listing WHERE status = 'active'), + (SELECT count(*) FROM mp_listing WHERE status = 'gone'), + (SELECT count(*) FROM discgod_seller_watch WHERE enabled), + (SELECT count(*) FROM mp_seller), + (SELECT count(*) FROM mp_page_log + WHERE page_type = 'api_inventory' AND scraped_at > now() - interval '24 hours'), + (SELECT max(scraped_at) FROM mp_page_log), + (SELECT count(*) FROM mp_listing_event + WHERE event = 'gone' AND recorded_at > now() - interval '7 days'), + (SELECT count(*) FROM mp_listing_event + WHERE event = 'price_change' AND recorded_at > now() - interval '7 days') + """) + (active, gone, watched, sellers, pages_24h, last_capture, + gone_7d, price_changes_7d) = cur.fetchone() + return {"ok": True, "active_listings": active, "gone_listings": gone, + "sellers_watched": watched, "sellers_known": sellers, + "pages_24h": pages_24h, + "last_capture": last_capture.isoformat() if last_capture else None, + "gone_7d": gone_7d, "price_changes_7d": price_changes_7d} + finally: + if owns: + conn.close() + + +# =========================================================================== +# HTTP server +# =========================================================================== + +def _log(msg): + print(f"[{datetime.now().strftime('%H:%M:%S')}] {msg}") + + +class Handler(BaseHTTPRequestHandler): + + def log_message(self, *_): + pass + + def _cors(self): + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Headers", "Content-Type") + self.send_header("Access-Control-Allow-Methods", "POST, GET, OPTIONS") + self.send_header("Access-Control-Allow-Private-Network", "true") + + def _send(self, obj, status=200): + body = json.dumps(obj).encode() + self.send_response(status) + self._cors() + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _read_json(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) if length else b"" + return json.loads(body) if body else {} + + def _safe(self, fn, label): + try: + self._send(fn()) + except Exception as e: + import traceback + traceback.print_exc() + _log(f"ERROR ({label}): {e}") + self._send({"ok": False, "error": str(e)}, status=500) + + def do_OPTIONS(self): + self.send_response(200) + self._cors() + self.end_headers() + + def do_GET(self): + if self.path in ("/discgod/health", "/discgod", "/"): + self._send({"ok": True, "server": "discgod", "db": DB_NAME}) + elif self.path == "/discgod/stats": + self._safe(stats, "stats") + elif self.path == "/discgod/tracked": + self._safe(lambda: {"ok": True, "sellers": list_tracked()}, "tracked") + else: + self.send_response(404) + self.end_headers() + + def do_POST(self): + if self.path == "/discgod/ingest": + def run(): + res = ingest_inventory_page(self._read_json()) + _log(f"[ingest] {res.get('username')} p{res.get('page')} " + f"{res.get('items')} items ({res.get('new')} new, " + f"{res.get('price_changes')} price changes, {res.get('snapshots')} snap)") + return res + self._safe(run, "ingest") + + elif self.path == "/discgod/crawl_complete": + def run(): + payload = self._read_json() + res = finish_crawl(payload) + if res.get("ok") and payload.get("username"): + mark_watch_crawled(payload["username"], payload.get("listing_count"), "ok") + _log(f"[crawl_complete] {payload.get('username')} -> " + f"{res.get('marked_gone', 0)} gone {res.get('skipped','')}") + return res + self._safe(run, "crawl_complete") + + elif self.path == "/discgod/track": + def run(): + p = self._read_json() + return track_seller(p.get("username"), p.get("note"), p.get("filter"), + p.get("enabled", True), p.get("seller_id")) + self._safe(run, "track") + + elif self.path == "/discgod/untrack": + def run(): + return untrack_seller(self._read_json().get("username")) + self._safe(run, "untrack") + + else: + self.send_response(404) + self.end_headers() + + +def main(): + conn = get_conn() + ensure_schema(conn) + conn.close() + _log(f"discgod server listening on 0.0.0.0:{PORT} (db={DB_NAME})") + ThreadingHTTPServer(("0.0.0.0", PORT), Handler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/server/requirements.txt b/server/requirements.txt new file mode 100644 index 0000000..09078f9 --- /dev/null +++ b/server/requirements.txt @@ -0,0 +1,3 @@ +# discgod server deps. The Discogs API client + HTTP server use only the +# Python standard library; the only third-party need is the Postgres driver. +psycopg2-binary>=2.9 diff --git a/server/schema.sql b/server/schema.sql new file mode 100644 index 0000000..757fd42 --- /dev/null +++ b/server/schema.sql @@ -0,0 +1,115 @@ +-- discgod schema (reference copy — the server applies this itself via +-- discgod_ingest.ensure_schema() on startup). Additive: it reuses the existing +-- marketwatcher mp_* tables verbatim and adds one watchlist table. +-- +-- Apply manually if you like: psql discogs_full -f schema.sql + +-- Sellers ever seen, with latest reputation stats. +CREATE TABLE IF NOT EXISTS mp_seller ( + seller_id BIGINT PRIMARY KEY, + username TEXT NOT NULL, + rating NUMERIC(3,1), + rating_percent NUMERIC(5,2), + ratings_count INTEGER, + ships_from TEXT, + first_seen TIMESTAMPTZ DEFAULT now(), + last_seen TIMESTAMPTZ DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_mp_seller_username ON mp_seller (username); + +-- One row per marketplace listing (unique physical copy). release_id joins +-- straight to your XML `release` table. +CREATE TABLE IF NOT EXISTS mp_listing ( + item_id BIGINT PRIMARY KEY, + release_id BIGINT NOT NULL, + seller_id BIGINT, + seller_username TEXT, + title TEXT, + label TEXT, + cat_no TEXT, + media_condition TEXT, + sleeve_condition TEXT, + notes TEXT, + price_value NUMERIC(12,2), + price_currency TEXT, + shipping_value NUMERIC(12,2), + converted_value NUMERIC(12,2), + have_count INTEGER, + want_count INTEGER, + ships_from TEXT, + status TEXT NOT NULL DEFAULT 'active', -- active | gone + gone_at TIMESTAMPTZ, + first_seen TIMESTAMPTZ DEFAULT now(), + last_seen TIMESTAMPTZ DEFAULT now(), + times_seen INTEGER DEFAULT 1, + last_crawl_id TEXT +); +CREATE INDEX IF NOT EXISTS idx_mp_listing_release ON mp_listing (release_id); +CREATE INDEX IF NOT EXISTS idx_mp_listing_seller ON mp_listing (seller_username, status); +CREATE INDEX IF NOT EXISTS idx_mp_listing_seen ON mp_listing (last_seen DESC); + +-- Append-only state-change history per listing. +-- event: listed | price_change | condition_change | gone | relisted +CREATE TABLE IF NOT EXISTS mp_listing_event ( + id BIGSERIAL PRIMARY KEY, + item_id BIGINT NOT NULL, + release_id BIGINT, + seller_username TEXT, + event TEXT NOT NULL, + old_price NUMERIC(12,2), + new_price NUMERIC(12,2), + currency TEXT, + details JSONB, + recorded_at TIMESTAMPTZ DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_mp_event_item ON mp_listing_event (item_id, recorded_at DESC); +CREATE INDEX IF NOT EXISTS idx_mp_event_type ON mp_listing_event (event, recorded_at DESC); +CREATE INDEX IF NOT EXISTS idx_mp_event_release ON mp_listing_event (release_id); + +-- Inventory-size time series per seller. source = 'api_inventory' for discgod +-- (the authoritative pagination total from the Discogs API). +CREATE TABLE IF NOT EXISTS mp_seller_snapshot ( + id BIGSERIAL PRIMARY KEY, + seller_id BIGINT, + username TEXT NOT NULL, + total_listings BIGINT, + source TEXT NOT NULL, + filter_key TEXT, + filter_params JSONB, + captured_at TIMESTAMPTZ DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_mp_snapshot_series + ON mp_seller_snapshot (username, source, filter_key, captured_at DESC); + +-- Provenance: every captured inventory page. +CREATE TABLE IF NOT EXISTS mp_page_log ( + id BIGSERIAL PRIMARY KEY, + url TEXT, + page_type TEXT, -- 'api_inventory' for discgod + seller_page TEXT, + params JSONB, + page INTEGER, + total_results BIGINT, + item_count INTEGER, + item_ids BIGINT[], + new_items INTEGER, + price_changes INTEGER, + crawl_id TEXT, + scraped_at TIMESTAMPTZ DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_mp_page_log_time ON mp_page_log (scraped_at DESC); + +-- discgod watchlist: who to auto-crawl. Shared by the extension popup and the +-- headless cron crawler. +CREATE TABLE IF NOT EXISTS discgod_seller_watch ( + username TEXT PRIMARY KEY, + seller_id BIGINT, + enabled BOOLEAN DEFAULT TRUE, + note TEXT, + filter_params JSONB, + added_at TIMESTAMPTZ DEFAULT now(), + last_crawled_at TIMESTAMPTZ, + last_listing_count INTEGER, + last_status TEXT +); +CREATE INDEX IF NOT EXISTS idx_discgod_watch_enabled ON discgod_seller_watch (enabled);