#!/usr/bin/env python3 """ discgod headless crawler. Pulls full seller inventories from the Discogs API and writes straight into `discogs_full` via discgod_ingest — no browser required. Use it for 24/7 tracking on the box (cron or --loop). The extension and this script share the same `discgod_seller_watch` watchlist, so a seller added in either shows up in both. Token (a Discogs personal access token) is read from, in order: 1. $DISCGOD_TOKEN 2. the file pointed to by $DISCGOD_TOKEN_FILE 3. ~/.discgod_token Usage python discgod_crawl.py # crawl every enabled watched seller python discgod_crawl.py NAME [NAME...] # crawl just these sellers python discgod_crawl.py --add NAME... # add sellers to the watchlist, exit python discgod_crawl.py --seed FILE # add every username in FILE, exit python discgod_crawl.py --loop 21600 # crawl all, then repeat every 6 h """ import os import sys import time import json import urllib.parse import urllib.request from pathlib import Path # The standalone server doubles as the importable engine (schema, mapping, # upserts, watchlist) — one source of truth, shared with the HTTP server. import discgod_server as ing API = "https://api.discogs.com" USER_AGENT = "discgod/1.0 +https://discogs.com" PER_PAGE = 100 BASE_DELAY = float(os.environ.get("DISCGOD_DELAY", "1.1")) # seconds between API calls (~55/min) MAX_RETRIES = 5 def get_token(): tok = os.environ.get("DISCGOD_TOKEN") if tok: return tok.strip() path = os.environ.get("DISCGOD_TOKEN_FILE") or str(Path.home() / ".discgod_token") try: return Path(path).read_text().strip() except OSError: return None def api_get(token, path, params=None): """GET a Discogs API resource with auth, rate-limit awareness and backoff.""" url = API + path if params: url += "?" + urllib.parse.urlencode(params) req = urllib.request.Request(url, headers={ "Authorization": f"Discogs token={token}", "User-Agent": USER_AGENT, "Accept": "application/json", }) for attempt in range(MAX_RETRIES): try: with urllib.request.urlopen(req, timeout=45) as resp: remaining = resp.headers.get("X-Discogs-Ratelimit-Remaining") data = json.loads(resp.read().decode()) # Ease off as we approach the per-minute ceiling. if remaining is not None and remaining.isdigit() and int(remaining) <= 2: time.sleep(5) else: time.sleep(BASE_DELAY) return data except urllib.error.HTTPError as e: if e.code == 429: wait = min(60, 2 ** (attempt + 1)) print(f" 429 rate-limited; backing off {wait}s") time.sleep(wait) continue if e.code in (404, 401, 403): raise RuntimeError(f"HTTP {e.code} for {path}") from e time.sleep(2 ** attempt) except (urllib.error.URLError, TimeoutError) as e: print(f" network error ({e}); retrying") time.sleep(2 ** attempt) raise RuntimeError(f"giving up on {path} after {MAX_RETRIES} attempts") def crawl_seller(token, username, filter_params=None, conn=None): """Pull a seller's whole inventory and ingest it. Returns total listing count.""" filter_params = filter_params or {} crawl_id = f"{username}-{int(time.time())}" owns = conn is None conn = conn or ing.get_conn() try: # Profile first: gives seller_id and a reliable num_for_sale. profile = None seller_id = None try: profile = api_get(token, f"/users/{urllib.parse.quote(username)}") seller_id = profile.get("id") except RuntimeError as e: print(f" {username}: profile fetch failed ({e}); continuing") page, pages, total = 1, 1, None params = {"per_page": PER_PAGE, "status": "For Sale", "sort": "listed", "sort_order": "desc"} params.update(filter_params) while page <= pages: params["page"] = page data = api_get(token, f"/users/{urllib.parse.quote(username)}/inventory", params) pg = data.get("pagination", {}) pages = pg.get("pages", 1) or 1 total = pg.get("items", total) listings = data.get("listings", []) res = ing.ingest_inventory_page({ "username": username, "seller_id": seller_id, "seller": profile if page == 1 else None, "page": page, "total": total, "crawl_id": crawl_id, "filter": filter_params, "listings": listings, }, conn=conn) print(f" {username} p{page}/{pages}: {res['items']} items " f"({res['new']} new, {res['price_changes']} price changes)") page += 1 filtered = bool(ing.filter_key_from_params(filter_params)) done = ing.finish_crawl({ "username": username, "crawl_id": crawl_id, "filter": filter_params, "treat_complete": not filtered, }, conn=conn) ing.mark_watch_crawled(username, total, "ok", conn=conn) print(f" {username}: done — {total} listed, {done.get('marked_gone', 0)} marked gone") return total except Exception as e: ing.mark_watch_crawled(username, None, f"error:{e}", conn=conn) print(f" {username}: ERROR {e}") return None finally: if owns: conn.close() def crawl_all(token, only=None): conn = ing.get_conn() try: ing.ensure_schema(conn) watched = ing.list_tracked(conn=conn, enabled_only=True) if only: wanted = {u.lower() for u in only} watched = [w for w in watched if w["username"].lower() in wanted] # Allow crawling sellers passed on the CLI even if not yet watched. known = {w["username"].lower() for w in watched} for u in only: if u.lower() not in known: watched.append({"username": u, "filter_params": None}) print(f"discgod crawl: {len(watched)} seller(s)") for w in watched: crawl_seller(token, w["username"], w.get("filter_params"), conn=conn) finally: conn.close() def main(argv): token = get_token() if argv and argv[0] == "--add": for u in argv[1:]: ing.track_seller(u) print(f"tracking {u}") return if argv and argv[0] == "--seed": path = argv[1] added = 0 for line in Path(path).read_text().splitlines(): name = line.strip() if name and not name.startswith(("#", "http")): ing.track_seller(name) added += 1 print(f"seeded {added} sellers from {path}") return if not token: print("No Discogs token. Set $DISCGOD_TOKEN or write ~/.discgod_token", file=sys.stderr) sys.exit(1) loop_secs = None only = [] i = 0 while i < len(argv): if argv[i] == "--loop": loop_secs = int(argv[i + 1]); i += 2 else: only.append(argv[i]); i += 1 while True: crawl_all(token, only or None) if not loop_secs: break print(f"sleeping {loop_secs}s…") time.sleep(loop_secs) if __name__ == "__main__": main(sys.argv[1:])