discgod/server/discgod_crawl.py

#!/usr/bin/env python3
"""
discgod headless crawler.

Pulls full seller inventories from the Discogs API and writes straight into
`discogs_full` via discgod_ingest — no browser required. Use it for 24/7
tracking on the box (cron or --loop). The extension and this script share the
same `discgod_seller_watch` watchlist, so a seller added in either shows up in
both.

Token (a Discogs personal access token) is read from, in order:
  1. $DISCGOD_TOKEN
  2. the file pointed to by $DISCGOD_TOKEN_FILE
  3. ~/.discgod_token

Usage
  python discgod_crawl.py                 # crawl every enabled watched seller
  python discgod_crawl.py NAME [NAME...]  # crawl just these sellers
  python discgod_crawl.py --add NAME...   # add sellers to the watchlist, exit
  python discgod_crawl.py --seed FILE     # add every username in FILE, exit
  python discgod_crawl.py --loop 21600    # crawl all, then repeat every 6 h
"""

import os
import sys
import time
import json
import urllib.parse
import urllib.request
from pathlib import Path

# The standalone server doubles as the importable engine (schema, mapping,
# upserts, watchlist) — one source of truth, shared with the HTTP server.
import discgod_server as ing

API = "https://api.discogs.com"
USER_AGENT = "discgod/1.0 +https://discogs.com"
PER_PAGE = 100
BASE_DELAY = float(os.environ.get("DISCGOD_DELAY", "1.1"))   # seconds between API calls (~55/min)
MAX_RETRIES = 5


def get_token():
    tok = os.environ.get("DISCGOD_TOKEN")
    if tok:
        return tok.strip()
    path = os.environ.get("DISCGOD_TOKEN_FILE") or str(Path.home() / ".discgod_token")
    try:
        return Path(path).read_text().strip()
    except OSError:
        return None


def api_get(token, path, params=None):
    """GET a Discogs API resource with auth, rate-limit awareness and backoff."""
    url = API + path
    if params:
        url += "?" + urllib.parse.urlencode(params)
    req = urllib.request.Request(url, headers={
        "Authorization": f"Discogs token={token}",
        "User-Agent": USER_AGENT,
        "Accept": "application/json",
    })
    for attempt in range(MAX_RETRIES):
        try:
            with urllib.request.urlopen(req, timeout=45) as resp:
                remaining = resp.headers.get("X-Discogs-Ratelimit-Remaining")
                data = json.loads(resp.read().decode())
                # Ease off as we approach the per-minute ceiling.
                if remaining is not None and remaining.isdigit() and int(remaining) <= 2:
                    time.sleep(5)
                else:
                    time.sleep(BASE_DELAY)
                return data
        except urllib.error.HTTPError as e:
            if e.code == 429:
                wait = min(60, 2 ** (attempt + 1))
                print(f"    429 rate-limited; backing off {wait}s")
                time.sleep(wait)
                continue
            if e.code in (404, 401, 403):
                raise RuntimeError(f"HTTP {e.code} for {path}") from e
            time.sleep(2 ** attempt)
        except (urllib.error.URLError, TimeoutError) as e:
            print(f"    network error ({e}); retrying")
            time.sleep(2 ** attempt)
    raise RuntimeError(f"giving up on {path} after {MAX_RETRIES} attempts")


def crawl_seller(token, username, filter_params=None, conn=None):
    """Pull a seller's whole inventory and ingest it. Returns total listing count."""
    filter_params = filter_params or {}
    crawl_id = f"{username}-{int(time.time())}"
    owns = conn is None
    conn = conn or ing.get_conn()
    try:
        # Profile first: gives seller_id and a reliable num_for_sale.
        profile = None
        seller_id = None
        try:
            profile = api_get(token, f"/users/{urllib.parse.quote(username)}")
            seller_id = profile.get("id")
        except RuntimeError as e:
            print(f"  {username}: profile fetch failed ({e}); continuing")

        page, pages, total = 1, 1, None
        params = {"per_page": PER_PAGE, "status": "For Sale",
                  "sort": "listed", "sort_order": "desc"}
        params.update(filter_params)

        while page <= pages:
            params["page"] = page
            data = api_get(token, f"/users/{urllib.parse.quote(username)}/inventory", params)
            pg = data.get("pagination", {})
            pages = pg.get("pages", 1) or 1
            total = pg.get("items", total)
            listings = data.get("listings", [])

            res = ing.ingest_inventory_page({
                "username": username,
                "seller_id": seller_id,
                "seller": profile if page == 1 else None,
                "page": page,
                "total": total,
                "crawl_id": crawl_id,
                "filter": filter_params,
                "listings": listings,
            }, conn=conn)
            print(f"  {username} p{page}/{pages}: {res['items']} items "
                  f"({res['new']} new, {res['price_changes']} price changes)")
            page += 1

        filtered = bool(ing.filter_key_from_params(filter_params))
        done = ing.finish_crawl({
            "username": username, "crawl_id": crawl_id,
            "filter": filter_params, "treat_complete": not filtered,
        }, conn=conn)
        ing.mark_watch_crawled(username, total, "ok", conn=conn)
        print(f"  {username}: done — {total} listed, {done.get('marked_gone', 0)} marked gone")
        return total
    except Exception as e:
        ing.mark_watch_crawled(username, None, f"error:{e}", conn=conn)
        print(f"  {username}: ERROR {e}")
        return None
    finally:
        if owns:
            conn.close()


def crawl_all(token, only=None):
    conn = ing.get_conn()
    try:
        ing.ensure_schema(conn)
        watched = ing.list_tracked(conn=conn, enabled_only=True)
        if only:
            wanted = {u.lower() for u in only}
            watched = [w for w in watched if w["username"].lower() in wanted]
            # Allow crawling sellers passed on the CLI even if not yet watched.
            known = {w["username"].lower() for w in watched}
            for u in only:
                if u.lower() not in known:
                    watched.append({"username": u, "filter_params": None})
        print(f"discgod crawl: {len(watched)} seller(s)")
        for w in watched:
            crawl_seller(token, w["username"], w.get("filter_params"), conn=conn)
    finally:
        conn.close()


def main(argv):
    token = get_token()

    if argv and argv[0] == "--add":
        for u in argv[1:]:
            ing.track_seller(u)
            print(f"tracking {u}")
        return
    if argv and argv[0] == "--seed":
        path = argv[1]
        added = 0
        for line in Path(path).read_text().splitlines():
            name = line.strip()
            if name and not name.startswith(("#", "http")):
                ing.track_seller(name)
                added += 1
        print(f"seeded {added} sellers from {path}")
        return

    if not token:
        print("No Discogs token. Set $DISCGOD_TOKEN or write ~/.discgod_token", file=sys.stderr)
        sys.exit(1)

    loop_secs = None
    only = []
    i = 0
    while i < len(argv):
        if argv[i] == "--loop":
            loop_secs = int(argv[i + 1]); i += 2
        else:
            only.append(argv[i]); i += 1

    while True:
        crawl_all(token, only or None)
        if not loop_secs:
            break
        print(f"sleeping {loop_secs}s…")
        time.sleep(loop_secs)


if __name__ == "__main__":
    main(sys.argv[1:])